From 05ec4414e3047a00ce6d92eb35bb3814a226238d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 25 Jul 2026 14:30:45 +0300 Subject: [PATCH 01/80] Add built-in on-device AI support --- .github/workflows/ai-cn1lib-native-check.yml | 27 +- .github/workflows/pr.yml | 22 - .../ai/inference/InferenceException.java | 16 + .../ai/inference/InferenceOptions.java | 43 ++ .../ai/inference/InferenceSession.java | 84 +++ .../codename1/ai/inference/ModelCache.java | 180 +++++ .../codename1/ai/inference/ModelSource.java | 61 ++ .../com/codename1/ai/inference/Tensor.java | 126 ++++ .../codename1/ai/inference/TensorInfo.java | 45 ++ .../codename1/ai/inference/TensorType.java | 15 + .../codename1/ai/inference/package-info.java | 9 + .../ai/language/LanguageBackend.java | 10 + .../ai/language/LanguageBackends.java | 39 ++ .../ai/language/LanguageCandidate.java | 24 + .../ai/language/LanguageIdentifier.java | 42 ++ .../ai/language/LanguageOptions.java | 29 + .../com/codename1/ai/language/SmartReply.java | 41 ++ .../ai/language/SmartReplyMessage.java | 37 + .../com/codename1/ai/language/Translator.java | 40 ++ .../codename1/ai/language/package-info.java | 11 + .../ai/vision/AbstractVisionAnalyzer.java | 58 ++ .../src/com/codename1/ai/vision/Barcode.java | 71 ++ .../codename1/ai/vision/BarcodeScanner.java | 16 + .../ai/vision/DocumentScanResult.java | 47 ++ .../codename1/ai/vision/DocumentScanner.java | 16 + .../src/com/codename1/ai/vision/Face.java | 82 +++ .../com/codename1/ai/vision/FaceDetector.java | 16 + .../com/codename1/ai/vision/ImageLabel.java | 41 ++ .../com/codename1/ai/vision/ImageLabeler.java | 16 + .../src/com/codename1/ai/vision/Pose.java | 59 ++ .../com/codename1/ai/vision/PoseDetector.java | 16 + .../codename1/ai/vision/SegmentationMask.java | 48 ++ .../codename1/ai/vision/SelfieSegmenter.java | 16 + .../ai/vision/TextRecognitionResult.java | 75 +++ .../codename1/ai/vision/TextRecognizer.java | 16 + .../codename1/ai/vision/VisionAnalyzer.java | 14 + .../codename1/ai/vision/VisionBackend.java | 18 + .../codename1/ai/vision/VisionBackends.java | 51 ++ .../codename1/ai/vision/VisionException.java | 30 + .../codename1/ai/vision/VisionFeature.java | 16 + .../com/codename1/ai/vision/VisionImage.java | 99 +++ .../codename1/ai/vision/VisionMetadata.java | 41 ++ .../codename1/ai/vision/VisionOptions.java | 39 ++ .../codename1/ai/vision/VisionPipeline.java | 115 ++++ .../ai/vision/VisionPipelineListener.java | 11 + .../com/codename1/ai/vision/VisionPoint.java | 24 + .../com/codename1/ai/vision/VisionRect.java | 38 ++ .../com/codename1/ai/vision/package-info.java | 14 + .../impl/CodenameOneImplementation.java | 21 + .../src/com/codename1/impl/InferenceImpl.java | 22 + .../src/com/codename1/impl/LanguageImpl.java | 23 + .../src/com/codename1/impl/VisionImpl.java | 18 + CodenameOne/src/com/codename1/ui/Display.java | 21 + .../impl/android/AndroidImplementation.java | 26 + .../ai/AndroidBarcodeScanningAdapter.java | 90 +++ .../ai/AndroidFaceDetectionAdapter.java | 87 +++ .../ai/AndroidImageLabelingAdapter.java | 46 ++ .../impl/android/ai/AndroidInferenceImpl.java | 292 ++++++++ .../android/ai/AndroidLanguageAdapter.java | 62 ++ .../android/ai/AndroidLanguageIdAdapter.java | 45 ++ .../impl/android/ai/AndroidLanguageImpl.java | 86 +++ .../ai/AndroidPoseDetectionAdapter.java | 51 ++ .../ai/AndroidSelfieSegmentationAdapter.java | 50 ++ .../android/ai/AndroidSmartReplyAdapter.java | 52 ++ .../ai/AndroidTextRecognitionAdapter.java | 48 ++ .../android/ai/AndroidTranslationAdapter.java | 42 ++ .../impl/android/ai/AndroidVisionAdapter.java | 61 ++ .../impl/android/ai/AndroidVisionImpl.java | 159 +++++ Ports/iOSPort/nativeSources/CN1Inference.m | 336 ++++++++++ Ports/iOSPort/nativeSources/CN1Language.m | 231 +++++++ Ports/iOSPort/nativeSources/CN1Vision.m | 630 ++++++++++++++++++ .../CodenameOne_GLViewController.h | 11 + .../codename1/impl/ios/IOSImplementation.java | 25 + .../codename1/impl/ios/IOSInferenceImpl.java | 377 +++++++++++ .../codename1/impl/ios/IOSLanguageImpl.java | 179 +++++ .../src/com/codename1/impl/ios/IOSNative.java | 18 + .../com/codename1/impl/ios/IOSVisionImpl.java | 219 ++++++ docs/ai-on-device-architecture.md | 186 ++++++ .../developer-guide/ai-and-speech.xml | 10 - docs/developer-guide/Ai-And-Speech.asciidoc | 208 +++++- docs/website/data/port_status.json | 21 + .../data/port_status_reports/android.json | 16 +- .../data/port_status_reports/ios-gl.json | 16 +- .../data/port_status_reports/ios-metal.json | 16 +- .../data/port_status_reports/javascript.json | 16 +- .../data/port_status_reports/linux-arm64.json | 16 +- .../data/port_status_reports/linux-x64.json | 16 +- .../data/port_status_reports/mac-native.json | 16 +- .../data/port_status_reports/tvos.json | 16 +- .../data/port_status_reports/watchos.json | 16 +- .../port_status_reports/windows-arm64.json | 16 +- .../data/port_status_reports/windows-x64.json | 16 +- maven/android/pom.xml | 4 + maven/cn1-ai-mlkit-barcode/android/pom.xml | 59 -- .../barcode/NativeBarcodeScannerImpl.java | 38 -- .../codenameone_library_appended.properties | 1 - .../codenameone_library_required.properties | 7 - maven/cn1-ai-mlkit-barcode/common/pom.xml | 86 --- .../ai/mlkit/barcode/BarcodeScanner.java | 79 --- .../mlkit/barcode/NativeBarcodeScanner.java | 32 - .../ai/mlkit/barcode/package-info.java | 40 -- .../ai/mlkit/barcode/BarcodeScannerTest.java | 29 - maven/cn1-ai-mlkit-barcode/ios/pom.xml | 53 -- ...i_mlkit_barcode_NativeBarcodeScannerImpl.h | 8 - ...i_mlkit_barcode_NativeBarcodeScannerImpl.m | 48 -- maven/cn1-ai-mlkit-barcode/javascript/pom.xml | 53 -- ...1_ai_mlkit_barcode_NativeBarcodeScanner.js | 15 - maven/cn1-ai-mlkit-barcode/javase/pom.xml | 59 -- .../barcode/NativeBarcodeScannerImpl.java | 30 - maven/cn1-ai-mlkit-barcode/lib/pom.xml | 79 --- maven/cn1-ai-mlkit-barcode/pom.xml | 33 - maven/cn1-ai-mlkit-docscan/android/pom.xml | 59 -- .../docscan/NativeDocumentScannerImpl.java | 20 - .../codenameone_library_appended.properties | 1 - .../codenameone_library_required.properties | 6 - maven/cn1-ai-mlkit-docscan/common/pom.xml | 86 --- .../ai/mlkit/docscan/DocumentScanner.java | 71 -- .../mlkit/docscan/NativeDocumentScanner.java | 32 - .../ai/mlkit/docscan/package-info.java | 38 -- .../ai/mlkit/docscan/DocumentScannerTest.java | 20 - maven/cn1-ai-mlkit-docscan/ios/pom.xml | 53 -- ..._mlkit_docscan_NativeDocumentScannerImpl.h | 8 - ..._mlkit_docscan_NativeDocumentScannerImpl.m | 42 -- maven/cn1-ai-mlkit-docscan/javascript/pom.xml | 53 -- ..._ai_mlkit_docscan_NativeDocumentScanner.js | 15 - maven/cn1-ai-mlkit-docscan/javase/pom.xml | 59 -- .../docscan/NativeDocumentScannerImpl.java | 17 - maven/cn1-ai-mlkit-docscan/lib/pom.xml | 79 --- maven/cn1-ai-mlkit-docscan/pom.xml | 33 - maven/cn1-ai-mlkit-face/android/pom.xml | 59 -- .../ai/mlkit/face/NativeFaceDetectorImpl.java | 40 -- .../codenameone_library_appended.properties | 1 - .../codenameone_library_required.properties | 6 - maven/cn1-ai-mlkit-face/common/pom.xml | 86 --- .../codename1/ai/mlkit/face/FaceDetector.java | 75 --- .../ai/mlkit/face/NativeFaceDetector.java | 32 - .../codename1/ai/mlkit/face/package-info.java | 40 -- .../ai/mlkit/face/FaceDetectorTest.java | 23 - maven/cn1-ai-mlkit-face/ios/pom.xml | 53 -- ...me1_ai_mlkit_face_NativeFaceDetectorImpl.h | 8 - ...me1_ai_mlkit_face_NativeFaceDetectorImpl.m | 36 - maven/cn1-ai-mlkit-face/javascript/pom.xml | 53 -- ...ename1_ai_mlkit_face_NativeFaceDetector.js | 15 - maven/cn1-ai-mlkit-face/javase/pom.xml | 59 -- .../ai/mlkit/face/NativeFaceDetectorImpl.java | 30 - maven/cn1-ai-mlkit-face/lib/pom.xml | 79 --- maven/cn1-ai-mlkit-face/pom.xml | 33 - maven/cn1-ai-mlkit-labeling/android/pom.xml | 59 -- .../labeling/NativeImageLabelerImpl.java | 34 - .../codenameone_library_appended.properties | 1 - .../codenameone_library_required.properties | 6 - maven/cn1-ai-mlkit-labeling/common/pom.xml | 86 --- .../ai/mlkit/labeling/ImageLabeler.java | 73 -- .../ai/mlkit/labeling/NativeImageLabeler.java | 32 - .../ai/mlkit/labeling/package-info.java | 40 -- .../ai/mlkit/labeling/ImageLabelerTest.java | 20 - maven/cn1-ai-mlkit-labeling/ios/pom.xml | 53 -- ...ai_mlkit_labeling_NativeImageLabelerImpl.h | 8 - ...ai_mlkit_labeling_NativeImageLabelerImpl.m | 45 -- .../cn1-ai-mlkit-labeling/javascript/pom.xml | 53 -- ...e1_ai_mlkit_labeling_NativeImageLabeler.js | 15 - maven/cn1-ai-mlkit-labeling/javase/pom.xml | 59 -- .../labeling/NativeImageLabelerImpl.java | 12 - maven/cn1-ai-mlkit-labeling/lib/pom.xml | 79 --- maven/cn1-ai-mlkit-labeling/pom.xml | 33 - maven/cn1-ai-mlkit-langid/android/pom.xml | 59 -- .../langid/NativeLanguageIdentifierImpl.java | 25 - .../codenameone_library_appended.properties | 1 - .../codenameone_library_required.properties | 6 - maven/cn1-ai-mlkit-langid/common/pom.xml | 86 --- .../ai/mlkit/langid/LanguageIdentifier.java | 71 -- .../langid/NativeLanguageIdentifier.java | 32 - .../ai/mlkit/langid/package-info.java | 38 -- .../mlkit/langid/LanguageIdentifierTest.java | 20 - maven/cn1-ai-mlkit-langid/ios/pom.xml | 53 -- ...lkit_langid_NativeLanguageIdentifierImpl.h | 8 - ...lkit_langid_NativeLanguageIdentifierImpl.m | 23 - maven/cn1-ai-mlkit-langid/javascript/pom.xml | 53 -- ...i_mlkit_langid_NativeLanguageIdentifier.js | 15 - maven/cn1-ai-mlkit-langid/javase/pom.xml | 59 -- .../langid/NativeLanguageIdentifierImpl.java | 13 - maven/cn1-ai-mlkit-langid/lib/pom.xml | 79 --- maven/cn1-ai-mlkit-langid/pom.xml | 33 - maven/cn1-ai-mlkit-pose/android/pom.xml | 59 -- .../ai/mlkit/pose/NativePoseDetectorImpl.java | 40 -- .../codenameone_library_appended.properties | 1 - .../codenameone_library_required.properties | 6 - maven/cn1-ai-mlkit-pose/common/pom.xml | 86 --- .../ai/mlkit/pose/NativePoseDetector.java | 32 - .../codename1/ai/mlkit/pose/PoseDetector.java | 73 -- .../codename1/ai/mlkit/pose/package-info.java | 38 -- .../ai/mlkit/pose/PoseDetectorTest.java | 20 - maven/cn1-ai-mlkit-pose/ios/pom.xml | 53 -- ...me1_ai_mlkit_pose_NativePoseDetectorImpl.h | 8 - ...me1_ai_mlkit_pose_NativePoseDetectorImpl.m | 39 -- maven/cn1-ai-mlkit-pose/javascript/pom.xml | 53 -- ...ename1_ai_mlkit_pose_NativePoseDetector.js | 15 - maven/cn1-ai-mlkit-pose/javase/pom.xml | 59 -- .../ai/mlkit/pose/NativePoseDetectorImpl.java | 13 - maven/cn1-ai-mlkit-pose/lib/pom.xml | 79 --- maven/cn1-ai-mlkit-pose/pom.xml | 33 - .../cn1-ai-mlkit-segmentation/android/pom.xml | 59 -- .../NativeSelfieSegmenterImpl.java | 46 -- .../codenameone_library_appended.properties | 1 - .../codenameone_library_required.properties | 6 - .../cn1-ai-mlkit-segmentation/common/pom.xml | 86 --- .../segmentation/NativeSelfieSegmenter.java | 32 - .../mlkit/segmentation/SelfieSegmenter.java | 73 -- .../ai/mlkit/segmentation/package-info.java | 38 -- .../segmentation/SelfieSegmenterTest.java | 20 - maven/cn1-ai-mlkit-segmentation/ios/pom.xml | 53 -- ...t_segmentation_NativeSelfieSegmenterImpl.h | 8 - ...t_segmentation_NativeSelfieSegmenterImpl.m | 48 -- .../javascript/pom.xml | 53 -- ...lkit_segmentation_NativeSelfieSegmenter.js | 15 - .../cn1-ai-mlkit-segmentation/javase/pom.xml | 59 -- .../NativeSelfieSegmenterImpl.java | 14 - maven/cn1-ai-mlkit-segmentation/lib/pom.xml | 79 --- maven/cn1-ai-mlkit-segmentation/pom.xml | 33 - maven/cn1-ai-mlkit-smartreply/android/pom.xml | 59 -- .../smartreply/NativeSmartReplyImpl.java | 49 -- .../codenameone_library_appended.properties | 1 - .../codenameone_library_required.properties | 6 - maven/cn1-ai-mlkit-smartreply/common/pom.xml | 86 --- .../ai/mlkit/smartreply/NativeSmartReply.java | 32 - .../ai/mlkit/smartreply/SmartReply.java | 71 -- .../ai/mlkit/smartreply/package-info.java | 38 -- .../ai/mlkit/smartreply/SmartReplyTest.java | 20 - maven/cn1-ai-mlkit-smartreply/ios/pom.xml | 53 -- ...ai_mlkit_smartreply_NativeSmartReplyImpl.h | 8 - ...ai_mlkit_smartreply_NativeSmartReplyImpl.m | 61 -- .../javascript/pom.xml | 53 -- ...e1_ai_mlkit_smartreply_NativeSmartReply.js | 15 - maven/cn1-ai-mlkit-smartreply/javase/pom.xml | 59 -- .../smartreply/NativeSmartReplyImpl.java | 11 - maven/cn1-ai-mlkit-smartreply/lib/pom.xml | 79 --- maven/cn1-ai-mlkit-smartreply/pom.xml | 33 - maven/cn1-ai-mlkit-text/android/pom.xml | 59 -- .../mlkit/text/NativeTextRecognizerImpl.java | 35 - .../codenameone_library_appended.properties | 1 - .../codenameone_library_required.properties | 6 - maven/cn1-ai-mlkit-text/common/pom.xml | 86 --- .../ai/mlkit/text/NativeTextRecognizer.java | 32 - .../ai/mlkit/text/TextRecognizer.java | 83 --- .../codename1/ai/mlkit/text/package-info.java | 40 -- .../ai/mlkit/text/TextRecognizerTest.java | 36 - maven/cn1-ai-mlkit-text/ios/pom.xml | 53 -- ...1_ai_mlkit_text_NativeTextRecognizerImpl.h | 8 - ...1_ai_mlkit_text_NativeTextRecognizerImpl.m | 33 - maven/cn1-ai-mlkit-text/javascript/pom.xml | 53 -- ...ame1_ai_mlkit_text_NativeTextRecognizer.js | 15 - maven/cn1-ai-mlkit-text/javase/pom.xml | 59 -- .../mlkit/text/NativeTextRecognizerImpl.java | 29 - maven/cn1-ai-mlkit-text/lib/pom.xml | 79 --- maven/cn1-ai-mlkit-text/pom.xml | 33 - maven/cn1-ai-mlkit-translate/android/pom.xml | 59 -- .../mlkit/translate/NativeTranslatorImpl.java | 38 -- .../codenameone_library_appended.properties | 1 - .../codenameone_library_required.properties | 6 - maven/cn1-ai-mlkit-translate/common/pom.xml | 86 --- .../ai/mlkit/translate/NativeTranslator.java | 32 - .../ai/mlkit/translate/Translator.java | 73 -- .../ai/mlkit/translate/package-info.java | 38 -- .../ai/mlkit/translate/TranslatorTest.java | 22 - maven/cn1-ai-mlkit-translate/ios/pom.xml | 53 -- ..._ai_mlkit_translate_NativeTranslatorImpl.h | 8 - ..._ai_mlkit_translate_NativeTranslatorImpl.m | 33 - .../cn1-ai-mlkit-translate/javascript/pom.xml | 53 -- ...me1_ai_mlkit_translate_NativeTranslator.js | 15 - maven/cn1-ai-mlkit-translate/javase/pom.xml | 59 -- .../mlkit/translate/NativeTranslatorImpl.java | 12 - maven/cn1-ai-mlkit-translate/lib/pom.xml | 79 --- maven/cn1-ai-mlkit-translate/pom.xml | 33 - maven/cn1-ai-tflite/android/pom.xml | 59 -- .../ai/tflite/NativeInterpreterImpl.java | 20 - .../codenameone_library_appended.properties | 1 - .../codenameone_library_required.properties | 6 - maven/cn1-ai-tflite/common/pom.xml | 86 --- .../com/codename1/ai/tflite/Interpreter.java | 80 --- .../ai/tflite/NativeInterpreter.java | 32 - .../com/codename1/ai/tflite/package-info.java | 40 -- .../codename1/ai/tflite/InterpreterTest.java | 26 - maven/cn1-ai-tflite/ios/pom.xml | 53 -- ...odename1_ai_tflite_NativeInterpreterImpl.h | 8 - ...odename1_ai_tflite_NativeInterpreterImpl.m | 28 - maven/cn1-ai-tflite/javascript/pom.xml | 53 -- ...m_codename1_ai_tflite_NativeInterpreter.js | 15 - maven/cn1-ai-tflite/javase/pom.xml | 59 -- .../ai/tflite/NativeInterpreterImpl.java | 16 - maven/cn1-ai-tflite/lib/pom.xml | 79 --- maven/cn1-ai-tflite/pom.xml | 33 - maven/codenameone-maven-plugin/pom.xml | 5 + .../builders/AndroidGradleBuilder.java | 112 +++- .../com/codename1/builders/IPhoneBuilder.java | 109 ++- .../codename1/builders/MacNativeBuilder.java | 2 +- .../com/codename1/maven/CN1BuildMojo.java | 7 + .../AndroidAiSourceSelectionTest.java | 51 ++ .../ai/inference/InferenceApiTest.java | 107 +++ .../ai/language/LanguageApiTest.java | 102 +++ .../codename1/ai/vision/VisionApiTest.java | 118 ++++ .../TestCodenameOneImplementation.java | 31 + maven/platform-feature-catalog/pom.xml | 31 + .../build/shared/PlatformFeatureCatalog.java} | 237 ++++--- .../shared/PlatformFeatureCatalogTest.java} | 169 +++-- maven/pom.xml | 12 +- scripts/gen-ai-cn1libs.py | 24 +- scripts/hellocodenameone/README.adoc | 9 + .../tests/Cn1ssDeviceRunner.java | 10 + .../tests/InferenceOnDeviceApiTest.java | 145 ++++ .../tests/LanguageOnDeviceApiTest.java | 130 ++++ .../tests/VisionOnDeviceApiTest.java | 160 +++++ .../conformance/port_status.py | 13 + .../conformance/test_port_status.py | 35 +- .../common/src/main/resources/skill/SKILL.md | 2 +- .../skill/references/ai-and-speech.md | 95 ++- .../extensions/MavenCentralSearch.java | 8 - 316 files changed, 7315 insertions(+), 8195 deletions(-) create mode 100644 CodenameOne/src/com/codename1/ai/inference/InferenceException.java create mode 100644 CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java create mode 100644 CodenameOne/src/com/codename1/ai/inference/InferenceSession.java create mode 100644 CodenameOne/src/com/codename1/ai/inference/ModelCache.java create mode 100644 CodenameOne/src/com/codename1/ai/inference/ModelSource.java create mode 100644 CodenameOne/src/com/codename1/ai/inference/Tensor.java create mode 100644 CodenameOne/src/com/codename1/ai/inference/TensorInfo.java create mode 100644 CodenameOne/src/com/codename1/ai/inference/TensorType.java create mode 100644 CodenameOne/src/com/codename1/ai/inference/package-info.java create mode 100644 CodenameOne/src/com/codename1/ai/language/LanguageBackend.java create mode 100644 CodenameOne/src/com/codename1/ai/language/LanguageBackends.java create mode 100644 CodenameOne/src/com/codename1/ai/language/LanguageCandidate.java create mode 100644 CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java create mode 100644 CodenameOne/src/com/codename1/ai/language/LanguageOptions.java create mode 100644 CodenameOne/src/com/codename1/ai/language/SmartReply.java create mode 100644 CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java create mode 100644 CodenameOne/src/com/codename1/ai/language/Translator.java create mode 100644 CodenameOne/src/com/codename1/ai/language/package-info.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/Barcode.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/DocumentScanResult.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/Face.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/FaceDetector.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/ImageLabel.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/Pose.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/PoseDetector.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/TextRecognitionResult.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/VisionBackend.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/VisionBackends.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/VisionException.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/VisionFeature.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/VisionImage.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/VisionMetadata.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/VisionOptions.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/VisionPipelineListener.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/VisionPoint.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/VisionRect.java create mode 100644 CodenameOne/src/com/codename1/ai/vision/package-info.java create mode 100644 CodenameOne/src/com/codename1/impl/InferenceImpl.java create mode 100644 CodenameOne/src/com/codename1/impl/LanguageImpl.java create mode 100644 CodenameOne/src/com/codename1/impl/VisionImpl.java create mode 100644 Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java create mode 100644 Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java create mode 100644 Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java create mode 100644 Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java create mode 100644 Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageAdapter.java create mode 100644 Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageIdAdapter.java create mode 100644 Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageImpl.java create mode 100644 Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java create mode 100644 Ports/Android/src/com/codename1/impl/android/ai/AndroidSelfieSegmentationAdapter.java create mode 100644 Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java create mode 100644 Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java create mode 100644 Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java create mode 100644 Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionAdapter.java create mode 100644 Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java create mode 100644 Ports/iOSPort/nativeSources/CN1Inference.m create mode 100644 Ports/iOSPort/nativeSources/CN1Language.m create mode 100644 Ports/iOSPort/nativeSources/CN1Vision.m create mode 100644 Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java create mode 100644 Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java create mode 100644 Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java create mode 100644 docs/ai-on-device-architecture.md delete mode 100644 docs/demos/common/src/main/snippets/developer-guide/ai-and-speech.xml delete mode 100644 maven/cn1-ai-mlkit-barcode/android/pom.xml delete mode 100644 maven/cn1-ai-mlkit-barcode/android/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScannerImpl.java delete mode 100644 maven/cn1-ai-mlkit-barcode/common/codenameone_library_appended.properties delete mode 100644 maven/cn1-ai-mlkit-barcode/common/codenameone_library_required.properties delete mode 100644 maven/cn1-ai-mlkit-barcode/common/pom.xml delete mode 100644 maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/BarcodeScanner.java delete mode 100644 maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScanner.java delete mode 100644 maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/package-info.java delete mode 100644 maven/cn1-ai-mlkit-barcode/common/src/test/java/com/codename1/ai/mlkit/barcode/BarcodeScannerTest.java delete mode 100644 maven/cn1-ai-mlkit-barcode/ios/pom.xml delete mode 100644 maven/cn1-ai-mlkit-barcode/ios/src/main/objectivec/com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.h delete mode 100644 maven/cn1-ai-mlkit-barcode/ios/src/main/objectivec/com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.m delete mode 100644 maven/cn1-ai-mlkit-barcode/javascript/pom.xml delete mode 100644 maven/cn1-ai-mlkit-barcode/javascript/src/main/javascript/com_codename1_ai_mlkit_barcode_NativeBarcodeScanner.js delete mode 100644 maven/cn1-ai-mlkit-barcode/javase/pom.xml delete mode 100644 maven/cn1-ai-mlkit-barcode/javase/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScannerImpl.java delete mode 100644 maven/cn1-ai-mlkit-barcode/lib/pom.xml delete mode 100644 maven/cn1-ai-mlkit-barcode/pom.xml delete mode 100644 maven/cn1-ai-mlkit-docscan/android/pom.xml delete mode 100644 maven/cn1-ai-mlkit-docscan/android/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScannerImpl.java delete mode 100644 maven/cn1-ai-mlkit-docscan/common/codenameone_library_appended.properties delete mode 100644 maven/cn1-ai-mlkit-docscan/common/codenameone_library_required.properties delete mode 100644 maven/cn1-ai-mlkit-docscan/common/pom.xml delete mode 100644 maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/DocumentScanner.java delete mode 100644 maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScanner.java delete mode 100644 maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/package-info.java delete mode 100644 maven/cn1-ai-mlkit-docscan/common/src/test/java/com/codename1/ai/mlkit/docscan/DocumentScannerTest.java delete mode 100644 maven/cn1-ai-mlkit-docscan/ios/pom.xml delete mode 100644 maven/cn1-ai-mlkit-docscan/ios/src/main/objectivec/com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.h delete mode 100644 maven/cn1-ai-mlkit-docscan/ios/src/main/objectivec/com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.m delete mode 100644 maven/cn1-ai-mlkit-docscan/javascript/pom.xml delete mode 100644 maven/cn1-ai-mlkit-docscan/javascript/src/main/javascript/com_codename1_ai_mlkit_docscan_NativeDocumentScanner.js delete mode 100644 maven/cn1-ai-mlkit-docscan/javase/pom.xml delete mode 100644 maven/cn1-ai-mlkit-docscan/javase/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScannerImpl.java delete mode 100644 maven/cn1-ai-mlkit-docscan/lib/pom.xml delete mode 100644 maven/cn1-ai-mlkit-docscan/pom.xml delete mode 100644 maven/cn1-ai-mlkit-face/android/pom.xml delete mode 100644 maven/cn1-ai-mlkit-face/android/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetectorImpl.java delete mode 100644 maven/cn1-ai-mlkit-face/common/codenameone_library_appended.properties delete mode 100644 maven/cn1-ai-mlkit-face/common/codenameone_library_required.properties delete mode 100644 maven/cn1-ai-mlkit-face/common/pom.xml delete mode 100644 maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/FaceDetector.java delete mode 100644 maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetector.java delete mode 100644 maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/package-info.java delete mode 100644 maven/cn1-ai-mlkit-face/common/src/test/java/com/codename1/ai/mlkit/face/FaceDetectorTest.java delete mode 100644 maven/cn1-ai-mlkit-face/ios/pom.xml delete mode 100644 maven/cn1-ai-mlkit-face/ios/src/main/objectivec/com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.h delete mode 100644 maven/cn1-ai-mlkit-face/ios/src/main/objectivec/com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.m delete mode 100644 maven/cn1-ai-mlkit-face/javascript/pom.xml delete mode 100644 maven/cn1-ai-mlkit-face/javascript/src/main/javascript/com_codename1_ai_mlkit_face_NativeFaceDetector.js delete mode 100644 maven/cn1-ai-mlkit-face/javase/pom.xml delete mode 100644 maven/cn1-ai-mlkit-face/javase/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetectorImpl.java delete mode 100644 maven/cn1-ai-mlkit-face/lib/pom.xml delete mode 100644 maven/cn1-ai-mlkit-face/pom.xml delete mode 100644 maven/cn1-ai-mlkit-labeling/android/pom.xml delete mode 100644 maven/cn1-ai-mlkit-labeling/android/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabelerImpl.java delete mode 100644 maven/cn1-ai-mlkit-labeling/common/codenameone_library_appended.properties delete mode 100644 maven/cn1-ai-mlkit-labeling/common/codenameone_library_required.properties delete mode 100644 maven/cn1-ai-mlkit-labeling/common/pom.xml delete mode 100644 maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/ImageLabeler.java delete mode 100644 maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabeler.java delete mode 100644 maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/package-info.java delete mode 100644 maven/cn1-ai-mlkit-labeling/common/src/test/java/com/codename1/ai/mlkit/labeling/ImageLabelerTest.java delete mode 100644 maven/cn1-ai-mlkit-labeling/ios/pom.xml delete mode 100644 maven/cn1-ai-mlkit-labeling/ios/src/main/objectivec/com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.h delete mode 100644 maven/cn1-ai-mlkit-labeling/ios/src/main/objectivec/com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.m delete mode 100644 maven/cn1-ai-mlkit-labeling/javascript/pom.xml delete mode 100644 maven/cn1-ai-mlkit-labeling/javascript/src/main/javascript/com_codename1_ai_mlkit_labeling_NativeImageLabeler.js delete mode 100644 maven/cn1-ai-mlkit-labeling/javase/pom.xml delete mode 100644 maven/cn1-ai-mlkit-labeling/javase/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabelerImpl.java delete mode 100644 maven/cn1-ai-mlkit-labeling/lib/pom.xml delete mode 100644 maven/cn1-ai-mlkit-labeling/pom.xml delete mode 100644 maven/cn1-ai-mlkit-langid/android/pom.xml delete mode 100644 maven/cn1-ai-mlkit-langid/android/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifierImpl.java delete mode 100644 maven/cn1-ai-mlkit-langid/common/codenameone_library_appended.properties delete mode 100644 maven/cn1-ai-mlkit-langid/common/codenameone_library_required.properties delete mode 100644 maven/cn1-ai-mlkit-langid/common/pom.xml delete mode 100644 maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/LanguageIdentifier.java delete mode 100644 maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifier.java delete mode 100644 maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/package-info.java delete mode 100644 maven/cn1-ai-mlkit-langid/common/src/test/java/com/codename1/ai/mlkit/langid/LanguageIdentifierTest.java delete mode 100644 maven/cn1-ai-mlkit-langid/ios/pom.xml delete mode 100644 maven/cn1-ai-mlkit-langid/ios/src/main/objectivec/com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.h delete mode 100644 maven/cn1-ai-mlkit-langid/ios/src/main/objectivec/com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.m delete mode 100644 maven/cn1-ai-mlkit-langid/javascript/pom.xml delete mode 100644 maven/cn1-ai-mlkit-langid/javascript/src/main/javascript/com_codename1_ai_mlkit_langid_NativeLanguageIdentifier.js delete mode 100644 maven/cn1-ai-mlkit-langid/javase/pom.xml delete mode 100644 maven/cn1-ai-mlkit-langid/javase/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifierImpl.java delete mode 100644 maven/cn1-ai-mlkit-langid/lib/pom.xml delete mode 100644 maven/cn1-ai-mlkit-langid/pom.xml delete mode 100644 maven/cn1-ai-mlkit-pose/android/pom.xml delete mode 100644 maven/cn1-ai-mlkit-pose/android/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetectorImpl.java delete mode 100644 maven/cn1-ai-mlkit-pose/common/codenameone_library_appended.properties delete mode 100644 maven/cn1-ai-mlkit-pose/common/codenameone_library_required.properties delete mode 100644 maven/cn1-ai-mlkit-pose/common/pom.xml delete mode 100644 maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetector.java delete mode 100644 maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/PoseDetector.java delete mode 100644 maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/package-info.java delete mode 100644 maven/cn1-ai-mlkit-pose/common/src/test/java/com/codename1/ai/mlkit/pose/PoseDetectorTest.java delete mode 100644 maven/cn1-ai-mlkit-pose/ios/pom.xml delete mode 100644 maven/cn1-ai-mlkit-pose/ios/src/main/objectivec/com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.h delete mode 100644 maven/cn1-ai-mlkit-pose/ios/src/main/objectivec/com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.m delete mode 100644 maven/cn1-ai-mlkit-pose/javascript/pom.xml delete mode 100644 maven/cn1-ai-mlkit-pose/javascript/src/main/javascript/com_codename1_ai_mlkit_pose_NativePoseDetector.js delete mode 100644 maven/cn1-ai-mlkit-pose/javase/pom.xml delete mode 100644 maven/cn1-ai-mlkit-pose/javase/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetectorImpl.java delete mode 100644 maven/cn1-ai-mlkit-pose/lib/pom.xml delete mode 100644 maven/cn1-ai-mlkit-pose/pom.xml delete mode 100644 maven/cn1-ai-mlkit-segmentation/android/pom.xml delete mode 100644 maven/cn1-ai-mlkit-segmentation/android/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenterImpl.java delete mode 100644 maven/cn1-ai-mlkit-segmentation/common/codenameone_library_appended.properties delete mode 100644 maven/cn1-ai-mlkit-segmentation/common/codenameone_library_required.properties delete mode 100644 maven/cn1-ai-mlkit-segmentation/common/pom.xml delete mode 100644 maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenter.java delete mode 100644 maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/SelfieSegmenter.java delete mode 100644 maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/package-info.java delete mode 100644 maven/cn1-ai-mlkit-segmentation/common/src/test/java/com/codename1/ai/mlkit/segmentation/SelfieSegmenterTest.java delete mode 100644 maven/cn1-ai-mlkit-segmentation/ios/pom.xml delete mode 100644 maven/cn1-ai-mlkit-segmentation/ios/src/main/objectivec/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.h delete mode 100644 maven/cn1-ai-mlkit-segmentation/ios/src/main/objectivec/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.m delete mode 100644 maven/cn1-ai-mlkit-segmentation/javascript/pom.xml delete mode 100644 maven/cn1-ai-mlkit-segmentation/javascript/src/main/javascript/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenter.js delete mode 100644 maven/cn1-ai-mlkit-segmentation/javase/pom.xml delete mode 100644 maven/cn1-ai-mlkit-segmentation/javase/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenterImpl.java delete mode 100644 maven/cn1-ai-mlkit-segmentation/lib/pom.xml delete mode 100644 maven/cn1-ai-mlkit-segmentation/pom.xml delete mode 100644 maven/cn1-ai-mlkit-smartreply/android/pom.xml delete mode 100644 maven/cn1-ai-mlkit-smartreply/android/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReplyImpl.java delete mode 100644 maven/cn1-ai-mlkit-smartreply/common/codenameone_library_appended.properties delete mode 100644 maven/cn1-ai-mlkit-smartreply/common/codenameone_library_required.properties delete mode 100644 maven/cn1-ai-mlkit-smartreply/common/pom.xml delete mode 100644 maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReply.java delete mode 100644 maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/SmartReply.java delete mode 100644 maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/package-info.java delete mode 100644 maven/cn1-ai-mlkit-smartreply/common/src/test/java/com/codename1/ai/mlkit/smartreply/SmartReplyTest.java delete mode 100644 maven/cn1-ai-mlkit-smartreply/ios/pom.xml delete mode 100644 maven/cn1-ai-mlkit-smartreply/ios/src/main/objectivec/com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.h delete mode 100644 maven/cn1-ai-mlkit-smartreply/ios/src/main/objectivec/com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.m delete mode 100644 maven/cn1-ai-mlkit-smartreply/javascript/pom.xml delete mode 100644 maven/cn1-ai-mlkit-smartreply/javascript/src/main/javascript/com_codename1_ai_mlkit_smartreply_NativeSmartReply.js delete mode 100644 maven/cn1-ai-mlkit-smartreply/javase/pom.xml delete mode 100644 maven/cn1-ai-mlkit-smartreply/javase/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReplyImpl.java delete mode 100644 maven/cn1-ai-mlkit-smartreply/lib/pom.xml delete mode 100644 maven/cn1-ai-mlkit-smartreply/pom.xml delete mode 100644 maven/cn1-ai-mlkit-text/android/pom.xml delete mode 100644 maven/cn1-ai-mlkit-text/android/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizerImpl.java delete mode 100644 maven/cn1-ai-mlkit-text/common/codenameone_library_appended.properties delete mode 100644 maven/cn1-ai-mlkit-text/common/codenameone_library_required.properties delete mode 100644 maven/cn1-ai-mlkit-text/common/pom.xml delete mode 100644 maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizer.java delete mode 100644 maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/TextRecognizer.java delete mode 100644 maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/package-info.java delete mode 100644 maven/cn1-ai-mlkit-text/common/src/test/java/com/codename1/ai/mlkit/text/TextRecognizerTest.java delete mode 100644 maven/cn1-ai-mlkit-text/ios/pom.xml delete mode 100644 maven/cn1-ai-mlkit-text/ios/src/main/objectivec/com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.h delete mode 100644 maven/cn1-ai-mlkit-text/ios/src/main/objectivec/com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.m delete mode 100644 maven/cn1-ai-mlkit-text/javascript/pom.xml delete mode 100644 maven/cn1-ai-mlkit-text/javascript/src/main/javascript/com_codename1_ai_mlkit_text_NativeTextRecognizer.js delete mode 100644 maven/cn1-ai-mlkit-text/javase/pom.xml delete mode 100644 maven/cn1-ai-mlkit-text/javase/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizerImpl.java delete mode 100644 maven/cn1-ai-mlkit-text/lib/pom.xml delete mode 100644 maven/cn1-ai-mlkit-text/pom.xml delete mode 100644 maven/cn1-ai-mlkit-translate/android/pom.xml delete mode 100644 maven/cn1-ai-mlkit-translate/android/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslatorImpl.java delete mode 100644 maven/cn1-ai-mlkit-translate/common/codenameone_library_appended.properties delete mode 100644 maven/cn1-ai-mlkit-translate/common/codenameone_library_required.properties delete mode 100644 maven/cn1-ai-mlkit-translate/common/pom.xml delete mode 100644 maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslator.java delete mode 100644 maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/Translator.java delete mode 100644 maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/package-info.java delete mode 100644 maven/cn1-ai-mlkit-translate/common/src/test/java/com/codename1/ai/mlkit/translate/TranslatorTest.java delete mode 100644 maven/cn1-ai-mlkit-translate/ios/pom.xml delete mode 100644 maven/cn1-ai-mlkit-translate/ios/src/main/objectivec/com_codename1_ai_mlkit_translate_NativeTranslatorImpl.h delete mode 100644 maven/cn1-ai-mlkit-translate/ios/src/main/objectivec/com_codename1_ai_mlkit_translate_NativeTranslatorImpl.m delete mode 100644 maven/cn1-ai-mlkit-translate/javascript/pom.xml delete mode 100644 maven/cn1-ai-mlkit-translate/javascript/src/main/javascript/com_codename1_ai_mlkit_translate_NativeTranslator.js delete mode 100644 maven/cn1-ai-mlkit-translate/javase/pom.xml delete mode 100644 maven/cn1-ai-mlkit-translate/javase/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslatorImpl.java delete mode 100644 maven/cn1-ai-mlkit-translate/lib/pom.xml delete mode 100644 maven/cn1-ai-mlkit-translate/pom.xml delete mode 100644 maven/cn1-ai-tflite/android/pom.xml delete mode 100644 maven/cn1-ai-tflite/android/src/main/java/com/codename1/ai/tflite/NativeInterpreterImpl.java delete mode 100644 maven/cn1-ai-tflite/common/codenameone_library_appended.properties delete mode 100644 maven/cn1-ai-tflite/common/codenameone_library_required.properties delete mode 100644 maven/cn1-ai-tflite/common/pom.xml delete mode 100644 maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/Interpreter.java delete mode 100644 maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/NativeInterpreter.java delete mode 100644 maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/package-info.java delete mode 100644 maven/cn1-ai-tflite/common/src/test/java/com/codename1/ai/tflite/InterpreterTest.java delete mode 100644 maven/cn1-ai-tflite/ios/pom.xml delete mode 100644 maven/cn1-ai-tflite/ios/src/main/objectivec/com_codename1_ai_tflite_NativeInterpreterImpl.h delete mode 100644 maven/cn1-ai-tflite/ios/src/main/objectivec/com_codename1_ai_tflite_NativeInterpreterImpl.m delete mode 100644 maven/cn1-ai-tflite/javascript/pom.xml delete mode 100644 maven/cn1-ai-tflite/javascript/src/main/javascript/com_codename1_ai_tflite_NativeInterpreter.js delete mode 100644 maven/cn1-ai-tflite/javase/pom.xml delete mode 100644 maven/cn1-ai-tflite/javase/src/main/java/com/codename1/ai/tflite/NativeInterpreterImpl.java delete mode 100644 maven/cn1-ai-tflite/lib/pom.xml delete mode 100644 maven/cn1-ai-tflite/pom.xml create mode 100644 maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java create mode 100644 maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java create mode 100644 maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java create mode 100644 maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java create mode 100644 maven/platform-feature-catalog/pom.xml rename maven/{codenameone-maven-plugin/src/main/java/com/codename1/builders/AiDependencyTable.java => platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java} (68%) rename maven/{codenameone-maven-plugin/src/test/java/com/codename1/builders/AiDependencyTableTest.java => platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java} (59%) create mode 100644 scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/InferenceOnDeviceApiTest.java create mode 100644 scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java create mode 100644 scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java diff --git a/.github/workflows/ai-cn1lib-native-check.yml b/.github/workflows/ai-cn1lib-native-check.yml index 929947c6328..2d41910c144 100644 --- a/.github/workflows/ai-cn1lib-native-check.yml +++ b/.github/workflows/ai-cn1lib-native-check.yml @@ -1,23 +1,23 @@ -name: AI cn1lib iOS xcodebuild check +name: Large AI cn1lib iOS xcodebuild check -# Per cn1lib: synthesise a tiny Xcode project that links the bundled -# `nativeios/*.m` against the real `GoogleMLKit/*` (or TFLite / whisper) -# pod, run `pod install`, then `xcodebuild build`. Catches API drift, -# missing imports, and missing pod hints early -- on the same macOS -# runners we already use for build-ios. +# The lightweight vision, language, and LiteRT bridges now live in core and +# are exercised by the normal port/build tests. Keep this isolated native +# check for the two deliberately large cn1libs. on: workflow_dispatch: pull_request: branches: [master] paths: - - 'maven/cn1-ai-**' + - 'maven/cn1-ai-whisper/**' + - 'maven/cn1-ai-stablediffusion/**' - 'scripts/gen-ai-cn1libs.py' - '.github/workflows/ai-cn1lib-native-check.yml' push: branches: [master] paths: - - 'maven/cn1-ai-**' + - 'maven/cn1-ai-whisper/**' + - 'maven/cn1-ai-stablediffusion/**' - 'scripts/gen-ai-cn1libs.py' jobs: @@ -32,17 +32,6 @@ jobs: fail-fast: false matrix: include: - - { lib: cn1-ai-mlkit-text, pod: 'GoogleMLKit/TextRecognition' } - - { lib: cn1-ai-mlkit-barcode, pod: 'GoogleMLKit/BarcodeScanning' } - - { lib: cn1-ai-mlkit-face, pod: 'GoogleMLKit/FaceDetection' } - - { lib: cn1-ai-mlkit-labeling, pod: 'GoogleMLKit/ImageLabeling' } - - { lib: cn1-ai-mlkit-translate, pod: 'GoogleMLKit/Translate' } - - { lib: cn1-ai-mlkit-smartreply, pod: 'GoogleMLKit/SmartReply' } - - { lib: cn1-ai-mlkit-langid, pod: 'GoogleMLKit/LanguageID' } - - { lib: cn1-ai-mlkit-pose, pod: 'GoogleMLKit/PoseDetection' } - - { lib: cn1-ai-mlkit-segmentation, pod: 'GoogleMLKit/SegmentationSelfie' } - - { lib: cn1-ai-mlkit-docscan, pod: '' } # VisionKit/CoreImage only - - { lib: cn1-ai-tflite, pod: 'TensorFlowLiteObjC' } - { lib: cn1-ai-whisper, pod: '' } # links static libwhisper.a - { lib: cn1-ai-stablediffusion, pod: '' } # links Swift runner steps: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index bf3190c7d73..a2819144e71 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -184,17 +184,6 @@ jobs: git config --global --add safe.directory "$GITHUB_WORKSPACE" all_libs=( - cn1-ai-mlkit-text - cn1-ai-mlkit-barcode - cn1-ai-mlkit-face - cn1-ai-mlkit-labeling - cn1-ai-mlkit-translate - cn1-ai-mlkit-smartreply - cn1-ai-mlkit-langid - cn1-ai-mlkit-pose - cn1-ai-mlkit-segmentation - cn1-ai-mlkit-docscan - cn1-ai-tflite cn1-ai-whisper cn1-ai-stablediffusion cn1-admob @@ -203,17 +192,6 @@ jobs: cn1-ads-mock ) ai_libs=( - cn1-ai-mlkit-text - cn1-ai-mlkit-barcode - cn1-ai-mlkit-face - cn1-ai-mlkit-labeling - cn1-ai-mlkit-translate - cn1-ai-mlkit-smartreply - cn1-ai-mlkit-langid - cn1-ai-mlkit-pose - cn1-ai-mlkit-segmentation - cn1-ai-mlkit-docscan - cn1-ai-tflite cn1-ai-whisper cn1-ai-stablediffusion ) diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceException.java b/CodenameOne/src/com/codename1/ai/inference/InferenceException.java new file mode 100644 index 00000000000..45c953e64ec --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceException.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.inference; + +/** Failure while loading or executing a LiteRT model. */ +public class InferenceException extends RuntimeException { + public InferenceException(String message) { + super(message); + } + + public InferenceException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java new file mode 100644 index 00000000000..38b5bfb7e17 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.inference; + +/** LiteRT session configuration. */ +public final class InferenceOptions { + public enum Accelerator { + AUTO, CPU, GPU, NPU, CORE_ML + } + + private Accelerator accelerator = Accelerator.AUTO; + private int threads; + private boolean allowFallback = true; + + public InferenceOptions accelerator(Accelerator value) { + accelerator = value == null ? Accelerator.AUTO : value; + return this; + } + + public InferenceOptions threads(int value) { + threads = Math.max(0, value); + return this; + } + + public InferenceOptions allowFallback(boolean value) { + allowFallback = value; + return this; + } + + public Accelerator getAccelerator() { + return accelerator; + } + + public int getThreads() { + return threads; + } + + public boolean isFallbackAllowed() { + return allowFallback; + } +} diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java new file mode 100644 index 00000000000..8f9fc027ed3 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.inference; + +import com.codename1.impl.InferenceImpl; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; + +/** Reusable LiteRT model session. */ +public final class InferenceSession implements AutoCloseable { + private final InferenceImpl implementation; + private final Object handle; + private boolean closed; + + private InferenceSession(InferenceImpl implementation, Object handle) { + this.implementation = implementation; + this.handle = handle; + } + + public static boolean isSupported() { + InferenceImpl impl = Display.getInstance().getInferenceBackend(); + return impl != null && impl.isSupported(); + } + + public static AsyncResource open(ModelSource source, + InferenceOptions options) { + if (source == null) { + throw new NullPointerException("source"); + } + final AsyncResource out = new AsyncResource(); + final InferenceImpl impl = Display.getInstance().getInferenceBackend(); + if (impl == null || !impl.isSupported()) { + out.error(new InferenceException("LiteRT inference is not supported")); + return out; + } + impl.open(source, options == null ? new InferenceOptions() : options) + .ready(new SuccessCallback() { + public void onSucess(Object value) { + out.complete(new InferenceSession(impl, value)); + } + }).except(new SuccessCallback() { + public void onSucess(Throwable error) { + out.error(error); + } + }); + return out; + } + + public TensorInfo[] getInputs() { + ensureOpen(); + return implementation.getInputs(handle); + } + + public TensorInfo[] getOutputs() { + ensureOpen(); + return implementation.getOutputs(handle); + } + + public AsyncResource run(Tensor[] inputs) { + ensureOpen(); + return implementation.run(handle, inputs == null ? new Tensor[0] : inputs); + } + + public void resizeInput(String name, int[] shape) { + ensureOpen(); + implementation.resizeInput(handle, name, shape); + } + + public void close() { + if (!closed) { + closed = true; + implementation.close(handle); + } + } + + private void ensureOpen() { + if (closed) { + throw new IllegalStateException("Inference session is closed"); + } + } +} diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java new file mode 100644 index 00000000000..c23a5f6654b --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.inference; + +import com.codename1.io.ConnectionRequest; +import com.codename1.io.FileSystemStorage; +import com.codename1.io.NetworkEvent; +import com.codename1.io.NetworkManager; +import com.codename1.security.Hash; +import com.codename1.ui.Display; +import com.codename1.ui.events.ActionListener; +import com.codename1.util.AsyncResource; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Downloads large LiteRT models once and exposes the cached file as a + * {@link ModelSource}. Small models can instead be packaged directly with + * {@link ModelSource#resource(String)}. + */ +public final class ModelCache { + private ModelCache() { + } + + /** + * Fetches a model into the app-private {@code ai-models} directory. + * + * @param url HTTPS URL of the model + * @param cacheKey stable cache name, independent of the URL + * @param sha256 optional lowercase or uppercase SHA-256 hex digest + * @return asynchronous cached model source + */ + public static AsyncResource fetch( + final String url, final String cacheKey, final String sha256) { + if (url == null || !url.startsWith("https://")) { + throw new IllegalArgumentException("Model URL must use HTTPS"); + } + if (cacheKey == null || cacheKey.length() == 0) { + throw new IllegalArgumentException("cacheKey must not be empty"); + } + if (sha256 != null && sha256.length() != 64) { + throw new IllegalArgumentException("SHA-256 must contain 64 hex characters"); + } + + final AsyncResource out = new AsyncResource(); + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + final FileSystemStorage fs = FileSystemStorage.getInstance(); + final String directory = fs.getAppHomePath() + "ai-models/"; + fs.mkdir(directory); + final String fileName = safeName(cacheKey) + ".tflite"; + final String target = directory + fileName; + try { + if (fs.exists(target) && verify(target, sha256)) { + complete(out, ModelSource.file(target)); + return; + } + if (fs.exists(target)) { + fs.delete(target); + } + } catch (IOException error) { + fail(out, error); + return; + } + Display.getInstance().callSerially(new Runnable() { + public void run() { + download(out, url, sha256, target, fileName); + } + }); + } + }); + return out; + } + + public static AsyncResource fetch(String url, String cacheKey) { + return fetch(url, cacheKey, null); + } + + private static void download(final AsyncResource out, + String url, final String sha256, + final String target, final String fileName) { + final FileSystemStorage fs = FileSystemStorage.getInstance(); + final String temporary = target + ".download"; + if (fs.exists(temporary)) { + fs.delete(temporary); + } + final ConnectionRequest request = new ConnectionRequest(); + request.setPost(false); + request.setFailSilently(true); + request.setReadResponseForErrors(false); + request.setDuplicateSupported(true); + request.setUrl(url); + request.setDestinationFile(temporary); + request.addResponseListener(new ActionListener() { + public void actionPerformed(NetworkEvent event) { + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + try { + if (!verify(temporary, sha256)) { + fs.delete(temporary); + throw new IOException("Downloaded model SHA-256 does not match"); + } + if (fs.exists(target)) { + fs.delete(target); + } + fs.rename(temporary, fileName); + complete(out, ModelSource.file(target)); + } catch (IOException error) { + fail(out, error); + } + } + }); + } + }); + ActionListener failure = new ActionListener() { + public void actionPerformed(NetworkEvent event) { + if (fs.exists(temporary)) { + fs.delete(temporary); + } + Throwable error = event.getError(); + fail(out, error == null + ? new IOException("Model download failed with HTTP " + + event.getResponseCode()) + : error); + } + }; + request.addExceptionListener(failure); + request.addResponseCodeListener(failure); + NetworkManager.getInstance().addToQueue(request); + } + + private static boolean verify(String path, String expected) throws IOException { + if (expected == null || expected.length() == 0) { + return true; + } + InputStream input = FileSystemStorage.getInstance().openInputStream(path); + try { + Hash hash = Hash.create(Hash.SHA256); + byte[] buffer = new byte[16384]; + int read; + while ((read = input.read(buffer)) >= 0) { + hash.update(buffer, 0, read); + } + return expected.equalsIgnoreCase(Hash.toHex(hash.digest())); + } finally { + input.close(); + } + } + + private static String safeName(String value) { + StringBuilder out = new StringBuilder(); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + out.append((c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') + || c == '-' || c == '_' ? c : '_'); + } + return out.toString(); + } + + private static void complete(final AsyncResource out, final T value) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(value); + } + }); + } + + private static void fail(final AsyncResource out, final Throwable error) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.error(new InferenceException("Could not cache LiteRT model", error)); + } + }); + } +} diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelSource.java b/CodenameOne/src/com/codename1/ai/inference/ModelSource.java new file mode 100644 index 00000000000..c7bf7ba28e3 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/ModelSource.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.inference; + +/** A `.tflite` model supplied as bytes, a filesystem path, or an app resource. */ +public final class ModelSource { + public static final int BYTES = 1; + public static final int FILE = 2; + public static final int RESOURCE = 3; + + private final int kind; + private final byte[] bytes; + private final String path; + + private ModelSource(int kind, byte[] bytes, String path) { + this.kind = kind; + this.bytes = bytes; + this.path = path; + } + + public static ModelSource bytes(byte[] value) { + if (value == null || value.length == 0) { + throw new IllegalArgumentException("Model bytes must not be empty"); + } + byte[] copy = new byte[value.length]; + System.arraycopy(value, 0, copy, 0, value.length); + return new ModelSource(BYTES, copy, null); + } + + public static ModelSource file(String path) { + return named(FILE, path); + } + + public static ModelSource resource(String path) { + return named(RESOURCE, path); + } + + private static ModelSource named(int kind, String path) { + if (path == null || path.length() == 0) { + throw new IllegalArgumentException("Model path must not be empty"); + } + return new ModelSource(kind, null, path); + } + + public int getKind() { + return kind; + } + + public byte[] getBytes() { + if (bytes == null) return null; + byte[] out = new byte[bytes.length]; + System.arraycopy(bytes, 0, out, 0, bytes.length); + return out; + } + + public String getPath() { + return path; + } +} diff --git a/CodenameOne/src/com/codename1/ai/inference/Tensor.java b/CodenameOne/src/com/codename1/ai/inference/Tensor.java new file mode 100644 index 00000000000..67836a93146 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/Tensor.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.inference; + +/** Named tensor value passed to or returned from an inference session. */ +public final class Tensor { + private final String name; + private final TensorType type; + private final int[] shape; + private final Object data; + + public Tensor(String name, TensorType type, int[] shape, Object data) { + if (type == null || data == null) { + throw new NullPointerException("type and data are required"); + } + validateData(type, data); + int count = elementCount(shape); + if (count >= 0 && count != dataLength(data)) { + throw new IllegalArgumentException("Tensor shape does not match data length"); + } + this.name = name; + this.type = type; + this.shape = copy(shape); + this.data = copyData(data); + } + + public static Tensor floats(String name, int[] shape, float[] data) { + return new Tensor(name, TensorType.FLOAT32, shape, data); + } + + public static Tensor ints(String name, int[] shape, int[] data) { + return new Tensor(name, TensorType.INT32, shape, data); + } + + public static Tensor bytes(String name, TensorType type, int[] shape, byte[] data) { + return new Tensor(name, type, shape, data); + } + + public String getName() { + return name; + } + + public TensorType getType() { + return type; + } + + public int[] getShape() { + return copy(shape); + } + + public Object getData() { + return copyData(data); + } + + private static void validateData(TensorType type, Object data) { + boolean valid; + switch (type) { + case FLOAT32: valid = data instanceof float[]; break; + case INT32: valid = data instanceof int[]; break; + case INT64: valid = data instanceof long[]; break; + case UINT8: + case INT8: + case BOOL: valid = data instanceof byte[]; break; + default: valid = false; + } + if (!valid) { + throw new IllegalArgumentException("Data array does not match " + type); + } + } + + private static int elementCount(int[] shape) { + if (shape == null || shape.length == 0) { + return 1; + } + int out = 1; + for (int i = 0; i < shape.length; i++) { + if (shape[i] < 0) { + return -1; + } + out *= shape[i]; + } + return out; + } + + private static int dataLength(Object value) { + if (value instanceof float[]) return ((float[]) value).length; + if (value instanceof int[]) return ((int[]) value).length; + if (value instanceof long[]) return ((long[]) value).length; + if (value instanceof byte[]) return ((byte[]) value).length; + throw new IllegalArgumentException("Unsupported tensor data array"); + } + + private static int[] copy(int[] value) { + if (value == null) return new int[0]; + int[] out = new int[value.length]; + System.arraycopy(value, 0, out, 0, value.length); + return out; + } + + private static Object copyData(Object value) { + int length = dataLength(value); + if (value instanceof float[]) { + float[] out = new float[length]; + System.arraycopy(value, 0, out, 0, length); + return out; + } + if (value instanceof int[]) { + int[] out = new int[length]; + System.arraycopy(value, 0, out, 0, length); + return out; + } + if (value instanceof long[]) { + long[] out = new long[length]; + System.arraycopy(value, 0, out, 0, length); + return out; + } + if (value instanceof byte[]) { + byte[] out = new byte[length]; + System.arraycopy(value, 0, out, 0, length); + return out; + } + throw new IllegalArgumentException("Unsupported tensor data array"); + } +} diff --git a/CodenameOne/src/com/codename1/ai/inference/TensorInfo.java b/CodenameOne/src/com/codename1/ai/inference/TensorInfo.java new file mode 100644 index 00000000000..49774d930ea --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/TensorInfo.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.inference; + +/** Immutable model input/output metadata. */ +public final class TensorInfo { + private final String name; + private final TensorType type; + private final int[] shape; + private final int index; + + public TensorInfo(String name, TensorType type, int[] shape, int index) { + this.name = name; + this.type = type; + this.shape = copy(shape); + this.index = index; + } + + public String getName() { + return name; + } + + public TensorType getType() { + return type; + } + + public int[] getShape() { + return copy(shape); + } + + public int getIndex() { + return index; + } + + private static int[] copy(int[] value) { + if (value == null) { + return new int[0]; + } + int[] out = new int[value.length]; + System.arraycopy(value, 0, out, 0, value.length); + return out; + } +} diff --git a/CodenameOne/src/com/codename1/ai/inference/TensorType.java b/CodenameOne/src/com/codename1/ai/inference/TensorType.java new file mode 100644 index 00000000000..04f22652ea3 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/TensorType.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.inference; + +/** Tensor element types supported by the portable LiteRT contract. */ +public enum TensorType { + FLOAT32, + INT32, + INT64, + UINT8, + INT8, + BOOL +} diff --git a/CodenameOne/src/com/codename1/ai/inference/package-info.java b/CodenameOne/src/com/codename1/ai/inference/package-info.java new file mode 100644 index 00000000000..b62037c5e30 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/package-info.java @@ -0,0 +1,9 @@ +/** + * Reusable on-device inference sessions for {@code .tflite} models. + * + *

Android uses LiteRT. iOS uses TensorFlow Lite Objective-C and may select + * the Core ML delegate. The builder links the runtime only when an application + * references {@link com.codename1.ai.inference.InferenceSession}. Other + * targets expose the same API with an explicit unsupported fallback.

+ */ +package com.codename1.ai.inference; diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageBackend.java b/CodenameOne/src/com/codename1/ai/language/LanguageBackend.java new file mode 100644 index 00000000000..854de1d0dc8 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/LanguageBackend.java @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.language; + +/** Identifies an on-device language implementation. */ +public interface LanguageBackend { + String getId(); +} diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java b/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java new file mode 100644 index 00000000000..0f3b8707ad8 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.language; + +/** Backend selectors for language identification, translation, and smart reply. */ +public final class LanguageBackends { + private static final LanguageBackend AUTO = new Named("auto"); + private static final LanguageBackend ML_KIT = new Named("ml-kit"); + private static final LanguageBackend LITE_RT = new Named("litert"); + + private LanguageBackends() { + } + + public static LanguageBackend auto() { + return AUTO; + } + + public static LanguageBackend mlKit() { + return ML_KIT; + } + + public static LanguageBackend liteRt() { + return LITE_RT; + } + + private static final class Named implements LanguageBackend { + private final String id; + + private Named(String id) { + this.id = id; + } + + public String getId() { + return id; + } + } +} diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageCandidate.java b/CodenameOne/src/com/codename1/ai/language/LanguageCandidate.java new file mode 100644 index 00000000000..31fe5176bba --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/LanguageCandidate.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.language; + +/** Language identification candidate. */ +public final class LanguageCandidate { + private final String languageTag; + private final float confidence; + + public LanguageCandidate(String languageTag, float confidence) { + this.languageTag = languageTag; + this.confidence = confidence; + } + + public String getLanguageTag() { + return languageTag; + } + + public float getConfidence() { + return confidence; + } +} diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java new file mode 100644 index 00000000000..438842a19d9 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.language; + +import com.codename1.impl.LanguageImpl; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; + +/** On-device language identification. */ +public final class LanguageIdentifier { + private LanguageIdentifier() { + } + + public static boolean isSupported() { + return isSupported(new LanguageOptions()); + } + + public static boolean isSupported(LanguageOptions options) { + LanguageImpl impl = Display.getInstance().getLanguageBackend(); + LanguageOptions actual = options == null ? new LanguageOptions() : options; + return impl != null && impl.isSupported("language-id", + actual.getBackend().getId()); + } + + public static AsyncResource identify( + String text, LanguageOptions options) { + LanguageOptions actual = options == null ? new LanguageOptions() : options; + LanguageImpl impl = Display.getInstance().getLanguageBackend(); + if (impl == null || !impl.isSupported("language-id", + actual.getBackend().getId())) { + AsyncResource out = + new AsyncResource(); + out.error(new UnsupportedOperationException( + "language-id is not supported")); + return out; + } + return impl.identify(text == null ? "" : text, + actual.getBackend().getId(), actual); + } +} diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java b/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java new file mode 100644 index 00000000000..afeb1299ffb --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.language; + +/** Common options for on-device language services. */ +public final class LanguageOptions { + private LanguageBackend backend = LanguageBackends.auto(); + private float minimumConfidence; + + public LanguageOptions backend(LanguageBackend value) { + backend = value == null ? LanguageBackends.auto() : value; + return this; + } + + public LanguageOptions minimumConfidence(float value) { + minimumConfidence = Math.max(0, Math.min(1, value)); + return this; + } + + public LanguageBackend getBackend() { + return backend; + } + + public float getMinimumConfidence() { + return minimumConfidence; + } +} diff --git a/CodenameOne/src/com/codename1/ai/language/SmartReply.java b/CodenameOne/src/com/codename1/ai/language/SmartReply.java new file mode 100644 index 00000000000..613e55aa7a4 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/SmartReply.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.language; + +import com.codename1.impl.LanguageImpl; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; + +/** On-device short reply suggestions for a conversation. */ +public final class SmartReply { + private SmartReply() { + } + + public static boolean isSupported() { + return isSupported(new LanguageOptions()); + } + + public static boolean isSupported(LanguageOptions options) { + LanguageOptions actual = options == null + ? new LanguageOptions() : options; + LanguageImpl impl = Display.getInstance().getLanguageBackend(); + return impl != null && impl.isSupported( + "smart-reply", actual.getBackend().getId()); + } + + public static AsyncResource suggest(SmartReplyMessage[] conversation, + LanguageOptions options) { + LanguageOptions actual = options == null ? new LanguageOptions() : options; + LanguageImpl impl = Display.getInstance().getLanguageBackend(); + if (impl == null || !impl.isSupported("smart-reply", actual.getBackend().getId())) { + AsyncResource out = new AsyncResource(); + out.error(new UnsupportedOperationException("smart reply is not supported")); + return out; + } + return impl.suggestReplies(conversation == null + ? new SmartReplyMessage[0] : conversation, + actual.getBackend().getId(), actual); + } +} diff --git a/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java b/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java new file mode 100644 index 00000000000..1bda62dcea1 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.language; + +/** One message supplied to the on-device smart-reply model. */ +public final class SmartReplyMessage { + private final String text; + private final String participantId; + private final boolean localUser; + private final long timestampMillis; + + public SmartReplyMessage(String text, String participantId, + boolean localUser, long timestampMillis) { + this.text = text == null ? "" : text; + this.participantId = participantId; + this.localUser = localUser; + this.timestampMillis = timestampMillis; + } + + public String getText() { + return text; + } + + public String getParticipantId() { + return participantId; + } + + public boolean isLocalUser() { + return localUser; + } + + public long getTimestampMillis() { + return timestampMillis; + } +} diff --git a/CodenameOne/src/com/codename1/ai/language/Translator.java b/CodenameOne/src/com/codename1/ai/language/Translator.java new file mode 100644 index 00000000000..85d8a7b9c92 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/Translator.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.language; + +import com.codename1.impl.LanguageImpl; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; + +/** On-device translation with lazily installed language-pair models. */ +public final class Translator { + private Translator() { + } + + public static boolean isSupported() { + return isSupported(new LanguageOptions()); + } + + public static boolean isSupported(LanguageOptions options) { + LanguageOptions actual = options == null ? new LanguageOptions() : options; + LanguageImpl impl = Display.getInstance().getLanguageBackend(); + return impl != null && impl.isSupported("translation", + actual.getBackend().getId()); + } + + public static AsyncResource translate(String text, String sourceLanguage, + String targetLanguage, + LanguageOptions options) { + LanguageOptions actual = options == null ? new LanguageOptions() : options; + LanguageImpl impl = Display.getInstance().getLanguageBackend(); + if (impl == null || !impl.isSupported("translation", actual.getBackend().getId())) { + AsyncResource out = new AsyncResource(); + out.error(new UnsupportedOperationException("translation is not supported")); + return out; + } + return impl.translate(text == null ? "" : text, sourceLanguage, targetLanguage, + actual.getBackend().getId(), actual); + } +} diff --git a/CodenameOne/src/com/codename1/ai/language/package-info.java b/CodenameOne/src/com/codename1/ai/language/package-info.java new file mode 100644 index 00000000000..2b711adb17e --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/package-info.java @@ -0,0 +1,11 @@ +/** + * Vendor-neutral on-device language identification, translation, and smart + * reply. + * + *

Android and iOS builds use feature-scoped ML Kit components. Each + * public entry point is scanned independently, so language identification + * does not bundle translation or Smart Reply. Translation model payloads are + * installed lazily by ML Kit. Other targets expose the same API with an + * explicit unsupported fallback.

+ */ +package com.codename1.ai.language; diff --git a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java new file mode 100644 index 00000000000..48896135a22 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +import com.codename1.impl.VisionImpl; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; + +abstract class AbstractVisionAnalyzer implements VisionAnalyzer { + private final VisionFeature feature; + private final VisionOptions options; + private VisionImpl implementation; + private boolean closed; + + AbstractVisionAnalyzer(VisionFeature feature, VisionOptions options) { + this.feature = feature; + this.options = options == null ? new VisionOptions() : options; + } + + public final boolean isSupported() { + VisionImpl impl = implementation(); + return impl != null && impl.isSupported(feature, options.getBackend().getId()); + } + + public final AsyncResource process(VisionImage image) { + if (image == null) { + throw new NullPointerException("image"); + } + if (closed) { + throw new IllegalStateException("Analyzer is closed"); + } + VisionImpl impl = implementation(); + if (impl == null || !impl.isSupported(feature, options.getBackend().getId())) { + AsyncResource out = new AsyncResource(); + out.error(new VisionException(VisionException.UNSUPPORTED, + feature + " is not supported by " + options.getBackend().getId())); + return out; + } + return impl.analyze(feature, options.getBackend().getId(), image, options); + } + + public final void close() { + closed = true; + if (implementation != null) { + implementation.close(); + implementation = null; + } + } + + private VisionImpl implementation() { + if (!closed && implementation == null) { + implementation = Display.getInstance().getVisionBackend(); + } + return implementation; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/Barcode.java b/CodenameOne/src/com/codename1/ai/vision/Barcode.java new file mode 100644 index 00000000000..da2625cb9e9 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/Barcode.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Portable barcode observation. */ +public final class Barcode { + private final String value; + private final String format; + private final byte[] rawBytes; + private final VisionRect bounds; + private final VisionPoint[] corners; + private final VisionMetadata metadata; + + public Barcode(String value, String format, byte[] rawBytes, + VisionRect bounds, VisionPoint[] corners) { + this(value, format, rawBytes, bounds, corners, null); + } + + public Barcode(String value, String format, byte[] rawBytes, + VisionRect bounds, VisionPoint[] corners, + VisionMetadata metadata) { + this.value = value; + this.format = format; + this.rawBytes = copy(rawBytes); + this.bounds = bounds == null ? VisionRect.EMPTY : bounds; + if (corners == null) { + this.corners = new VisionPoint[0]; + } else { + this.corners = new VisionPoint[corners.length]; + System.arraycopy(corners, 0, this.corners, 0, corners.length); + } + this.metadata = metadata; + } + + public String getValue() { + return value; + } + + public String getFormat() { + return format; + } + + public byte[] getRawBytes() { + return copy(rawBytes); + } + + public VisionRect getBounds() { + return bounds; + } + + public VisionPoint[] getCorners() { + VisionPoint[] out = new VisionPoint[corners.length]; + System.arraycopy(corners, 0, out, 0, corners.length); + return out; + } + + public VisionMetadata getMetadata() { + return metadata; + } + + private static byte[] copy(byte[] value) { + if (value == null) { + return null; + } + byte[] out = new byte[value.length]; + System.arraycopy(value, 0, out, 0, value.length); + return out; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java b/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java new file mode 100644 index 00000000000..53644a89e1b --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Creates reusable still-image or live-frame barcode analyzers. */ +public final class BarcodeScanner extends AbstractVisionAnalyzer { + public BarcodeScanner() { + this(null); + } + + public BarcodeScanner(VisionOptions options) { + super(VisionFeature.BARCODE_SCANNING, options); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/DocumentScanResult.java b/CodenameOne/src/com/codename1/ai/vision/DocumentScanResult.java new file mode 100644 index 00000000000..85f4ce6acbb --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/DocumentScanResult.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Corrected document pages returned as encoded images. */ +public final class DocumentScanResult { + private final byte[][] pages; + private final VisionMetadata metadata; + + public DocumentScanResult(byte[][] pages) { + this(pages, null); + } + + public DocumentScanResult(byte[][] pages, VisionMetadata metadata) { + if (pages == null) { + this.pages = new byte[0][]; + } else { + this.pages = new byte[pages.length][]; + for (int i = 0; i < pages.length; i++) { + byte[] page = pages[i]; + if (page == null) { + throw new NullPointerException("pages[" + i + "]"); + } + this.pages[i] = new byte[page.length]; + System.arraycopy(page, 0, this.pages[i], 0, page.length); + } + } + this.metadata = metadata; + } + + public int getPageCount() { + return pages.length; + } + + public byte[] getPage(int index) { + byte[] page = pages[index]; + byte[] out = new byte[page.length]; + System.arraycopy(page, 0, out, 0, page.length); + return out; + } + + public VisionMetadata getMetadata() { + return metadata; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java b/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java new file mode 100644 index 00000000000..63c1cb3f808 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Creates reusable document-boundary and perspective-correction analyzers. */ +public final class DocumentScanner extends AbstractVisionAnalyzer { + public DocumentScanner() { + this(null); + } + + public DocumentScanner(VisionOptions options) { + super(VisionFeature.DOCUMENT_SCANNING, options); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/Face.java b/CodenameOne/src/com/codename1/ai/vision/Face.java new file mode 100644 index 00000000000..b6387d4fae0 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/Face.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Portable face observation. + * + *

Euler angles are expressed in degrees. Bounds and landmark points use + * the normalized, top-left-origin coordinate space defined by + * {@link VisionRect} and {@link VisionPoint}.

+ */ +public final class Face { + private final VisionRect bounds; + private final Map landmarks; + private final float yaw; + private final float pitch; + private final float roll; + private final float smilingProbability; + private final int trackingId; + private final VisionMetadata metadata; + + public Face(VisionRect bounds, Map landmarks, + float yaw, float pitch, float roll, + float smilingProbability, int trackingId) { + this(bounds, landmarks, yaw, pitch, roll, smilingProbability, + trackingId, null); + } + + public Face(VisionRect bounds, Map landmarks, + float yaw, float pitch, float roll, + float smilingProbability, int trackingId, + VisionMetadata metadata) { + this.bounds = bounds == null ? VisionRect.EMPTY : bounds; + this.landmarks = landmarks == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(new HashMap(landmarks)); + this.yaw = yaw; + this.pitch = pitch; + this.roll = roll; + this.smilingProbability = smilingProbability; + this.trackingId = trackingId; + this.metadata = metadata; + } + + public VisionRect getBounds() { + return bounds; + } + + public Map getLandmarks() { + return landmarks; + } + + public float getYaw() { + return yaw; + } + + public float getPitch() { + return pitch; + } + + public float getRoll() { + return roll; + } + + public float getSmilingProbability() { + return smilingProbability; + } + + public int getTrackingId() { + return trackingId; + } + + public VisionMetadata getMetadata() { + return metadata; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java b/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java new file mode 100644 index 00000000000..a7f5c98f7c9 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Creates reusable on-device face analyzers. */ +public final class FaceDetector extends AbstractVisionAnalyzer { + public FaceDetector() { + this(null); + } + + public FaceDetector(VisionOptions options) { + super(VisionFeature.FACE_DETECTION, options); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java b/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java new file mode 100644 index 00000000000..ac91a89de65 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Portable image classification label. */ +public final class ImageLabel { + private final String text; + private final float confidence; + private final int index; + private final VisionMetadata metadata; + + public ImageLabel(String text, float confidence, int index) { + this(text, confidence, index, null); + } + + public ImageLabel(String text, float confidence, int index, + VisionMetadata metadata) { + this.text = text == null ? "" : text; + this.confidence = confidence; + this.index = index; + this.metadata = metadata; + } + + public String getText() { + return text; + } + + public float getConfidence() { + return confidence; + } + + public int getIndex() { + return index; + } + + public VisionMetadata getMetadata() { + return metadata; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java b/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java new file mode 100644 index 00000000000..598249da9eb --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Creates reusable on-device image classifiers. */ +public final class ImageLabeler extends AbstractVisionAnalyzer { + public ImageLabeler() { + this(null); + } + + public ImageLabeler(VisionOptions options) { + super(VisionFeature.IMAGE_LABELING, options); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/Pose.java b/CodenameOne/src/com/codename1/ai/vision/Pose.java new file mode 100644 index 00000000000..32b1b352270 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/Pose.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Portable body-pose result. */ +public final class Pose { + private final Landmark[] landmarks; + private final VisionMetadata metadata; + + public Pose(Landmark[] landmarks) { + this(landmarks, null); + } + + public Pose(Landmark[] landmarks, VisionMetadata metadata) { + if (landmarks == null) { + this.landmarks = new Landmark[0]; + } else { + this.landmarks = new Landmark[landmarks.length]; + System.arraycopy(landmarks, 0, this.landmarks, 0, landmarks.length); + } + this.metadata = metadata; + } + + public Landmark[] getLandmarks() { + Landmark[] out = new Landmark[landmarks.length]; + System.arraycopy(landmarks, 0, out, 0, landmarks.length); + return out; + } + + public VisionMetadata getMetadata() { + return metadata; + } + + public static final class Landmark { + private final String name; + private final VisionPoint position; + private final float confidence; + + public Landmark(String name, VisionPoint position, float confidence) { + this.name = name; + this.position = position; + this.confidence = confidence; + } + + public String getName() { + return name; + } + + public VisionPoint getPosition() { + return position; + } + + public float getConfidence() { + return confidence; + } + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java b/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java new file mode 100644 index 00000000000..cd07c52970c --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Creates reusable body-pose analyzers. */ +public final class PoseDetector extends AbstractVisionAnalyzer { + public PoseDetector() { + this(null); + } + + public PoseDetector(VisionOptions options) { + super(VisionFeature.POSE_DETECTION, options); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java b/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java new file mode 100644 index 00000000000..cd1e4b0791f --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Per-pixel foreground confidence mask. */ +public final class SegmentationMask { + private final int width; + private final int height; + private final float[] confidence; + private final VisionMetadata metadata; + + public SegmentationMask(int width, int height, float[] confidence) { + this(width, height, confidence, null); + } + + public SegmentationMask(int width, int height, float[] confidence, + VisionMetadata metadata) { + if (width < 0 || height < 0 + || confidence == null || confidence.length != width * height) { + throw new IllegalArgumentException("Mask dimensions do not match its data"); + } + this.width = width; + this.height = height; + this.confidence = new float[confidence.length]; + System.arraycopy(confidence, 0, this.confidence, 0, confidence.length); + this.metadata = metadata; + } + + public int getWidth() { + return width; + } + + public int getHeight() { + return height; + } + + public float[] getConfidence() { + float[] out = new float[confidence.length]; + System.arraycopy(confidence, 0, out, 0, confidence.length); + return out; + } + + public VisionMetadata getMetadata() { + return metadata; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java b/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java new file mode 100644 index 00000000000..9e4254bfddc --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Creates reusable foreground/person segmentation analyzers. */ +public final class SelfieSegmenter extends AbstractVisionAnalyzer { + public SelfieSegmenter() { + this(null); + } + + public SelfieSegmenter(VisionOptions options) { + super(VisionFeature.SELFIE_SEGMENTATION, options); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/TextRecognitionResult.java b/CodenameOne/src/com/codename1/ai/vision/TextRecognitionResult.java new file mode 100644 index 00000000000..c3df7d0dad9 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/TextRecognitionResult.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** OCR output with a portable block-level structure. */ +public final class TextRecognitionResult { + public static final TextRecognitionResult EMPTY = + new TextRecognitionResult("", new TextBlock[0]); + + private final String text; + private final TextBlock[] blocks; + private final VisionMetadata metadata; + + public TextRecognitionResult(String text, TextBlock[] blocks) { + this(text, blocks, null); + } + + public TextRecognitionResult(String text, TextBlock[] blocks, + VisionMetadata metadata) { + this.text = text == null ? "" : text; + if (blocks == null) { + this.blocks = new TextBlock[0]; + } else { + this.blocks = new TextBlock[blocks.length]; + System.arraycopy(blocks, 0, this.blocks, 0, blocks.length); + } + this.metadata = metadata; + } + + public String getText() { + return text; + } + + public TextBlock[] getBlocks() { + TextBlock[] out = new TextBlock[blocks.length]; + System.arraycopy(blocks, 0, out, 0, blocks.length); + return out; + } + + public VisionMetadata getMetadata() { + return metadata; + } + + public static final class TextBlock { + private final String text; + private final float confidence; + private final VisionRect bounds; + private final String languageTag; + + public TextBlock(String text, float confidence, VisionRect bounds, String languageTag) { + this.text = text == null ? "" : text; + this.confidence = confidence; + this.bounds = bounds == null ? VisionRect.EMPTY : bounds; + this.languageTag = languageTag; + } + + public String getText() { + return text; + } + + public float getConfidence() { + return confidence; + } + + public VisionRect getBounds() { + return bounds; + } + + public String getLanguageTag() { + return languageTag; + } + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java b/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java new file mode 100644 index 00000000000..244303fda23 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Creates reusable on-device OCR analyzers. */ +public final class TextRecognizer extends AbstractVisionAnalyzer { + public TextRecognizer() { + this(null); + } + + public TextRecognizer(VisionOptions options) { + super(VisionFeature.TEXT_RECOGNITION, options); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java b/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java new file mode 100644 index 00000000000..135c779a224 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +import com.codename1.util.AsyncResource; + +/** Reusable, closable analyzer for still images or camera frames. */ +public interface VisionAnalyzer extends AutoCloseable { + boolean isSupported(); + AsyncResource process(VisionImage image); + void close(); +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionBackend.java b/CodenameOne/src/com/codename1/ai/vision/VisionBackend.java new file mode 100644 index 00000000000..0e0a7d44149 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionBackend.java @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + */ +package com.codename1.ai.vision; + +/** + * Identifies a vision implementation. Use {@link VisionBackends} to obtain + * instances so builders can detect explicit optional-backend use. + */ +public interface VisionBackend { + String getId(); +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java b/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java new file mode 100644 index 00000000000..7a820d04306 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + */ +package com.codename1.ai.vision; + +/** + * Vision backend selectors. {@code auto()} chooses Apple Vision on iOS and + * ML Kit on Android. Unsupported ports report that through the analyzer. + */ +public final class VisionBackends { + private static final VisionBackend AUTO = new NamedBackend("auto"); + private static final VisionBackend APPLE = new NamedBackend("apple-vision"); + private static final VisionBackend ML_KIT = new NamedBackend("ml-kit"); + + private VisionBackends() { + } + + public static VisionBackend auto() { + return AUTO; + } + + public static VisionBackend appleVision() { + return APPLE; + } + + public static VisionBackend mlKit() { + return ML_KIT; + } + + private static final class NamedBackend implements VisionBackend { + private final String id; + + private NamedBackend(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public String toString() { + return id; + } + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionException.java b/CodenameOne/src/com/codename1/ai/vision/VisionException.java new file mode 100644 index 00000000000..8fcaab2acc0 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionException.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Failure reported by an on-device vision backend. */ +public class VisionException extends RuntimeException { + public static final int UNSUPPORTED = 1; + public static final int INVALID_IMAGE = 2; + public static final int MODEL_UNAVAILABLE = 3; + public static final int CANCELLED = 4; + public static final int BACKEND_ERROR = 5; + + private final int code; + + public VisionException(int code, String message) { + super(message); + this.code = code; + } + + public VisionException(int code, String message, Throwable cause) { + super(message, cause); + this.code = code; + } + + public int getCode() { + return code; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionFeature.java b/CodenameOne/src/com/codename1/ai/vision/VisionFeature.java new file mode 100644 index 00000000000..d9460fcfded --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionFeature.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Features understood by the shared vision backend. */ +public enum VisionFeature { + TEXT_RECOGNITION, + BARCODE_SCANNING, + FACE_DETECTION, + IMAGE_LABELING, + POSE_DETECTION, + SELFIE_SEGMENTATION, + DOCUMENT_SCANNING +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java new file mode 100644 index 00000000000..9d82a8877e7 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +import com.codename1.camera.CameraFrame; +import com.codename1.camera.FrameFormat; + +/** + * Immutable vision input. Camera-frame factories copy the callback-owned bytes + * so asynchronous analyzers never retain recycled camera buffers. + */ +public final class VisionImage { + private final byte[] encodedBytes; + private final byte[] pixels; + private final int width; + private final int height; + private final int rotationDegrees; + private final long timestampNanos; + private final FrameFormat format; + + private VisionImage(byte[] encodedBytes, byte[] pixels, int width, int height, + int rotationDegrees, long timestampNanos, FrameFormat format) { + this.encodedBytes = copy(encodedBytes); + this.pixels = copy(pixels); + this.width = width; + this.height = height; + this.rotationDegrees = normalizeRotation(rotationDegrees); + this.timestampNanos = timestampNanos; + this.format = format == null ? FrameFormat.JPEG : format; + } + + public static VisionImage encoded(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + throw new IllegalArgumentException("Image bytes must not be empty"); + } + return new VisionImage(bytes, null, 0, 0, 0, 0, FrameFormat.JPEG); + } + + public static VisionImage pixels(byte[] bytes, int width, int height, + FrameFormat format, int rotationDegrees) { + if (bytes == null || bytes.length == 0 || width <= 0 || height <= 0) { + throw new IllegalArgumentException("Pixels and positive dimensions are required"); + } + return new VisionImage(null, bytes, width, height, rotationDegrees, 0, format); + } + + public static VisionImage fromCameraFrame(CameraFrame frame) { + if (frame == null) { + throw new NullPointerException("frame"); + } + return new VisionImage(frame.getJpegBytes(), frame.getRawBytes(), + frame.getWidth(), frame.getHeight(), frame.getRotationDegrees(), + frame.getTimestampNanos(), frame.getFormat()); + } + + public byte[] getEncodedBytes() { + return copy(encodedBytes); + } + + public byte[] getPixels() { + return copy(pixels); + } + + public int getWidth() { + return width; + } + + public int getHeight() { + return height; + } + + public int getRotationDegrees() { + return rotationDegrees; + } + + public long getTimestampNanos() { + return timestampNanos; + } + + public FrameFormat getFormat() { + return format; + } + + private static byte[] copy(byte[] value) { + if (value == null) { + return null; + } + byte[] out = new byte[value.length]; + System.arraycopy(value, 0, out, 0, value.length); + return out; + } + + private static int normalizeRotation(int value) { + int out = value % 360; + return out < 0 ? out + 360 : out; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionMetadata.java b/CodenameOne/src/com/codename1/ai/vision/VisionMetadata.java new file mode 100644 index 00000000000..8c0ee8c6e2a --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionMetadata.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Optional backend identity and backend-specific string values attached to a + * vision result. Portable code should use the typed result fields first. + */ +public final class VisionMetadata { + private final String backendId; + private final Map values; + + public VisionMetadata(String backendId, Map values) { + this.backendId = backendId; + this.values = values == null || values.isEmpty() + ? Collections.emptyMap() + : Collections.unmodifiableMap(new HashMap(values)); + } + + public VisionMetadata(String backendId) { + this(backendId, null); + } + + public String getBackendId() { + return backendId; + } + + public Map getValues() { + return values; + } + + public String get(String key) { + return values.get(key); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java b/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java new file mode 100644 index 00000000000..778f0263495 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Common analyzer options; feature-specific options may extend this class. */ +public class VisionOptions { + private VisionBackend backend = VisionBackends.auto(); + private float minimumConfidence; + private int maximumResults; + + public VisionOptions backend(VisionBackend value) { + backend = value == null ? VisionBackends.auto() : value; + return this; + } + + public VisionOptions minimumConfidence(float value) { + minimumConfidence = Math.max(0, Math.min(1, value)); + return this; + } + + public VisionOptions maximumResults(int value) { + maximumResults = Math.max(0, value); + return this; + } + + public VisionBackend getBackend() { + return backend; + } + + public float getMinimumConfidence() { + return minimumConfidence; + } + + public int getMaximumResults() { + return maximumResults; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java new file mode 100644 index 00000000000..08f9731f427 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +import com.codename1.camera.CameraFrame; +import com.codename1.camera.CameraSession; +import com.codename1.camera.FrameListener; +import com.codename1.ui.Display; +import com.codename1.util.SuccessCallback; + +/** + * Connects a camera frame stream to an analyzer with keep-only-latest + * backpressure. The pipeline owns the analyzer and closes it on close. + */ +public final class VisionPipeline implements AutoCloseable { + private final CameraSession session; + private final VisionAnalyzer analyzer; + private final VisionPipelineListener listener; + private final FrameListener frameListener; + private VisionImage pending; + private boolean busy; + private boolean closed; + + public VisionPipeline(CameraSession session, VisionAnalyzer analyzer, + VisionPipelineListener listener) { + if (session == null || analyzer == null || listener == null) { + throw new NullPointerException("session, analyzer, and listener are required"); + } + this.session = session; + this.analyzer = analyzer; + this.listener = listener; + frameListener = new FrameListener() { + public void onFrame(CameraFrame frame) { + accept(VisionImage.fromCameraFrame(frame)); + } + }; + session.setFrameListener(frameListener); + } + + private void accept(VisionImage image) { + synchronized (this) { + if (closed) { + return; + } + if (busy) { + pending = image; + return; + } + busy = true; + } + process(image); + } + + private void process(final VisionImage image) { + analyzer.process(image).ready(new SuccessCallback() { + public void onSucess(final T value) { + onFinished(new Runnable() { + public void run() { + listener.result(value, image); + } + }); + } + }).except(new SuccessCallback() { + public void onSucess(final Throwable error) { + onFinished(new Runnable() { + public void run() { + listener.error(error); + } + }); + } + }); + } + + private void onFinished(final Runnable notification) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + VisionImage next; + synchronized (VisionPipeline.this) { + if (closed) { + busy = false; + pending = null; + return; + } + next = pending; + pending = null; + busy = next != null; + } + notification.run(); + if (next != null) { + process(next); + } + } + }); + } + + public boolean isBusy() { + synchronized (this) { + return busy; + } + } + + public void close() { + synchronized (this) { + if (closed) { + return; + } + closed = true; + pending = null; + } + session.removeFrameListener(frameListener); + analyzer.close(); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionPipelineListener.java b/CodenameOne/src/com/codename1/ai/vision/VisionPipelineListener.java new file mode 100644 index 00000000000..332e018db3b --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionPipelineListener.java @@ -0,0 +1,11 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Receives live vision results and recoverable analysis failures on the EDT. */ +public interface VisionPipelineListener { + void result(T value, VisionImage source); + void error(Throwable error); +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionPoint.java b/CodenameOne/src/com/codename1/ai/vision/VisionPoint.java new file mode 100644 index 00000000000..581f19e56ef --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionPoint.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Immutable point normalized to a top-left-origin 0..1 coordinate space. */ +public final class VisionPoint { + private final float x; + private final float y; + + public VisionPoint(float x, float y) { + this.x = x; + this.y = y; + } + + public float getX() { + return x; + } + + public float getY() { + return y; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionRect.java b/CodenameOne/src/com/codename1/ai/vision/VisionRect.java new file mode 100644 index 00000000000..b708081a95a --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionRect.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +/** Immutable normalized rectangle using a top-left origin. */ +public final class VisionRect { + public static final VisionRect EMPTY = new VisionRect(0, 0, 0, 0); + + private final float x; + private final float y; + private final float width; + private final float height; + + public VisionRect(float x, float y, float width, float height) { + this.x = x; + this.y = y; + this.width = width; + this.height = height; + } + + public float getX() { + return x; + } + + public float getY() { + return y; + } + + public float getWidth() { + return width; + } + + public float getHeight() { + return height; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/package-info.java b/CodenameOne/src/com/codename1/ai/vision/package-info.java new file mode 100644 index 00000000000..131830dc56c --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/package-info.java @@ -0,0 +1,14 @@ +/** + * Vendor-neutral on-device vision APIs for still images and live camera frames. + * + *

The automatic backend uses Apple Vision/Core Image on iOS and Mac + * Catalyst and ML Kit on Android. Unsupported ports report that through + * {@code isSupported()}. Optional backends are selected with + * {@link com.codename1.ai.vision.VisionBackends}.

+ * + *

Each analyzer is a separate build-time feature. Referencing one analyzer + * causes the builder to retain only its platform adapter and native + * dependency. {@link com.codename1.ai.vision.VisionPipeline} safely copies + * callback-owned camera frames and drops stale pending frames under load.

+ */ +package com.codename1.ai.vision; diff --git a/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java b/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java index 329c97724ee..3af7f9c7ad0 100644 --- a/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java +++ b/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java @@ -7400,6 +7400,27 @@ public ARImpl createARImpl() { return null; } + /// Factory for the built-in on-device vision API. + /// + /// @hidden + public VisionImpl createVisionImpl() { + return null; + } + + /// Factory for the built-in LiteRT inference API. + /// + /// @hidden + public InferenceImpl createInferenceImpl() { + return null; + } + + /// Factory for built-in on-device language services. + /// + /// @hidden + public LanguageImpl createLanguageImpl() { + return null; + } + /// Captures a screenshot of the screen. /// /// #### Returns diff --git a/CodenameOne/src/com/codename1/impl/InferenceImpl.java b/CodenameOne/src/com/codename1/impl/InferenceImpl.java new file mode 100644 index 00000000000..25936de936a --- /dev/null +++ b/CodenameOne/src/com/codename1/impl/InferenceImpl.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl; + +import com.codename1.ai.inference.InferenceOptions; +import com.codename1.ai.inference.ModelSource; +import com.codename1.ai.inference.Tensor; +import com.codename1.ai.inference.TensorInfo; +import com.codename1.util.AsyncResource; + +/** Port contract behind the built-in LiteRT inference API. @hidden */ +public abstract class InferenceImpl { + public abstract boolean isSupported(); + public abstract AsyncResource open(ModelSource source, InferenceOptions options); + public abstract TensorInfo[] getInputs(Object handle); + public abstract TensorInfo[] getOutputs(Object handle); + public abstract AsyncResource run(Object handle, Tensor[] inputs); + public abstract void resizeInput(Object handle, String name, int[] shape); + public abstract void close(Object handle); +} diff --git a/CodenameOne/src/com/codename1/impl/LanguageImpl.java b/CodenameOne/src/com/codename1/impl/LanguageImpl.java new file mode 100644 index 00000000000..9eab6f3919d --- /dev/null +++ b/CodenameOne/src/com/codename1/impl/LanguageImpl.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl; + +import com.codename1.ai.language.LanguageCandidate; +import com.codename1.ai.language.LanguageOptions; +import com.codename1.ai.language.SmartReplyMessage; +import com.codename1.util.AsyncResource; + +/** Port contract behind the built-in on-device language APIs. @hidden */ +public abstract class LanguageImpl { + public abstract boolean isSupported(String feature, String backendId); + public abstract AsyncResource identify( + String text, String backendId, LanguageOptions options); + public abstract AsyncResource translate( + String text, String sourceLanguage, String targetLanguage, + String backendId, LanguageOptions options); + public abstract AsyncResource suggestReplies( + SmartReplyMessage[] conversation, String backendId, LanguageOptions options); + public abstract void close(); +} diff --git a/CodenameOne/src/com/codename1/impl/VisionImpl.java b/CodenameOne/src/com/codename1/impl/VisionImpl.java new file mode 100644 index 00000000000..007803dac7b --- /dev/null +++ b/CodenameOne/src/com/codename1/impl/VisionImpl.java @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl; + +import com.codename1.ai.vision.VisionFeature; +import com.codename1.ai.vision.VisionImage; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.util.AsyncResource; + +/** Port contract behind {@code com.codename1.ai.vision}. @hidden */ +public abstract class VisionImpl { + public abstract boolean isSupported(VisionFeature feature, String backendId); + public abstract AsyncResource analyze(VisionFeature feature, String backendId, + VisionImage image, VisionOptions options); + public abstract void close(); +} diff --git a/CodenameOne/src/com/codename1/ui/Display.java b/CodenameOne/src/com/codename1/ui/Display.java index 83e61ccb8e3..fe149ef78c6 100644 --- a/CodenameOne/src/com/codename1/ui/Display.java +++ b/CodenameOne/src/com/codename1/ui/Display.java @@ -6237,6 +6237,27 @@ public com.codename1.impl.ARImpl getARBackend() { return impl.createARImpl(); } + /// Creates a fresh backend for `com.codename1.ai.vision`. + /// + /// @hidden + public com.codename1.impl.VisionImpl getVisionBackend() { + return impl.createVisionImpl(); + } + + /// Creates a fresh backend for `com.codename1.ai.inference`. + /// + /// @hidden + public com.codename1.impl.InferenceImpl getInferenceBackend() { + return impl.createInferenceImpl(); + } + + /// Creates a fresh backend for `com.codename1.ai.language`. + /// + /// @hidden + public com.codename1.impl.LanguageImpl getLanguageBackend() { + return impl.createLanguageImpl(); + } + /// Indicates whether the native picker dialog is supported for the given type /// which can include one of PICKER_TYPE_DATE_AND_TIME, PICKER_TYPE_TIME, PICKER_TYPE_DATE /// diff --git a/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java b/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java index be652ba5258..94209dead70 100644 --- a/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java +++ b/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java @@ -11247,6 +11247,32 @@ public com.codename1.impl.ARImpl createARImpl() { } } + @Override + public com.codename1.impl.VisionImpl createVisionImpl() { + return (com.codename1.impl.VisionImpl) createOptionalAiBackend( + "com.codename1.impl.android.ai.AndroidVisionImpl"); + } + + @Override + public com.codename1.impl.InferenceImpl createInferenceImpl() { + return (com.codename1.impl.InferenceImpl) createOptionalAiBackend( + "com.codename1.impl.android.ai.AndroidInferenceImpl"); + } + + @Override + public com.codename1.impl.LanguageImpl createLanguageImpl() { + return (com.codename1.impl.LanguageImpl) createOptionalAiBackend( + "com.codename1.impl.android.ai.AndroidLanguageImpl"); + } + + private Object createOptionalAiBackend(String className) { + try { + return Class.forName(className).newInstance(); + } catch (Throwable t) { + return null; + } + } + // Deeper-network connectivity platform factories. Each returns a small // platform-specific class living under // com.codename1.impl.android.connectivity. Those classes are loaded diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java new file mode 100644 index 00000000000..e02b4aa9ca6 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.android.ai; + +import android.graphics.Point; + +import com.codename1.ai.vision.Barcode; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.ai.vision.VisionPoint; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.vision.barcode.BarcodeScanner; +import com.google.mlkit.vision.barcode.BarcodeScanning; +import com.google.mlkit.vision.common.InputImage; + +import java.util.List; + +/** ML Kit barcode scanning; retained only for {@code BarcodeScanner} users. */ +final class AndroidBarcodeScanningAdapter extends AndroidVisionAdapter { + @Override + @SuppressWarnings("unchecked") + void analyze(InputImage input, final int imageWidth, + final int imageHeight, VisionOptions options, + AsyncResource resource) { + final AsyncResource out = (AsyncResource) resource; + final BarcodeScanner client = BarcodeScanning.getClient(); + client.process(input).addOnSuccessListener( + new OnSuccessListener>() { + public void onSuccess( + List values) { + Barcode[] result = new Barcode[values.size()]; + for (int i = 0; i < result.length; i++) { + com.google.mlkit.vision.barcode.common.Barcode value = + values.get(i); + Point[] points = value.getCornerPoints(); + VisionPoint[] corners = points == null + ? new VisionPoint[0] : new VisionPoint[points.length]; + for (int p = 0; p < corners.length; p++) { + corners[p] = new VisionPoint( + points[p].x / (float) imageWidth, + points[p].y / (float) imageHeight); + } + result[i] = new Barcode(value.getRawValue(), + barcodeFormat(value.getFormat()), + value.getRawBytes(), + normalized(value.getBoundingBox(), + imageWidth, imageHeight), + corners, METADATA); + } + complete(out, result); + client.close(); + } + }).addOnFailureListener(failure(out, client)); + } + + private static String barcodeFormat(int format) { + switch (format) { + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_AZTEC: + return "AZTEC"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_CODABAR: + return "CODABAR"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_CODE_39: + return "CODE_39"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_CODE_93: + return "CODE_93"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_CODE_128: + return "CODE_128"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_DATA_MATRIX: + return "DATA_MATRIX"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_EAN_8: + return "EAN_8"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_EAN_13: + return "EAN_13"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_ITF: + return "ITF"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_PDF417: + return "PDF417"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_QR_CODE: + return "QR_CODE"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_UPC_A: + return "UPC_A"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_UPC_E: + return "UPC_E"; + default: + return "UNKNOWN"; + } + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java new file mode 100644 index 00000000000..99654d5a3ab --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.vision.Face; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.ai.vision.VisionPoint; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.face.FaceDetection; +import com.google.mlkit.vision.face.FaceDetector; +import com.google.mlkit.vision.face.FaceDetectorOptions; +import com.google.mlkit.vision.face.FaceLandmark; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** ML Kit face detection; retained only for {@code FaceDetector} users. */ +final class AndroidFaceDetectionAdapter extends AndroidVisionAdapter { + @Override + @SuppressWarnings("unchecked") + void analyze(InputImage input, final int imageWidth, + final int imageHeight, VisionOptions options, + AsyncResource resource) { + final AsyncResource out = (AsyncResource) resource; + FaceDetectorOptions detectorOptions = + new FaceDetectorOptions.Builder() + .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_ALL) + .setClassificationMode( + FaceDetectorOptions.CLASSIFICATION_MODE_ALL) + .enableTracking().build(); + final FaceDetector client = + FaceDetection.getClient(detectorOptions); + client.process(input).addOnSuccessListener( + new OnSuccessListener>() { + public void onSuccess( + List values) { + Face[] result = new Face[values.size()]; + for (int i = 0; i < result.length; i++) { + com.google.mlkit.vision.face.Face value = values.get(i); + Map landmarks = + new HashMap(); + addLandmark(landmarks, "leftEye", + value.getLandmark(FaceLandmark.LEFT_EYE), + imageWidth, imageHeight); + addLandmark(landmarks, "rightEye", + value.getLandmark(FaceLandmark.RIGHT_EYE), + imageWidth, imageHeight); + addLandmark(landmarks, "noseBase", + value.getLandmark(FaceLandmark.NOSE_BASE), + imageWidth, imageHeight); + addLandmark(landmarks, "mouthLeft", + value.getLandmark(FaceLandmark.MOUTH_LEFT), + imageWidth, imageHeight); + addLandmark(landmarks, "mouthRight", + value.getLandmark(FaceLandmark.MOUTH_RIGHT), + imageWidth, imageHeight); + Float smile = value.getSmilingProbability(); + Integer tracking = value.getTrackingId(); + result[i] = new Face( + normalized(value.getBoundingBox(), + imageWidth, imageHeight), + landmarks, value.getHeadEulerAngleY(), 0, + value.getHeadEulerAngleZ(), + smile == null ? -1 : smile, + tracking == null ? -1 : tracking, METADATA); + } + complete(out, result); + client.close(); + } + }).addOnFailureListener(failure(out, client)); + } + + private static void addLandmark(Map out, String name, + FaceLandmark landmark, int imageWidth, + int imageHeight) { + if (landmark != null) { + out.put(name, new VisionPoint( + landmark.getPosition().x / imageWidth, + landmark.getPosition().y / imageHeight)); + } + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java new file mode 100644 index 00000000000..91a134a1f1d --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.vision.ImageLabel; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.label.ImageLabeler; +import com.google.mlkit.vision.label.ImageLabeling; +import com.google.mlkit.vision.label.defaults.ImageLabelerOptions; + +import java.util.List; + +/** ML Kit image labeling; retained only for {@code ImageLabeler} users. */ +final class AndroidImageLabelingAdapter extends AndroidVisionAdapter { + @Override + @SuppressWarnings("unchecked") + void analyze(InputImage input, int imageWidth, int imageHeight, + VisionOptions options, AsyncResource resource) { + final AsyncResource out = + (AsyncResource) resource; + final ImageLabeler client = ImageLabeling.getClient( + new ImageLabelerOptions.Builder() + .setConfidenceThreshold(options.getMinimumConfidence()) + .build()); + client.process(input).addOnSuccessListener( + new OnSuccessListener>() { + public void onSuccess( + List values) { + ImageLabel[] result = new ImageLabel[values.size()]; + for (int i = 0; i < result.length; i++) { + com.google.mlkit.vision.label.ImageLabel value = + values.get(i); + result[i] = new ImageLabel(value.getText(), + value.getConfidence(), value.getIndex(), METADATA); + } + complete(out, result); + client.close(); + } + }).addOnFailureListener(failure(out, client)); + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java new file mode 100644 index 00000000000..c8f8ca6594a --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java @@ -0,0 +1,292 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.inference.InferenceException; +import com.codename1.ai.inference.InferenceOptions; +import com.codename1.ai.inference.ModelSource; +import com.codename1.ai.inference.Tensor; +import com.codename1.ai.inference.TensorInfo; +import com.codename1.ai.inference.TensorType; +import com.codename1.impl.InferenceImpl; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; + +import org.tensorflow.lite.DataType; +import org.tensorflow.lite.Interpreter; + +import java.io.ByteArrayOutputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.HashMap; +import java.util.Map; + +/** Android LiteRT backend. */ +public final class AndroidInferenceImpl extends InferenceImpl { + private static final class Handle { + final Interpreter interpreter; + + Handle(Interpreter interpreter) { + this.interpreter = interpreter; + } + } + + @Override + public boolean isSupported() { + return true; + } + + @Override + public AsyncResource open(final ModelSource source, + final InferenceOptions options) { + final AsyncResource out = new AsyncResource(); + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + try { + ByteBuffer model = loadModel(source); + Interpreter.Options nativeOptions = new Interpreter.Options(); + if (options.getThreads() > 0) { + nativeOptions.setNumThreads(options.getThreads()); + } + if (options.getAccelerator() == InferenceOptions.Accelerator.NPU) { + nativeOptions.setUseNNAPI(true); + } + final Handle handle = new Handle(new Interpreter(model, nativeOptions)); + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(handle); + } + }); + } catch (final Throwable error) { + fail(out, "Could not open LiteRT model", error); + } + } + }); + return out; + } + + @Override + public TensorInfo[] getInputs(Object handle) { + Interpreter interpreter = checked(handle).interpreter; + TensorInfo[] result = new TensorInfo[interpreter.getInputTensorCount()]; + for (int i = 0; i < result.length; i++) { + org.tensorflow.lite.Tensor tensor = interpreter.getInputTensor(i); + result[i] = info(tensor, i); + } + return result; + } + + @Override + public TensorInfo[] getOutputs(Object handle) { + Interpreter interpreter = checked(handle).interpreter; + TensorInfo[] result = new TensorInfo[interpreter.getOutputTensorCount()]; + for (int i = 0; i < result.length; i++) { + org.tensorflow.lite.Tensor tensor = interpreter.getOutputTensor(i); + result[i] = info(tensor, i); + } + return result; + } + + @Override + public AsyncResource run(Object handle, final Tensor[] inputs) { + final AsyncResource out = new AsyncResource(); + final Interpreter interpreter = checked(handle).interpreter; + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + try { + Object[] nativeInputs = new Object[interpreter.getInputTensorCount()]; + for (int i = 0; i < nativeInputs.length; i++) { + org.tensorflow.lite.Tensor metadata = interpreter.getInputTensor(i); + Tensor value = find(inputs, metadata.name(), i); + nativeInputs[i] = toBuffer(value, metadata.dataType()); + } + Map nativeOutputs = new HashMap(); + ByteBuffer[] outputBuffers = + new ByteBuffer[interpreter.getOutputTensorCount()]; + for (int i = 0; i < outputBuffers.length; i++) { + org.tensorflow.lite.Tensor metadata = interpreter.getOutputTensor(i); + ByteBuffer buffer = ByteBuffer.allocateDirect(metadata.numBytes()) + .order(ByteOrder.nativeOrder()); + outputBuffers[i] = buffer; + nativeOutputs.put(Integer.valueOf(i), buffer); + } + interpreter.runForMultipleInputsOutputs(nativeInputs, nativeOutputs); + final Tensor[] result = new Tensor[outputBuffers.length]; + for (int i = 0; i < result.length; i++) { + org.tensorflow.lite.Tensor metadata = interpreter.getOutputTensor(i); + result[i] = fromBuffer(metadata.name(), metadata.shape(), + metadata.dataType(), outputBuffers[i]); + } + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(result); + } + }); + } catch (final Throwable error) { + fail(out, "LiteRT inference failed", error); + } + } + }); + return out; + } + + @Override + public void resizeInput(Object handle, String name, int[] shape) { + Interpreter interpreter = checked(handle).interpreter; + int index = inputIndex(interpreter, name); + interpreter.resizeInput(index, shape); + interpreter.allocateTensors(); + } + + @Override + public void close(Object handle) { + checked(handle).interpreter.close(); + } + + private static Handle checked(Object value) { + if (!(value instanceof Handle)) { + throw new IllegalArgumentException("Invalid LiteRT session handle"); + } + return (Handle) value; + } + + private static Tensor find(Tensor[] values, String name, int index) { + if (values == null) { + throw new IllegalArgumentException("No input tensors supplied"); + } + for (int i = 0; i < values.length; i++) { + if (name != null && name.equals(values[i].getName())) { + return values[i]; + } + } + if (index < values.length) { + return values[index]; + } + throw new IllegalArgumentException("Missing model input " + name); + } + + private static int inputIndex(Interpreter interpreter, String name) { + for (int i = 0; i < interpreter.getInputTensorCount(); i++) { + if (name == null || name.equals(interpreter.getInputTensor(i).name())) { + return i; + } + } + throw new IllegalArgumentException("Unknown model input " + name); + } + + private static TensorInfo info(org.tensorflow.lite.Tensor value, int index) { + return new TensorInfo(value.name(), type(value.dataType()), value.shape(), index); + } + + private static TensorType type(DataType type) { + if (type == DataType.FLOAT32) return TensorType.FLOAT32; + if (type == DataType.INT32) return TensorType.INT32; + if (type == DataType.INT64) return TensorType.INT64; + if (type == DataType.UINT8) return TensorType.UINT8; + if (type == DataType.INT8) return TensorType.INT8; + if (type == DataType.BOOL) return TensorType.BOOL; + throw new InferenceException("Unsupported LiteRT tensor type " + type); + } + + private static ByteBuffer toBuffer(Tensor value, DataType nativeType) { + if (value.getType() != type(nativeType)) { + throw new IllegalArgumentException("Input " + value.getName() + + " type does not match the model"); + } + Object data = value.getData(); + int byteCount; + if (data instanceof float[]) byteCount = ((float[]) data).length * 4; + else if (data instanceof int[]) byteCount = ((int[]) data).length * 4; + else if (data instanceof long[]) byteCount = ((long[]) data).length * 8; + else if (data instanceof byte[]) byteCount = ((byte[]) data).length; + else throw new InferenceException("Unsupported input tensor data"); + ByteBuffer out = ByteBuffer.allocateDirect(byteCount).order(ByteOrder.nativeOrder()); + if (data instanceof float[]) out.asFloatBuffer().put((float[]) data); + else if (data instanceof int[]) out.asIntBuffer().put((int[]) data); + else if (data instanceof long[]) out.asLongBuffer().put((long[]) data); + else out.put((byte[]) data); + out.rewind(); + return out; + } + + private static Tensor fromBuffer(String name, int[] shape, DataType nativeType, + ByteBuffer value) { + value.rewind(); + TensorType type = type(nativeType); + int count = elementCount(shape); + if (type == TensorType.FLOAT32) { + float[] data = new float[count]; + value.asFloatBuffer().get(data); + return new Tensor(name, type, shape, data); + } + if (type == TensorType.INT32) { + int[] data = new int[count]; + value.asIntBuffer().get(data); + return new Tensor(name, type, shape, data); + } + if (type == TensorType.INT64) { + long[] data = new long[count]; + value.asLongBuffer().get(data); + return new Tensor(name, type, shape, data); + } + byte[] data = new byte[count]; + value.get(data); + return new Tensor(name, type, shape, data); + } + + private static int elementCount(int[] shape) { + int count = 1; + for (int i = 0; i < shape.length; i++) { + count *= shape[i]; + } + return count; + } + + private static ByteBuffer loadModel(ModelSource source) throws IOException { + byte[] bytes; + if (source.getKind() == ModelSource.BYTES) { + bytes = source.getBytes(); + } else { + InputStream input; + if (source.getKind() == ModelSource.FILE) { + input = new FileInputStream(source.getPath()); + } else { + input = Display.getInstance().getResourceAsStream( + AndroidInferenceImpl.class, source.getPath()); + if (input == null) { + throw new IOException("Model resource not found: " + source.getPath()); + } + } + try { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[16384]; + int read; + while ((read = input.read(buffer)) >= 0) { + output.write(buffer, 0, read); + } + bytes = output.toByteArray(); + } finally { + input.close(); + } + } + ByteBuffer result = ByteBuffer.allocateDirect(bytes.length) + .order(ByteOrder.nativeOrder()); + result.put(bytes); + result.rewind(); + return result; + } + + private static void fail(final AsyncResource out, final String message, + final Throwable cause) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.error(new InferenceException(message, cause)); + } + }); + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageAdapter.java new file mode 100644 index 00000000000..e08029fe522 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageAdapter.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.language.LanguageCandidate; +import com.codename1.ai.language.LanguageOptions; +import com.codename1.ai.language.SmartReplyMessage; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnFailureListener; + +/** Shared contract and completion helpers for Android language adapters. */ +abstract class AndroidLanguageAdapter { + AsyncResource identify( + String text, LanguageOptions options) { + return unsupported("Language identification is not supported"); + } + + AsyncResource translate( + String text, String sourceLanguage, String targetLanguage, + LanguageOptions options) { + return unsupported("Translation is not supported"); + } + + AsyncResource suggestReplies( + SmartReplyMessage[] conversation, LanguageOptions options) { + return unsupported("Smart Reply is not supported"); + } + + static AsyncResource unsupported(String message) { + AsyncResource out = new AsyncResource(); + out.error(new UnsupportedOperationException(message)); + return out; + } + + static void complete(final AsyncResource out, final T value) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(value); + } + }); + } + + static OnFailureListener failure(final AsyncResource out, + final AutoCloseable client) { + return new OnFailureListener() { + public void onFailure(final Exception error) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.error(error); + } + }); + try { + client.close(); + } catch (Exception ignored) { + } + } + }; + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageIdAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageIdAdapter.java new file mode 100644 index 00000000000..1a6ed6ad163 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageIdAdapter.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.language.LanguageCandidate; +import com.codename1.ai.language.LanguageOptions; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.nl.languageid.IdentifiedLanguage; +import com.google.mlkit.nl.languageid.LanguageIdentification; +import com.google.mlkit.nl.languageid.LanguageIdentificationOptions; +import com.google.mlkit.nl.languageid.LanguageIdentifier; + +import java.util.List; + +/** ML Kit language ID; retained only for {@code LanguageIdentifier} users. */ +final class AndroidLanguageIdAdapter extends AndroidLanguageAdapter { + @Override + AsyncResource identify( + String text, LanguageOptions options) { + final AsyncResource out = + new AsyncResource(); + final LanguageIdentifier client = LanguageIdentification.getClient( + new LanguageIdentificationOptions.Builder() + .setConfidenceThreshold(options.getMinimumConfidence()) + .build()); + client.identifyPossibleLanguages(text).addOnSuccessListener( + new OnSuccessListener>() { + public void onSuccess(List values) { + LanguageCandidate[] result = + new LanguageCandidate[values.size()]; + for (int i = 0; i < result.length; i++) { + IdentifiedLanguage value = values.get(i); + result[i] = new LanguageCandidate( + value.getLanguageTag(), value.getConfidence()); + } + complete(out, result); + client.close(); + } + }).addOnFailureListener(failure(out, client)); + return out; + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageImpl.java new file mode 100644 index 00000000000..02486ceeb2f --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.language.LanguageCandidate; +import com.codename1.ai.language.LanguageOptions; +import com.codename1.ai.language.SmartReplyMessage; +import com.codename1.impl.LanguageImpl; +import com.codename1.util.AsyncResource; + +/** + * Android language-service dispatcher. The builder retains only the adapter + * source and ML Kit artifact for each language API referenced by the app. + */ +public final class AndroidLanguageImpl extends LanguageImpl { + private volatile boolean closed; + + @Override + public boolean isSupported(String feature, String backendId) { + return !closed + && ("auto".equals(backendId) || "ml-kit".equals(backendId)) + && adapter(feature) != null; + } + + @Override + public AsyncResource identify( + String text, String backendId, LanguageOptions options) { + AndroidLanguageAdapter adapter = adapter("language-id"); + return adapter == null + ? AndroidLanguageAdapter.unsupported( + "Language identification is not included in this build") + : adapter.identify(text, options); + } + + @Override + public AsyncResource translate( + String text, String sourceLanguage, String targetLanguage, + String backendId, LanguageOptions options) { + AndroidLanguageAdapter adapter = adapter("translation"); + return adapter == null + ? AndroidLanguageAdapter.unsupported( + "Translation is not included in this build") + : adapter.translate(text, sourceLanguage, targetLanguage, + options); + } + + @Override + public AsyncResource suggestReplies( + SmartReplyMessage[] conversation, String backendId, + LanguageOptions options) { + AndroidLanguageAdapter adapter = adapter("smart-reply"); + return adapter == null + ? AndroidLanguageAdapter.unsupported( + "Smart Reply is not included in this build") + : adapter.suggestReplies(conversation, options); + } + + private static AndroidLanguageAdapter adapter(String feature) { + String className; + if ("language-id".equals(feature)) { + className = + "com.codename1.impl.android.ai.AndroidLanguageIdAdapter"; + } else if ("translation".equals(feature)) { + className = + "com.codename1.impl.android.ai.AndroidTranslationAdapter"; + } else if ("smart-reply".equals(feature)) { + className = + "com.codename1.impl.android.ai.AndroidSmartReplyAdapter"; + } else { + return null; + } + try { + return (AndroidLanguageAdapter) + Class.forName(className).newInstance(); + } catch (Throwable ignored) { + return null; + } + } + + @Override + public void close() { + closed = true; + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java new file mode 100644 index 00000000000..3cd1de05ed6 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.vision.Pose; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.ai.vision.VisionPoint; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.pose.PoseDetection; +import com.google.mlkit.vision.pose.PoseDetector; +import com.google.mlkit.vision.pose.PoseLandmark; +import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions; + +import java.util.List; + +/** ML Kit pose detection; retained only for {@code PoseDetector} users. */ +final class AndroidPoseDetectionAdapter extends AndroidVisionAdapter { + @Override + @SuppressWarnings("unchecked") + void analyze(InputImage input, final int imageWidth, + final int imageHeight, VisionOptions options, + AsyncResource resource) { + final AsyncResource out = (AsyncResource) resource; + final PoseDetector client = PoseDetection.getClient( + new PoseDetectorOptions.Builder() + .setDetectorMode(PoseDetectorOptions.SINGLE_IMAGE_MODE) + .build()); + client.process(input).addOnSuccessListener( + new OnSuccessListener() { + public void onSuccess(com.google.mlkit.vision.pose.Pose value) { + List source = value.getAllPoseLandmarks(); + Pose.Landmark[] landmarks = new Pose.Landmark[source.size()]; + for (int i = 0; i < landmarks.length; i++) { + PoseLandmark point = source.get(i); + landmarks[i] = new Pose.Landmark( + String.valueOf(point.getLandmarkType()), + new VisionPoint( + point.getPosition().x / imageWidth, + point.getPosition().y / imageHeight), + point.getInFrameLikelihood()); + } + complete(out, new Pose(landmarks, METADATA)); + client.close(); + } + }).addOnFailureListener(failure(out, client)); + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidSelfieSegmentationAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSelfieSegmentationAdapter.java new file mode 100644 index 00000000000..decfc4265e1 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSelfieSegmentationAdapter.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.vision.SegmentationMask; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.segmentation.Segmentation; +import com.google.mlkit.vision.segmentation.Segmenter; +import com.google.mlkit.vision.segmentation.selfie.SelfieSegmenterOptions; + +import java.nio.ByteBuffer; + +/** + * ML Kit selfie segmentation; retained only for {@code SelfieSegmenter} users. + */ +final class AndroidSelfieSegmentationAdapter extends AndroidVisionAdapter { + @Override + @SuppressWarnings("unchecked") + void analyze(InputImage input, int imageWidth, int imageHeight, + VisionOptions options, AsyncResource resource) { + final AsyncResource out = + (AsyncResource) resource; + final Segmenter client = Segmentation.getClient( + new SelfieSegmenterOptions.Builder() + .setDetectorMode( + SelfieSegmenterOptions.SINGLE_IMAGE_MODE) + .build()); + client.process(input).addOnSuccessListener( + new OnSuccessListener< + com.google.mlkit.vision.segmentation.SegmentationMask>() { + public void onSuccess( + com.google.mlkit.vision.segmentation.SegmentationMask value) { + ByteBuffer buffer = value.getBuffer(); + buffer.rewind(); + float[] confidence = + new float[value.getWidth() * value.getHeight()]; + buffer.asFloatBuffer().get(confidence); + complete(out, new SegmentationMask( + value.getWidth(), value.getHeight(), confidence, + METADATA)); + client.close(); + } + }).addOnFailureListener(failure(out, client)); + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java new file mode 100644 index 00000000000..728236c0a0b --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.language.LanguageOptions; +import com.codename1.ai.language.SmartReplyMessage; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.nl.smartreply.SmartReply; +import com.google.mlkit.nl.smartreply.SmartReplyGenerator; +import com.google.mlkit.nl.smartreply.SmartReplySuggestion; +import com.google.mlkit.nl.smartreply.SmartReplySuggestionResult; +import com.google.mlkit.nl.smartreply.TextMessage; + +import java.util.ArrayList; +import java.util.List; + +/** ML Kit Smart Reply; retained only for {@code SmartReply} users. */ +final class AndroidSmartReplyAdapter extends AndroidLanguageAdapter { + @Override + AsyncResource suggestReplies( + SmartReplyMessage[] conversation, LanguageOptions options) { + final AsyncResource out = new AsyncResource(); + final SmartReplyGenerator client = SmartReply.getClient(); + List messages = new ArrayList(); + for (int i = 0; i < conversation.length; i++) { + SmartReplyMessage message = conversation[i]; + messages.add(message.isLocalUser() + ? TextMessage.createForLocalUser( + message.getText(), message.getTimestampMillis()) + : TextMessage.createForRemoteUser( + message.getText(), message.getTimestampMillis(), + message.getParticipantId())); + } + client.suggestReplies(messages).addOnSuccessListener( + new OnSuccessListener() { + public void onSuccess(SmartReplySuggestionResult value) { + List suggestions = + value.getSuggestions(); + String[] result = new String[suggestions.size()]; + for (int i = 0; i < result.length; i++) { + result[i] = suggestions.get(i).getText(); + } + complete(out, result); + client.close(); + } + }).addOnFailureListener(failure(out, client)); + return out; + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java new file mode 100644 index 00000000000..ebfc61fdce3 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.vision.TextRecognitionResult; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.text.Text; +import com.google.mlkit.vision.text.TextRecognition; +import com.google.mlkit.vision.text.TextRecognizer; +import com.google.mlkit.vision.text.latin.TextRecognizerOptions; + +import java.util.List; + +/** ML Kit text recognition; retained only for {@code TextRecognizer} users. */ +final class AndroidTextRecognitionAdapter extends AndroidVisionAdapter { + @Override + @SuppressWarnings("unchecked") + void analyze(InputImage input, final int imageWidth, + final int imageHeight, VisionOptions options, + AsyncResource resource) { + final AsyncResource out = + (AsyncResource) resource; + final TextRecognizer client = TextRecognition.getClient( + TextRecognizerOptions.DEFAULT_OPTIONS); + client.process(input).addOnSuccessListener(new OnSuccessListener() { + public void onSuccess(Text text) { + List source = text.getTextBlocks(); + TextRecognitionResult.TextBlock[] blocks = + new TextRecognitionResult.TextBlock[source.size()]; + for (int i = 0; i < blocks.length; i++) { + Text.TextBlock block = source.get(i); + blocks[i] = new TextRecognitionResult.TextBlock( + block.getText(), 1f, + normalized(block.getBoundingBox(), + imageWidth, imageHeight), null); + } + complete(out, new TextRecognitionResult( + text.getText(), blocks, METADATA)); + client.close(); + } + }).addOnFailureListener(failure(out, client)); + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java new file mode 100644 index 00000000000..ec3f74818f3 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.language.LanguageOptions; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.Continuation; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.android.gms.tasks.Task; +import com.google.mlkit.nl.translate.Translation; +import com.google.mlkit.nl.translate.Translator; +import com.google.mlkit.nl.translate.TranslatorOptions; + +/** ML Kit translation; retained only for {@code Translator} users. */ +final class AndroidTranslationAdapter extends AndroidLanguageAdapter { + @Override + AsyncResource translate( + final String text, String sourceLanguage, String targetLanguage, + LanguageOptions options) { + final AsyncResource out = new AsyncResource(); + TranslatorOptions translatorOptions = + new TranslatorOptions.Builder() + .setSourceLanguage(sourceLanguage) + .setTargetLanguage(targetLanguage).build(); + final Translator client = + Translation.getClient(translatorOptions); + client.downloadModelIfNeeded().continueWithTask( + new Continuation>() { + public Task then(Task ignored) { + return client.translate(text); + } + }).addOnSuccessListener(new OnSuccessListener() { + public void onSuccess(String value) { + complete(out, value); + client.close(); + } + }).addOnFailureListener(failure(out, client)); + return out; + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionAdapter.java new file mode 100644 index 00000000000..8d2fc880523 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionAdapter.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.android.ai; + +import android.graphics.Rect; + +import com.codename1.ai.vision.VisionException; +import com.codename1.ai.vision.VisionMetadata; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.ai.vision.VisionRect; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnFailureListener; +import com.google.mlkit.vision.common.InputImage; + +/** Shared contract and result helpers for feature-scoped Android adapters. */ +abstract class AndroidVisionAdapter { + static final VisionMetadata METADATA = new VisionMetadata("ml-kit"); + + abstract void analyze(InputImage input, int imageWidth, int imageHeight, + VisionOptions options, AsyncResource out); + + static VisionRect normalized(Rect rect, int imageWidth, int imageHeight) { + if (rect == null) { + return VisionRect.EMPTY; + } + return new VisionRect(rect.left / (float) imageWidth, + rect.top / (float) imageHeight, + rect.width() / (float) imageWidth, + rect.height() / (float) imageHeight); + } + + static void complete(final AsyncResource out, final T value) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(value); + } + }); + } + + static OnFailureListener failure(final AsyncResource out, + final AutoCloseable client) { + return new OnFailureListener() { + public void onFailure(final Exception error) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.error(new VisionException( + VisionException.BACKEND_ERROR, + error.getMessage(), error)); + } + }); + try { + client.close(); + } catch (Exception ignored) { + } + } + }; + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java new file mode 100644 index 00000000000..4b41fb1078a --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.android.ai; + +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; + +import com.codename1.ai.vision.VisionException; +import com.codename1.ai.vision.VisionFeature; +import com.codename1.ai.vision.VisionImage; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.camera.FrameFormat; +import com.codename1.impl.VisionImpl; +import com.codename1.util.AsyncResource; +import com.google.mlkit.vision.common.InputImage; + +/** + * Android vision dispatcher. Feature implementations live in separate source + * files so the Android builder can retain one adapter and one ML Kit artifact + * per analyzer used by the application. + */ +public final class AndroidVisionImpl extends VisionImpl { + private volatile boolean closed; + + @Override + public boolean isSupported(VisionFeature feature, String backendId) { + return !closed + && ("auto".equals(backendId) || "ml-kit".equals(backendId)) + && adapterClass(feature) != null + && isClassPresent(adapterClass(feature)); + } + + @Override + @SuppressWarnings({"rawtypes", "unchecked"}) + public AsyncResource analyze(VisionFeature feature, String backendId, + VisionImage image, VisionOptions options) { + AsyncResource out = new AsyncResource(); + if (!isSupported(feature, backendId)) { + out.error(new VisionException(VisionException.UNSUPPORTED, + feature + " is not included in this Android build")); + return out; + } + DecodedInput decoded = decode(image); + if (decoded == null) { + out.error(new VisionException(VisionException.INVALID_IMAGE, + "Vision input is not valid JPEG, PNG, NV21, or RGBA8888 data")); + return out; + } + try { + AndroidVisionAdapter adapter = (AndroidVisionAdapter) + Class.forName(adapterClass(feature)).newInstance(); + adapter.analyze(decoded.input, decoded.width, decoded.height, + options, (AsyncResource) out); + } catch (Throwable error) { + out.error(new VisionException(VisionException.BACKEND_ERROR, + "Could not start the Android " + feature + " adapter", + error)); + } + return out; + } + + private static DecodedInput decode(VisionImage image) { + byte[] encoded = image.getEncodedBytes(); + if (encoded != null) { + Bitmap bitmap = BitmapFactory.decodeByteArray( + encoded, 0, encoded.length); + return bitmap == null ? null : new DecodedInput( + InputImage.fromBitmap(bitmap, image.getRotationDegrees()), + bitmap.getWidth(), bitmap.getHeight()); + } + byte[] pixels = image.getPixels(); + int width = image.getWidth(); + int height = image.getHeight(); + if (pixels == null || width <= 0 || height <= 0) { + return null; + } + if (image.getFormat() == FrameFormat.NV21) { + if (pixels.length < width * height * 3 / 2) { + return null; + } + return new DecodedInput(InputImage.fromByteArray( + pixels, width, height, image.getRotationDegrees(), + InputImage.IMAGE_FORMAT_NV21), width, height); + } + if (image.getFormat() == FrameFormat.RGBA8888) { + if (pixels.length < width * height * 4) { + return null; + } + int[] argb = new int[width * height]; + for (int i = 0, p = 0; i < argb.length; i++, p += 4) { + argb[i] = ((pixels[p + 3] & 255) << 24) + | ((pixels[p] & 255) << 16) + | ((pixels[p + 1] & 255) << 8) + | (pixels[p + 2] & 255); + } + Bitmap bitmap = Bitmap.createBitmap( + argb, width, height, Bitmap.Config.ARGB_8888); + return new DecodedInput(InputImage.fromBitmap( + bitmap, image.getRotationDegrees()), width, height); + } + Bitmap bitmap = BitmapFactory.decodeByteArray( + pixels, 0, pixels.length); + return bitmap == null ? null : new DecodedInput( + InputImage.fromBitmap(bitmap, image.getRotationDegrees()), + bitmap.getWidth(), bitmap.getHeight()); + } + + private static final class DecodedInput { + final InputImage input; + final int width; + final int height; + + DecodedInput(InputImage input, int width, int height) { + this.input = input; + this.width = width; + this.height = height; + } + } + + private static boolean isClassPresent(String className) { + try { + Class.forName(className); + return true; + } catch (Throwable ignored) { + return false; + } + } + + private static String adapterClass(VisionFeature feature) { + if (feature == null) { + return null; + } + switch (feature) { + case TEXT_RECOGNITION: + return "com.codename1.impl.android.ai.AndroidTextRecognitionAdapter"; + case BARCODE_SCANNING: + return "com.codename1.impl.android.ai.AndroidBarcodeScanningAdapter"; + case FACE_DETECTION: + return "com.codename1.impl.android.ai.AndroidFaceDetectionAdapter"; + case IMAGE_LABELING: + return "com.codename1.impl.android.ai.AndroidImageLabelingAdapter"; + case POSE_DETECTION: + return "com.codename1.impl.android.ai.AndroidPoseDetectionAdapter"; + case SELFIE_SEGMENTATION: + return "com.codename1.impl.android.ai.AndroidSelfieSegmentationAdapter"; + default: + // ML Kit's document scanner owns an Activity camera flow; it + // cannot implement the still-image VisionAnalyzer contract. + return null; + } + } + + @Override + public void close() { + closed = true; + } +} diff --git a/Ports/iOSPort/nativeSources/CN1Inference.m b/Ports/iOSPort/nativeSources/CN1Inference.m new file mode 100644 index 00000000000..c33d7baf1c0 --- /dev/null +++ b/Ports/iOSPort/nativeSources/CN1Inference.m @@ -0,0 +1,336 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +#import "CodenameOne_GLViewController.h" +#import "xmlvm.h" + +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV +#import +#import "java_lang_String.h" + +#if __has_include() +#import +#define CN1_HAS_LITERT 1 +#endif + +#if defined(CN1_HAS_LITERT) +@interface CN1InferenceHandle : NSObject +@property(nonatomic, strong) TFLInterpreter *interpreter; +@property(nonatomic, strong) TFLDelegate *delegate; +@property(nonatomic, copy) NSString *modelPath; +@end + +@implementation CN1InferenceHandle +@end + +static NSMutableDictionary *cn1InferenceHandles; +static int cn1InferenceNextHandle = 1; + +static void cn1InferenceEnsureHandles(void) { + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + cn1InferenceHandles = [NSMutableDictionary dictionary]; + }); +} + +static NSString *cn1InferenceJSON(NSDictionary *value) { + NSError *error = nil; + NSData *data = [NSJSONSerialization dataWithJSONObject:value options:0 + error:&error]; + if (data == nil) { + return [NSString stringWithFormat:@"{\"error\":\"%@\"}", + error.localizedDescription ?: @"Could not encode inference result"]; + } + return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]; +} + +static NSString *cn1InferenceError(NSError *error, NSString *fallback) { + return cn1InferenceJSON(@{ + @"error": error.localizedDescription ?: fallback + }); +} + +static NSString *cn1InferenceType(TFLTensorDataType type) { + switch (type) { + case TFLTensorDataTypeFloat32: return @"FLOAT32"; + case TFLTensorDataTypeInt32: return @"INT32"; + case TFLTensorDataTypeUInt8: return @"UINT8"; + case TFLTensorDataTypeInt64: return @"INT64"; + case TFLTensorDataTypeBool: return @"BOOL"; + case TFLTensorDataTypeInt8: return @"INT8"; + default: return nil; + } +} + +static CN1InferenceHandle *cn1InferenceHandle(int handle) { + cn1InferenceEnsureHandles(); + @synchronized (cn1InferenceHandles) { + return cn1InferenceHandles[@(handle)]; + } +} + +static NSString *cn1InferenceOpen(NSData *model, int threads, int accelerator, + BOOL allowFallback) { + NSString *path = [NSTemporaryDirectory() stringByAppendingPathComponent: + [NSString stringWithFormat:@"cn1-litert-%@.tflite", + NSUUID.UUID.UUIDString]]; + if (![model writeToFile:path atomically:YES]) { + return cn1InferenceJSON(@{@"error": @"Could not stage LiteRT model"}); + } + + TFLInterpreterOptions *options = [[TFLInterpreterOptions alloc] init]; + if (threads > 0) options.numberOfThreads = (NSUInteger)threads; + NSMutableArray *delegates = [NSMutableArray array]; + TFLDelegate *delegate = nil; + + // InferenceOptions.Accelerator ordinal: + // AUTO=0, CPU=1, GPU=2, NPU=3, CORE_ML=4. + if (accelerator == 0 || accelerator == 3 || accelerator == 4) { +#if __has_include() + delegate = [[TFLCoreMLDelegate alloc] init]; + if (delegate != nil) [delegates addObject:delegate]; +#endif + if (delegate == nil && !allowFallback && accelerator != 0) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + return cn1InferenceJSON(@{ + @"error": @"Core ML delegate is unavailable on this device" + }); + } + } else if (accelerator == 2 && !allowFallback) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + return cn1InferenceJSON(@{ + @"error": @"The iOS backend does not provide a GPU delegate" + }); + } + + NSError *error = nil; + TFLInterpreter *interpreter = [[TFLInterpreter alloc] + initWithModelPath:path options:options delegates:delegates error:&error]; + if (interpreter == nil || error != nil) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + return cn1InferenceError(error, @"Could not create LiteRT interpreter"); + } + if (![interpreter allocateTensorsWithError:&error]) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + return cn1InferenceError(error, @"Could not allocate LiteRT tensors"); + } + + CN1InferenceHandle *value = [[CN1InferenceHandle alloc] init]; + value.interpreter = interpreter; + value.delegate = delegate; + value.modelPath = path; + cn1InferenceEnsureHandles(); + int handle; + @synchronized (cn1InferenceHandles) { + handle = cn1InferenceNextHandle++; + cn1InferenceHandles[@(handle)] = value; + } + return cn1InferenceJSON(@{@"handle": @(handle)}); +} + +static NSString *cn1InferenceMetadata(int handle, BOOL outputs) { + CN1InferenceHandle *value = cn1InferenceHandle(handle); + if (value == nil) { + return cn1InferenceJSON(@{@"error": @"Unknown LiteRT handle"}); + } + TFLInterpreter *interpreter = value.interpreter; + NSUInteger count = outputs ? interpreter.outputTensorCount + : interpreter.inputTensorCount; + NSMutableArray *items = [NSMutableArray array]; + for (NSUInteger i = 0; i < count; i++) { + NSError *error = nil; + TFLTensor *tensor = outputs + ? [interpreter outputTensorAtIndex:i error:&error] + : [interpreter inputTensorAtIndex:i error:&error]; + if (tensor == nil || error != nil) { + return cn1InferenceError(error, @"Could not read LiteRT tensor"); + } + NSString *type = cn1InferenceType(tensor.dataType); + if (type == nil) { + return cn1InferenceJSON(@{ + @"error": [NSString stringWithFormat: + @"Unsupported LiteRT tensor type %lu", + (unsigned long)tensor.dataType] + }); + } + NSArray *shape = [tensor shapeWithError:&error]; + if (shape == nil || error != nil) { + return cn1InferenceError(error, @"Could not read LiteRT tensor shape"); + } + [items addObject:@{ + @"index": @(i), + @"name": tensor.name ?: @"", + @"type": type, + @"shape": shape + }]; + } + return cn1InferenceJSON(@{@"items": items}); +} + +static NSString *cn1InferenceRun(int handle, NSString *inputsJSON) { + CN1InferenceHandle *value = cn1InferenceHandle(handle); + if (value == nil) { + return cn1InferenceJSON(@{@"error": @"Unknown LiteRT handle"}); + } + NSError *error = nil; + NSDictionary *root = [NSJSONSerialization JSONObjectWithData: + [inputsJSON dataUsingEncoding:NSUTF8StringEncoding] + options:0 error:&error]; + if (root == nil || error != nil) { + return cn1InferenceError(error, @"Invalid LiteRT input JSON"); + } + TFLInterpreter *interpreter = value.interpreter; + for (NSDictionary *item in root[@"items"] ?: @[]) { + NSUInteger index = [item[@"index"] unsignedIntegerValue]; + if (index >= interpreter.inputTensorCount) { + return cn1InferenceJSON(@{@"error": @"Invalid LiteRT input index"}); + } + TFLTensor *tensor = [interpreter inputTensorAtIndex:index error:&error]; + if (tensor == nil || error != nil) { + return cn1InferenceError(error, @"Could not read LiteRT input"); + } + NSString *expectedType = cn1InferenceType(tensor.dataType); + if (![expectedType isEqualToString:item[@"type"]]) { + return cn1InferenceJSON(@{@"error": @"LiteRT input type mismatch"}); + } + NSData *data = [[NSData alloc] initWithBase64EncodedString:item[@"data"] + options:0]; + if (data == nil || ![tensor copyData:data error:&error]) { + return cn1InferenceError(error, @"Could not copy LiteRT input"); + } + } + if (![interpreter invokeWithError:&error]) { + return cn1InferenceError(error, @"LiteRT invocation failed"); + } + NSMutableArray *items = [NSMutableArray array]; + for (NSUInteger i = 0; i < interpreter.outputTensorCount; i++) { + TFLTensor *tensor = [interpreter outputTensorAtIndex:i error:&error]; + if (tensor == nil || error != nil) { + return cn1InferenceError(error, @"Could not read LiteRT output"); + } + NSString *type = cn1InferenceType(tensor.dataType); + if (type == nil) { + return cn1InferenceJSON(@{@"error": @"Unsupported output tensor type"}); + } + NSArray *shape = [tensor shapeWithError:&error]; + NSData *data = [tensor dataWithError:&error]; + if (shape == nil || data == nil || error != nil) { + return cn1InferenceError(error, @"Could not copy LiteRT output"); + } + [items addObject:@{ + @"index": @(i), + @"name": tensor.name ?: @"", + @"type": type, + @"shape": shape, + @"data": [data base64EncodedStringWithOptions:0] + }]; + } + return cn1InferenceJSON(@{@"items": items}); +} + +static NSString *cn1InferenceResize(int handle, int index, + NSArray *shape) { + CN1InferenceHandle *value = cn1InferenceHandle(handle); + if (value == nil) { + return cn1InferenceJSON(@{@"error": @"Unknown LiteRT handle"}); + } + NSError *error = nil; + if (![value.interpreter resizeInputTensorAtIndex:(NSUInteger)index + toShape:shape error:&error]) { + return cn1InferenceError(error, @"Could not resize LiteRT input"); + } + if (![value.interpreter allocateTensorsWithError:&error]) { + return cn1InferenceError(error, @"Could not reallocate LiteRT tensors"); + } + return cn1InferenceJSON(@{@"ok": @(YES)}); +} +#endif +#endif + +JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_cn1InferenceIsSupported___R_boolean( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + return 1; +#else + return 0; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceOpen___byte_1ARRAY_int_int_boolean_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT model, + JAVA_INT threads, JAVA_INT accelerator, JAVA_BOOLEAN allowFallback) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + if (model == JAVA_NULL) { + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + @"{\"error\":\"Model data is null\"}"); + } + JAVA_ARRAY bytes = (JAVA_ARRAY)model; + NSData *data = [NSData dataWithBytes:bytes->data + length:(NSUInteger)bytes->length]; + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1InferenceOpen(data, threads, accelerator, allowFallback)); +#else + return JAVA_NULL; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceMetadata___int_boolean_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT handle, + JAVA_BOOLEAN outputs) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1InferenceMetadata(handle, outputs)); +#else + return JAVA_NULL; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceRun___int_java_lang_String_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT handle, + JAVA_OBJECT inputsJSON) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1InferenceRun(handle, + toNSString(CN1_THREAD_GET_STATE_PASS_ARG inputsJSON))); +#else + return JAVA_NULL; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceResize___int_int_int_1ARRAY_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT handle, + JAVA_INT index, JAVA_OBJECT shape) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + if (shape == JAVA_NULL) { + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + @"{\"error\":\"Shape is null\"}"); + } + JAVA_ARRAY array = (JAVA_ARRAY)shape; + JAVA_ARRAY_INT *values = (JAVA_ARRAY_INT *)array->data; + NSMutableArray *nativeShape = + [NSMutableArray arrayWithCapacity:(NSUInteger)array->length]; + for (int i = 0; i < array->length; i++) { + [nativeShape addObject:@(values[i])]; + } + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1InferenceResize(handle, index, nativeShape)); +#else + return JAVA_NULL; +#endif +} + +void com_codename1_impl_ios_IOSNative_cn1InferenceClose___int( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT handle) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + cn1InferenceEnsureHandles(); + CN1InferenceHandle *value; + @synchronized (cn1InferenceHandles) { + value = cn1InferenceHandles[@(handle)]; + [cn1InferenceHandles removeObjectForKey:@(handle)]; + } + if (value.modelPath != nil) { + [[NSFileManager defaultManager] removeItemAtPath:value.modelPath error:nil]; + } +#endif +} diff --git a/Ports/iOSPort/nativeSources/CN1Language.m b/Ports/iOSPort/nativeSources/CN1Language.m new file mode 100644 index 00000000000..7bd07f87b90 --- /dev/null +++ b/Ports/iOSPort/nativeSources/CN1Language.m @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +#import "CodenameOne_GLViewController.h" +#import "xmlvm.h" + +#if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV +#import +#import "java_lang_String.h" + +#if __has_include() +#import +#define CN1_HAS_MLKIT_LANGUAGE_ID 1 +#endif +#if __has_include() +#import +#import +#define CN1_HAS_MLKIT_TRANSLATE 1 +#endif +#if __has_include() +#import +#define CN1_HAS_MLKIT_SMART_REPLY 1 +#endif + +static NSString *cn1LanguageJSON(NSDictionary *value) { + NSError *error = nil; + NSData *data = [NSJSONSerialization dataWithJSONObject:value options:0 + error:&error]; + if (data == nil) { + return [NSString stringWithFormat:@"{\"error\":\"%@\"}", + error.localizedDescription ?: @"Could not encode language result"]; + } + return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]; +} + +static NSString *cn1LanguageError(NSError *error, NSString *fallback) { + return cn1LanguageJSON(@{ + @"error": error.localizedDescription ?: fallback + }); +} + +static NSString *cn1IdentifyLanguage(NSString *text, float threshold) { +#if defined(CN1_HAS_MLKIT_LANGUAGE_ID) + MLKLanguageIdentificationOptions *options = + [[MLKLanguageIdentificationOptions alloc] + initWithConfidenceThreshold:threshold]; + MLKLanguageIdentification *identifier = + [MLKLanguageIdentification languageIdentificationWithOptions:options]; + __block NSArray *languages = nil; + __block NSError *requestError = nil; + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + [identifier identifyPossibleLanguagesForText:text ?: @"" + completion:^(NSArray *result, + NSError *error) { + languages = result; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) { + return cn1LanguageError(requestError, @"Language identification failed"); + } + NSMutableArray *items = [NSMutableArray array]; + for (MLKIdentifiedLanguage *language in languages ?: @[]) { + [items addObject:@{ + @"language": language.languageTag ?: @"und", + @"confidence": @(language.confidence) + }]; + } + return cn1LanguageJSON(@{@"items": items}); +#else + return cn1LanguageJSON(@{@"error": @"ML Kit Language ID is not linked"}); +#endif +} + +static NSString *cn1TranslateLanguage(NSString *text, NSString *source, + NSString *target) { +#if defined(CN1_HAS_MLKIT_TRANSLATE) + MLKTranslateLanguage sourceLanguage = source.lowercaseString; + MLKTranslateLanguage targetLanguage = target.lowercaseString; + NSSet *supported = MLKTranslateAllLanguages(); + if (sourceLanguage == nil || targetLanguage == nil || + ![supported containsObject:sourceLanguage] || + ![supported containsObject:targetLanguage]) { + return cn1LanguageJSON(@{@"error": @"Unsupported translation language"}); + } + MLKTranslatorOptions *options = [[MLKTranslatorOptions alloc] + initWithSourceLanguage:sourceLanguage targetLanguage:targetLanguage]; + MLKTranslator *translator = [MLKTranslator translatorWithOptions:options]; + MLKModelDownloadConditions *conditions = [[MLKModelDownloadConditions alloc] + initWithAllowsCellularAccess:YES allowsBackgroundDownloading:YES]; + __block NSString *translation = nil; + __block NSError *requestError = nil; + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + [translator downloadModelIfNeededWithConditions:conditions + completion:^(NSError *error) { + if (error != nil) { + requestError = error; + dispatch_semaphore_signal(semaphore); + return; + } + [translator translateText:text ?: @"" + completion:^(NSString *result, NSError *error) { + translation = result; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) { + return cn1LanguageError(requestError, @"Translation failed"); + } + return cn1LanguageJSON(@{@"text": translation ?: @""}); +#else + return cn1LanguageJSON(@{@"error": @"ML Kit Translate is not linked"}); +#endif +} + +static NSString *cn1SmartReply(NSString *conversationJSON) { +#if defined(CN1_HAS_MLKIT_SMART_REPLY) + NSError *jsonError = nil; + NSDictionary *root = [NSJSONSerialization JSONObjectWithData: + [conversationJSON dataUsingEncoding:NSUTF8StringEncoding] + options:0 error:&jsonError]; + if (jsonError != nil || ![root isKindOfClass:[NSDictionary class]]) { + return cn1LanguageError(jsonError, @"Invalid smart-reply conversation"); + } + NSMutableArray *messages = [NSMutableArray array]; + for (NSDictionary *value in root[@"items"] ?: @[]) { + if (![value isKindOfClass:[NSDictionary class]]) continue; + MLKTextMessage *message = [[MLKTextMessage alloc] + initWithText:value[@"text"] ?: @"" + timestamp:[value[@"timestamp"] doubleValue] + userID:value[@"participant"] ?: @"remote" + isLocalUser:[value[@"local"] boolValue]]; + [messages addObject:message]; + } + __block MLKSmartReplySuggestionResult *suggestions = nil; + __block NSError *requestError = nil; + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + [[MLKSmartReply smartReply] suggestRepliesForMessages:messages + completion:^(MLKSmartReplySuggestionResult *result, + NSError *error) { + suggestions = result; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) { + return cn1LanguageError(requestError, @"Smart Reply failed"); + } + NSMutableArray *items = [NSMutableArray array]; + for (MLKSmartReplySuggestion *suggestion in suggestions.suggestions ?: @[]) { + if (suggestion.text != nil) [items addObject:suggestion.text]; + } + return cn1LanguageJSON(@{@"items": items}); +#else + return cn1LanguageJSON(@{@"error": @"ML Kit Smart Reply is not linked"}); +#endif +} +#endif + +JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_cn1LanguageIsSupported___int_R_boolean( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT feature) { +#if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV + switch (feature) { + case 0: +#if defined(CN1_HAS_MLKIT_LANGUAGE_ID) + return 1; +#else + return 0; +#endif + case 1: +#if defined(CN1_HAS_MLKIT_TRANSLATE) + return 1; +#else + return 0; +#endif + case 2: +#if defined(CN1_HAS_MLKIT_SMART_REPLY) + return 1; +#else + return 0; +#endif + default: + return 0; + } +#else + return 0; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1LanguageIdentify___java_lang_String_float_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT text, + JAVA_FLOAT minimumConfidence) { +#if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1IdentifyLanguage( + toNSString(CN1_THREAD_GET_STATE_PASS_ARG text), + minimumConfidence)); +#else + return JAVA_NULL; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1LanguageTranslate___java_lang_String_java_lang_String_java_lang_String_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT text, + JAVA_OBJECT sourceLanguage, JAVA_OBJECT targetLanguage) { +#if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1TranslateLanguage( + toNSString(CN1_THREAD_GET_STATE_PASS_ARG text), + toNSString(CN1_THREAD_GET_STATE_PASS_ARG sourceLanguage), + toNSString(CN1_THREAD_GET_STATE_PASS_ARG targetLanguage))); +#else + return JAVA_NULL; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1LanguageSmartReply___java_lang_String_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, + JAVA_OBJECT conversationJSON) { +#if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1SmartReply(toNSString(CN1_THREAD_GET_STATE_PASS_ARG + conversationJSON))); +#else + return JAVA_NULL; +#endif +} diff --git a/Ports/iOSPort/nativeSources/CN1Vision.m b/Ports/iOSPort/nativeSources/CN1Vision.m new file mode 100644 index 00000000000..a3d73dedbaf --- /dev/null +++ b/Ports/iOSPort/nativeSources/CN1Vision.m @@ -0,0 +1,630 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +#import "CodenameOne_GLViewController.h" +#import "xmlvm.h" + +#if defined(INCLUDE_CN1_VISION) && !TARGET_OS_WATCH && !TARGET_OS_TV +#import +#import +#import +#import +#import +#import +#import +#import "java_lang_String.h" + +#if __has_include() +#import +#define CN1_HAS_MLKIT_VISION 1 +#endif +#if __has_include() +#import +#import +#define CN1_HAS_MLKIT_TEXT 1 +#endif +#if __has_include() +#import +#define CN1_HAS_MLKIT_BARCODE 1 +#endif +#if __has_include() +#import +#define CN1_HAS_MLKIT_FACE 1 +#endif +#if __has_include() +#import +#import +#define CN1_HAS_MLKIT_LABEL 1 +#endif +#if __has_include() +#import +#import +#define CN1_HAS_MLKIT_POSE 1 +#endif +#if __has_include() +#import +#import +#define CN1_HAS_MLKIT_SEGMENTATION 1 +#endif + +static NSDictionary *cn1VisionRect(CGRect rect) { + return @{ + @"x": @(rect.origin.x), + @"y": @(1.0 - CGRectGetMaxY(rect)), + @"width": @(rect.size.width), + @"height": @(rect.size.height) + }; +} + +static NSDictionary *cn1MLKitRect(CGRect rect, CGSize size) { + if (size.width <= 0 || size.height <= 0) { + return @{@"x": @0, @"y": @0, @"width": @0, @"height": @0}; + } + return @{ + @"x": @(rect.origin.x / size.width), + @"y": @(rect.origin.y / size.height), + @"width": @(rect.size.width / size.width), + @"height": @(rect.size.height / size.height) + }; +} + +static NSString *cn1VisionJSON(NSDictionary *value) { + NSError *error = nil; + NSData *data = [NSJSONSerialization dataWithJSONObject:value options:0 error:&error]; + if (data == nil) { + return [NSString stringWithFormat:@"{\"error\":\"%@\"}", + error.localizedDescription ?: @"Could not encode result"]; + } + return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]; +} + +static NSString *cn1VisionError(NSError *error) { + return cn1VisionJSON(@{ + @"error": error.localizedDescription ?: @"Apple Vision request failed" + }); +} + +#if defined(CN1_HAS_MLKIT_VISION) +static UIImageOrientation cn1UIImageOrientation(int rotation) { + switch (rotation) { + case 90: return UIImageOrientationRight; + case 180: return UIImageOrientationDown; + case 270: return UIImageOrientationLeft; + default: return UIImageOrientationUp; + } +} + +static MLKVisionImage *cn1MLKitImage(UIImage *image, int rotation) { + MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; + vision.orientation = cn1UIImageOrientation(rotation); + return vision; +} +#endif + +static NSString *cn1MLKitVisionPerform(NSData *data, int feature, int rotation) { +#if defined(CN1_HAS_MLKIT_VISION) + UIImage *image = [UIImage imageWithData:data]; + if (image == nil) { + return cn1VisionJSON(@{@"error": @"Could not decode ML Kit image"}); + } + MLKVisionImage *vision = cn1MLKitImage(image, rotation); + __block NSError *requestError = nil; + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + + if (feature == 0) { +#if defined(CN1_HAS_MLKIT_TEXT) + MLKTextRecognizer *recognizer = [MLKTextRecognizer textRecognizerWithOptions: + [[MLKTextRecognizerOptions alloc] init]]; + __block MLKText *result = nil; + [recognizer processImage:vision completion:^(MLKText *text, NSError *error) { + result = text; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) return cn1VisionError(requestError); + NSMutableArray *items = [NSMutableArray array]; + for (MLKTextBlock *block in result.blocks ?: @[]) { + NSMutableDictionary *item = [NSMutableDictionary + dictionaryWithDictionary:cn1MLKitRect(block.frame, image.size)]; + item[@"text"] = block.text ?: @""; + item[@"confidence"] = @1; + [items addObject:item]; + } + return cn1VisionJSON(@{ + @"text": result.text ?: @"", + @"items": items + }); +#endif + } else if (feature == 1) { +#if defined(CN1_HAS_MLKIT_BARCODE) + MLKBarcodeScannerOptions *options = [[MLKBarcodeScannerOptions alloc] init]; + MLKBarcodeScanner *scanner = + [MLKBarcodeScanner barcodeScannerWithOptions:options]; + __block NSArray *result = nil; + [scanner processImage:vision completion:^(NSArray *barcodes, + NSError *error) { + result = barcodes; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) return cn1VisionError(requestError); + NSMutableArray *items = [NSMutableArray array]; + for (MLKBarcode *barcode in result ?: @[]) { + NSMutableDictionary *item = [NSMutableDictionary + dictionaryWithDictionary:cn1MLKitRect(barcode.frame, image.size)]; + item[@"value"] = barcode.rawValue ?: [NSNull null]; + item[@"format"] = [NSString stringWithFormat:@"%ld", + (long)barcode.format]; + [items addObject:item]; + } + return cn1VisionJSON(@{@"items": items}); +#endif + } else if (feature == 2) { +#if defined(CN1_HAS_MLKIT_FACE) + MLKFaceDetectorOptions *options = [[MLKFaceDetectorOptions alloc] init]; + options.landmarkMode = MLKFaceDetectorLandmarkModeAll; + options.classificationMode = MLKFaceDetectorClassificationModeAll; + options.trackingEnabled = YES; + MLKFaceDetector *detector = [MLKFaceDetector faceDetectorWithOptions:options]; + __block NSArray *result = nil; + [detector processImage:vision completion:^(NSArray *faces, + NSError *error) { + result = faces; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) return cn1VisionError(requestError); + NSMutableArray *items = [NSMutableArray array]; + for (MLKFace *face in result ?: @[]) { + NSMutableDictionary *item = [NSMutableDictionary + dictionaryWithDictionary:cn1MLKitRect(face.frame, image.size)]; + item[@"yaw"] = @(face.headEulerAngleY); + item[@"pitch"] = @(face.headEulerAngleX); + item[@"roll"] = @(face.headEulerAngleZ); + [items addObject:item]; + } + return cn1VisionJSON(@{@"items": items}); +#endif + } else if (feature == 3) { +#if defined(CN1_HAS_MLKIT_LABEL) + MLKImageLabelerOptions *options = [[MLKImageLabelerOptions alloc] init]; + MLKImageLabeler *labeler = [MLKImageLabeler imageLabelerWithOptions:options]; + __block NSArray *result = nil; + [labeler processImage:vision completion:^(NSArray *labels, + NSError *error) { + result = labels; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) return cn1VisionError(requestError); + NSMutableArray *items = [NSMutableArray array]; + for (MLKImageLabel *label in result ?: @[]) { + [items addObject:@{ + @"text": label.text ?: @"", + @"confidence": @(label.confidence) + }]; + } + return cn1VisionJSON(@{@"items": items}); +#endif + } else if (feature == 4) { +#if defined(CN1_HAS_MLKIT_POSE) + MLKPoseDetectorOptions *options = [[MLKPoseDetectorOptions alloc] init]; + options.detectorMode = MLKPoseDetectorModeSingleImage; + MLKPoseDetector *detector = [MLKPoseDetector poseDetectorWithOptions:options]; + __block NSArray *result = nil; + [detector processImage:vision completion:^(NSArray *poses, + NSError *error) { + result = poses; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) return cn1VisionError(requestError); + NSMutableArray *items = [NSMutableArray array]; + MLKPose *pose = result.firstObject; + for (MLKPoseLandmark *landmark in pose.landmarks ?: @[]) { + [items addObject:@{ + @"name": [NSString stringWithFormat:@"%ld", + (long)landmark.type], + @"x": @(landmark.position.x / image.size.width), + @"y": @(landmark.position.y / image.size.height), + @"confidence": @(landmark.inFrameLikelihood) + }]; + } + return cn1VisionJSON(@{@"items": items}); +#endif + } else if (feature == 5) { +#if defined(CN1_HAS_MLKIT_SEGMENTATION) + MLKSelfieSegmenterOptions *options = + [[MLKSelfieSegmenterOptions alloc] init]; + options.segmenterMode = MLKSegmenterModeSingleImage; + MLKSegmenter *segmenter = [MLKSegmenter segmenterWithOptions:options]; + __block MLKSegmentationMask *result = nil; + [segmenter processImage:vision completion:^(MLKSegmentationMask *mask, + NSError *error) { + result = mask; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) return cn1VisionError(requestError); + CVPixelBufferRef buffer = result.buffer; + if (buffer == nil) { + return cn1VisionJSON(@{@"error": @"No segmentation mask returned"}); + } + CVPixelBufferLockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly); + size_t width = CVPixelBufferGetWidth(buffer); + size_t height = CVPixelBufferGetHeight(buffer); + size_t stride = CVPixelBufferGetBytesPerRow(buffer); + const uint8_t *base = CVPixelBufferGetBaseAddress(buffer); + NSMutableData *packed = [NSMutableData dataWithLength:width * height]; + uint8_t *dest = packed.mutableBytes; + for (size_t y = 0; y < height; y++) { + const float *row = (const float *)(base + y * stride); + for (size_t x = 0; x < width; x++) { + float confidence = fmaxf(0, fminf(1, row[x])); + dest[y * width + x] = (uint8_t)lrintf(confidence * 255); + } + } + CVPixelBufferUnlockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly); + return cn1VisionJSON(@{ + @"width": @(width), + @"height": @(height), + @"data": [packed base64EncodedStringWithOptions:0] + }); +#endif + } + return cn1VisionJSON(@{ + @"error": @"Requested ML Kit vision component is not linked" + }); +#else + return cn1VisionJSON(@{@"error": @"ML Kit Vision is not linked"}); +#endif +} + +static CGImagePropertyOrientation cn1CGOrientation(int rotation) { + switch (rotation) { + case 90: return kCGImagePropertyOrientationRight; + case 180: return kCGImagePropertyOrientationDown; + case 270: return kCGImagePropertyOrientationLeft; + default: return kCGImagePropertyOrientationUp; + } +} + +static NSString *cn1VisionPerform(NSData *data, int feature, int rotation) { + NSError *error = nil; + VNImageRequestHandler *handler = + [[VNImageRequestHandler alloc] initWithData:data + orientation:cn1CGOrientation(rotation) options:@{}]; + + if (feature == 0) { + if (@available(iOS 13.0, *)) { + VNRecognizeTextRequest *request = [[VNRecognizeTextRequest alloc] init]; + request.recognitionLevel = VNRequestTextRecognitionLevelAccurate; + if (![handler performRequests:@[request] error:&error]) { + return cn1VisionError(error); + } + NSMutableArray *items = [NSMutableArray array]; + NSMutableArray *lines = [NSMutableArray array]; + for (VNRecognizedTextObservation *observation in request.results) { + VNRecognizedText *candidate = [observation topCandidates:1].firstObject; + if (candidate == nil) continue; + NSMutableDictionary *item = + [NSMutableDictionary dictionaryWithDictionary: + cn1VisionRect(observation.boundingBox)]; + item[@"text"] = candidate.string ?: @""; + item[@"confidence"] = @(candidate.confidence); + [items addObject:item]; + [lines addObject:candidate.string ?: @""]; + } + return cn1VisionJSON(@{ + @"text": [lines componentsJoinedByString:@"\n"], + @"items": items + }); + } + } else if (feature == 1) { + VNDetectBarcodesRequest *request = [[VNDetectBarcodesRequest alloc] init]; + if (![handler performRequests:@[request] error:&error]) { + return cn1VisionError(error); + } + NSMutableArray *items = [NSMutableArray array]; + for (VNBarcodeObservation *observation in request.results) { + NSMutableDictionary *item = + [NSMutableDictionary dictionaryWithDictionary: + cn1VisionRect(observation.boundingBox)]; + item[@"value"] = observation.payloadStringValue ?: [NSNull null]; + item[@"format"] = observation.symbology ?: @"UNKNOWN"; + [items addObject:item]; + } + return cn1VisionJSON(@{@"items": items}); + } else if (feature == 2) { + VNDetectFaceLandmarksRequest *request = [[VNDetectFaceLandmarksRequest alloc] init]; + if (![handler performRequests:@[request] error:&error]) { + return cn1VisionError(error); + } + NSMutableArray *items = [NSMutableArray array]; + for (VNFaceObservation *observation in request.results) { + NSMutableDictionary *item = + [NSMutableDictionary dictionaryWithDictionary: + cn1VisionRect(observation.boundingBox)]; + item[@"yaw"] = @((observation.yaw ?: @0).doubleValue + * 180.0 / M_PI); + item[@"pitch"] = @0; + item[@"roll"] = @((observation.roll ?: @0).doubleValue + * 180.0 / M_PI); + [items addObject:item]; + } + return cn1VisionJSON(@{@"items": items}); + } else if (feature == 3) { + if (@available(iOS 15.0, *)) { + VNClassifyImageRequest *request = [[VNClassifyImageRequest alloc] init]; + if (![handler performRequests:@[request] error:&error]) { + return cn1VisionError(error); + } + NSMutableArray *items = [NSMutableArray array]; + for (VNClassificationObservation *observation in request.results) { + [items addObject:@{ + @"text": observation.identifier ?: @"", + @"confidence": @(observation.confidence) + }]; + } + return cn1VisionJSON(@{@"items": items}); + } + } else if (feature == 4) { + if (@available(iOS 14.0, *)) { + VNDetectHumanBodyPoseRequest *request = + [[VNDetectHumanBodyPoseRequest alloc] init]; + if (![handler performRequests:@[request] error:&error]) { + return cn1VisionError(error); + } + NSMutableArray *items = [NSMutableArray array]; + VNHumanBodyPoseObservation *pose = request.results.firstObject; + NSDictionary *points = + [pose recognizedPointsForGroupKey: + VNHumanBodyPoseObservationJointsGroupNameAll error:&error]; + if (error != nil) { + return cn1VisionError(error); + } + for (NSString *name in points) { + VNRecognizedPoint *point = points[name]; + [items addObject:@{ + @"name": name, + @"x": @(point.location.x), + @"y": @(1.0 - point.location.y), + @"confidence": @(point.confidence) + }]; + } + return cn1VisionJSON(@{@"items": items}); + } + } else if (feature == 5) { + if (@available(iOS 15.0, *)) { + VNGeneratePersonSegmentationRequest *request = + [[VNGeneratePersonSegmentationRequest alloc] init]; + request.qualityLevel = VNGeneratePersonSegmentationRequestQualityLevelBalanced; + request.outputPixelFormat = kCVPixelFormatType_OneComponent8; + if (![handler performRequests:@[request] error:&error]) { + return cn1VisionError(error); + } + VNPixelBufferObservation *observation = request.results.firstObject; + CVPixelBufferRef buffer = observation.pixelBuffer; + if (buffer == nil) { + return cn1VisionJSON(@{@"error": @"No segmentation mask returned"}); + } + CVPixelBufferLockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly); + size_t width = CVPixelBufferGetWidth(buffer); + size_t height = CVPixelBufferGetHeight(buffer); + size_t stride = CVPixelBufferGetBytesPerRow(buffer); + const uint8_t *base = CVPixelBufferGetBaseAddress(buffer); + NSMutableData *packed = [NSMutableData dataWithLength:width * height]; + uint8_t *dest = packed.mutableBytes; + for (size_t y = 0; y < height; y++) { + memcpy(dest + y * width, base + y * stride, width); + } + CVPixelBufferUnlockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly); + return cn1VisionJSON(@{ + @"width": @(width), + @"height": @(height), + @"data": [packed base64EncodedStringWithOptions:0] + }); + } + } else if (feature == 6) { + UIImage *image = [UIImage imageWithData:data]; + if (image == nil) { + return cn1VisionJSON(@{@"error": @"Could not decode document image"}); + } + CIImage *source = [CIImage imageWithCGImage:image.CGImage]; + CIContext *context = [CIContext context]; + CIDetector *detector = [CIDetector detectorOfType:CIDetectorTypeRectangle + context:context + options:@{ + CIDetectorAccuracy: CIDetectorAccuracyHigh + }]; + NSArray *features = [detector featuresInImage:source]; + CIImage *corrected = source; + if (features.count > 0) { + CIRectangleFeature *rectangle = features.firstObject; + corrected = [source imageByApplyingFilter:@"CIPerspectiveCorrection" + withInputParameters:@{ + @"inputTopLeft": [CIVector vectorWithCGPoint:rectangle.topLeft], + @"inputTopRight": [CIVector vectorWithCGPoint:rectangle.topRight], + @"inputBottomLeft": [CIVector vectorWithCGPoint:rectangle.bottomLeft], + @"inputBottomRight": [CIVector vectorWithCGPoint:rectangle.bottomRight] + }]; + } + CGImageRef outputImage = [context createCGImage:corrected + fromRect:corrected.extent]; + UIImage *output = [UIImage imageWithCGImage:outputImage]; + CGImageRelease(outputImage); + NSData *jpeg = UIImageJPEGRepresentation(output, 0.92); + return cn1VisionJSON(@{ + @"data": [jpeg base64EncodedStringWithOptions:0] + }); + } + return cn1VisionJSON(@{@"error": @"Vision feature is unavailable on this OS"}); +} +#endif + +JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_cn1VisionIsSupported___int_boolean_R_boolean( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT feature, + JAVA_BOOLEAN mlKit) { +#if defined(INCLUDE_CN1_VISION) && !TARGET_OS_WATCH && !TARGET_OS_TV + if (mlKit) { + switch (feature) { + case 0: +#if defined(CN1_HAS_MLKIT_TEXT) + return 1; +#else + return 0; +#endif + case 1: +#if defined(CN1_HAS_MLKIT_BARCODE) + return 1; +#else + return 0; +#endif + case 2: +#if defined(CN1_HAS_MLKIT_FACE) + return 1; +#else + return 0; +#endif + case 3: +#if defined(CN1_HAS_MLKIT_LABEL) + return 1; +#else + return 0; +#endif + case 4: +#if defined(CN1_HAS_MLKIT_POSE) + return 1; +#else + return 0; +#endif + case 5: +#if defined(CN1_HAS_MLKIT_SEGMENTATION) + return 1; +#else + return 0; +#endif + default: + return 0; + } + } + switch (feature) { + case 0: + if (@available(iOS 13.0, *)) return 1; + return 0; + case 3: + case 5: + if (@available(iOS 15.0, *)) return 1; + return 0; + case 4: + if (@available(iOS 14.0, *)) return 1; + return 0; + default: + return 1; + } +#else + return 0; +#endif +} + +#if defined(INCLUDE_CN1_VISION) && !TARGET_OS_WATCH && !TARGET_OS_TV +static uint8_t cn1VisionClamp(int value) { + return (uint8_t)(value < 0 ? 0 : (value > 255 ? 255 : value)); +} + +/* + * Converts the two raw CameraFrame formats to JPEG so both Apple Vision and + * the optional ML Kit path consume identical, orientation-neutral data. + * format mirrors FrameFormat: 0=JPEG/encoded, 1=NV21, 2=RGBA8888. + */ +static NSData *cn1VisionEncodeInput(NSData *data, int width, int height, + int format) { + if (format == 0) { + return data; + } + if (width <= 0 || height <= 0) { + return nil; + } + NSUInteger pixelCount = (NSUInteger)width * (NSUInteger)height; + const uint8_t *source = data.bytes; + NSMutableData *rgba = [NSMutableData dataWithLength:pixelCount * 4]; + uint8_t *dest = rgba.mutableBytes; + if (format == 2) { + if (data.length < pixelCount * 4) { + return nil; + } + memcpy(dest, source, pixelCount * 4); + } else if (format == 1) { + if (data.length < pixelCount + pixelCount / 2) { + return nil; + } + for (int y = 0; y < height; y++) { + int uvRow = width * height + (y >> 1) * width; + for (int x = 0; x < width; x++) { + int yy = source[y * width + x] & 255; + int uv = uvRow + (x & ~1); + int v = (source[uv] & 255) - 128; + int u = (source[uv + 1] & 255) - 128; + int c = yy - 16; + if (c < 0) c = 0; + NSUInteger p = ((NSUInteger)y * width + x) * 4; + dest[p] = cn1VisionClamp((298 * c + 409 * v + 128) >> 8); + dest[p + 1] = cn1VisionClamp( + (298 * c - 100 * u - 208 * v + 128) >> 8); + dest[p + 2] = cn1VisionClamp( + (298 * c + 516 * u + 128) >> 8); + dest[p + 3] = 255; + } + } + } else { + return nil; + } + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + CGDataProviderRef provider = CGDataProviderCreateWithCFData( + (CFDataRef)rgba); + CGImageRef cgImage = CGImageCreate(width, height, 8, 32, width * 4, + colorSpace, kCGBitmapByteOrderDefault | kCGImageAlphaLast, + provider, NULL, false, kCGRenderingIntentDefault); + UIImage *image = cgImage == NULL ? nil + : [UIImage imageWithCGImage:cgImage]; + NSData *encoded = image == nil ? nil + : UIImageJPEGRepresentation(image, 0.95); + if (cgImage != NULL) CGImageRelease(cgImage); + CGDataProviderRelease(provider); + CGColorSpaceRelease(colorSpace); + return encoded; +} +#endif + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1VisionAnalyze___byte_1ARRAY_int_boolean_int_int_int_int_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, + JAVA_OBJECT encodedImage, JAVA_INT feature, JAVA_BOOLEAN mlKit, + JAVA_INT rotation, JAVA_INT width, JAVA_INT height, + JAVA_INT frameFormat) { +#if defined(INCLUDE_CN1_VISION) && !TARGET_OS_WATCH && !TARGET_OS_TV + if (encodedImage == JAVA_NULL) { + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + @"{\"error\":\"Image data is null\"}"); + } + JAVA_ARRAY bytes = (JAVA_ARRAY) encodedImage; + NSData *data = [NSData dataWithBytes:bytes->data length:(NSUInteger) bytes->length]; + data = cn1VisionEncodeInput(data, width, height, frameFormat); + if (data == nil) { + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + @"{\"error\":\"Invalid raw vision image\"}"); + } + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + mlKit ? cn1MLKitVisionPerform(data, feature, rotation) + : cn1VisionPerform(data, feature, rotation)); +#else + return JAVA_NULL; +#endif +} diff --git a/Ports/iOSPort/nativeSources/CodenameOne_GLViewController.h b/Ports/iOSPort/nativeSources/CodenameOne_GLViewController.h index b22fa97e66e..964a27f8cd9 100644 --- a/Ports/iOSPort/nativeSources/CodenameOne_GLViewController.h +++ b/Ports/iOSPort/nativeSources/CodenameOne_GLViewController.h @@ -122,6 +122,17 @@ void cn1RunSyncOnMainQueue(void (^block)(void)); #undef INCLUDE_CN1_AR #endif +// INCLUDE_CN1_VISION gates Apple Vision/Core Image analysis. It is enabled +// only when the app references com.codename1.ai.vision. +//#define INCLUDE_CN1_VISION +//#define INCLUDE_CN1_LANGUAGE +//#define INCLUDE_CN1_INFERENCE +#if TARGET_OS_WATCH || TARGET_OS_TV +#undef INCLUDE_CN1_VISION +#undef INCLUDE_CN1_LANGUAGE +#undef INCLUDE_CN1_INFERENCE +#endif + // CN1_USE_CARPLAY gates the Apple CarPlay native bridge // (CodenameOne_CarPlaySceneDelegate.{h,m} + the IOSNative carPlay* trampolines: // CarPlay.framework, the CPTemplate translation). IPhoneBuilder uncomments this diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java index d8033ec0b07..222d03ade86 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java @@ -4764,6 +4764,31 @@ public com.codename1.impl.ARImpl createARImpl() { return new IOSARImpl(); } + @Override + public com.codename1.impl.VisionImpl createVisionImpl() { + for (int feature = 0; feature <= 6; feature++) { + if (nativeInstance.cn1VisionIsSupported(feature, false) + || nativeInstance.cn1VisionIsSupported(feature, true)) { + return new IOSVisionImpl(); + } + } + return null; + } + + @Override + public com.codename1.impl.LanguageImpl createLanguageImpl() { + return nativeInstance.cn1LanguageIsSupported(0) + || nativeInstance.cn1LanguageIsSupported(1) + || nativeInstance.cn1LanguageIsSupported(2) + ? new IOSLanguageImpl() : null; + } + + @Override + public com.codename1.impl.InferenceImpl createInferenceImpl() { + return nativeInstance.cn1InferenceIsSupported() + ? new IOSInferenceImpl() : null; + } + @Override public String [] getAvailableRecordingMimeTypes() { // All of these amount to the same thing. diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java new file mode 100644 index 00000000000..3462dda4453 --- /dev/null +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java @@ -0,0 +1,377 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.ios; + +import com.codename1.ai.inference.InferenceException; +import com.codename1.ai.inference.InferenceOptions; +import com.codename1.ai.inference.ModelSource; +import com.codename1.ai.inference.Tensor; +import com.codename1.ai.inference.TensorInfo; +import com.codename1.ai.inference.TensorType; +import com.codename1.impl.InferenceImpl; +import com.codename1.io.JSONParser; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; +import com.codename1.util.Base64; + +import java.io.ByteArrayOutputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** LiteRT Objective-C backend with optional Core ML delegation. */ +public final class IOSInferenceImpl extends InferenceImpl { + private static final class Handle { + final int id; + + Handle(int id) { + this.id = id; + } + } + + @Override + public boolean isSupported() { + return IOSImplementation.nativeInstance.cn1InferenceIsSupported(); + } + + @Override + public AsyncResource open(final ModelSource source, + final InferenceOptions options) { + final AsyncResource out = new AsyncResource(); + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + try { + Map root = parse(IOSImplementation.nativeInstance.cn1InferenceOpen( + loadModel(source), options.getThreads(), + options.getAccelerator().ordinal(), + options.isFallbackAllowed())); + final Handle handle = new Handle(integer(root, "handle")); + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(handle); + } + }); + } catch (final Throwable error) { + fail(out, "Could not open LiteRT model", error); + } + } + }); + return out; + } + + @Override + public TensorInfo[] getInputs(Object handle) { + return metadata(checked(handle), false); + } + + @Override + public TensorInfo[] getOutputs(Object handle) { + return metadata(checked(handle), true); + } + + @Override + public AsyncResource run(Object handle, final Tensor[] inputs) { + final AsyncResource out = new AsyncResource(); + final Handle checked = checked(handle); + final int id = checked.id; + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + try { + TensorInfo[] metadata = metadata(checked, false); + Map root = parse(IOSImplementation.nativeInstance.cn1InferenceRun( + id, inputsJson(inputs, metadata))); + List items = list(root, "items"); + final Tensor[] result = new Tensor[items.size()]; + for (int i = 0; i < result.length; i++) { + Map value = (Map) items.get(i); + TensorType type = TensorType.valueOf(string(value, "type")); + int[] shape = intArray(list(value, "shape")); + byte[] bytes = Base64.decode( + string(value, "data").getBytes("UTF-8")); + result[i] = new Tensor(string(value, "name"), type, shape, + decodeData(type, bytes, elementCount(shape))); + } + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(result); + } + }); + } catch (final Throwable error) { + fail(out, "LiteRT inference failed", error); + } + } + }); + return out; + } + + @Override + public void resizeInput(Object handle, String name, int[] shape) { + Handle checked = checked(handle); + TensorInfo[] inputs = metadata(checked, false); + int index = -1; + for (int i = 0; i < inputs.length; i++) { + if (name == null || name.equals(inputs[i].getName())) { + index = inputs[i].getIndex(); + break; + } + } + if (index < 0) { + throw new IllegalArgumentException("Unknown model input " + name); + } + try { + parse(IOSImplementation.nativeInstance.cn1InferenceResize( + checked.id, index, shape)); + } catch (Exception error) { + throw new InferenceException("Could not resize input " + name, error); + } + } + + @Override + public void close(Object handle) { + IOSImplementation.nativeInstance.cn1InferenceClose(checked(handle).id); + } + + private static TensorInfo[] metadata(Handle handle, boolean outputs) { + try { + Map root = parse(IOSImplementation.nativeInstance.cn1InferenceMetadata( + handle.id, outputs)); + List items = list(root, "items"); + TensorInfo[] result = new TensorInfo[items.size()]; + for (int i = 0; i < result.length; i++) { + Map value = (Map) items.get(i); + result[i] = new TensorInfo(string(value, "name"), + TensorType.valueOf(string(value, "type")), + intArray(list(value, "shape")), + integer(value, "index")); + } + return result; + } catch (Exception error) { + throw new InferenceException("Could not read LiteRT metadata", error); + } + } + + private static Handle checked(Object value) { + if (!(value instanceof Handle)) { + throw new IllegalArgumentException("Invalid LiteRT session handle"); + } + return (Handle) value; + } + + private static String inputsJson(Tensor[] inputs, TensorInfo[] metadata) { + if (inputs.length != metadata.length) { + throw new IllegalArgumentException("Expected " + metadata.length + + " input tensors but received " + inputs.length); + } + List> items = new ArrayList>(); + for (int i = 0; i < inputs.length; i++) { + Tensor tensor = inputs[i]; + int index = inputIndex(tensor, i, metadata); + Map value = new HashMap(); + value.put("index", Integer.valueOf(index)); + value.put("name", tensor.getName()); + value.put("type", tensor.getType().name()); + value.put("shape", integerList(tensor.getShape())); + value.put("data", Base64.encode(encodeData( + tensor.getType(), tensor.getData()))); + items.add(value); + } + Map root = new HashMap(); + root.put("items", items); + return JSONParser.mapToJson(root); + } + + private static int inputIndex(Tensor tensor, int fallback, + TensorInfo[] metadata) { + if (tensor.getName() != null) { + for (int i = 0; i < metadata.length; i++) { + if (tensor.getName().equals(metadata[i].getName())) { + return metadata[i].getIndex(); + } + } + throw new IllegalArgumentException("Unknown model input " + + tensor.getName()); + } + return metadata[fallback].getIndex(); + } + + private static byte[] encodeData(TensorType type, Object value) { + if (value instanceof byte[]) { + return (byte[]) value; + } + if (value instanceof float[]) { + float[] values = (float[]) value; + byte[] out = new byte[values.length * 4]; + for (int i = 0; i < values.length; i++) { + writeInt(out, i * 4, Float.floatToIntBits(values[i])); + } + return out; + } + if (value instanceof int[]) { + int[] values = (int[]) value; + byte[] out = new byte[values.length * 4]; + for (int i = 0; i < values.length; i++) { + writeInt(out, i * 4, values[i]); + } + return out; + } + if (value instanceof long[]) { + long[] values = (long[]) value; + byte[] out = new byte[values.length * 8]; + for (int i = 0; i < values.length; i++) { + writeLong(out, i * 8, values[i]); + } + return out; + } + throw new InferenceException("Unsupported input tensor data"); + } + + private static Object decodeData(TensorType type, byte[] bytes, int count) { + if (type == TensorType.FLOAT32) { + float[] out = new float[count]; + for (int i = 0; i < count; i++) { + out[i] = Float.intBitsToFloat(readInt(bytes, i * 4)); + } + return out; + } + if (type == TensorType.INT32) { + int[] out = new int[count]; + for (int i = 0; i < count; i++) { + out[i] = readInt(bytes, i * 4); + } + return out; + } + if (type == TensorType.INT64) { + long[] out = new long[count]; + for (int i = 0; i < count; i++) { + out[i] = readLong(bytes, i * 8); + } + return out; + } + if (bytes.length != count) { + throw new InferenceException("Unexpected output tensor byte count"); + } + return bytes; + } + + private static void writeInt(byte[] out, int offset, int value) { + out[offset] = (byte) value; + out[offset + 1] = (byte) (value >>> 8); + out[offset + 2] = (byte) (value >>> 16); + out[offset + 3] = (byte) (value >>> 24); + } + + private static int readInt(byte[] value, int offset) { + return (value[offset] & 255) + | ((value[offset + 1] & 255) << 8) + | ((value[offset + 2] & 255) << 16) + | ((value[offset + 3] & 255) << 24); + } + + private static void writeLong(byte[] out, int offset, long value) { + for (int i = 0; i < 8; i++) { + out[offset + i] = (byte) (value >>> (i * 8)); + } + } + + private static long readLong(byte[] value, int offset) { + long out = 0; + for (int i = 0; i < 8; i++) { + out |= (long) (value[offset + i] & 255) << (i * 8); + } + return out; + } + + private static int elementCount(int[] shape) { + int out = 1; + for (int i = 0; i < shape.length; i++) { + out *= shape[i]; + } + return out; + } + + private static List integerList(int[] values) { + List out = new ArrayList(); + for (int i = 0; i < values.length; i++) { + out.add(Integer.valueOf(values[i])); + } + return out; + } + + private static int[] intArray(List values) { + int[] out = new int[values.size()]; + for (int i = 0; i < out.length; i++) { + out[i] = ((Number) values.get(i)).intValue(); + } + return out; + } + + private static Map parse(String json) throws Exception { + if (json == null || json.length() == 0) { + throw new InferenceException("LiteRT returned no result"); + } + Map root = new JSONParser().parseJSON(new StringReader(json)); + Object error = root.get("error"); + if (error != null) { + throw new InferenceException(String.valueOf(error)); + } + return root; + } + + private static List list(Map value, String key) { + Object out = value.get(key); + return out instanceof List ? (List) out : java.util.Collections.EMPTY_LIST; + } + + private static String string(Map value, String key) { + Object out = value.get(key); + return out == null ? "" : String.valueOf(out); + } + + private static int integer(Map value, String key) { + Object out = value.get(key); + return out instanceof Number ? ((Number) out).intValue() : 0; + } + + private static byte[] loadModel(ModelSource source) throws IOException { + if (source.getKind() == ModelSource.BYTES) { + return source.getBytes(); + } + InputStream input; + if (source.getKind() == ModelSource.FILE) { + input = new FileInputStream(source.getPath()); + } else { + input = Display.getInstance().getResourceAsStream( + IOSInferenceImpl.class, source.getPath()); + if (input == null) { + throw new IOException("Model resource not found: " + source.getPath()); + } + } + try { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[16384]; + int read; + while ((read = input.read(buffer)) >= 0) { + output.write(buffer, 0, read); + } + return output.toByteArray(); + } finally { + input.close(); + } + } + + private static void fail(final AsyncResource out, final String message, + final Throwable cause) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.error(new InferenceException(message, cause)); + } + }); + } +} diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java new file mode 100644 index 00000000000..53177239793 --- /dev/null +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.ios; + +import com.codename1.ai.language.LanguageCandidate; +import com.codename1.ai.language.LanguageOptions; +import com.codename1.ai.language.SmartReplyMessage; +import com.codename1.impl.LanguageImpl; +import com.codename1.io.JSONParser; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** iOS ML Kit language services. */ +public final class IOSLanguageImpl extends LanguageImpl { + private static final int LANGUAGE_ID = 0; + private static final int TRANSLATION = 1; + private static final int SMART_REPLY = 2; + + private volatile boolean closed; + + @Override + public boolean isSupported(String feature, String backendId) { + if (closed || (!"auto".equals(backendId) && !"ml-kit".equals(backendId))) { + return false; + } + return IOSImplementation.nativeInstance.cn1LanguageIsSupported( + featureId(feature)); + } + + @Override + public AsyncResource identify( + final String text, String backendId, final LanguageOptions options) { + final AsyncResource out = + new AsyncResource(); + run(out, new NativeCall() { + public LanguageCandidate[] call() throws Exception { + String json = IOSImplementation.nativeInstance.cn1LanguageIdentify( + text, options.getMinimumConfidence()); + Map root = parse(json); + List values = list(root, "items"); + LanguageCandidate[] result = new LanguageCandidate[values.size()]; + for (int i = 0; i < result.length; i++) { + Map value = (Map) values.get(i); + result[i] = new LanguageCandidate( + string(value, "language"), + number(value, "confidence")); + } + return result; + } + }); + return out; + } + + @Override + public AsyncResource translate( + final String text, final String sourceLanguage, + final String targetLanguage, String backendId, + LanguageOptions options) { + final AsyncResource out = new AsyncResource(); + run(out, new NativeCall() { + public String call() throws Exception { + Map root = parse(IOSImplementation.nativeInstance.cn1LanguageTranslate( + text, sourceLanguage, targetLanguage)); + return string(root, "text"); + } + }); + return out; + } + + @Override + public AsyncResource suggestReplies( + SmartReplyMessage[] conversation, String backendId, + LanguageOptions options) { + final AsyncResource out = new AsyncResource(); + final String json = conversationJson(conversation); + run(out, new NativeCall() { + public String[] call() throws Exception { + Map root = parse(IOSImplementation.nativeInstance + .cn1LanguageSmartReply(json)); + List values = list(root, "items"); + String[] result = new String[values.size()]; + for (int i = 0; i < result.length; i++) { + result[i] = String.valueOf(values.get(i)); + } + return result; + } + }); + return out; + } + + private static int featureId(String feature) { + if ("language-id".equals(feature)) return LANGUAGE_ID; + if ("translation".equals(feature)) return TRANSLATION; + if ("smart-reply".equals(feature)) return SMART_REPLY; + return -1; + } + + private static String conversationJson(SmartReplyMessage[] conversation) { + List> values = new ArrayList>(); + for (int i = 0; i < conversation.length; i++) { + SmartReplyMessage message = conversation[i]; + Map value = new HashMap(); + value.put("text", message.getText()); + value.put("participant", message.getParticipantId()); + value.put("local", Boolean.valueOf(message.isLocalUser())); + value.put("timestamp", Long.valueOf(message.getTimestampMillis())); + values.add(value); + } + Map root = new HashMap(); + root.put("items", values); + return JSONParser.mapToJson(root); + } + + private static Map parse(String json) throws Exception { + if (json == null || json.length() == 0) { + throw new IllegalStateException("ML Kit returned no result"); + } + Map root = new JSONParser().parseJSON(new StringReader(json)); + Object error = root.get("error"); + if (error != null) { + throw new IllegalStateException(String.valueOf(error)); + } + return root; + } + + private static List list(Map value, String key) { + Object out = value.get(key); + return out instanceof List ? (List) out : java.util.Collections.EMPTY_LIST; + } + + private static String string(Map value, String key) { + Object out = value.get(key); + return out == null ? "" : String.valueOf(out); + } + + private static float number(Map value, String key) { + Object out = value.get(key); + return out instanceof Number ? ((Number) out).floatValue() : 0; + } + + private static void run(final AsyncResource out, + final NativeCall call) { + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + try { + final T value = call.call(); + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(value); + } + }); + } catch (final Throwable error) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.error(error); + } + }); + } + } + }); + } + + private interface NativeCall { + T call() throws Exception; + } + + @Override + public void close() { + closed = true; + } +} diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java index 4e8afbc50ac..99880a36586 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java @@ -538,6 +538,24 @@ native void fillGradient(int kind, int stopCount, float[] positions, float[] pre native void cn1CameraResume(long sessionPeer); native void cn1CameraClose(long sessionPeer); + // On-device image analysis backed by Apple Vision/Core Image. + native boolean cn1VisionIsSupported(int feature, boolean mlKit); + native String cn1VisionAnalyze(byte[] imageData, int feature, boolean mlKit, + int rotationDegrees, int width, int height, + int frameFormat); + native boolean cn1LanguageIsSupported(int feature); + native String cn1LanguageIdentify(String text, float minimumConfidence); + native String cn1LanguageTranslate(String text, String sourceLanguage, + String targetLanguage); + native String cn1LanguageSmartReply(String conversationJson); + native boolean cn1InferenceIsSupported(); + native String cn1InferenceOpen(byte[] model, int threads, int accelerator, + boolean allowFallback); + native String cn1InferenceMetadata(int handle, boolean outputs); + native String cn1InferenceRun(int handle, String inputsJson); + native String cn1InferenceResize(int handle, int index, int[] shape); + native void cn1InferenceClose(int handle); + // Augmented reality API (com.codename1.ar). Backed by CN1AR.m which wraps // an ARKit ARSession composited through an ARSCNView. The IOSARImpl class // on the Java side routes static callbacks delivered from the session and diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java new file mode 100644 index 00000000000..3198515df56 --- /dev/null +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.impl.ios; + +import com.codename1.ai.vision.Barcode; +import com.codename1.ai.vision.DocumentScanResult; +import com.codename1.ai.vision.Face; +import com.codename1.ai.vision.ImageLabel; +import com.codename1.ai.vision.Pose; +import com.codename1.ai.vision.SegmentationMask; +import com.codename1.ai.vision.TextRecognitionResult; +import com.codename1.ai.vision.VisionException; +import com.codename1.ai.vision.VisionFeature; +import com.codename1.ai.vision.VisionImage; +import com.codename1.ai.vision.VisionMetadata; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.ai.vision.VisionPoint; +import com.codename1.ai.vision.VisionRect; +import com.codename1.impl.VisionImpl; +import com.codename1.io.JSONParser; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; +import com.codename1.util.Base64; + +import java.io.StringReader; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Apple Vision/Core Image and optional ML Kit implementation. */ +public final class IOSVisionImpl extends VisionImpl { + private volatile boolean closed; + + @Override + public boolean isSupported(VisionFeature feature, String backendId) { + if (closed || (!"auto".equals(backendId) + && !"apple-vision".equals(backendId) + && !"ml-kit".equals(backendId))) { + return false; + } + return IOSImplementation.nativeInstance.cn1VisionIsSupported( + feature.ordinal(), "ml-kit".equals(backendId)); + } + + @Override + public AsyncResource analyze(final VisionFeature feature, + String backendId, final VisionImage image, + VisionOptions options) { + final AsyncResource out = new AsyncResource(); + final boolean mlKit = "ml-kit".equals(backendId); + if (!isSupported(feature, backendId)) { + out.error(new VisionException(VisionException.UNSUPPORTED, + feature + " is unavailable in " + + (mlKit ? "ML Kit" : "Apple Vision"))); + return out; + } + byte[] encoded = image.getEncodedBytes(); + final byte[] input = encoded == null ? image.getPixels() : encoded; + final int width = encoded == null ? image.getWidth() : 0; + final int height = encoded == null ? image.getHeight() : 0; + final int frameFormat = encoded == null + ? image.getFormat().ordinal() : 0; + if (input == null || input.length == 0) { + out.error(new VisionException(VisionException.INVALID_IMAGE, + "Apple Vision requires encoded, NV21, or RGBA8888 input")); + return out; + } + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + try { + String json = IOSImplementation.nativeInstance.cn1VisionAnalyze( + input, feature.ordinal(), mlKit, + image.getRotationDegrees(), width, height, + frameFormat); + if (json == null || json.length() == 0) { + throw new VisionException(VisionException.BACKEND_ERROR, + "Apple Vision returned no result"); + } + final T value = parse(feature, json, + mlKit ? "ml-kit" : "apple-vision"); + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(value); + } + }); + } catch (final Throwable error) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.error(error instanceof VisionException ? error + : new VisionException(VisionException.BACKEND_ERROR, + error.getMessage(), error)); + } + }); + } + } + }); + return out; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static T parse(VisionFeature feature, String json, + String backendId) throws Exception { + Map root = new JSONParser().parseJSON(new StringReader(json)); + Object error = root.get("error"); + if (error != null) { + throw new VisionException(VisionException.BACKEND_ERROR, String.valueOf(error)); + } + List items = (List) root.get("items"); + if (items == null) { + items = java.util.Collections.EMPTY_LIST; + } + VisionMetadata metadata = new VisionMetadata(backendId); + switch (feature) { + case TEXT_RECOGNITION: { + TextRecognitionResult.TextBlock[] blocks = + new TextRecognitionResult.TextBlock[items.size()]; + for (int i = 0; i < blocks.length; i++) { + Map value = (Map) items.get(i); + blocks[i] = new TextRecognitionResult.TextBlock( + string(value, "text"), number(value, "confidence"), + rect(value), stringOrNull(value, "language")); + } + return (T) new TextRecognitionResult( + string(root, "text"), blocks, metadata); + } + case BARCODE_SCANNING: { + Barcode[] values = new Barcode[items.size()]; + for (int i = 0; i < values.length; i++) { + Map value = (Map) items.get(i); + values[i] = new Barcode(stringOrNull(value, "value"), + string(value, "format"), null, rect(value), + new VisionPoint[0], metadata); + } + return (T) values; + } + case FACE_DETECTION: { + Face[] values = new Face[items.size()]; + for (int i = 0; i < values.length; i++) { + Map value = (Map) items.get(i); + values[i] = new Face(rect(value), + new HashMap(), + number(value, "yaw"), number(value, "pitch"), + number(value, "roll"), -1, -1, metadata); + } + return (T) values; + } + case IMAGE_LABELING: { + ImageLabel[] values = new ImageLabel[items.size()]; + for (int i = 0; i < values.length; i++) { + Map value = (Map) items.get(i); + values[i] = new ImageLabel(string(value, "text"), + number(value, "confidence"), i, metadata); + } + return (T) values; + } + case POSE_DETECTION: { + Pose.Landmark[] values = new Pose.Landmark[items.size()]; + for (int i = 0; i < values.length; i++) { + Map value = (Map) items.get(i); + values[i] = new Pose.Landmark(string(value, "name"), + new VisionPoint(number(value, "x"), number(value, "y")), + number(value, "confidence")); + } + return (T) new Pose(values, metadata); + } + case SELFIE_SEGMENTATION: { + String base64 = string(root, "data"); + byte[] bytes = Base64.decode(base64.getBytes("UTF-8")); + float[] confidence = new float[bytes.length]; + for (int i = 0; i < bytes.length; i++) { + confidence[i] = (bytes[i] & 0xff) / 255f; + } + return (T) new SegmentationMask(integer(root, "width"), + integer(root, "height"), confidence, metadata); + } + case DOCUMENT_SCANNING: { + String base64 = string(root, "data"); + return (T) new DocumentScanResult(new byte[][] { + Base64.decode(base64.getBytes("UTF-8")) + }, metadata); + } + default: + throw new VisionException(VisionException.UNSUPPORTED, + "Unknown Apple Vision feature " + feature); + } + } + + private static VisionRect rect(Map value) { + return new VisionRect(number(value, "x"), number(value, "y"), + number(value, "width"), number(value, "height")); + } + + private static float number(Map value, String key) { + Object out = value.get(key); + return out instanceof Number ? ((Number) out).floatValue() : 0; + } + + private static int integer(Map value, String key) { + Object out = value.get(key); + return out instanceof Number ? ((Number) out).intValue() : 0; + } + + private static String string(Map value, String key) { + String out = stringOrNull(value, key); + return out == null ? "" : out; + } + + private static String stringOrNull(Map value, String key) { + Object out = value.get(key); + return out == null ? null : String.valueOf(out); + } + + @Override + public void close() { + closed = true; + } +} diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md new file mode 100644 index 00000000000..c24c58871b2 --- /dev/null +++ b/docs/ai-on-device-architecture.md @@ -0,0 +1,186 @@ +# Built-in on-device AI architecture + +This document is the maintainer reference for the built-in vision, +language, and LiteRT APIs. User-facing examples live in +`docs/developer-guide/Ai-And-Speech.asciidoc`. + +## Scope + +The public surface is in: + +- `CodenameOne/src/com/codename1/ai/vision` +- `CodenameOne/src/com/codename1/ai/language` +- `CodenameOne/src/com/codename1/ai/inference` + +The API is part of `codenameone-core` on every target. The default +`CodenameOneImplementation` factories return `null`, so an unsupported +target has deterministic `isSupported() == false` behavior and never falls +back to a cloud service. + +Native backends are: + +| Target | Vision | Language | `.tflite` inference | +| --- | --- | --- | --- | +| Android | ML Kit | ML Kit | LiteRT | +| iOS | Apple Vision/Core Image; optional ML Kit | ML Kit | TensorFlow Lite Objective-C; optional Core ML delegate | +| Mac native | Apple Vision/Core Image through Mac Catalyst | Unsupported fallback | Unsupported fallback | +| JavaSE, JavaScript, native Windows/Linux | Unsupported fallback | Unsupported fallback | Unsupported fallback | +| watchOS, tvOS | Unsupported fallback | Unsupported fallback | Unsupported fallback | + +Document correction is intentionally Apple-only. The Google ML Kit document +scanner is an interactive Activity camera flow, while the core +`DocumentScanner` contract analyzes an existing `VisionImage`. + +## Build-time selection + +`maven/platform-feature-catalog` is the authoritative dependency registry. +`PlatformFeatureCatalog.Accumulator` consumes exact class and method +references from the existing bytecode scanners. + +The local Maven plugin and BuildDaemon both consume the same source contract: + +- `maven/codenameone-maven-plugin/.../AndroidGradleBuilder.java` +- `maven/codenameone-maven-plugin/.../IPhoneBuilder.java` +- `BuildDaemon/src/com/codename1/build/daemon/AndroidGradleBuilder.java` +- `BuildDaemon/src/com/codename1/build/daemon/IPhoneBuilder.java` + +BuildDaemon carries a source copy of `PlatformFeatureCatalog.java` because it +is built and deployed separately. Keep that copy byte-for-byte equal to the +Maven module source and run `cmp` as part of validation. + +Local source builds replace the generated iOS/Mac project directory before +copying the new result. This is required for granular removal: a plain +directory merge would retain an old Podfile, workspace, Pods directory, or +optional native source after the application stops referencing a feature. + +### Android source granularity + +The optional Android sources are excluded from `maven/android` compilation +and compiled in the generated application after dependencies are selected. +The port bundle contains: + +- one reflection-loaded group dispatcher (`AndroidVisionImpl` or + `AndroidLanguageImpl`); +- one dependency-neutral group adapter base; +- one source file per concrete ML Kit feature; +- one `AndroidInferenceImpl`, because LiteRT is already a single dependency. + +`AndroidGradleBuilder.androidAiAdapterSource()` maps each exact public entry +point to one source. `pruneOptionalAiSources()` deletes every unselected +source before Gradle compiles the generated application. R8 keeps only the +remaining `com.codename1.impl.android.ai` classes so reflection cannot rename +the selected dispatcher or adapter. + +| Core entry point | Retained Android source | Gradle dependency | +| --- | --- | --- | +| `TextRecognizer` | `AndroidTextRecognitionAdapter.java` | `com.google.mlkit:text-recognition:16.0.0` | +| `BarcodeScanner` | `AndroidBarcodeScanningAdapter.java` | `com.google.mlkit:barcode-scanning:17.2.0` | +| `FaceDetector` | `AndroidFaceDetectionAdapter.java` | `com.google.mlkit:face-detection:16.1.5` | +| `ImageLabeler` | `AndroidImageLabelingAdapter.java` | `com.google.mlkit:image-labeling:17.0.7` | +| `PoseDetector` | `AndroidPoseDetectionAdapter.java` | `com.google.mlkit:pose-detection:18.0.0-beta3` | +| `SelfieSegmenter` | `AndroidSelfieSegmentationAdapter.java` | `com.google.mlkit:segmentation-selfie:16.0.0-beta5` | +| `LanguageIdentifier` | `AndroidLanguageIdAdapter.java` | `com.google.mlkit:language-id:17.0.6` | +| `Translator` | `AndroidTranslationAdapter.java` | `com.google.mlkit:translate:17.0.3` | +| `SmartReply` | `AndroidSmartReplyAdapter.java` | `com.google.mlkit:smart-reply:17.0.4` | +| `InferenceSession` | `AndroidInferenceImpl.java` | `com.google.ai.edge.litert:litert:1.0.1` | + +### Apple method granularity + +`CN1Vision.m` and `CN1Language.m` use `__has_include` around each ML Kit +component. The class scanner always links the small Apple system frameworks +needed by a referenced analyzer. It adds a Google ML Kit vision pod only +when it observes both: + +1. the concrete analyzer class; and +2. a call to `VisionBackends.mlKit()`. + +On iOS, language entry points map directly to independent +`GoogleMLKit/LanguageID`, `GoogleMLKit/Translate`, and +`GoogleMLKit/SmartReply` pods. `InferenceSession` selects +`TensorFlowLiteObjC/CoreML`. + +The native implementation is disabled for watchOS and tvOS. Mac native uses +the Catalyst slice of the framework-only Apple Vision implementation. The +official Google ML Kit and TensorFlow Lite Objective-C packages do not ship +Mac Catalyst slices, so their catalog entries are marked incompatible with +Catalyst. The builders omit those package dependencies from a Mac-native +build; language and inference consequently use their explicit unsupported +stubs there. + +## Image and camera contract + +`VisionImage` owns defensive copies of its input. It accepts encoded JPEG or +PNG, NV21, and RGBA8888. Android feeds raw NV21 directly to ML Kit and +converts RGBA to a bitmap. Apple converts the raw formats to an encoded image +inside `CN1Vision.m`. + +`VisionImage.fromCameraFrame()` is safe beyond the +`FrameListener.onFrame()` callback because it copies the callback-owned +arrays. `VisionPipeline` allows one request to run and retains only the newest +pending frame. This bounds memory and latency for live OCR, barcode, face, +pose, labeling, or segmentation. + +Native results are converted to stable Codename One value types. Geometry is +normalized to the top-left coordinate system. `VisionMetadata` carries the +actual backend id without exposing platform classes. + +## Inference contract + +`InferenceSession` supports named typed tensors, multiple inputs and outputs, +input resizing, and reusable native sessions. Model sources are bytes, +resources, private files, or the HTTPS-only `ModelCache`. The cache can verify +a SHA-256 digest before publishing a downloaded model. + +All native sessions and analyzers must be closed. Expensive open, analysis, +and inference work runs off the EDT; completion and error delivery return to +the EDT. + +## Permanent cross-platform coverage + +`scripts/hellocodenameone` registers three non-screenshot conformance tests: + +- `VisionOnDeviceApiTest` covers `VisionImage.fromCameraFrame()` ownership, + option normalization, capability queries for every analyzer, and close + semantics. +- `LanguageOnDeviceApiTest` covers language value/options contracts, + capability queries for identification, translation, and smart reply, and + immediate unsupported resources. +- `InferenceOnDeviceApiTest` covers immutable tensors and model sources, + validation and options, runtime capability reporting, and the unsupported + session-open contract. + +The tests avoid permission prompts, mutable model downloads, and large bundled +models. Their concrete API references are nevertheless part of the application +bytecode, so the Android and Apple source-build jobs exercise the same granular +builder selection used by applications. The tests map to independent +`on-device-vision`, `on-device-language`, and `on-device-inference` features in +`docs/website/data/port_status.json`; fresh per-port reports replace the +bootstrap `not-run` results after the updated suite runs on `master`. + +## Adding a feature + +1. Add the vendor-neutral API and result types to core. +2. Extend the appropriate implementation SPI under + `CodenameOne/src/com/codename1/impl`. +3. Add one Android adapter source and one exact + `androidAiAdapterSource()` mapping. +4. Add the smallest Android dependency to `PlatformFeatureCatalog`. +5. Add the Apple implementation behind a feature-specific compile guard. +6. Add the system framework and optional pod mapping to the catalog. +7. Copy the catalog into BuildDaemon and update both builders if source + selection changes. +8. Add core API tests, catalog tests, builder-selection tests, Java 8 Android + compilation, Objective-C syntax compilation, CocoaPods integration, and + Xcode device/Catalyst builds as applicable. Some Google ML Kit releases + cannot link an arm64 simulator even though the sources compile there. + +Use an isolated Maven repository for validation: + +```sh +mvn -Dmaven.repo.local=/private/tmp/cn1-ai-validation-m2 ... +``` + +Large model families and native runtimes such as Whisper and Stable Diffusion +remain opt-in cn1libs. The core/builder approach is intended for APIs whose +code can be selected cheaply and whose model payloads are supplied by the OS, +downloaded lazily by the native SDK, or supplied by the application. diff --git a/docs/demos/common/src/main/snippets/developer-guide/ai-and-speech.xml b/docs/demos/common/src/main/snippets/developer-guide/ai-and-speech.xml deleted file mode 100644 index 7098b1402ee..00000000000 --- a/docs/demos/common/src/main/snippets/developer-guide/ai-and-speech.xml +++ /dev/null @@ -1,10 +0,0 @@ -// Generated from docs/developer-guide source blocks. Edit the guide snippets here, not inline. - -// tag::ai-and-speech-xml-001[] - - com.codenameone - cn1-ai-mlkit-barcode-lib - ${cn1.version} - pom - -// end::ai-and-speech-xml-001[] diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index 0d5ffb29561..080fb65a821 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -2,10 +2,10 @@ [[ai-and-speech-section,AI And Speech Section]] Codename One ships a portable LLM client, a streaming chat component, -speech-to-text and text-to-speech APIs, and a family of cn1lib bridges -that wire ML Kit into iOS and Android builds. All the public surface -lives next to the rest of the framework: the simulator, the cloud -builder, and your CI pipeline all run the same code. +speech-to-text and text-to-speech APIs, built-in vision and language +services, and a portable LiteRT inference API. The builders add native +frameworks and dependencies when application bytecode references the +corresponding entry point. This chapter introduces every piece in turn: @@ -19,8 +19,12 @@ This chapter introduces every piece in turn: * `com.codename1.security.SecureStorage` non-prompting overloads -- silent reads for secrets the network layer needs on every call, such as LLM API keys. -* The `cn1-ai-mlkit-*` cn1libs -- ML Kit barcode, document, and face - detection bridges with the native build-time scanner already wired up. +* `com.codename1.ai.vision` -- OCR, barcode, face, labeling, pose, + segmentation, document correction, and camera-frame pipelines. +* `com.codename1.ai.inference` -- reusable LiteRT sessions for + `.tflite` models. +* `com.codename1.ai.language` -- language identification, translation, + and smart reply. NOTE: Each async call in this chapter returns an `com.codename1.util.AsyncResource`. Use `.ready(...)` for the success @@ -267,49 +271,183 @@ when the OS exposes them; `setVoiceId(...)` accepts any of those strings, or `null` to use the default voice for the configured language. -=== ML Kit cn1libs +=== On-device vision -Three cn1libs ship with the framework today, each backed by ML Kit on -both platforms. The build-time scanner picks up the `com.codename1.ai.*` -class references in your code and injects the matching Pods, Swift -Packages, Gradle dependencies, `Info.plist` usage strings, and Android -permissions; you don't edit build hints by hand. +The vision API accepts JPEG, PNG, NV21, or RGBA8888 data through +`VisionImage`. `VisionImage.fromCameraFrame(frame)` is normally the +best way to connect a camera because it copies the frame's +callback-owned buffers before analysis becomes asynchronous. Each +analyzer returns stable Codename One result types. Bounds and points +use normalized coordinates in the range 0 through 1. -[cols="1,1,2", options="header"] +Android uses ML Kit. iOS and Mac native builds default to Apple Vision +and Core Image. iOS can select ML Kit explicitly: + +[source,java] +---- +VisionOptions options = new VisionOptions() + .backend(VisionBackends.appleVision()) + .minimumConfidence(0.5f); + +new TextRecognizer(options) + .process(VisionImage.encoded(jpegBytes)) + .ready(result -> Log.p(result.getText())) + .except(error -> Log.e(error)); +---- + +Create and reuse an analyzer when processing a sequence. Call `close()` +when the analyzer is no longer needed. Before showing a feature, call +its `isSupported()` method: availability can depend on the target, +linked backend, and OS version. + +`VisionPipeline` attaches an analyzer to a `CameraSession` frame stream +and keeps only the newest pending frame. Live OCR and pose detection +therefore cannot build an unbounded queue when inference is slower than +the camera. Results and errors are delivered on the Codename One EDT. + +`DocumentScanner` currently performs still-image rectangle correction on +Apple platforms. Google's Android document scanner is an interactive +camera flow rather than an analyzer for an existing `VisionImage`, so the +Android backend reports it as unsupported instead of silently launching UI. + +==== Vision feature and dependency selection + +The Java package is always part of core. The builders scan the concrete +analyzer classes and retain only the native adapter and dependency for +each class the application actually uses: + +[cols="2,2,3",options="header"] |=== -| cn1lib | Public API | Provides - -| `cn1-ai-mlkit-barcode` | `BarcodeScanner.scan(byte[])` | Decode QR, -EAN, Code 128, and the other ML Kit-supported barcode formats from an -image. -| `cn1-ai-mlkit-docscan` | `DocumentScanner.scanToFile(byte[])` | -Capture and crop document photos. On iOS this routes through `VisionKit` -and on Android through the Google Play Services document scanner. -| `cn1-ai-mlkit-face` | `FaceDetector.detect(byte[])` | Detect faces and -return packed `int[]` bounding rectangles (four ints per face: x, y, -width, height). +| API | Android | Apple default / optional backend + +| `TextRecognizer` +| ML Kit text recognition +| Vision / `GoogleMLKit/TextRecognition` + +| `BarcodeScanner` +| ML Kit barcode scanning +| Vision / `GoogleMLKit/BarcodeScanning` + +| `FaceDetector` +| ML Kit face detection +| Vision / `GoogleMLKit/FaceDetection` + +| `ImageLabeler` +| ML Kit image labeling +| Vision / `GoogleMLKit/ImageLabeling` + +| `PoseDetector` +| ML Kit pose detection +| Vision / `GoogleMLKit/PoseDetection` + +| `SelfieSegmenter` +| ML Kit selfie segmentation +| Vision / `GoogleMLKit/SegmentationSelfie` + +| `DocumentScanner` +| Unsupported for still images +| Core Image perspective correction |=== -==== Adding a cn1lib +The optional iOS ML Kit pod is added only when both the analyzer and +`VisionBackends.mlKit()` are referenced. Referencing one Android +analyzer does not compile or package the other five adapters. + +=== Portable LiteRT inference -Add the dependency to `common/pom.xml` with `pom` so Maven -pulls in the per-platform classifier jars: +`InferenceSession` loads `.tflite` data from bytes, a bundled resource, +or a private file. Android uses LiteRT and can request NNAPI. iOS uses +TensorFlow Lite Objective-C and may request the Core ML delegate. Both +mobile implementations fall back to CPU unless +`allowFallback(false)` is set. The JavaSE simulator and targets without +a native LiteRT runtime, including Mac native, return `false` from +`InferenceSession.isSupported()`. -[source,xml] +[source,java] ---- -include::../demos/common/src/main/snippets/developer-guide/ai-and-speech.xml[tag=ai-and-speech-xml-001,indent=0] +InferenceSession.open( + ModelSource.resource("/models/classifier.tflite"), + new InferenceOptions().accelerator( + InferenceOptions.Accelerator.AUTO)) + .ready(session -> { + Tensor input = Tensor.floats("serving_default_input", + new int[] {1, 224, 224, 3}, pixels); + session.run(new Tensor[] {input}) + .ready(outputs -> consume(outputs[0])) + .except(error -> Log.e(error)); + }); ---- -==== Example: Scanning a barcode +Open is asynchronous because model initialization can allocate native +resources. A session is reusable and supports multiple named inputs, +multiple outputs, and `resizeInput(...)`. Tensor data is copied across +the port boundary. Always close the session. + +Bundle small models as resources. For larger models use +`ModelCache.fetch(url, cacheKey, sha256)`, which downloads over HTTPS, +verifies the optional SHA-256 digest, and reuses the app-private cached +file. The URL must use HTTPS. Treat the digest as required when the +model is not controlled by the same deployment pipeline as the app. +=== On-device language services -==== Example: Face detection +`LanguageIdentifier`, `Translator`, and `SmartReply` share +`LanguageOptions`. Translation models are downloaded and cached by ML +Kit when a language pair is first used. Android and iOS builders +select `language-id`, `translate`, and `smart-reply` independently, so +using language identification alone does not package the other two +SDKs. +[source,java] +---- +if (Translator.isSupported()) { + Translator.translate("Where is the station?", "en", "fr", + new LanguageOptions()) + .ready(value -> Log.p(value)) + .except(error -> Log.e(error)); +} +---- -NOTE: ML Kit barcode and face detection ship as small Pods or Gradle -dependencies that add a few megabytes to the binary. The document -scanner depends on Google Play Services on Android; on devices without -Play Services the call returns an error through `AsyncResource.except`. +`LanguageBackends.auto()` selects ML Kit on the supported Android and +iOS targets. `LanguageBackends.mlKit()` makes that selection explicit. +Mac native and the other unsupported targets return `false` from the +language service `isSupported()` checks. Completion and failure callbacks +arrive on the EDT. + +=== Platform availability + +The public API and the unsupported fallback are present on every +Codename One target. Vision has native implementations on Android, iOS, +and Mac native (Mac Catalyst); Apple Vision is the default on Apple +targets and Android uses ML Kit. Language services and LiteRT inference +have native implementations on Android and iOS. JavaSE, JavaScript, +native Windows, native Linux, watchOS, and tvOS report unsupported for +these new APIs; Mac native reports unsupported for language and LiteRT. +No target silently sends data to a cloud inference service. Large +portable runtimes or model payloads for the remaining targets remain +appropriate opt-in libraries. + +No build hints or cn1lib coordinates are required for the built-in +mobile implementations. The local Maven plugin and BuildDaemon use the +same `PlatformFeatureCatalog`, so local and cloud builds make the same +dependency decisions. + +=== Migrating from the AI cn1libs + +The old `com.codename1.ai.mlkit.*` and `com.codename1.ai.tflite` +packages are not compatibility aliases. Remove those dependencies and +move directly to `com.codename1.ai.vision`, +`com.codename1.ai.language`, or `com.codename1.ai.inference`. Result +coordinates are normalized floats rather than packed +platform-specific integer arrays. `cn1-ai-whisper` and +`cn1-ai-stablediffusion` remain cn1libs because their native payloads +and models are intentionally large. + +Remove the retired cn1lib dependency before adding the built-in class. +Keeping both packages can leave duplicate ML Kit versions in an +Android or CocoaPods dependency graph. The builders deliberately do +not ship compatibility aliases because those aliases would keep the +old native-interface and packaging contracts alive. ==== Putting it all together diff --git a/docs/website/data/port_status.json b/docs/website/data/port_status.json index 65efde54555..5da97844775 100644 --- a/docs/website/data/port_status.json +++ b/docs/website/data/port_status.json @@ -377,6 +377,27 @@ "description": "Checks camera discovery and capture where hardware and runtime permissions are available.", "tests": ["CameraApiTest"] }, + { + "id": "on-device-vision", + "category": "On-device AI", + "name": "Vision analysis", + "description": "Checks camera-frame image ownership, backend capability reporting, and analyzer lifecycle for OCR, barcodes, faces, image labels, pose, segmentation, and document scanning.", + "tests": ["VisionOnDeviceApiTest"] + }, + { + "id": "on-device-language", + "category": "On-device AI", + "name": "Language services", + "description": "Checks language identification, translation, and smart-reply capability reporting plus deterministic unsupported fallbacks.", + "tests": ["LanguageOnDeviceApiTest"] + }, + { + "id": "on-device-inference", + "category": "On-device AI", + "name": "Model inference", + "description": "Checks immutable tensor and model-source contracts, inference options, runtime capability reporting, and deterministic unsupported fallback behavior.", + "tests": ["InferenceOnDeviceApiTest"] + }, { "id": "calendar-integration", "category": "Platform-dependent APIs", diff --git a/docs/website/data/port_status_reports/android.json b/docs/website/data/port_status_reports/android.json index 8da69261ae6..fb044632f8f 100644 --- a/docs/website/data/port_status_reports/android.json +++ b/docs/website/data/port_status_reports/android.json @@ -7,8 +7,8 @@ "suite_finished": true, "summary": { "fail": 0, - "not-run": 0, - "pass": 163, + "not-run": 5, + "pass": 164, "skip": 1 }, "tests": { @@ -682,6 +682,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/ios-gl.json b/docs/website/data/port_status_reports/ios-gl.json index a17b7de0cbf..56dceeb7689 100644 --- a/docs/website/data/port_status_reports/ios-gl.json +++ b/docs/website/data/port_status_reports/ios-gl.json @@ -7,8 +7,8 @@ "suite_finished": true, "summary": { "fail": 0, - "not-run": 0, - "pass": 162, + "not-run": 5, + "pass": 163, "skip": 2 }, "tests": { @@ -685,6 +685,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/ios-metal.json b/docs/website/data/port_status_reports/ios-metal.json index dd8910cd5e8..41e6d8bb21c 100644 --- a/docs/website/data/port_status_reports/ios-metal.json +++ b/docs/website/data/port_status_reports/ios-metal.json @@ -7,8 +7,8 @@ "suite_finished": true, "summary": { "fail": 0, - "not-run": 0, - "pass": 162, + "not-run": 5, + "pass": 163, "skip": 2 }, "tests": { @@ -685,6 +685,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/javascript.json b/docs/website/data/port_status_reports/javascript.json index c48cde4c8c1..a31881d039b 100644 --- a/docs/website/data/port_status_reports/javascript.json +++ b/docs/website/data/port_status_reports/javascript.json @@ -7,8 +7,8 @@ "suite_finished": true, "summary": { "fail": 0, - "not-run": 0, - "pass": 163, + "not-run": 5, + "pass": 164, "skip": 1 }, "tests": { @@ -682,6 +682,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/linux-arm64.json b/docs/website/data/port_status_reports/linux-arm64.json index b7e1e9e0acf..e2858afe2d2 100644 --- a/docs/website/data/port_status_reports/linux-arm64.json +++ b/docs/website/data/port_status_reports/linux-arm64.json @@ -7,8 +7,8 @@ "suite_finished": false, "summary": { "fail": 0, - "not-run": 0, - "pass": 164, + "not-run": 5, + "pass": 165, "skip": 0 }, "tests": { @@ -679,6 +679,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/linux-x64.json b/docs/website/data/port_status_reports/linux-x64.json index 782e551b720..91949b47702 100644 --- a/docs/website/data/port_status_reports/linux-x64.json +++ b/docs/website/data/port_status_reports/linux-x64.json @@ -7,8 +7,8 @@ "suite_finished": false, "summary": { "fail": 0, - "not-run": 0, - "pass": 164, + "not-run": 5, + "pass": 165, "skip": 0 }, "tests": { @@ -679,6 +679,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/mac-native.json b/docs/website/data/port_status_reports/mac-native.json index 0ff436a2839..d7792f7aa4b 100644 --- a/docs/website/data/port_status_reports/mac-native.json +++ b/docs/website/data/port_status_reports/mac-native.json @@ -7,8 +7,8 @@ "suite_finished": true, "summary": { "fail": 0, - "not-run": 0, - "pass": 162, + "not-run": 5, + "pass": 163, "skip": 2 }, "tests": { @@ -685,6 +685,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/tvos.json b/docs/website/data/port_status_reports/tvos.json index 5a608484997..d9c49c0ec83 100644 --- a/docs/website/data/port_status_reports/tvos.json +++ b/docs/website/data/port_status_reports/tvos.json @@ -7,8 +7,8 @@ "suite_finished": true, "summary": { "fail": 0, - "not-run": 0, - "pass": 162, + "not-run": 5, + "pass": 163, "skip": 2 }, "tests": { @@ -685,6 +685,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/watchos.json b/docs/website/data/port_status_reports/watchos.json index d0a5fc21f7a..b2bdbfad26b 100644 --- a/docs/website/data/port_status_reports/watchos.json +++ b/docs/website/data/port_status_reports/watchos.json @@ -7,8 +7,8 @@ "suite_finished": true, "summary": { "fail": 0, - "not-run": 0, - "pass": 162, + "not-run": 5, + "pass": 163, "skip": 2 }, "tests": { @@ -685,6 +685,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/windows-arm64.json b/docs/website/data/port_status_reports/windows-arm64.json index 636a57e5cbb..591628f26f5 100644 --- a/docs/website/data/port_status_reports/windows-arm64.json +++ b/docs/website/data/port_status_reports/windows-arm64.json @@ -7,8 +7,8 @@ "suite_finished": false, "summary": { "fail": 0, - "not-run": 0, - "pass": 164, + "not-run": 5, + "pass": 165, "skip": 0 }, "tests": { @@ -679,6 +679,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/windows-x64.json b/docs/website/data/port_status_reports/windows-x64.json index b0025155457..1530b2cc8e0 100644 --- a/docs/website/data/port_status_reports/windows-x64.json +++ b/docs/website/data/port_status_reports/windows-x64.json @@ -7,8 +7,8 @@ "suite_finished": false, "summary": { "fail": 0, - "not-run": 0, - "pass": 164, + "not-run": 5, + "pass": 165, "skip": 0 }, "tests": { @@ -679,6 +679,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/maven/android/pom.xml b/maven/android/pom.xml index 2afe8497b10..c56214a4553 100644 --- a/maven/android/pom.xml +++ b/maven/android/pom.xml @@ -89,6 +89,10 @@ below because that zip takes the whole source dir. --> com/codename1/impl/android/ar/** + + com/codename1/impl/android/ai/** diff --git a/maven/cn1-ai-mlkit-barcode/android/pom.xml b/maven/cn1-ai-mlkit-barcode/android/pom.xml deleted file mode 100644 index cdd53c44ed4..00000000000 --- a/maven/cn1-ai-mlkit-barcode/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-barcode - 8.0-SNAPSHOT - - - cn1-ai-mlkit-barcode-android - jar - Codename One AI: cn1-ai-mlkit-barcode-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-barcode-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-barcode/android/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScannerImpl.java b/maven/cn1-ai-mlkit-barcode/android/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScannerImpl.java deleted file mode 100644 index b9143cd3b2a..00000000000 --- a/maven/cn1-ai-mlkit-barcode/android/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScannerImpl.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.codename1.ai.mlkit.barcode; - - -public class NativeBarcodeScannerImpl { - public String[] scan(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new String[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.barcode.BarcodeScannerOptions o = - new com.google.mlkit.vision.barcode.BarcodeScannerOptions.Builder().build(); - com.google.mlkit.vision.barcode.BarcodeScanner scanner = - com.google.mlkit.vision.barcode.BarcodeScanning.getClient(o); - final java.util.List out = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - scanner.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - java.util.List>() { - public void onSuccess(java.util.List rs) { - for (com.google.mlkit.vision.barcode.common.Barcode b : rs) { - String v = b.getRawValue(); - if (v != null) out.add(v); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.toArray(new String[0]); - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-barcode/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-barcode/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-barcode/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-barcode/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-barcode/common/codenameone_library_required.properties deleted file mode 100644 index 62c8e803db1..00000000000 --- a/maven/cn1-ai-mlkit-barcode/common/codenameone_library_required.properties +++ /dev/null @@ -1,7 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-barcode. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/BarcodeScanning -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:barcode-scanning:17.2.0' -codename1.arg.android.xpermissions= diff --git a/maven/cn1-ai-mlkit-barcode/common/pom.xml b/maven/cn1-ai-mlkit-barcode/common/pom.xml deleted file mode 100644 index fd6174a399c..00000000000 --- a/maven/cn1-ai-mlkit-barcode/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-barcode - 8.0-SNAPSHOT - - - cn1-ai-mlkit-barcode-common - jar - Codename One AI: cn1-ai-mlkit-barcode-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/BarcodeScanner.java b/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/BarcodeScanner.java deleted file mode 100644 index 64724c6cc01..00000000000 --- a/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/BarcodeScanner.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.barcode; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Barcode Scanning. -/// -/// Decodes barcodes (QR, EAN, UPC, Data Matrix, PDF417, etc.) from images. -/// Bridges to `MLKitBarcodeScanning` on iOS and -/// `com.google.mlkit:barcode-scanning` on Android. -/// -public final class BarcodeScanner { - private BarcodeScanner() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeBarcodeScanner bridge = NativeLookup.create(NativeBarcodeScanner.class); - return bridge != null && bridge.isSupported(); - } - - public static AsyncResource scan(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - if (imageBytes == null || imageBytes.length == 0) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(new String[0]); } - }); - return out; - } - final NativeBarcodeScanner bridge = NativeLookup.create(NativeBarcodeScanner.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("BarcodeScanner.scan is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String[] r = bridge.scan(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new String[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("BarcodeScanner.scan failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScanner.java b/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScanner.java deleted file mode 100644 index 84cafb7deee..00000000000 --- a/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScanner.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.barcode; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [BarcodeScanner]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeBarcodeScanner extends NativeInterface { - String[] scan(byte[] imageBytes); -} diff --git a/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/package-info.java b/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/package-info.java deleted file mode 100644 index 573acffadf5..00000000000 --- a/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/package-info.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit Barcode Scanning. -/// -/// Decodes barcodes (QR, EAN, UPC, Data Matrix, PDF417, etc.) from images. -/// Bridges to `MLKitBarcodeScanning` on iOS and -/// `com.google.mlkit:barcode-scanning` on Android. -/// -/// The single public class in this package is [BarcodeScanner], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeBarcodeScanner` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `BarcodeScanner.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.barcode; diff --git a/maven/cn1-ai-mlkit-barcode/common/src/test/java/com/codename1/ai/mlkit/barcode/BarcodeScannerTest.java b/maven/cn1-ai-mlkit-barcode/common/src/test/java/com/codename1/ai/mlkit/barcode/BarcodeScannerTest.java deleted file mode 100644 index aad5a6bf8d1..00000000000 --- a/maven/cn1-ai-mlkit-barcode/common/src/test/java/com/codename1/ai/mlkit/barcode/BarcodeScannerTest.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.codename1.ai.mlkit.barcode; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class BarcodeScannerTest { - - /** Mock implementation of NativeBarcodeScanner for headless JVM tests. */ - static class MockBridge implements NativeBarcodeScanner { - boolean supported = true; - public boolean isSupported() { return supported; } - public String[] scan(byte[] imageBytes) { - return new String[]{"x", "y"}; - } - } - - @Test - void mock_bridge_returns_two_codes() { - MockBridge b = new MockBridge(); - String[] r = b.scan(new byte[]{1, 2, 3}); - assertEquals(2, r.length); - assertEquals("x", r[0]); - } - - @Test - void bridge_reports_supported() { - assertTrue(new MockBridge().isSupported()); - } -} diff --git a/maven/cn1-ai-mlkit-barcode/ios/pom.xml b/maven/cn1-ai-mlkit-barcode/ios/pom.xml deleted file mode 100644 index d949a9c1c35..00000000000 --- a/maven/cn1-ai-mlkit-barcode/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-barcode - 8.0-SNAPSHOT - - - cn1-ai-mlkit-barcode-ios - jar - Codename One AI: cn1-ai-mlkit-barcode-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-barcode-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-barcode/ios/src/main/objectivec/com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.h b/maven/cn1-ai-mlkit-barcode/ios/src/main/objectivec/com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.h deleted file mode 100644 index aa55f016d55..00000000000 --- a/maven/cn1-ai-mlkit-barcode/ios/src/main/objectivec/com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl : NSObject { -} - --(NSData*)scan:(NSData*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-barcode/ios/src/main/objectivec/com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.m b/maven/cn1-ai-mlkit-barcode/ios/src/main/objectivec/com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.m deleted file mode 100644 index 2c93b7eb6e1..00000000000 --- a/maven/cn1-ai-mlkit-barcode/ios/src/main/objectivec/com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.m +++ /dev/null @@ -1,48 +0,0 @@ -#import "com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.h" -#import -#import -#import -#import - -@implementation com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl - --(NSData*)scan:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [self packStrings:@[]]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKBarcodeScannerOptions *opts = [[MLKBarcodeScannerOptions alloc] init]; - MLKBarcodeScanner *scanner = [MLKBarcodeScanner barcodeScannerWithOptions:opts]; - __block NSArray *values = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [scanner processImage:vision completion:^(NSArray * _Nullable barcodes, - NSError * _Nullable error) { - NSMutableArray *m = [NSMutableArray array]; - for (MLKBarcode *b in barcodes ?: @[]) { - if (b.rawValue) [m addObject:b.rawValue]; - } - values = m; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return [self packStrings:values]; -} - --(NSData*)packStrings:(NSArray *)strings { - // Encode as length-prefixed UTF-8 (network byte order int + bytes). - NSMutableData *out = [NSMutableData data]; - uint32_t count = htonl((uint32_t)strings.count); - [out appendBytes:&count length:sizeof(count)]; - for (NSString *s in strings) { - NSData *u = [s dataUsingEncoding:NSUTF8StringEncoding]; - uint32_t len = htonl((uint32_t)u.length); - [out appendBytes:&len length:sizeof(len)]; - [out appendData:u]; - } - return out; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-barcode/javascript/pom.xml b/maven/cn1-ai-mlkit-barcode/javascript/pom.xml deleted file mode 100644 index 9b47c753f93..00000000000 --- a/maven/cn1-ai-mlkit-barcode/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-barcode - 8.0-SNAPSHOT - - - cn1-ai-mlkit-barcode-javascript - jar - Codename One AI: cn1-ai-mlkit-barcode-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-barcode-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-barcode/javascript/src/main/javascript/com_codename1_ai_mlkit_barcode_NativeBarcodeScanner.js b/maven/cn1-ai-mlkit-barcode/javascript/src/main/javascript/com_codename1_ai_mlkit_barcode_NativeBarcodeScanner.js deleted file mode 100644 index 0c74196d599..00000000000 --- a/maven/cn1-ai-mlkit-barcode/javascript/src/main/javascript/com_codename1_ai_mlkit_barcode_NativeBarcodeScanner.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.scan__byte_1ARRAY = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_barcode_NativeBarcodeScanner= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-barcode/javase/pom.xml b/maven/cn1-ai-mlkit-barcode/javase/pom.xml deleted file mode 100644 index c398acd1b44..00000000000 --- a/maven/cn1-ai-mlkit-barcode/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-barcode - 8.0-SNAPSHOT - - - cn1-ai-mlkit-barcode-javase - jar - Codename One AI: cn1-ai-mlkit-barcode-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-barcode-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-barcode/javase/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScannerImpl.java b/maven/cn1-ai-mlkit-barcode/javase/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScannerImpl.java deleted file mode 100644 index 55e1343f022..00000000000 --- a/maven/cn1-ai-mlkit-barcode/javase/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScannerImpl.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.codename1.ai.mlkit.barcode; - -public class NativeBarcodeScannerImpl implements NativeBarcodeScanner { - - private static boolean hintsEnsured; - private static synchronized void ensureSimulatorHints() { - if (hintsEnsured) return; - hintsEnsured = true; - java.util.Map hints = - com.codename1.ui.Display.getInstance().getProjectBuildHints(); - if (hints == null) return; // not running in the simulator - if (!hints.containsKey("ios.NSCameraUsageDescription")) { - com.codename1.ui.Display.getInstance() - .setProjectBuildHint("ios.NSCameraUsageDescription", "This app uses the camera to scan barcodes."); - } - } - - public NativeBarcodeScannerImpl() { - ensureSimulatorHints(); - } - public String[] scan(byte[] imageBytes) { - if (imageBytes == null || imageBytes.length == 0) return new String[0]; - // Deterministic stub for simulator runs. - return new String[]{"SIMULATOR_BARCODE_" + imageBytes.length}; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-barcode/lib/pom.xml b/maven/cn1-ai-mlkit-barcode/lib/pom.xml deleted file mode 100644 index 5cb4277dbcb..00000000000 --- a/maven/cn1-ai-mlkit-barcode/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-barcode - 8.0-SNAPSHOT - - - cn1-ai-mlkit-barcode-lib - pom - Codename One AI: cn1-ai-mlkit-barcode-lib - - - - com.codenameone - cn1-ai-mlkit-barcode-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-barcode-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-barcode-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-barcode-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-barcode-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-barcode/pom.xml b/maven/cn1-ai-mlkit-barcode/pom.xml deleted file mode 100644 index e1f34b84bc1..00000000000 --- a/maven/cn1-ai-mlkit-barcode/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-barcode - pom - Codename One AI: cn1-ai-mlkit-barcode - ML Kit Barcode Scanning - - - cn1-ai-mlkit-barcode - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-docscan/android/pom.xml b/maven/cn1-ai-mlkit-docscan/android/pom.xml deleted file mode 100644 index d13a13c6b03..00000000000 --- a/maven/cn1-ai-mlkit-docscan/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-docscan - 8.0-SNAPSHOT - - - cn1-ai-mlkit-docscan-android - jar - Codename One AI: cn1-ai-mlkit-docscan-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-docscan-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-docscan/android/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScannerImpl.java b/maven/cn1-ai-mlkit-docscan/android/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScannerImpl.java deleted file mode 100644 index c29139f4819..00000000000 --- a/maven/cn1-ai-mlkit-docscan/android/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScannerImpl.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.mlkit.docscan; - - -public class NativeDocumentScannerImpl { - public String scanToFile(byte[] imageBytes) { - try { - java.io.File f = java.io.File.createTempFile("docscan-", ".jpg"); - java.io.FileOutputStream fos = new java.io.FileOutputStream(f); - fos.write(imageBytes); - fos.close(); - return f.getAbsolutePath(); - } catch (java.io.IOException ioe) { - return ""; - } - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-docscan/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-docscan/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-docscan/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-docscan/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-docscan/common/codenameone_library_required.properties deleted file mode 100644 index 59ed1294531..00000000000 --- a/maven/cn1-ai-mlkit-docscan/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-docscan. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.add_frameworks=VisionKit -codename1.arg.android.gradleDep=implementation 'com.google.android.gms:play-services-mlkit-document-scanner:16.0.0-beta1' diff --git a/maven/cn1-ai-mlkit-docscan/common/pom.xml b/maven/cn1-ai-mlkit-docscan/common/pom.xml deleted file mode 100644 index 23d18917299..00000000000 --- a/maven/cn1-ai-mlkit-docscan/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-docscan - 8.0-SNAPSHOT - - - cn1-ai-mlkit-docscan-common - jar - Codename One AI: cn1-ai-mlkit-docscan-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/DocumentScanner.java b/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/DocumentScanner.java deleted file mode 100644 index 2eb94018e20..00000000000 --- a/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/DocumentScanner.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.docscan; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit / VisionKit Document Scanner. -/// -/// Captures and crops document photos. On iOS uses Apple's VisionKit + Core Image rectangle detection (no extra pod). On Android uses the Google Play services document-scanner module. -/// -public final class DocumentScanner { - private DocumentScanner() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeDocumentScanner bridge = NativeLookup.create(NativeDocumentScanner.class); - return bridge != null && bridge.isSupported(); - } - - public static AsyncResource scanToFile(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativeDocumentScanner bridge = NativeLookup.create(NativeDocumentScanner.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("DocumentScanner.scanToFile is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String r = bridge.scanToFile(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? "" : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("DocumentScanner.scanToFile failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScanner.java b/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScanner.java deleted file mode 100644 index 3426aa7e59e..00000000000 --- a/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScanner.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.docscan; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [DocumentScanner]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeDocumentScanner extends NativeInterface { - String scanToFile(byte[] imageBytes); -} diff --git a/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/package-info.java b/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/package-info.java deleted file mode 100644 index 1dd15b5334f..00000000000 --- a/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/package-info.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit / VisionKit Document Scanner. -/// -/// Captures and crops document photos. On iOS uses Apple's VisionKit + Core Image rectangle detection (no extra pod). On Android uses the Google Play services document-scanner module. -/// -/// The single public class in this package is [DocumentScanner], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeDocumentScanner` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `DocumentScanner.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.docscan; diff --git a/maven/cn1-ai-mlkit-docscan/common/src/test/java/com/codename1/ai/mlkit/docscan/DocumentScannerTest.java b/maven/cn1-ai-mlkit-docscan/common/src/test/java/com/codename1/ai/mlkit/docscan/DocumentScannerTest.java deleted file mode 100644 index d348880c08a..00000000000 --- a/maven/cn1-ai-mlkit-docscan/common/src/test/java/com/codename1/ai/mlkit/docscan/DocumentScannerTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.mlkit.docscan; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class DocumentScannerTest { - - /** Mock implementation of NativeDocumentScanner for headless JVM tests. */ - static class MockBridge implements NativeDocumentScanner { - boolean supported = true; - public boolean isSupported() { return supported; } - public String scanToFile(byte[] imageBytes) { return "/tmp/x.jpg"; } - } - - @Test - void mock_returns_path() { - MockBridge b = new MockBridge(); - assertEquals("/tmp/x.jpg", b.scanToFile(new byte[]{1})); - } -} diff --git a/maven/cn1-ai-mlkit-docscan/ios/pom.xml b/maven/cn1-ai-mlkit-docscan/ios/pom.xml deleted file mode 100644 index f416f905a37..00000000000 --- a/maven/cn1-ai-mlkit-docscan/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-docscan - 8.0-SNAPSHOT - - - cn1-ai-mlkit-docscan-ios - jar - Codename One AI: cn1-ai-mlkit-docscan-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-docscan-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-docscan/ios/src/main/objectivec/com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.h b/maven/cn1-ai-mlkit-docscan/ios/src/main/objectivec/com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.h deleted file mode 100644 index c68dd8802ee..00000000000 --- a/maven/cn1-ai-mlkit-docscan/ios/src/main/objectivec/com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl : NSObject { -} - --(NSString*)scanToFile:(NSData*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-docscan/ios/src/main/objectivec/com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.m b/maven/cn1-ai-mlkit-docscan/ios/src/main/objectivec/com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.m deleted file mode 100644 index 23ca8e045d0..00000000000 --- a/maven/cn1-ai-mlkit-docscan/ios/src/main/objectivec/com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.m +++ /dev/null @@ -1,42 +0,0 @@ -#import "com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.h" -#import -#import - -@implementation com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl - -// VisionKit-based fallback: Apple's VNDocumentCameraViewController is -// interactive; this bridge accepts a pre-captured image and returns its -// cropped JPEG path. On iOS 13+ VisionKit handles the live UI flow; the -// sample app drives that flow and feeds the bytes into the cn1lib. --(NSString*)scanToFile:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return @""; - CIImage *ci = [CIImage imageWithCGImage:image.CGImage]; - CIContext *ctx = [CIContext context]; - CIDetector *det = [CIDetector detectorOfType:CIDetectorTypeRectangle context:ctx - options:@{CIDetectorAccuracy: CIDetectorAccuracyHigh}]; - NSArray *features = [det featuresInImage:ci]; - UIImage *cropped = image; - if (features.count > 0) { - CIRectangleFeature *rf = (CIRectangleFeature *)features.firstObject; - CIImage *flat = [ci imageByApplyingFilter:@"CIPerspectiveCorrection" withInputParameters:@{ - @"inputTopLeft": [CIVector vectorWithCGPoint:rf.topLeft], - @"inputTopRight": [CIVector vectorWithCGPoint:rf.topRight], - @"inputBottomLeft": [CIVector vectorWithCGPoint:rf.bottomLeft], - @"inputBottomRight": [CIVector vectorWithCGPoint:rf.bottomRight] - }]; - CGImageRef cg = [ctx createCGImage:flat fromRect:flat.extent]; - cropped = [UIImage imageWithCGImage:cg]; - CGImageRelease(cg); - } - NSString *path = [NSString stringWithFormat:@"%@/docscan-%@.jpg", - NSTemporaryDirectory(), [[NSUUID UUID] UUIDString]]; - [UIImageJPEGRepresentation(cropped, 0.92) writeToFile:path atomically:YES]; - return path; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-docscan/javascript/pom.xml b/maven/cn1-ai-mlkit-docscan/javascript/pom.xml deleted file mode 100644 index 459f61c8265..00000000000 --- a/maven/cn1-ai-mlkit-docscan/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-docscan - 8.0-SNAPSHOT - - - cn1-ai-mlkit-docscan-javascript - jar - Codename One AI: cn1-ai-mlkit-docscan-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-docscan-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-docscan/javascript/src/main/javascript/com_codename1_ai_mlkit_docscan_NativeDocumentScanner.js b/maven/cn1-ai-mlkit-docscan/javascript/src/main/javascript/com_codename1_ai_mlkit_docscan_NativeDocumentScanner.js deleted file mode 100644 index a57708226b0..00000000000 --- a/maven/cn1-ai-mlkit-docscan/javascript/src/main/javascript/com_codename1_ai_mlkit_docscan_NativeDocumentScanner.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.scanToFile__byte_1ARRAY = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_docscan_NativeDocumentScanner= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-docscan/javase/pom.xml b/maven/cn1-ai-mlkit-docscan/javase/pom.xml deleted file mode 100644 index f85a17983ea..00000000000 --- a/maven/cn1-ai-mlkit-docscan/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-docscan - 8.0-SNAPSHOT - - - cn1-ai-mlkit-docscan-javase - jar - Codename One AI: cn1-ai-mlkit-docscan-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-docscan-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-docscan/javase/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScannerImpl.java b/maven/cn1-ai-mlkit-docscan/javase/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScannerImpl.java deleted file mode 100644 index 573928cc411..00000000000 --- a/maven/cn1-ai-mlkit-docscan/javase/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScannerImpl.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.codename1.ai.mlkit.docscan; - -public class NativeDocumentScannerImpl implements NativeDocumentScanner { - public String scanToFile(byte[] imageBytes) { - try { - java.io.File f = java.io.File.createTempFile("docscan-stub-", ".jpg"); - java.nio.file.Files.write(f.toPath(), imageBytes); - return f.getAbsolutePath(); - } catch (java.io.IOException ioe) { - return ""; - } - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-docscan/lib/pom.xml b/maven/cn1-ai-mlkit-docscan/lib/pom.xml deleted file mode 100644 index 42e7ecbeb82..00000000000 --- a/maven/cn1-ai-mlkit-docscan/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-docscan - 8.0-SNAPSHOT - - - cn1-ai-mlkit-docscan-lib - pom - Codename One AI: cn1-ai-mlkit-docscan-lib - - - - com.codenameone - cn1-ai-mlkit-docscan-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-docscan-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-docscan-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-docscan-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-docscan-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-docscan/pom.xml b/maven/cn1-ai-mlkit-docscan/pom.xml deleted file mode 100644 index 4fc73d70b6f..00000000000 --- a/maven/cn1-ai-mlkit-docscan/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-docscan - pom - Codename One AI: cn1-ai-mlkit-docscan - ML Kit / VisionKit Document Scanner - - - cn1-ai-mlkit-docscan - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-face/android/pom.xml b/maven/cn1-ai-mlkit-face/android/pom.xml deleted file mode 100644 index e6d31b07afb..00000000000 --- a/maven/cn1-ai-mlkit-face/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-face - 8.0-SNAPSHOT - - - cn1-ai-mlkit-face-android - jar - Codename One AI: cn1-ai-mlkit-face-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-face-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-face/android/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetectorImpl.java b/maven/cn1-ai-mlkit-face/android/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetectorImpl.java deleted file mode 100644 index 959577903bc..00000000000 --- a/maven/cn1-ai-mlkit-face/android/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetectorImpl.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.codename1.ai.mlkit.face; - - -public class NativeFaceDetectorImpl { - public int[] detect(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new int[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.face.FaceDetector det = - com.google.mlkit.vision.face.FaceDetection.getClient( - new com.google.mlkit.vision.face.FaceDetectorOptions.Builder().build()); - final java.util.List rs = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - det.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - java.util.List>() { - public void onSuccess(java.util.List faces) { - for (com.google.mlkit.vision.face.Face f : faces) { - android.graphics.Rect r = f.getBoundingBox(); - rs.add(new int[]{r.left, r.top, r.width(), r.height()}); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - int[] flat = new int[rs.size() * 4]; - int i = 0; - for (int[] r : rs) { System.arraycopy(r, 0, flat, i, 4); i += 4; } - return flat; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-face/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-face/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-face/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-face/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-face/common/codenameone_library_required.properties deleted file mode 100644 index 7232c084e6a..00000000000 --- a/maven/cn1-ai-mlkit-face/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-face. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/FaceDetection -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:face-detection:16.1.5' diff --git a/maven/cn1-ai-mlkit-face/common/pom.xml b/maven/cn1-ai-mlkit-face/common/pom.xml deleted file mode 100644 index 987f71784b5..00000000000 --- a/maven/cn1-ai-mlkit-face/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-face - 8.0-SNAPSHOT - - - cn1-ai-mlkit-face-common - jar - Codename One AI: cn1-ai-mlkit-face-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/FaceDetector.java b/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/FaceDetector.java deleted file mode 100644 index a73e47dc29e..00000000000 --- a/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/FaceDetector.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.face; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Face Detection. -/// -/// Detects faces in images and returns bounding boxes. -/// Bridges to `MLKitFaceDetection` on iOS and -/// `com.google.mlkit:face-detection` on Android. -/// -public final class FaceDetector { - private FaceDetector() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeFaceDetector bridge = NativeLookup.create(NativeFaceDetector.class); - return bridge != null && bridge.isSupported(); - } - - /// Returns an array of face bounding-box quadruples - /// (x, y, width, height) packed as `int[4 * n]`. - public static AsyncResource detect(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativeFaceDetector bridge = NativeLookup.create(NativeFaceDetector.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("FaceDetector.detect is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final int[] r = bridge.detect(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new int[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("FaceDetector.detect failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetector.java b/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetector.java deleted file mode 100644 index 2e4d8e52eaf..00000000000 --- a/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetector.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.face; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [FaceDetector]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeFaceDetector extends NativeInterface { - int[] detect(byte[] imageBytes); -} diff --git a/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/package-info.java b/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/package-info.java deleted file mode 100644 index b6ce855ece6..00000000000 --- a/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/package-info.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit Face Detection. -/// -/// Detects faces in images and returns bounding boxes. -/// Bridges to `MLKitFaceDetection` on iOS and -/// `com.google.mlkit:face-detection` on Android. -/// -/// The single public class in this package is [FaceDetector], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeFaceDetector` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `FaceDetector.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.face; diff --git a/maven/cn1-ai-mlkit-face/common/src/test/java/com/codename1/ai/mlkit/face/FaceDetectorTest.java b/maven/cn1-ai-mlkit-face/common/src/test/java/com/codename1/ai/mlkit/face/FaceDetectorTest.java deleted file mode 100644 index 718c8fdf837..00000000000 --- a/maven/cn1-ai-mlkit-face/common/src/test/java/com/codename1/ai/mlkit/face/FaceDetectorTest.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.codename1.ai.mlkit.face; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class FaceDetectorTest { - - /** Mock implementation of NativeFaceDetector for headless JVM tests. */ - static class MockBridge implements NativeFaceDetector { - boolean supported = true; - public boolean isSupported() { return supported; } - public int[] detect(byte[] imageBytes) { - return new int[]{1, 2, 3, 4, 5, 6, 7, 8}; - } - } - - @Test - void mock_bridge_returns_two_faces() { - MockBridge b = new MockBridge(); - int[] r = b.detect(new byte[]{1}); - assertEquals(8, r.length); - } -} diff --git a/maven/cn1-ai-mlkit-face/ios/pom.xml b/maven/cn1-ai-mlkit-face/ios/pom.xml deleted file mode 100644 index f7187d44e74..00000000000 --- a/maven/cn1-ai-mlkit-face/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-face - 8.0-SNAPSHOT - - - cn1-ai-mlkit-face-ios - jar - Codename One AI: cn1-ai-mlkit-face-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-face-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-face/ios/src/main/objectivec/com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.h b/maven/cn1-ai-mlkit-face/ios/src/main/objectivec/com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.h deleted file mode 100644 index 9fa4ad92523..00000000000 --- a/maven/cn1-ai-mlkit-face/ios/src/main/objectivec/com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_face_NativeFaceDetectorImpl : NSObject { -} - --(NSData*)detect:(NSData*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-face/ios/src/main/objectivec/com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.m b/maven/cn1-ai-mlkit-face/ios/src/main/objectivec/com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.m deleted file mode 100644 index df6c511fdb2..00000000000 --- a/maven/cn1-ai-mlkit-face/ios/src/main/objectivec/com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.m +++ /dev/null @@ -1,36 +0,0 @@ -#import "com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.h" -#import -#import -#import -#import - -@implementation com_codename1_ai_mlkit_face_NativeFaceDetectorImpl - --(NSData*)detect:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [NSData data]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKFaceDetectorOptions *opts = [[MLKFaceDetectorOptions alloc] init]; - MLKFaceDetector *det = [MLKFaceDetector faceDetectorWithOptions:opts]; - __block NSArray *faces = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [det processImage:vision completion:^(NSArray * _Nullable f, NSError * _Nullable e) { - faces = f ?: @[]; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - NSMutableData *out = [NSMutableData data]; - for (MLKFace *face in faces) { - CGRect r = face.frame; - int32_t v[4] = { htonl((int32_t)r.origin.x), htonl((int32_t)r.origin.y), - htonl((int32_t)r.size.width), htonl((int32_t)r.size.height) }; - [out appendBytes:v length:sizeof(v)]; - } - return out; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-face/javascript/pom.xml b/maven/cn1-ai-mlkit-face/javascript/pom.xml deleted file mode 100644 index b0517a4dd53..00000000000 --- a/maven/cn1-ai-mlkit-face/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-face - 8.0-SNAPSHOT - - - cn1-ai-mlkit-face-javascript - jar - Codename One AI: cn1-ai-mlkit-face-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-face-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-face/javascript/src/main/javascript/com_codename1_ai_mlkit_face_NativeFaceDetector.js b/maven/cn1-ai-mlkit-face/javascript/src/main/javascript/com_codename1_ai_mlkit_face_NativeFaceDetector.js deleted file mode 100644 index 26de1134e7a..00000000000 --- a/maven/cn1-ai-mlkit-face/javascript/src/main/javascript/com_codename1_ai_mlkit_face_NativeFaceDetector.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.detect__byte_1ARRAY = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_face_NativeFaceDetector= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-face/javase/pom.xml b/maven/cn1-ai-mlkit-face/javase/pom.xml deleted file mode 100644 index 425750cadfe..00000000000 --- a/maven/cn1-ai-mlkit-face/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-face - 8.0-SNAPSHOT - - - cn1-ai-mlkit-face-javase - jar - Codename One AI: cn1-ai-mlkit-face-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-face-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-face/javase/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetectorImpl.java b/maven/cn1-ai-mlkit-face/javase/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetectorImpl.java deleted file mode 100644 index f9ab7c094cb..00000000000 --- a/maven/cn1-ai-mlkit-face/javase/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetectorImpl.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.codename1.ai.mlkit.face; - -public class NativeFaceDetectorImpl implements NativeFaceDetector { - - private static boolean hintsEnsured; - private static synchronized void ensureSimulatorHints() { - if (hintsEnsured) return; - hintsEnsured = true; - java.util.Map hints = - com.codename1.ui.Display.getInstance().getProjectBuildHints(); - if (hints == null) return; // not running in the simulator - if (!hints.containsKey("ios.NSCameraUsageDescription")) { - com.codename1.ui.Display.getInstance() - .setProjectBuildHint("ios.NSCameraUsageDescription", "This app uses the camera to detect faces."); - } - } - - public NativeFaceDetectorImpl() { - ensureSimulatorHints(); - } - public int[] detect(byte[] imageBytes) { - // Deterministic 1-face stub for simulator runs. - if (imageBytes == null || imageBytes.length == 0) return new int[0]; - return new int[]{10, 20, 100, 120}; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-face/lib/pom.xml b/maven/cn1-ai-mlkit-face/lib/pom.xml deleted file mode 100644 index 19f8679b8c4..00000000000 --- a/maven/cn1-ai-mlkit-face/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-face - 8.0-SNAPSHOT - - - cn1-ai-mlkit-face-lib - pom - Codename One AI: cn1-ai-mlkit-face-lib - - - - com.codenameone - cn1-ai-mlkit-face-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-face-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-face-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-face-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-face-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-face/pom.xml b/maven/cn1-ai-mlkit-face/pom.xml deleted file mode 100644 index cfab4a1dc37..00000000000 --- a/maven/cn1-ai-mlkit-face/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-face - pom - Codename One AI: cn1-ai-mlkit-face - ML Kit Face Detection - - - cn1-ai-mlkit-face - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-labeling/android/pom.xml b/maven/cn1-ai-mlkit-labeling/android/pom.xml deleted file mode 100644 index 259b93fbdfe..00000000000 --- a/maven/cn1-ai-mlkit-labeling/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-labeling - 8.0-SNAPSHOT - - - cn1-ai-mlkit-labeling-android - jar - Codename One AI: cn1-ai-mlkit-labeling-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-labeling-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-labeling/android/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabelerImpl.java b/maven/cn1-ai-mlkit-labeling/android/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabelerImpl.java deleted file mode 100644 index 484bd451305..00000000000 --- a/maven/cn1-ai-mlkit-labeling/android/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabelerImpl.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.codename1.ai.mlkit.labeling; - - -public class NativeImageLabelerImpl { - public String[] label(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new String[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.label.ImageLabeler labeler = - com.google.mlkit.vision.label.ImageLabeling.getClient( - com.google.mlkit.vision.label.defaults.ImageLabelerOptions.DEFAULT_OPTIONS); - final java.util.List out = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - labeler.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - java.util.List>() { - public void onSuccess(java.util.List rs) { - for (com.google.mlkit.vision.label.ImageLabel l : rs) out.add(l.getText()); - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.toArray(new String[0]); - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-labeling/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-labeling/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-labeling/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-labeling/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-labeling/common/codenameone_library_required.properties deleted file mode 100644 index e70e9c425cd..00000000000 --- a/maven/cn1-ai-mlkit-labeling/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-labeling. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/ImageLabeling -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:image-labeling:17.0.7' diff --git a/maven/cn1-ai-mlkit-labeling/common/pom.xml b/maven/cn1-ai-mlkit-labeling/common/pom.xml deleted file mode 100644 index 1d9a1b6adb5..00000000000 --- a/maven/cn1-ai-mlkit-labeling/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-labeling - 8.0-SNAPSHOT - - - cn1-ai-mlkit-labeling-common - jar - Codename One AI: cn1-ai-mlkit-labeling-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/ImageLabeler.java b/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/ImageLabeler.java deleted file mode 100644 index 7a634b32411..00000000000 --- a/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/ImageLabeler.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.labeling; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Image Labeling. -/// -/// Returns descriptive labels for the contents of an image. -/// Bridges to `MLKitImageLabeling` on iOS and -/// `com.google.mlkit:image-labeling` on Android. -/// -public final class ImageLabeler { - private ImageLabeler() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeImageLabeler bridge = NativeLookup.create(NativeImageLabeler.class); - return bridge != null && bridge.isSupported(); - } - - public static AsyncResource label(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativeImageLabeler bridge = NativeLookup.create(NativeImageLabeler.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("ImageLabeler.label is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String[] r = bridge.label(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new String[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("ImageLabeler.label failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabeler.java b/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabeler.java deleted file mode 100644 index 6f0753f07f0..00000000000 --- a/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabeler.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.labeling; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [ImageLabeler]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeImageLabeler extends NativeInterface { - String[] label(byte[] imageBytes); -} diff --git a/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/package-info.java b/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/package-info.java deleted file mode 100644 index da8f0a06a14..00000000000 --- a/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/package-info.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit Image Labeling. -/// -/// Returns descriptive labels for the contents of an image. -/// Bridges to `MLKitImageLabeling` on iOS and -/// `com.google.mlkit:image-labeling` on Android. -/// -/// The single public class in this package is [ImageLabeler], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeImageLabeler` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `ImageLabeler.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.labeling; diff --git a/maven/cn1-ai-mlkit-labeling/common/src/test/java/com/codename1/ai/mlkit/labeling/ImageLabelerTest.java b/maven/cn1-ai-mlkit-labeling/common/src/test/java/com/codename1/ai/mlkit/labeling/ImageLabelerTest.java deleted file mode 100644 index 900ceb7acdb..00000000000 --- a/maven/cn1-ai-mlkit-labeling/common/src/test/java/com/codename1/ai/mlkit/labeling/ImageLabelerTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.mlkit.labeling; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class ImageLabelerTest { - - /** Mock implementation of NativeImageLabeler for headless JVM tests. */ - static class MockBridge implements NativeImageLabeler { - boolean supported = true; - public boolean isSupported() { return supported; } - public String[] label(byte[] imageBytes) { return new String[]{"a", "b"}; } - } - - @Test - void mock_bridge_returns_labels() { - MockBridge b = new MockBridge(); - assertArrayEquals(new String[]{"a", "b"}, b.label(new byte[]{1})); - } -} diff --git a/maven/cn1-ai-mlkit-labeling/ios/pom.xml b/maven/cn1-ai-mlkit-labeling/ios/pom.xml deleted file mode 100644 index b943abc66bd..00000000000 --- a/maven/cn1-ai-mlkit-labeling/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-labeling - 8.0-SNAPSHOT - - - cn1-ai-mlkit-labeling-ios - jar - Codename One AI: cn1-ai-mlkit-labeling-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-labeling-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-labeling/ios/src/main/objectivec/com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.h b/maven/cn1-ai-mlkit-labeling/ios/src/main/objectivec/com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.h deleted file mode 100644 index a0cb28db0b2..00000000000 --- a/maven/cn1-ai-mlkit-labeling/ios/src/main/objectivec/com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl : NSObject { -} - --(NSData*)label:(NSData*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-labeling/ios/src/main/objectivec/com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.m b/maven/cn1-ai-mlkit-labeling/ios/src/main/objectivec/com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.m deleted file mode 100644 index 3b686a66c0a..00000000000 --- a/maven/cn1-ai-mlkit-labeling/ios/src/main/objectivec/com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.m +++ /dev/null @@ -1,45 +0,0 @@ -#import "com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.h" -#import -#import -#import -#import -#import - -@implementation com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl - --(NSData*)label:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [self packStrings:@[]]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKImageLabelerOptions *opts = [[MLKImageLabelerOptions alloc] init]; - MLKImageLabeler *labeler = [MLKImageLabeler imageLabelerWithOptions:opts]; - __block NSArray *labels = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [labeler processImage:vision completion:^(NSArray * _Nullable r, NSError * _Nullable e) { - labels = r ?: @[]; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - NSMutableArray *m = [NSMutableArray array]; - for (MLKImageLabel *l in labels) { if (l.text) [m addObject:l.text]; } - return [self packStrings:m]; -} - --(NSData*)packStrings:(NSArray *)strings { - NSMutableData *out = [NSMutableData data]; - uint32_t count = htonl((uint32_t)strings.count); - [out appendBytes:&count length:sizeof(count)]; - for (NSString *s in strings) { - NSData *u = [s dataUsingEncoding:NSUTF8StringEncoding]; - uint32_t len = htonl((uint32_t)u.length); - [out appendBytes:&len length:sizeof(len)]; - [out appendData:u]; - } - return out; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-labeling/javascript/pom.xml b/maven/cn1-ai-mlkit-labeling/javascript/pom.xml deleted file mode 100644 index 51273cbb35f..00000000000 --- a/maven/cn1-ai-mlkit-labeling/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-labeling - 8.0-SNAPSHOT - - - cn1-ai-mlkit-labeling-javascript - jar - Codename One AI: cn1-ai-mlkit-labeling-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-labeling-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-labeling/javascript/src/main/javascript/com_codename1_ai_mlkit_labeling_NativeImageLabeler.js b/maven/cn1-ai-mlkit-labeling/javascript/src/main/javascript/com_codename1_ai_mlkit_labeling_NativeImageLabeler.js deleted file mode 100644 index a3fab1d15ba..00000000000 --- a/maven/cn1-ai-mlkit-labeling/javascript/src/main/javascript/com_codename1_ai_mlkit_labeling_NativeImageLabeler.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.label__byte_1ARRAY = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_labeling_NativeImageLabeler= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-labeling/javase/pom.xml b/maven/cn1-ai-mlkit-labeling/javase/pom.xml deleted file mode 100644 index a240f1b8d07..00000000000 --- a/maven/cn1-ai-mlkit-labeling/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-labeling - 8.0-SNAPSHOT - - - cn1-ai-mlkit-labeling-javase - jar - Codename One AI: cn1-ai-mlkit-labeling-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-labeling-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-labeling/javase/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabelerImpl.java b/maven/cn1-ai-mlkit-labeling/javase/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabelerImpl.java deleted file mode 100644 index de44d817bc8..00000000000 --- a/maven/cn1-ai-mlkit-labeling/javase/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabelerImpl.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.codename1.ai.mlkit.labeling; - -public class NativeImageLabelerImpl implements NativeImageLabeler { - public String[] label(byte[] imageBytes) { - if (imageBytes == null || imageBytes.length == 0) return new String[0]; - return new String[]{"object", "stub", "simulator"}; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-labeling/lib/pom.xml b/maven/cn1-ai-mlkit-labeling/lib/pom.xml deleted file mode 100644 index cf99170089d..00000000000 --- a/maven/cn1-ai-mlkit-labeling/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-labeling - 8.0-SNAPSHOT - - - cn1-ai-mlkit-labeling-lib - pom - Codename One AI: cn1-ai-mlkit-labeling-lib - - - - com.codenameone - cn1-ai-mlkit-labeling-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-labeling-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-labeling-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-labeling-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-labeling-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-labeling/pom.xml b/maven/cn1-ai-mlkit-labeling/pom.xml deleted file mode 100644 index f9842d84290..00000000000 --- a/maven/cn1-ai-mlkit-labeling/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-labeling - pom - Codename One AI: cn1-ai-mlkit-labeling - ML Kit Image Labeling - - - cn1-ai-mlkit-labeling - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-langid/android/pom.xml b/maven/cn1-ai-mlkit-langid/android/pom.xml deleted file mode 100644 index 7b6b78367b6..00000000000 --- a/maven/cn1-ai-mlkit-langid/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-langid - 8.0-SNAPSHOT - - - cn1-ai-mlkit-langid-android - jar - Codename One AI: cn1-ai-mlkit-langid-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-langid-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-langid/android/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifierImpl.java b/maven/cn1-ai-mlkit-langid/android/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifierImpl.java deleted file mode 100644 index 56a88ba9106..00000000000 --- a/maven/cn1-ai-mlkit-langid/android/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifierImpl.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.codename1.ai.mlkit.langid; - - -public class NativeLanguageIdentifierImpl { - public String identify(String input) { - com.google.mlkit.nl.languageid.LanguageIdentifier id = - com.google.mlkit.nl.languageid.LanguageIdentification.getClient(); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference("und"); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - id.identifyLanguage(input) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener() { - public void onSuccess(String s) { if (s != null) out.set(s); latch.countDown(); } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-langid/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-langid/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-langid/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-langid/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-langid/common/codenameone_library_required.properties deleted file mode 100644 index 049e4fa23ec..00000000000 --- a/maven/cn1-ai-mlkit-langid/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-langid. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/LanguageID -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:language-id:17.0.6' diff --git a/maven/cn1-ai-mlkit-langid/common/pom.xml b/maven/cn1-ai-mlkit-langid/common/pom.xml deleted file mode 100644 index f6097b7208c..00000000000 --- a/maven/cn1-ai-mlkit-langid/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-langid - 8.0-SNAPSHOT - - - cn1-ai-mlkit-langid-common - jar - Codename One AI: cn1-ai-mlkit-langid-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/LanguageIdentifier.java b/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/LanguageIdentifier.java deleted file mode 100644 index d0ad135a207..00000000000 --- a/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/LanguageIdentifier.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.langid; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Language Identification. -/// -/// Identifies the language of a given text string. -/// -public final class LanguageIdentifier { - private LanguageIdentifier() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeLanguageIdentifier bridge = NativeLookup.create(NativeLanguageIdentifier.class); - return bridge != null && bridge.isSupported(); - } - - public static AsyncResource identify(final String input) { - final AsyncResource out = new AsyncResource(); - final NativeLanguageIdentifier bridge = NativeLookup.create(NativeLanguageIdentifier.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("LanguageIdentifier.identify is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String r = bridge.identify(input); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? "" : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("LanguageIdentifier.identify failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifier.java b/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifier.java deleted file mode 100644 index 21cf9a5a350..00000000000 --- a/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifier.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.langid; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [LanguageIdentifier]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeLanguageIdentifier extends NativeInterface { - String identify(String input); -} diff --git a/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/package-info.java b/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/package-info.java deleted file mode 100644 index ec5bd92c30c..00000000000 --- a/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/package-info.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit Language Identification. -/// -/// Identifies the language of a given text string. -/// -/// The single public class in this package is [LanguageIdentifier], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeLanguageIdentifier` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `LanguageIdentifier.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.langid; diff --git a/maven/cn1-ai-mlkit-langid/common/src/test/java/com/codename1/ai/mlkit/langid/LanguageIdentifierTest.java b/maven/cn1-ai-mlkit-langid/common/src/test/java/com/codename1/ai/mlkit/langid/LanguageIdentifierTest.java deleted file mode 100644 index 571211dec60..00000000000 --- a/maven/cn1-ai-mlkit-langid/common/src/test/java/com/codename1/ai/mlkit/langid/LanguageIdentifierTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.mlkit.langid; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class LanguageIdentifierTest { - - /** Mock implementation of NativeLanguageIdentifier for headless JVM tests. */ - static class MockBridge implements NativeLanguageIdentifier { - boolean supported = true; - public boolean isSupported() { return supported; } - public String identify(String input) { return "en"; } - } - - @Test - void mock_identifies_english() { - MockBridge b = new MockBridge(); - assertEquals("en", b.identify("hello")); - } -} diff --git a/maven/cn1-ai-mlkit-langid/ios/pom.xml b/maven/cn1-ai-mlkit-langid/ios/pom.xml deleted file mode 100644 index 1209768a42b..00000000000 --- a/maven/cn1-ai-mlkit-langid/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-langid - 8.0-SNAPSHOT - - - cn1-ai-mlkit-langid-ios - jar - Codename One AI: cn1-ai-mlkit-langid-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-langid-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-langid/ios/src/main/objectivec/com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.h b/maven/cn1-ai-mlkit-langid/ios/src/main/objectivec/com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.h deleted file mode 100644 index 921b3831354..00000000000 --- a/maven/cn1-ai-mlkit-langid/ios/src/main/objectivec/com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl : NSObject { -} - --(NSString*)identify:(NSString*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-langid/ios/src/main/objectivec/com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.m b/maven/cn1-ai-mlkit-langid/ios/src/main/objectivec/com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.m deleted file mode 100644 index 88c6de4cf81..00000000000 --- a/maven/cn1-ai-mlkit-langid/ios/src/main/objectivec/com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.m +++ /dev/null @@ -1,23 +0,0 @@ -#import "com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.h" -#import -#import - -@implementation com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl - --(NSString*)identify:(NSString*)param { - MLKLanguageIdentification *id = [MLKLanguageIdentification languageIdentification]; - __block NSString *result = @"und"; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [id identifyLanguageForText:param completion:^(NSString * _Nullable lang, NSError * _Nullable e) { - if (lang) result = lang; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-langid/javascript/pom.xml b/maven/cn1-ai-mlkit-langid/javascript/pom.xml deleted file mode 100644 index 5a976e413c8..00000000000 --- a/maven/cn1-ai-mlkit-langid/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-langid - 8.0-SNAPSHOT - - - cn1-ai-mlkit-langid-javascript - jar - Codename One AI: cn1-ai-mlkit-langid-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-langid-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-langid/javascript/src/main/javascript/com_codename1_ai_mlkit_langid_NativeLanguageIdentifier.js b/maven/cn1-ai-mlkit-langid/javascript/src/main/javascript/com_codename1_ai_mlkit_langid_NativeLanguageIdentifier.js deleted file mode 100644 index 76582a049e0..00000000000 --- a/maven/cn1-ai-mlkit-langid/javascript/src/main/javascript/com_codename1_ai_mlkit_langid_NativeLanguageIdentifier.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.identify__java_lang_String = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_langid_NativeLanguageIdentifier= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-langid/javase/pom.xml b/maven/cn1-ai-mlkit-langid/javase/pom.xml deleted file mode 100644 index b0b91e3b73e..00000000000 --- a/maven/cn1-ai-mlkit-langid/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-langid - 8.0-SNAPSHOT - - - cn1-ai-mlkit-langid-javase - jar - Codename One AI: cn1-ai-mlkit-langid-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-langid-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-langid/javase/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifierImpl.java b/maven/cn1-ai-mlkit-langid/javase/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifierImpl.java deleted file mode 100644 index d0b384aa78d..00000000000 --- a/maven/cn1-ai-mlkit-langid/javase/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifierImpl.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.codename1.ai.mlkit.langid; - -public class NativeLanguageIdentifierImpl implements NativeLanguageIdentifier { - public String identify(String input) { - // Crude language ID stub for simulator. - if (input == null || input.isEmpty()) return "und"; - return "en"; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-langid/lib/pom.xml b/maven/cn1-ai-mlkit-langid/lib/pom.xml deleted file mode 100644 index 40d01b6dfcd..00000000000 --- a/maven/cn1-ai-mlkit-langid/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-langid - 8.0-SNAPSHOT - - - cn1-ai-mlkit-langid-lib - pom - Codename One AI: cn1-ai-mlkit-langid-lib - - - - com.codenameone - cn1-ai-mlkit-langid-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-langid-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-langid-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-langid-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-langid-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-langid/pom.xml b/maven/cn1-ai-mlkit-langid/pom.xml deleted file mode 100644 index f4d79b963a2..00000000000 --- a/maven/cn1-ai-mlkit-langid/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-langid - pom - Codename One AI: cn1-ai-mlkit-langid - ML Kit Language Identification - - - cn1-ai-mlkit-langid - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-pose/android/pom.xml b/maven/cn1-ai-mlkit-pose/android/pom.xml deleted file mode 100644 index b86a5bc62bf..00000000000 --- a/maven/cn1-ai-mlkit-pose/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-pose - 8.0-SNAPSHOT - - - cn1-ai-mlkit-pose-android - jar - Codename One AI: cn1-ai-mlkit-pose-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-pose-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-pose/android/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetectorImpl.java b/maven/cn1-ai-mlkit-pose/android/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetectorImpl.java deleted file mode 100644 index 1e5fc6b2ab8..00000000000 --- a/maven/cn1-ai-mlkit-pose/android/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetectorImpl.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.codename1.ai.mlkit.pose; - - -public class NativePoseDetectorImpl { - public float[] detect(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new float[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.pose.PoseDetector det = - com.google.mlkit.vision.pose.PoseDetection.getClient( - new com.google.mlkit.vision.pose.defaults.PoseDetectorOptions.Builder().build()); - final float[] out = new float[33 * 3]; - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - det.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.vision.pose.Pose>() { - public void onSuccess(com.google.mlkit.vision.pose.Pose p) { - java.util.List lms = p.getAllPoseLandmarks(); - for (int i = 0; i < 33 && i < lms.size(); i++) { - com.google.mlkit.vision.pose.PoseLandmark lm = lms.get(i); - out[i * 3] = lm.getPosition().x; - out[i * 3 + 1] = lm.getPosition().y; - out[i * 3 + 2] = lm.getInFrameLikelihood(); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-pose/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-pose/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-pose/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-pose/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-pose/common/codenameone_library_required.properties deleted file mode 100644 index 3f7f45a924c..00000000000 --- a/maven/cn1-ai-mlkit-pose/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-pose. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/PoseDetection -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:pose-detection:18.0.0-beta3' diff --git a/maven/cn1-ai-mlkit-pose/common/pom.xml b/maven/cn1-ai-mlkit-pose/common/pom.xml deleted file mode 100644 index 8defd866f57..00000000000 --- a/maven/cn1-ai-mlkit-pose/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-pose - 8.0-SNAPSHOT - - - cn1-ai-mlkit-pose-common - jar - Codename One AI: cn1-ai-mlkit-pose-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetector.java b/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetector.java deleted file mode 100644 index 2a636176311..00000000000 --- a/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetector.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.pose; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [PoseDetector]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativePoseDetector extends NativeInterface { - float[] detect(byte[] imageBytes); -} diff --git a/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/PoseDetector.java b/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/PoseDetector.java deleted file mode 100644 index a273d87f561..00000000000 --- a/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/PoseDetector.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.pose; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Pose Detection. -/// -/// Returns skeletal landmarks for human bodies detected in an image. -/// -public final class PoseDetector { - private PoseDetector() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativePoseDetector bridge = NativeLookup.create(NativePoseDetector.class); - return bridge != null && bridge.isSupported(); - } - - /// Returns 33 landmark triples (x,y,confidence) per detected pose - /// packed as `float[3 * 33]`. - public static AsyncResource detect(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativePoseDetector bridge = NativeLookup.create(NativePoseDetector.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("PoseDetector.detect is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final float[] r = bridge.detect(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new float[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("PoseDetector.detect failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/package-info.java b/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/package-info.java deleted file mode 100644 index e9d8c6920d9..00000000000 --- a/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/package-info.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit Pose Detection. -/// -/// Returns skeletal landmarks for human bodies detected in an image. -/// -/// The single public class in this package is [PoseDetector], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativePoseDetector` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `PoseDetector.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.pose; diff --git a/maven/cn1-ai-mlkit-pose/common/src/test/java/com/codename1/ai/mlkit/pose/PoseDetectorTest.java b/maven/cn1-ai-mlkit-pose/common/src/test/java/com/codename1/ai/mlkit/pose/PoseDetectorTest.java deleted file mode 100644 index ba4f4a8f63a..00000000000 --- a/maven/cn1-ai-mlkit-pose/common/src/test/java/com/codename1/ai/mlkit/pose/PoseDetectorTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.mlkit.pose; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class PoseDetectorTest { - - /** Mock implementation of NativePoseDetector for headless JVM tests. */ - static class MockBridge implements NativePoseDetector { - boolean supported = true; - public boolean isSupported() { return supported; } - public float[] detect(byte[] imageBytes) { return new float[99]; } - } - - @Test - void mock_returns_33_landmarks() { - MockBridge b = new MockBridge(); - assertEquals(99, b.detect(new byte[]{1}).length); - } -} diff --git a/maven/cn1-ai-mlkit-pose/ios/pom.xml b/maven/cn1-ai-mlkit-pose/ios/pom.xml deleted file mode 100644 index 58cfd6232b1..00000000000 --- a/maven/cn1-ai-mlkit-pose/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-pose - 8.0-SNAPSHOT - - - cn1-ai-mlkit-pose-ios - jar - Codename One AI: cn1-ai-mlkit-pose-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-pose-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-pose/ios/src/main/objectivec/com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.h b/maven/cn1-ai-mlkit-pose/ios/src/main/objectivec/com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.h deleted file mode 100644 index 551df98fe57..00000000000 --- a/maven/cn1-ai-mlkit-pose/ios/src/main/objectivec/com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_pose_NativePoseDetectorImpl : NSObject { -} - --(NSData*)detect:(NSData*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-pose/ios/src/main/objectivec/com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.m b/maven/cn1-ai-mlkit-pose/ios/src/main/objectivec/com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.m deleted file mode 100644 index e6682ff6b38..00000000000 --- a/maven/cn1-ai-mlkit-pose/ios/src/main/objectivec/com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.m +++ /dev/null @@ -1,39 +0,0 @@ -#import "com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.h" -#import -#import -#import -#import - -@implementation com_codename1_ai_mlkit_pose_NativePoseDetectorImpl - --(NSData*)detect:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [NSData data]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKPoseDetectorOptions *opts = [[MLKPoseDetectorOptions alloc] init]; - MLKPoseDetector *det = [MLKPoseDetector poseDetectorWithOptions:opts]; - __block MLKPose *pose = nil; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [det processImage:vision completion:^(NSArray * _Nullable r, NSError * _Nullable e) { - if (r.count > 0) pose = r[0]; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - float buf[99] = {0}; - if (pose) { - for (NSInteger i = 0; i < 33 && i < pose.landmarks.count; i++) { - MLKPoseLandmark *lm = pose.landmarks[i]; - buf[i * 3] = (float)lm.position.x; - buf[i * 3 + 1] = (float)lm.position.y; - buf[i * 3 + 2] = (float)lm.inFrameLikelihood; - } - } - // Pack as big-endian float bytes (matches JAVA_ARRAY_FLOAT on iOS port). - return [NSData dataWithBytes:buf length:sizeof(buf)]; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-pose/javascript/pom.xml b/maven/cn1-ai-mlkit-pose/javascript/pom.xml deleted file mode 100644 index 12d00e75017..00000000000 --- a/maven/cn1-ai-mlkit-pose/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-pose - 8.0-SNAPSHOT - - - cn1-ai-mlkit-pose-javascript - jar - Codename One AI: cn1-ai-mlkit-pose-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-pose-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-pose/javascript/src/main/javascript/com_codename1_ai_mlkit_pose_NativePoseDetector.js b/maven/cn1-ai-mlkit-pose/javascript/src/main/javascript/com_codename1_ai_mlkit_pose_NativePoseDetector.js deleted file mode 100644 index f04d324c889..00000000000 --- a/maven/cn1-ai-mlkit-pose/javascript/src/main/javascript/com_codename1_ai_mlkit_pose_NativePoseDetector.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.detect__byte_1ARRAY = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_pose_NativePoseDetector= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-pose/javase/pom.xml b/maven/cn1-ai-mlkit-pose/javase/pom.xml deleted file mode 100644 index 93bdaa9a5fc..00000000000 --- a/maven/cn1-ai-mlkit-pose/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-pose - 8.0-SNAPSHOT - - - cn1-ai-mlkit-pose-javase - jar - Codename One AI: cn1-ai-mlkit-pose-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-pose-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-pose/javase/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetectorImpl.java b/maven/cn1-ai-mlkit-pose/javase/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetectorImpl.java deleted file mode 100644 index ac05c406fbb..00000000000 --- a/maven/cn1-ai-mlkit-pose/javase/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetectorImpl.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.codename1.ai.mlkit.pose; - -public class NativePoseDetectorImpl implements NativePoseDetector { - public float[] detect(byte[] imageBytes) { - float[] out = new float[99]; - for (int i = 0; i < 33; i++) { out[i * 3] = i; out[i * 3 + 2] = 0.5f; } - return out; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-pose/lib/pom.xml b/maven/cn1-ai-mlkit-pose/lib/pom.xml deleted file mode 100644 index dce8cf9e244..00000000000 --- a/maven/cn1-ai-mlkit-pose/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-pose - 8.0-SNAPSHOT - - - cn1-ai-mlkit-pose-lib - pom - Codename One AI: cn1-ai-mlkit-pose-lib - - - - com.codenameone - cn1-ai-mlkit-pose-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-pose-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-pose-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-pose-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-pose-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-pose/pom.xml b/maven/cn1-ai-mlkit-pose/pom.xml deleted file mode 100644 index 59b1266fa79..00000000000 --- a/maven/cn1-ai-mlkit-pose/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-pose - pom - Codename One AI: cn1-ai-mlkit-pose - ML Kit Pose Detection - - - cn1-ai-mlkit-pose - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-segmentation/android/pom.xml b/maven/cn1-ai-mlkit-segmentation/android/pom.xml deleted file mode 100644 index 3c1eab0d7a5..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-segmentation - 8.0-SNAPSHOT - - - cn1-ai-mlkit-segmentation-android - jar - Codename One AI: cn1-ai-mlkit-segmentation-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-segmentation-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-segmentation/android/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenterImpl.java b/maven/cn1-ai-mlkit-segmentation/android/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenterImpl.java deleted file mode 100644 index a832c4c323d..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/android/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenterImpl.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.codename1.ai.mlkit.segmentation; - - -public class NativeSelfieSegmenterImpl { - public byte[] segment(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new byte[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.segmentation.Segmenter seg = - com.google.mlkit.vision.segmentation.Segmentation.getClient( - new com.google.mlkit.vision.segmentation.SelfieSegmenterOptions.Builder() - .setDetectorMode( - com.google.mlkit.vision.segmentation.SelfieSegmenterOptions.SINGLE_IMAGE_MODE) - .build()); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference(new byte[0]); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - seg.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.vision.segmentation.SegmentationMask>() { - public void onSuccess(com.google.mlkit.vision.segmentation.SegmentationMask mask) { - int w = mask.getWidth(), h = mask.getHeight(); - java.nio.ByteBuffer buf = mask.getBuffer(); - buf.rewind(); - byte[] outb = new byte[w * h]; - for (int i = 0; i < w * h; i++) { - float v = buf.getFloat(); - outb[i] = (byte)(int)(v * 255); - } - out.set(outb); - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-segmentation/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-segmentation/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-segmentation/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-segmentation/common/codenameone_library_required.properties deleted file mode 100644 index 29e4a86796d..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-segmentation. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/SegmentationSelfie -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:segmentation-selfie:16.0.0-beta5' diff --git a/maven/cn1-ai-mlkit-segmentation/common/pom.xml b/maven/cn1-ai-mlkit-segmentation/common/pom.xml deleted file mode 100644 index 42a389f1738..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-segmentation - 8.0-SNAPSHOT - - - cn1-ai-mlkit-segmentation-common - jar - Codename One AI: cn1-ai-mlkit-segmentation-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenter.java b/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenter.java deleted file mode 100644 index 1fcb3fd96ec..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenter.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.segmentation; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [SelfieSegmenter]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeSelfieSegmenter extends NativeInterface { - byte[] segment(byte[] imageBytes); -} diff --git a/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/SelfieSegmenter.java b/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/SelfieSegmenter.java deleted file mode 100644 index 5af60615dd5..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/SelfieSegmenter.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.segmentation; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Selfie Segmentation. -/// -/// Returns a per-pixel mask separating a person in the foreground from the background. -/// -public final class SelfieSegmenter { - private SelfieSegmenter() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeSelfieSegmenter bridge = NativeLookup.create(NativeSelfieSegmenter.class); - return bridge != null && bridge.isSupported(); - } - - /// Returns a per-pixel mask separating foreground (person) from - /// background as `byte[width * height]` (0=background, 255=foreground). - public static AsyncResource segment(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativeSelfieSegmenter bridge = NativeLookup.create(NativeSelfieSegmenter.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("SelfieSegmenter.segment is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final byte[] r = bridge.segment(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new byte[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("SelfieSegmenter.segment failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/package-info.java b/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/package-info.java deleted file mode 100644 index 3f12b066b68..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/package-info.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit Selfie Segmentation. -/// -/// Returns a per-pixel mask separating a person in the foreground from the background. -/// -/// The single public class in this package is [SelfieSegmenter], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeSelfieSegmenter` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `SelfieSegmenter.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.segmentation; diff --git a/maven/cn1-ai-mlkit-segmentation/common/src/test/java/com/codename1/ai/mlkit/segmentation/SelfieSegmenterTest.java b/maven/cn1-ai-mlkit-segmentation/common/src/test/java/com/codename1/ai/mlkit/segmentation/SelfieSegmenterTest.java deleted file mode 100644 index 6ff2b587824..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/common/src/test/java/com/codename1/ai/mlkit/segmentation/SelfieSegmenterTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.mlkit.segmentation; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class SelfieSegmenterTest { - - /** Mock implementation of NativeSelfieSegmenter for headless JVM tests. */ - static class MockBridge implements NativeSelfieSegmenter { - boolean supported = true; - public boolean isSupported() { return supported; } - public byte[] segment(byte[] imageBytes) { return new byte[16]; } - } - - @Test - void mock_returns_mask_bytes() { - MockBridge b = new MockBridge(); - assertEquals(16, b.segment(new byte[]{1}).length); - } -} diff --git a/maven/cn1-ai-mlkit-segmentation/ios/pom.xml b/maven/cn1-ai-mlkit-segmentation/ios/pom.xml deleted file mode 100644 index 38d5951a239..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-segmentation - 8.0-SNAPSHOT - - - cn1-ai-mlkit-segmentation-ios - jar - Codename One AI: cn1-ai-mlkit-segmentation-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-segmentation-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-segmentation/ios/src/main/objectivec/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.h b/maven/cn1-ai-mlkit-segmentation/ios/src/main/objectivec/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.h deleted file mode 100644 index d9abad43f4e..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/ios/src/main/objectivec/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl : NSObject { -} - --(NSData*)segment:(NSData*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-segmentation/ios/src/main/objectivec/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.m b/maven/cn1-ai-mlkit-segmentation/ios/src/main/objectivec/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.m deleted file mode 100644 index 0274648353d..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/ios/src/main/objectivec/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.m +++ /dev/null @@ -1,48 +0,0 @@ -#import "com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.h" -#import -#import -#import -#import -#import - -@implementation com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl - --(NSData*)segment:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [NSData data]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKSelfieSegmenterOptions *opts = [[MLKSelfieSegmenterOptions alloc] init]; - opts.segmenterMode = MLKSegmenterModeSingleImage; - MLKSegmenter *seg = [MLKSegmenter segmenterWithOptions:opts]; - __block NSData *result = [NSData data]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [seg processImage:vision completion:^(MLKSegmentationMask * _Nullable mask, NSError * _Nullable e) { - if (mask) { - // MLKSegmentationMask exposes only `buffer` (a - // CVPixelBuffer); dimensions come from CVPixelBufferGet*. - CVPixelBufferRef buf = mask.buffer; - size_t w = CVPixelBufferGetWidth(buf); - size_t h = CVPixelBufferGetHeight(buf); - CVPixelBufferLockBaseAddress(buf, kCVPixelBufferLock_ReadOnly); - void *base = CVPixelBufferGetBaseAddress(buf); - NSMutableData *m = [NSMutableData dataWithLength:w * h]; - uint8_t *out = m.mutableBytes; - float *src = (float *)base; - for (size_t i = 0; i < w * h; i++) { - float v = src[i]; - out[i] = (uint8_t)(v * 255.0f); - } - CVPixelBufferUnlockBaseAddress(buf, kCVPixelBufferLock_ReadOnly); - result = m; - } - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-segmentation/javascript/pom.xml b/maven/cn1-ai-mlkit-segmentation/javascript/pom.xml deleted file mode 100644 index 797cc8c41c6..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-segmentation - 8.0-SNAPSHOT - - - cn1-ai-mlkit-segmentation-javascript - jar - Codename One AI: cn1-ai-mlkit-segmentation-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-segmentation-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-segmentation/javascript/src/main/javascript/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenter.js b/maven/cn1-ai-mlkit-segmentation/javascript/src/main/javascript/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenter.js deleted file mode 100644 index 2da08051d23..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/javascript/src/main/javascript/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenter.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.segment__byte_1ARRAY = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenter= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-segmentation/javase/pom.xml b/maven/cn1-ai-mlkit-segmentation/javase/pom.xml deleted file mode 100644 index 933039325a9..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-segmentation - 8.0-SNAPSHOT - - - cn1-ai-mlkit-segmentation-javase - jar - Codename One AI: cn1-ai-mlkit-segmentation-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-segmentation-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-segmentation/javase/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenterImpl.java b/maven/cn1-ai-mlkit-segmentation/javase/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenterImpl.java deleted file mode 100644 index edd293eed28..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/javase/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenterImpl.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.codename1.ai.mlkit.segmentation; - -public class NativeSelfieSegmenterImpl implements NativeSelfieSegmenter { - public byte[] segment(byte[] imageBytes) { - // 8x8 checkerboard stub. - byte[] out = new byte[64]; - for (int i = 0; i < 64; i++) out[i] = (byte)(((i / 8) + (i % 8)) % 2 == 0 ? 255 : 0); - return out; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-segmentation/lib/pom.xml b/maven/cn1-ai-mlkit-segmentation/lib/pom.xml deleted file mode 100644 index bbe5e0bd225..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-segmentation - 8.0-SNAPSHOT - - - cn1-ai-mlkit-segmentation-lib - pom - Codename One AI: cn1-ai-mlkit-segmentation-lib - - - - com.codenameone - cn1-ai-mlkit-segmentation-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-segmentation-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-segmentation-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-segmentation-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-segmentation-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-segmentation/pom.xml b/maven/cn1-ai-mlkit-segmentation/pom.xml deleted file mode 100644 index 25dcd7294fe..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-segmentation - pom - Codename One AI: cn1-ai-mlkit-segmentation - ML Kit Selfie Segmentation - - - cn1-ai-mlkit-segmentation - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-smartreply/android/pom.xml b/maven/cn1-ai-mlkit-smartreply/android/pom.xml deleted file mode 100644 index 2006c1b9b4a..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-smartreply - 8.0-SNAPSHOT - - - cn1-ai-mlkit-smartreply-android - jar - Codename One AI: cn1-ai-mlkit-smartreply-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-smartreply-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-smartreply/android/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReplyImpl.java b/maven/cn1-ai-mlkit-smartreply/android/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReplyImpl.java deleted file mode 100644 index f94269fe1e4..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/android/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReplyImpl.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.codename1.ai.mlkit.smartreply; - - -public class NativeSmartReplyImpl { - public String[] suggest(String conversationJson) { - java.util.List msgs = - new java.util.ArrayList(); - try { - org.json.JSONArray a = new org.json.JSONArray(conversationJson); - for (int i = 0; i < a.length(); i++) { - org.json.JSONObject o = a.getJSONObject(i); - String role = o.optString("role", "user"); - long ts = o.optLong("timestamp", 0); - String text = o.optString("message", ""); - String userId = o.optString("userId", "u"); - if ("user".equals(role)) { - msgs.add(com.google.mlkit.nl.smartreply.TextMessage.createForLocalUser(text, ts)); - } else { - msgs.add(com.google.mlkit.nl.smartreply.TextMessage.createForRemoteUser(text, ts, userId)); - } - } - } catch (org.json.JSONException jex) { - return new String[0]; - } - final java.util.List out = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - com.google.mlkit.nl.smartreply.SmartReplyGenerator gen = - com.google.mlkit.nl.smartreply.SmartReply.getClient(); - gen.suggestReplies(msgs) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.nl.smartreply.SmartReplySuggestionResult>() { - public void onSuccess(com.google.mlkit.nl.smartreply.SmartReplySuggestionResult r) { - for (com.google.mlkit.nl.smartreply.SmartReplySuggestion s : r.getSuggestions()) { - out.add(s.getText()); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.toArray(new String[0]); - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-smartreply/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-smartreply/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-smartreply/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-smartreply/common/codenameone_library_required.properties deleted file mode 100644 index c02fc2c10be..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-smartreply. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/SmartReply -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:smart-reply:17.0.4' diff --git a/maven/cn1-ai-mlkit-smartreply/common/pom.xml b/maven/cn1-ai-mlkit-smartreply/common/pom.xml deleted file mode 100644 index cfa4154d85f..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-smartreply - 8.0-SNAPSHOT - - - cn1-ai-mlkit-smartreply-common - jar - Codename One AI: cn1-ai-mlkit-smartreply-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReply.java b/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReply.java deleted file mode 100644 index 2161f2610eb..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReply.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.smartreply; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [SmartReply]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeSmartReply extends NativeInterface { - String[] suggest(String conversationJson); -} diff --git a/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/SmartReply.java b/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/SmartReply.java deleted file mode 100644 index e6330b1f3f5..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/SmartReply.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.smartreply; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Smart Reply. -/// -/// Generates short reply suggestions for chat conversations on-device. -/// -public final class SmartReply { - private SmartReply() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeSmartReply bridge = NativeLookup.create(NativeSmartReply.class); - return bridge != null && bridge.isSupported(); - } - - public static AsyncResource suggest(final String input) { - final AsyncResource out = new AsyncResource(); - final NativeSmartReply bridge = NativeLookup.create(NativeSmartReply.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("SmartReply.suggest is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String[] r = bridge.suggest(input); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new String[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("SmartReply.suggest failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/package-info.java b/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/package-info.java deleted file mode 100644 index e31db21b55d..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/package-info.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit Smart Reply. -/// -/// Generates short reply suggestions for chat conversations on-device. -/// -/// The single public class in this package is [SmartReply], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeSmartReply` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `SmartReply.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.smartreply; diff --git a/maven/cn1-ai-mlkit-smartreply/common/src/test/java/com/codename1/ai/mlkit/smartreply/SmartReplyTest.java b/maven/cn1-ai-mlkit-smartreply/common/src/test/java/com/codename1/ai/mlkit/smartreply/SmartReplyTest.java deleted file mode 100644 index 59f881fd758..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/common/src/test/java/com/codename1/ai/mlkit/smartreply/SmartReplyTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.mlkit.smartreply; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class SmartReplyTest { - - /** Mock implementation of NativeSmartReply for headless JVM tests. */ - static class MockBridge implements NativeSmartReply { - boolean supported = true; - public boolean isSupported() { return supported; } - public String[] suggest(String c) { return new String[]{"ok"}; } - } - - @Test - void mock_returns_single_suggestion() { - MockBridge b = new MockBridge(); - assertEquals(1, b.suggest("[]").length); - } -} diff --git a/maven/cn1-ai-mlkit-smartreply/ios/pom.xml b/maven/cn1-ai-mlkit-smartreply/ios/pom.xml deleted file mode 100644 index 29ecb190185..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-smartreply - 8.0-SNAPSHOT - - - cn1-ai-mlkit-smartreply-ios - jar - Codename One AI: cn1-ai-mlkit-smartreply-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-smartreply-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-smartreply/ios/src/main/objectivec/com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.h b/maven/cn1-ai-mlkit-smartreply/ios/src/main/objectivec/com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.h deleted file mode 100644 index 5ac5c16fea2..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/ios/src/main/objectivec/com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl : NSObject { -} - --(NSData*)suggest:(NSString*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-smartreply/ios/src/main/objectivec/com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.m b/maven/cn1-ai-mlkit-smartreply/ios/src/main/objectivec/com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.m deleted file mode 100644 index b5dd8ba2115..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/ios/src/main/objectivec/com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.m +++ /dev/null @@ -1,61 +0,0 @@ -#import "com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.h" -#import -#import -#import - -@implementation com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl - --(NSData*)suggest:(NSString*)param { - // param is a JSON array of {role,message,timestamp,userId}. - NSError *err = nil; - NSArray *items = [NSJSONSerialization JSONObjectWithData: - [param dataUsingEncoding:NSUTF8StringEncoding] - options:0 error:&err]; - NSMutableArray *messages = [NSMutableArray array]; - if ([items isKindOfClass:[NSArray class]]) { - for (NSDictionary *d in items) { - if (![d isKindOfClass:[NSDictionary class]]) continue; - NSString *role = d[@"role"] ?: @"user"; - BOOL isLocalUser = [role isEqualToString:@"user"]; - NSString *text = d[@"message"] ?: @""; - NSNumber *ts = d[@"timestamp"] ?: @0; - MLKTextMessage *m = [[MLKTextMessage alloc] - initWithText:text timestamp:[ts doubleValue] - userID:(d[@"userId"] ?: @"u") - isLocalUser:isLocalUser]; - [messages addObject:m]; - } - } - __block NSArray *out = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [[MLKSmartReply smartReply] suggestRepliesForMessages:messages - completion:^(MLKSmartReplySuggestionResult * _Nullable result, NSError * _Nullable e) { - NSMutableArray *m = [NSMutableArray array]; - for (MLKSmartReplySuggestion *s in result.suggestions ?: @[]) { - if (s.text) [m addObject:s.text]; - } - out = m; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return [self packStrings:out]; -} - --(NSData*)packStrings:(NSArray *)strings { - NSMutableData *out = [NSMutableData data]; - uint32_t count = htonl((uint32_t)strings.count); - [out appendBytes:&count length:sizeof(count)]; - for (NSString *s in strings) { - NSData *u = [s dataUsingEncoding:NSUTF8StringEncoding]; - uint32_t len = htonl((uint32_t)u.length); - [out appendBytes:&len length:sizeof(len)]; - [out appendData:u]; - } - return out; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-smartreply/javascript/pom.xml b/maven/cn1-ai-mlkit-smartreply/javascript/pom.xml deleted file mode 100644 index 7c4de49b0f3..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-smartreply - 8.0-SNAPSHOT - - - cn1-ai-mlkit-smartreply-javascript - jar - Codename One AI: cn1-ai-mlkit-smartreply-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-smartreply-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-smartreply/javascript/src/main/javascript/com_codename1_ai_mlkit_smartreply_NativeSmartReply.js b/maven/cn1-ai-mlkit-smartreply/javascript/src/main/javascript/com_codename1_ai_mlkit_smartreply_NativeSmartReply.js deleted file mode 100644 index 9d0a9c0a594..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/javascript/src/main/javascript/com_codename1_ai_mlkit_smartreply_NativeSmartReply.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.suggest__java_lang_String = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_smartreply_NativeSmartReply= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-smartreply/javase/pom.xml b/maven/cn1-ai-mlkit-smartreply/javase/pom.xml deleted file mode 100644 index de53fe91c38..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-smartreply - 8.0-SNAPSHOT - - - cn1-ai-mlkit-smartreply-javase - jar - Codename One AI: cn1-ai-mlkit-smartreply-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-smartreply-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-smartreply/javase/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReplyImpl.java b/maven/cn1-ai-mlkit-smartreply/javase/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReplyImpl.java deleted file mode 100644 index 0190707fdad..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/javase/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReplyImpl.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.codename1.ai.mlkit.smartreply; - -public class NativeSmartReplyImpl implements NativeSmartReply { - public String[] suggest(String conversationJson) { - return new String[]{"Sounds good", "Thanks!", "Got it"}; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-smartreply/lib/pom.xml b/maven/cn1-ai-mlkit-smartreply/lib/pom.xml deleted file mode 100644 index b5d8bf097c6..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-smartreply - 8.0-SNAPSHOT - - - cn1-ai-mlkit-smartreply-lib - pom - Codename One AI: cn1-ai-mlkit-smartreply-lib - - - - com.codenameone - cn1-ai-mlkit-smartreply-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-smartreply-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-smartreply-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-smartreply-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-smartreply-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-smartreply/pom.xml b/maven/cn1-ai-mlkit-smartreply/pom.xml deleted file mode 100644 index 1bd869f426e..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-smartreply - pom - Codename One AI: cn1-ai-mlkit-smartreply - ML Kit Smart Reply - - - cn1-ai-mlkit-smartreply - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-text/android/pom.xml b/maven/cn1-ai-mlkit-text/android/pom.xml deleted file mode 100644 index 4997f4164f7..00000000000 --- a/maven/cn1-ai-mlkit-text/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-text - 8.0-SNAPSHOT - - - cn1-ai-mlkit-text-android - jar - Codename One AI: cn1-ai-mlkit-text-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-text-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-text/android/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizerImpl.java b/maven/cn1-ai-mlkit-text/android/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizerImpl.java deleted file mode 100644 index b3fa0212158..00000000000 --- a/maven/cn1-ai-mlkit-text/android/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizerImpl.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.codename1.ai.mlkit.text; - - -public class NativeTextRecognizerImpl { - public String recognize(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return ""; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.text.TextRecognizer rec = - com.google.mlkit.vision.text.TextRecognition.getClient( - com.google.mlkit.vision.text.latin.TextRecognizerOptions.DEFAULT_OPTIONS); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference(""); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - rec.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.vision.text.Text>() { - public void onSuccess(com.google.mlkit.vision.text.Text t) { - out.set(t.getText() == null ? "" : t.getText()); - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-text/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-text/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-text/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-text/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-text/common/codenameone_library_required.properties deleted file mode 100644 index ba65d8437d0..00000000000 --- a/maven/cn1-ai-mlkit-text/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-text. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/TextRecognition -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:text-recognition:16.0.0' diff --git a/maven/cn1-ai-mlkit-text/common/pom.xml b/maven/cn1-ai-mlkit-text/common/pom.xml deleted file mode 100644 index cbe95b4cb73..00000000000 --- a/maven/cn1-ai-mlkit-text/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-text - 8.0-SNAPSHOT - - - cn1-ai-mlkit-text-common - jar - Codename One AI: cn1-ai-mlkit-text-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizer.java b/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizer.java deleted file mode 100644 index 80203101333..00000000000 --- a/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizer.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.text; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [TextRecognizer]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeTextRecognizer extends NativeInterface { - String recognize(byte[] imageBytes); -} diff --git a/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/TextRecognizer.java b/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/TextRecognizer.java deleted file mode 100644 index ac689727410..00000000000 --- a/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/TextRecognizer.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.text; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Text Recognition (OCR). -/// -/// Extracts text strings from images entirely on-device via Google's ML Kit. -/// Bridges to `GoogleMLKit/TextRecognition` on iOS and -/// `com.google.mlkit:text-recognition` on Android. -/// -public final class TextRecognizer { - private TextRecognizer() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeTextRecognizer bridge = NativeLookup.create(NativeTextRecognizer.class); - return bridge != null && bridge.isSupported(); - } - - /// Runs OCR on the supplied image bytes (JPEG or PNG). Completes with - /// the recognised text. Empty image -> empty string. No text -> empty - /// string. Hard errors fire `AsyncResource.error(...)`. - public static AsyncResource recognize(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - if (imageBytes == null || imageBytes.length == 0) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(""); } - }); - return out; - } - final NativeTextRecognizer bridge = NativeLookup.create(NativeTextRecognizer.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException( - "TextRecognizer.recognize is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String result = bridge.recognize(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(result == null ? "" : result); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("TextRecognizer.recognize failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/package-info.java b/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/package-info.java deleted file mode 100644 index 06da696570b..00000000000 --- a/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/package-info.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit Text Recognition (OCR). -/// -/// Extracts text strings from images entirely on-device via Google's ML Kit. -/// Bridges to `GoogleMLKit/TextRecognition` on iOS and -/// `com.google.mlkit:text-recognition` on Android. -/// -/// The single public class in this package is [TextRecognizer], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeTextRecognizer` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `TextRecognizer.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.text; diff --git a/maven/cn1-ai-mlkit-text/common/src/test/java/com/codename1/ai/mlkit/text/TextRecognizerTest.java b/maven/cn1-ai-mlkit-text/common/src/test/java/com/codename1/ai/mlkit/text/TextRecognizerTest.java deleted file mode 100644 index d5e7cbde6f2..00000000000 --- a/maven/cn1-ai-mlkit-text/common/src/test/java/com/codename1/ai/mlkit/text/TextRecognizerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.codename1.ai.mlkit.text; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class TextRecognizerTest { - - /** Mock implementation of NativeTextRecognizer for headless JVM tests. */ - static class MockBridge implements NativeTextRecognizer { - boolean supported = true; - public boolean isSupported() { return supported; } - String response = "hello"; - public String recognize(byte[] imageBytes) { - if (imageBytes == null) throw new NullPointerException(); - return response; - } - } - - @Test - void bridge_returns_canned_string() { - MockBridge b = new MockBridge(); - assertEquals("hello", b.recognize(new byte[]{1, 2, 3})); - } - - @Test - void bridge_reports_supported() { - MockBridge b = new MockBridge(); - assertTrue(b.isSupported()); - } - - @Test - void bridge_rejects_null_input() { - MockBridge b = new MockBridge(); - assertThrows(NullPointerException.class, () -> b.recognize(null)); - } -} diff --git a/maven/cn1-ai-mlkit-text/ios/pom.xml b/maven/cn1-ai-mlkit-text/ios/pom.xml deleted file mode 100644 index ad02c249737..00000000000 --- a/maven/cn1-ai-mlkit-text/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-text - 8.0-SNAPSHOT - - - cn1-ai-mlkit-text-ios - jar - Codename One AI: cn1-ai-mlkit-text-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-text-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-text/ios/src/main/objectivec/com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.h b/maven/cn1-ai-mlkit-text/ios/src/main/objectivec/com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.h deleted file mode 100644 index 70bce8ba280..00000000000 --- a/maven/cn1-ai-mlkit-text/ios/src/main/objectivec/com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_text_NativeTextRecognizerImpl : NSObject { -} - --(NSString*)recognize:(NSData*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-text/ios/src/main/objectivec/com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.m b/maven/cn1-ai-mlkit-text/ios/src/main/objectivec/com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.m deleted file mode 100644 index b3bac461d66..00000000000 --- a/maven/cn1-ai-mlkit-text/ios/src/main/objectivec/com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.m +++ /dev/null @@ -1,33 +0,0 @@ -#import "com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.h" -#import -#import -#import -#import - -@implementation com_codename1_ai_mlkit_text_NativeTextRecognizerImpl - --(NSString*)recognize:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return @""; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKTextRecognizerOptions *opts = [[MLKTextRecognizerOptions alloc] init]; - MLKTextRecognizer *recognizer = [MLKTextRecognizer textRecognizerWithOptions:opts]; - __block NSString *result = @""; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [recognizer processImage:vision completion:^(MLKText * _Nullable text, NSError * _Nullable err) { - if (text && !err) { - result = text.text ?: @""; - } else if (err) { - result = @""; - } - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-text/javascript/pom.xml b/maven/cn1-ai-mlkit-text/javascript/pom.xml deleted file mode 100644 index 03df3541787..00000000000 --- a/maven/cn1-ai-mlkit-text/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-text - 8.0-SNAPSHOT - - - cn1-ai-mlkit-text-javascript - jar - Codename One AI: cn1-ai-mlkit-text-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-text-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-text/javascript/src/main/javascript/com_codename1_ai_mlkit_text_NativeTextRecognizer.js b/maven/cn1-ai-mlkit-text/javascript/src/main/javascript/com_codename1_ai_mlkit_text_NativeTextRecognizer.js deleted file mode 100644 index de4cb7d3e12..00000000000 --- a/maven/cn1-ai-mlkit-text/javascript/src/main/javascript/com_codename1_ai_mlkit_text_NativeTextRecognizer.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.recognize__byte_1ARRAY = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_text_NativeTextRecognizer= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-text/javase/pom.xml b/maven/cn1-ai-mlkit-text/javase/pom.xml deleted file mode 100644 index 1b2dc7c3783..00000000000 --- a/maven/cn1-ai-mlkit-text/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-text - 8.0-SNAPSHOT - - - cn1-ai-mlkit-text-javase - jar - Codename One AI: cn1-ai-mlkit-text-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-text-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-text/javase/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizerImpl.java b/maven/cn1-ai-mlkit-text/javase/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizerImpl.java deleted file mode 100644 index 97802c2f919..00000000000 --- a/maven/cn1-ai-mlkit-text/javase/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizerImpl.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.codename1.ai.mlkit.text; - -public class NativeTextRecognizerImpl implements NativeTextRecognizer { - - private static boolean hintsEnsured; - private static synchronized void ensureSimulatorHints() { - if (hintsEnsured) return; - hintsEnsured = true; - java.util.Map hints = - com.codename1.ui.Display.getInstance().getProjectBuildHints(); - if (hints == null) return; // not running in the simulator - if (!hints.containsKey("ios.NSCameraUsageDescription")) { - com.codename1.ui.Display.getInstance() - .setProjectBuildHint("ios.NSCameraUsageDescription", "This app uses the camera to recognise text."); - } - } - - public NativeTextRecognizerImpl() { - ensureSimulatorHints(); - } - public String recognize(byte[] imageBytes) { - if (imageBytes == null || imageBytes.length == 0) return ""; - return "[mlkit-text simulator stub] " + imageBytes.length + " bytes"; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-text/lib/pom.xml b/maven/cn1-ai-mlkit-text/lib/pom.xml deleted file mode 100644 index f5d94e33e9c..00000000000 --- a/maven/cn1-ai-mlkit-text/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-text - 8.0-SNAPSHOT - - - cn1-ai-mlkit-text-lib - pom - Codename One AI: cn1-ai-mlkit-text-lib - - - - com.codenameone - cn1-ai-mlkit-text-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-text-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-text-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-text-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-text-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-text/pom.xml b/maven/cn1-ai-mlkit-text/pom.xml deleted file mode 100644 index 6552a73bba3..00000000000 --- a/maven/cn1-ai-mlkit-text/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-text - pom - Codename One AI: cn1-ai-mlkit-text - ML Kit Text Recognition (OCR) - - - cn1-ai-mlkit-text - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-translate/android/pom.xml b/maven/cn1-ai-mlkit-translate/android/pom.xml deleted file mode 100644 index 3e9154f5b2d..00000000000 --- a/maven/cn1-ai-mlkit-translate/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-translate - 8.0-SNAPSHOT - - - cn1-ai-mlkit-translate-android - jar - Codename One AI: cn1-ai-mlkit-translate-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-translate-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-translate/android/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslatorImpl.java b/maven/cn1-ai-mlkit-translate/android/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslatorImpl.java deleted file mode 100644 index 8ee87b28c7a..00000000000 --- a/maven/cn1-ai-mlkit-translate/android/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslatorImpl.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.codename1.ai.mlkit.translate; - - -public class NativeTranslatorImpl { - public String translate(String text, String sourceLang, String targetLang) { - com.google.mlkit.nl.translate.TranslatorOptions opts = - new com.google.mlkit.nl.translate.TranslatorOptions.Builder() - .setSourceLanguage(sourceLang) - .setTargetLanguage(targetLang) - .build(); - com.google.mlkit.nl.translate.Translator t = - com.google.mlkit.nl.translate.Translation.getClient(opts); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference(""); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - t.downloadModelIfNeeded() - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener() { - public void onSuccess(Void v) { - t.translate(text) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener() { - public void onSuccess(String r) { out.set(r); latch.countDown(); } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-translate/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-translate/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-translate/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-translate/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-translate/common/codenameone_library_required.properties deleted file mode 100644 index 758e2eca244..00000000000 --- a/maven/cn1-ai-mlkit-translate/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-translate. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/Translate -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:translate:17.0.3' diff --git a/maven/cn1-ai-mlkit-translate/common/pom.xml b/maven/cn1-ai-mlkit-translate/common/pom.xml deleted file mode 100644 index 04aa7fecf2b..00000000000 --- a/maven/cn1-ai-mlkit-translate/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-translate - 8.0-SNAPSHOT - - - cn1-ai-mlkit-translate-common - jar - Codename One AI: cn1-ai-mlkit-translate-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslator.java b/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslator.java deleted file mode 100644 index 84c232bd8fe..00000000000 --- a/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslator.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.translate; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [Translator]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeTranslator extends NativeInterface { - String translate(String text, String sourceLang, String targetLang); -} diff --git a/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/Translator.java b/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/Translator.java deleted file mode 100644 index 0a8fc8aac05..00000000000 --- a/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/Translator.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.translate; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit on-device Translation. -/// -/// Translates short text between language pairs entirely on-device. -/// -public final class Translator { - private Translator() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeTranslator bridge = NativeLookup.create(NativeTranslator.class); - return bridge != null && bridge.isSupported(); - } - - public static AsyncResource translate(final String text, - final String sourceLang, - final String targetLang) { - final AsyncResource out = new AsyncResource(); - final NativeTranslator bridge = NativeLookup.create(NativeTranslator.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("Translator.translate is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String r = bridge.translate(text, sourceLang, targetLang); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? "" : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("Translator.translate failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/package-info.java b/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/package-info.java deleted file mode 100644 index f0509240c9a..00000000000 --- a/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/package-info.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit on-device Translation. -/// -/// Translates short text between language pairs entirely on-device. -/// -/// The single public class in this package is [Translator], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeTranslator` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `Translator.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.translate; diff --git a/maven/cn1-ai-mlkit-translate/common/src/test/java/com/codename1/ai/mlkit/translate/TranslatorTest.java b/maven/cn1-ai-mlkit-translate/common/src/test/java/com/codename1/ai/mlkit/translate/TranslatorTest.java deleted file mode 100644 index 4299d695667..00000000000 --- a/maven/cn1-ai-mlkit-translate/common/src/test/java/com/codename1/ai/mlkit/translate/TranslatorTest.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.codename1.ai.mlkit.translate; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class TranslatorTest { - - /** Mock implementation of NativeTranslator for headless JVM tests. */ - static class MockBridge implements NativeTranslator { - boolean supported = true; - public boolean isSupported() { return supported; } - public String translate(String text, String sourceLang, String targetLang) { - return text + "@" + sourceLang + "->" + targetLang; - } - } - - @Test - void mock_translate_round_trip() { - MockBridge b = new MockBridge(); - assertEquals("hi@en->fr", b.translate("hi", "en", "fr")); - } -} diff --git a/maven/cn1-ai-mlkit-translate/ios/pom.xml b/maven/cn1-ai-mlkit-translate/ios/pom.xml deleted file mode 100644 index 6fbc9ae87cb..00000000000 --- a/maven/cn1-ai-mlkit-translate/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-translate - 8.0-SNAPSHOT - - - cn1-ai-mlkit-translate-ios - jar - Codename One AI: cn1-ai-mlkit-translate-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-translate-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-translate/ios/src/main/objectivec/com_codename1_ai_mlkit_translate_NativeTranslatorImpl.h b/maven/cn1-ai-mlkit-translate/ios/src/main/objectivec/com_codename1_ai_mlkit_translate_NativeTranslatorImpl.h deleted file mode 100644 index 437112805f1..00000000000 --- a/maven/cn1-ai-mlkit-translate/ios/src/main/objectivec/com_codename1_ai_mlkit_translate_NativeTranslatorImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_translate_NativeTranslatorImpl : NSObject { -} - --(NSString*)translate:(NSString*)param param1:(NSString*)param1 param2:(NSString*)param2; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-translate/ios/src/main/objectivec/com_codename1_ai_mlkit_translate_NativeTranslatorImpl.m b/maven/cn1-ai-mlkit-translate/ios/src/main/objectivec/com_codename1_ai_mlkit_translate_NativeTranslatorImpl.m deleted file mode 100644 index 56aadf63d22..00000000000 --- a/maven/cn1-ai-mlkit-translate/ios/src/main/objectivec/com_codename1_ai_mlkit_translate_NativeTranslatorImpl.m +++ /dev/null @@ -1,33 +0,0 @@ -#import "com_codename1_ai_mlkit_translate_NativeTranslatorImpl.h" -#import -#import -#import - -@implementation com_codename1_ai_mlkit_translate_NativeTranslatorImpl - --(NSString*)translate:(NSString*)param param1:(NSString*)param1 param2:(NSString*)param2 { - MLKTranslatorOptions *opts = [[MLKTranslatorOptions alloc] - initWithSourceLanguage:param1 - targetLanguage:param2]; - MLKTranslator *t = [MLKTranslator translatorWithOptions:opts]; - MLKModelDownloadConditions *cond = [[MLKModelDownloadConditions alloc] - initWithAllowsCellularAccess:YES - allowsBackgroundDownloading:YES]; - __block NSString *result = @""; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [t downloadModelIfNeededWithConditions:cond completion:^(NSError * _Nullable err) { - if (err) { dispatch_semaphore_signal(sem); return; } - [t translateText:param completion:^(NSString * _Nullable r, NSError * _Nullable e) { - if (r && !e) result = r; - dispatch_semaphore_signal(sem); - }]; - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-translate/javascript/pom.xml b/maven/cn1-ai-mlkit-translate/javascript/pom.xml deleted file mode 100644 index 526a14678fc..00000000000 --- a/maven/cn1-ai-mlkit-translate/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-translate - 8.0-SNAPSHOT - - - cn1-ai-mlkit-translate-javascript - jar - Codename One AI: cn1-ai-mlkit-translate-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-translate-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-translate/javascript/src/main/javascript/com_codename1_ai_mlkit_translate_NativeTranslator.js b/maven/cn1-ai-mlkit-translate/javascript/src/main/javascript/com_codename1_ai_mlkit_translate_NativeTranslator.js deleted file mode 100644 index 3b00f5db46c..00000000000 --- a/maven/cn1-ai-mlkit-translate/javascript/src/main/javascript/com_codename1_ai_mlkit_translate_NativeTranslator.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.translate__java_lang_String_java_lang_String_java_lang_String = function(param1, param2, param3, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_translate_NativeTranslator= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-translate/javase/pom.xml b/maven/cn1-ai-mlkit-translate/javase/pom.xml deleted file mode 100644 index fd867181408..00000000000 --- a/maven/cn1-ai-mlkit-translate/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-translate - 8.0-SNAPSHOT - - - cn1-ai-mlkit-translate-javase - jar - Codename One AI: cn1-ai-mlkit-translate-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-translate-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-translate/javase/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslatorImpl.java b/maven/cn1-ai-mlkit-translate/javase/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslatorImpl.java deleted file mode 100644 index eef89f81388..00000000000 --- a/maven/cn1-ai-mlkit-translate/javase/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslatorImpl.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.codename1.ai.mlkit.translate; - -public class NativeTranslatorImpl implements NativeTranslator { - public String translate(String text, String sourceLang, String targetLang) { - if (text == null) return ""; - return "[" + sourceLang + "->" + targetLang + "] " + text; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-translate/lib/pom.xml b/maven/cn1-ai-mlkit-translate/lib/pom.xml deleted file mode 100644 index 37784d38cf6..00000000000 --- a/maven/cn1-ai-mlkit-translate/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-translate - 8.0-SNAPSHOT - - - cn1-ai-mlkit-translate-lib - pom - Codename One AI: cn1-ai-mlkit-translate-lib - - - - com.codenameone - cn1-ai-mlkit-translate-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-translate-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-translate-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-translate-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-translate-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-translate/pom.xml b/maven/cn1-ai-mlkit-translate/pom.xml deleted file mode 100644 index e7dc9267bde..00000000000 --- a/maven/cn1-ai-mlkit-translate/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-translate - pom - Codename One AI: cn1-ai-mlkit-translate - ML Kit on-device Translation - - - cn1-ai-mlkit-translate - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-tflite/android/pom.xml b/maven/cn1-ai-tflite/android/pom.xml deleted file mode 100644 index 3e33e0aec67..00000000000 --- a/maven/cn1-ai-tflite/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-tflite - 8.0-SNAPSHOT - - - cn1-ai-tflite-android - jar - Codename One AI: cn1-ai-tflite-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-tflite-common - ${project.version} - - - - diff --git a/maven/cn1-ai-tflite/android/src/main/java/com/codename1/ai/tflite/NativeInterpreterImpl.java b/maven/cn1-ai-tflite/android/src/main/java/com/codename1/ai/tflite/NativeInterpreterImpl.java deleted file mode 100644 index 18e20fa4699..00000000000 --- a/maven/cn1-ai-tflite/android/src/main/java/com/codename1/ai/tflite/NativeInterpreterImpl.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.tflite; - - -public class NativeInterpreterImpl { - public float[] run(byte[] modelBytes, float[] input, int outputLength) { - java.nio.ByteBuffer bb = java.nio.ByteBuffer.allocateDirect(modelBytes.length); - bb.order(java.nio.ByteOrder.nativeOrder()); - bb.put(modelBytes); - bb.rewind(); - org.tensorflow.lite.Interpreter interp = new org.tensorflow.lite.Interpreter(bb); - float[][] out = new float[1][outputLength]; - interp.run(new float[][]{input}, out); - interp.close(); - return out[0]; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-tflite/common/codenameone_library_appended.properties b/maven/cn1-ai-tflite/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-tflite/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-tflite/common/codenameone_library_required.properties b/maven/cn1-ai-tflite/common/codenameone_library_required.properties deleted file mode 100644 index f6e175a63fe..00000000000 --- a/maven/cn1-ai-tflite/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-tflite. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=TensorFlowLiteObjC -codename1.arg.android.gradleDep=implementation 'org.tensorflow:tensorflow-lite:2.14.0' diff --git a/maven/cn1-ai-tflite/common/pom.xml b/maven/cn1-ai-tflite/common/pom.xml deleted file mode 100644 index 1a2412f35d8..00000000000 --- a/maven/cn1-ai-tflite/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-tflite - 8.0-SNAPSHOT - - - cn1-ai-tflite-common - jar - Codename One AI: cn1-ai-tflite-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/Interpreter.java b/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/Interpreter.java deleted file mode 100644 index 8fcaa4a8ddc..00000000000 --- a/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/Interpreter.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.tflite; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// TensorFlow Lite on-device inference. -/// -/// Loads a `.tflite` model and runs inference against `float[]` inputs. -/// Bridges to `TensorFlowLiteObjC` on iOS and `org.tensorflow:tensorflow-lite` -/// on Android. -/// -public final class Interpreter { - private Interpreter() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeInterpreter bridge = NativeLookup.create(NativeInterpreter.class); - return bridge != null && bridge.isSupported(); - } - - /// Loads a TensorFlow Lite model from the supplied bytes and runs - /// inference against a float32 input tensor. Returns the output as - /// `float[]`. The model file is held in a native handle keyed by - /// the SHA-1 of the input bytes; repeated calls reuse the loaded - /// model. - public static AsyncResource run(final byte[] modelBytes, - final float[] input, - final int outputLength) { - final AsyncResource out = new AsyncResource(); - final NativeInterpreter bridge = NativeLookup.create(NativeInterpreter.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("Interpreter.run is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final float[] r = bridge.run(modelBytes, input, outputLength); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new float[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("Interpreter.run failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/NativeInterpreter.java b/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/NativeInterpreter.java deleted file mode 100644 index 75bfac35172..00000000000 --- a/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/NativeInterpreter.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.tflite; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [Interpreter]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeInterpreter extends NativeInterface { - float[] run(byte[] modelBytes, float[] input, int outputLength); -} diff --git a/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/package-info.java b/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/package-info.java deleted file mode 100644 index 7bf52a592d0..00000000000 --- a/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/package-info.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// TensorFlow Lite on-device inference. -/// -/// Loads a `.tflite` model and runs inference against `float[]` inputs. -/// Bridges to `TensorFlowLiteObjC` on iOS and `org.tensorflow:tensorflow-lite` -/// on Android. -/// -/// The single public class in this package is [Interpreter], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeInterpreter` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `Interpreter.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.tflite; diff --git a/maven/cn1-ai-tflite/common/src/test/java/com/codename1/ai/tflite/InterpreterTest.java b/maven/cn1-ai-tflite/common/src/test/java/com/codename1/ai/tflite/InterpreterTest.java deleted file mode 100644 index 246690c5a9f..00000000000 --- a/maven/cn1-ai-tflite/common/src/test/java/com/codename1/ai/tflite/InterpreterTest.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.codename1.ai.tflite; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class InterpreterTest { - - /** Mock implementation of NativeInterpreter for headless JVM tests. */ - static class MockBridge implements NativeInterpreter { - boolean supported = true; - public boolean isSupported() { return supported; } - public float[] run(byte[] modelBytes, float[] input, int outputLength) { - float[] r = new float[outputLength]; - for (int i = 0; i < r.length; i++) r[i] = i; - return r; - } - } - - @Test - void mock_returns_increasing_vector() { - MockBridge b = new MockBridge(); - float[] r = b.run(new byte[0], new float[]{1f, 2f}, 4); - assertEquals(4, r.length); - assertEquals(3.0f, r[3], 1e-6); - } -} diff --git a/maven/cn1-ai-tflite/ios/pom.xml b/maven/cn1-ai-tflite/ios/pom.xml deleted file mode 100644 index a00bcd00cc4..00000000000 --- a/maven/cn1-ai-tflite/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-tflite - 8.0-SNAPSHOT - - - cn1-ai-tflite-ios - jar - Codename One AI: cn1-ai-tflite-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-tflite-common - ${project.version} - - - diff --git a/maven/cn1-ai-tflite/ios/src/main/objectivec/com_codename1_ai_tflite_NativeInterpreterImpl.h b/maven/cn1-ai-tflite/ios/src/main/objectivec/com_codename1_ai_tflite_NativeInterpreterImpl.h deleted file mode 100644 index f76a7e48606..00000000000 --- a/maven/cn1-ai-tflite/ios/src/main/objectivec/com_codename1_ai_tflite_NativeInterpreterImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_tflite_NativeInterpreterImpl : NSObject { -} - --(NSData*)run:(NSData*)param param1:(NSData*)param1 param2:(int)param2; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-tflite/ios/src/main/objectivec/com_codename1_ai_tflite_NativeInterpreterImpl.m b/maven/cn1-ai-tflite/ios/src/main/objectivec/com_codename1_ai_tflite_NativeInterpreterImpl.m deleted file mode 100644 index f6be4c4d80c..00000000000 --- a/maven/cn1-ai-tflite/ios/src/main/objectivec/com_codename1_ai_tflite_NativeInterpreterImpl.m +++ /dev/null @@ -1,28 +0,0 @@ -#import "com_codename1_ai_tflite_NativeInterpreterImpl.h" -#import -#import - -@implementation com_codename1_ai_tflite_NativeInterpreterImpl - --(NSData*)run:(NSData*)param param1:(NSData*)param1 param2:(int)param2 { - NSError *err = nil; - NSString *modelPath = [NSString stringWithFormat:@"%@/tflite-%@.tflite", - NSTemporaryDirectory(), [[NSUUID UUID] UUIDString]]; - [param writeToFile:modelPath atomically:YES]; - TFLInterpreter *interp = [[TFLInterpreter alloc] initWithModelPath:modelPath error:&err]; - if (err) return [NSData data]; - [interp allocateTensorsWithError:&err]; - if (err) return [NSData data]; - TFLTensor *in0 = [interp inputTensorAtIndex:0 error:&err]; - [in0 copyData:param1 error:&err]; - [interp invokeWithError:&err]; - TFLTensor *out0 = [interp outputTensorAtIndex:0 error:&err]; - NSData *outBytes = [out0 dataWithError:&err]; - return outBytes ?: [NSData data]; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-tflite/javascript/pom.xml b/maven/cn1-ai-tflite/javascript/pom.xml deleted file mode 100644 index 89211e280d4..00000000000 --- a/maven/cn1-ai-tflite/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-tflite - 8.0-SNAPSHOT - - - cn1-ai-tflite-javascript - jar - Codename One AI: cn1-ai-tflite-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-tflite-common - ${project.version} - - - diff --git a/maven/cn1-ai-tflite/javascript/src/main/javascript/com_codename1_ai_tflite_NativeInterpreter.js b/maven/cn1-ai-tflite/javascript/src/main/javascript/com_codename1_ai_tflite_NativeInterpreter.js deleted file mode 100644 index 141325d49cc..00000000000 --- a/maven/cn1-ai-tflite/javascript/src/main/javascript/com_codename1_ai_tflite_NativeInterpreter.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.run__byte_1ARRAY_float_1ARRAY_int = function(param1, param2, param3, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_tflite_NativeInterpreter= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-tflite/javase/pom.xml b/maven/cn1-ai-tflite/javase/pom.xml deleted file mode 100644 index 5f90217b0f1..00000000000 --- a/maven/cn1-ai-tflite/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-tflite - 8.0-SNAPSHOT - - - cn1-ai-tflite-javase - jar - Codename One AI: cn1-ai-tflite-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-tflite-common - ${project.version} - - - - diff --git a/maven/cn1-ai-tflite/javase/src/main/java/com/codename1/ai/tflite/NativeInterpreterImpl.java b/maven/cn1-ai-tflite/javase/src/main/java/com/codename1/ai/tflite/NativeInterpreterImpl.java deleted file mode 100644 index 03f151c0868..00000000000 --- a/maven/cn1-ai-tflite/javase/src/main/java/com/codename1/ai/tflite/NativeInterpreterImpl.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.codename1.ai.tflite; - -public class NativeInterpreterImpl implements NativeInterpreter { - public float[] run(byte[] modelBytes, float[] input, int outputLength) { - // Identity stub: returns first outputLength entries of input - // (or zero-padded if input shorter). Lets simulator test plumbing. - float[] out = new float[outputLength]; - int n = Math.min(input.length, outputLength); - System.arraycopy(input, 0, out, 0, n); - return out; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-tflite/lib/pom.xml b/maven/cn1-ai-tflite/lib/pom.xml deleted file mode 100644 index 5ecb3fab5cd..00000000000 --- a/maven/cn1-ai-tflite/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-tflite - 8.0-SNAPSHOT - - - cn1-ai-tflite-lib - pom - Codename One AI: cn1-ai-tflite-lib - - - - com.codenameone - cn1-ai-tflite-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-tflite-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-tflite-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-tflite-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-tflite-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-tflite/pom.xml b/maven/cn1-ai-tflite/pom.xml deleted file mode 100644 index 699c6ac8d75..00000000000 --- a/maven/cn1-ai-tflite/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-tflite - pom - Codename One AI: cn1-ai-tflite - TensorFlow Lite on-device inference - - - cn1-ai-tflite - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/codenameone-maven-plugin/pom.xml b/maven/codenameone-maven-plugin/pom.xml index 21e1f626816..4aabedc6fd1 100644 --- a/maven/codenameone-maven-plugin/pom.xml +++ b/maven/codenameone-maven-plugin/pom.xml @@ -34,6 +34,11 @@ + + ${project.groupId} + codenameone-platform-feature-catalog + ${project.version} + org.jdom jdom2 diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java index 1c6d1f58071..d5b4d87c097 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java @@ -22,7 +22,7 @@ */ package com.codename1.builders; - +import com.codename1.build.shared.PlatformFeatureCatalog; import static com.codename1.maven.PathUtil.path; @@ -467,6 +467,11 @@ public File getGradleProjectDirectory() { private boolean migrateToAndroidX; private boolean shouldIncludeGoogleImpl; private boolean arSupport; + private boolean visionSupport; + private boolean inferenceSupport; + private boolean languageSupport; + private final Set includedAiAdapterSources = + new HashSet(); static { isMac = System.getProperty("os.name").toLowerCase().contains("mac"); @@ -1313,12 +1318,16 @@ public boolean build(File sourceZip, final BuildRequest request) throws BuildExc wakeLock = true; } mediaPlaybackPermission = false; + visionSupport = false; + inferenceSupport = false; + languageSupport = false; + includedAiAdapterSources.clear(); // Accumulator for AI/ML class hits. After the scan we apply - // every matched AiDependencyTable.Entry -- appending Gradle + // every matched PlatformFeatureCatalog.Entry -- appending Gradle // deps to additionalDependencies (later) and permissions/ // features to xPermissions right now. - final AiDependencyTable.Accumulator aiAcc = new AiDependencyTable.Accumulator(); + final PlatformFeatureCatalog.Accumulator aiAcc = new PlatformFeatureCatalog.Accumulator(); try { scanClassesForPermissions(dummyClassesDir, new Executor.ClassScanner() { @@ -1327,6 +1336,21 @@ public boolean build(File sourceZip, final BuildRequest request) throws BuildExc @Override public void usesClass(String cls) { aiAcc.consume(cls); + String aiAdapter = androidAiAdapterSource(cls); + if (aiAdapter != null) { + includedAiAdapterSources.add(aiAdapter); + } + if (aiAdapter != null + && cls.indexOf("com/codename1/ai/vision/") == 0) { + visionSupport = true; + } + if ("com/codename1/ai/inference/InferenceSession".equals(cls)) { + inferenceSupport = true; + } + if (aiAdapter != null + && cls.indexOf("com/codename1/ai/language/") == 0) { + languageSupport = true; + } if (cls.indexOf("com/codename1/ar/") == 0) { // Keeps the ARCore-backed impl sources (deleted for // non-AR apps below) and bumps minSdk to the ARCore @@ -1501,6 +1525,7 @@ public void usesClass(String cls) { @Override public void usesClassMethod(String cls, String method) { + aiAcc.consumeMethod(cls, method); if (cls.indexOf("com/codename1/calendar/LocalCalendarSource") == 0 || (cls.indexOf("com/codename1/calendar/CalendarManager") == 0 && (method.indexOf("getLocalSource") >= 0 @@ -1666,7 +1691,7 @@ public void usesClassMethod(String cls, String method) { // additionalDependencies is written to build.gradle below. StringBuilder aiExtraGradleDependencies = new StringBuilder(); String aiApplicationMetaData = ""; - for (AiDependencyTable.Entry entry : aiAcc.hits()) { + for (PlatformFeatureCatalog.Entry entry : aiAcc.hits()) { for (String perm : entry.androidPermissions()) { String addString = " \n"; xPermissions += permissionAdd(request, perm, addString); @@ -2454,6 +2479,8 @@ public void usesClassMethod(String cls, String method) { arPackage.delete(); } + pruneOptionalAiSources(srcDir); + final String moPubAdUnitId = request.getArg("android.mopubId", null); if (moPubAdUnitId != null && moPubAdUnitId.length() > 0) { integrateMoPub = true; @@ -4605,6 +4632,9 @@ public void usesClassMethod(String cls, String method) { String keepFirebase = "-keep class com.google.android.gms.** { *; }\n\n" + "-keep class com.google.firebase.** { *; }\n\n"; + String keepAi = visionSupport || languageSupport || inferenceSupport + ? "-keep class com.codename1.impl.android.ai.** { *; }\n\n" + : ""; // workaround broken optimizer in proguard String proguardConfigOverride = "-dontusemixedcaseclassnames\n" + "-dontskipnonpubliclibraryclasses\n" @@ -4615,6 +4645,7 @@ public void usesClassMethod(String cls, String method) { + "\n" + "-dontwarn com.google.android.gms.**\n" + keepFirebase + + keepAi + "-keep class com.codename1.impl.android.AndroidBrowserComponentCallback {\n" + "*;\n" + "}\n\n" @@ -5255,6 +5286,79 @@ public void usesClassMethod(String cls, String method) { return true; } + private void pruneOptionalAiSources(File srcDir) { + File aiDir = new File(srcDir, + "com/codename1/impl/android/ai"); + String[] vision = { + "AndroidTextRecognitionAdapter.java", + "AndroidBarcodeScanningAdapter.java", + "AndroidFaceDetectionAdapter.java", + "AndroidImageLabelingAdapter.java", + "AndroidPoseDetectionAdapter.java", + "AndroidSelfieSegmentationAdapter.java" + }; + String[] language = { + "AndroidLanguageIdAdapter.java", + "AndroidTranslationAdapter.java", + "AndroidSmartReplyAdapter.java" + }; + pruneAiGroup(aiDir, visionSupport, + new String[] {"AndroidVisionImpl.java", + "AndroidVisionAdapter.java"}, vision); + pruneAiGroup(aiDir, languageSupport, + new String[] {"AndroidLanguageImpl.java", + "AndroidLanguageAdapter.java"}, language); + if (!inferenceSupport) { + new File(aiDir, "AndroidInferenceImpl.java").delete(); + } + } + + private void pruneAiGroup(File aiDir, boolean groupIncluded, + String[] sharedSources, String[] adapters) { + if (!groupIncluded) { + for (int i = 0; i < sharedSources.length; i++) { + new File(aiDir, sharedSources[i]).delete(); + } + } + for (int i = 0; i < adapters.length; i++) { + if (!groupIncluded + || !includedAiAdapterSources.contains(adapters[i])) { + new File(aiDir, adapters[i]).delete(); + } + } + } + + static String androidAiAdapterSource(String cls) { + if ("com/codename1/ai/vision/TextRecognizer".equals(cls)) { + return "AndroidTextRecognitionAdapter.java"; + } + if ("com/codename1/ai/vision/BarcodeScanner".equals(cls)) { + return "AndroidBarcodeScanningAdapter.java"; + } + if ("com/codename1/ai/vision/FaceDetector".equals(cls)) { + return "AndroidFaceDetectionAdapter.java"; + } + if ("com/codename1/ai/vision/ImageLabeler".equals(cls)) { + return "AndroidImageLabelingAdapter.java"; + } + if ("com/codename1/ai/vision/PoseDetector".equals(cls)) { + return "AndroidPoseDetectionAdapter.java"; + } + if ("com/codename1/ai/vision/SelfieSegmenter".equals(cls)) { + return "AndroidSelfieSegmentationAdapter.java"; + } + if ("com/codename1/ai/language/LanguageIdentifier".equals(cls)) { + return "AndroidLanguageIdAdapter.java"; + } + if ("com/codename1/ai/language/Translator".equals(cls)) { + return "AndroidTranslationAdapter.java"; + } + if ("com/codename1/ai/language/SmartReply".equals(cls)) { + return "AndroidSmartReplyAdapter.java"; + } + return null; + } + static String xmlize(String s) { s = s.replace("&", "&"); s = s.replace("<", "<"); diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java index c74849d71de..940ec0ad3e5 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java @@ -22,6 +22,7 @@ */ package com.codename1.builders; +import com.codename1.build.shared.PlatformFeatureCatalog; import com.codename1.util.IOSWalletExtensionBuilder; import com.codename1.util.IOSWidgetExtensionBuilder; import org.w3c.dom.Document; @@ -117,6 +118,9 @@ public class IPhoneBuilder extends Executor { private boolean usesBluetoothPeripheral; private boolean usesCn1Camera; private boolean usesCn1Ar; + private boolean usesCn1Vision; + private boolean usesCn1Language; + private boolean usesCn1Inference; // Set when the app references com.codename1.car.* (Apple CarPlay support). Gates the // CN1_USE_CARPLAY native define, CarPlay.framework linkage, the carplay entitlement and the // CarPlay scene in the Info.plist scene manifest. Apps that never touch the API see no change. @@ -791,10 +795,10 @@ public boolean build(File sourceZip, BuildRequest request) throws BuildException } // Accumulator for AI/ML class hits. After the scan we apply - // every matched AiDependencyTable.Entry -- appending pods, + // every matched PlatformFeatureCatalog.Entry -- appending pods, // SPM specs, plist defaults and Android perms -- so the user // doesn't have to declare them by hand. - final AiDependencyTable.Accumulator aiAcc = new AiDependencyTable.Accumulator(); + final PlatformFeatureCatalog.Accumulator aiAcc = new PlatformFeatureCatalog.Accumulator(); try { scanClassesForPermissions(classesDir, new Executor.ClassScanner() { @@ -874,6 +878,16 @@ public void usesClass(String cls) { if (!usesCn1Ar && cls.indexOf("com/codename1/ar/") == 0) { usesCn1Ar = true; } + if (!usesCn1Vision && isVisionAnalyzerClass(cls)) { + usesCn1Vision = true; + } + if (!usesCn1Language && isLanguageFeatureClass(cls)) { + usesCn1Language = true; + } + if (!usesCn1Inference + && "com/codename1/ai/inference/InferenceSession".equals(cls)) { + usesCn1Inference = true; + } // Apple CarPlay (com.codename1.car.*). Gated on actual usage so the // CarPlay scene/entitlement/framework are only added for apps that // build an in-car experience. @@ -927,6 +941,7 @@ public void usesClass(String cls) { @Override public void usesClassMethod(String cls, String method) { + aiAcc.consumeMethod(cls, method); if (cls.indexOf("com/codename1/calendar/LocalCalendarSource") == 0 || (cls.indexOf("com/codename1/calendar/CalendarManager") == 0 && (method.indexOf("getLocalSource") >= 0 @@ -1018,10 +1033,20 @@ public void usesClassMethod(String cls, String method) { // upgrade the effective mode to BOTH below). boolean projectPrefersSpm = dependencyConfig.usesSwiftPackages() && !dependencyConfig.usesCocoaPods(); StringBuilder spmPackages = new StringBuilder(request.getArg("ios.spm.packages", "")); - for (AiDependencyTable.Entry entry : aiAcc.hits()) { + for (PlatformFeatureCatalog.Entry entry : aiAcc.hits()) { + // Google ML Kit and TensorFlowLiteObjC publish iOS/device and + // simulator slices, but not Mac Catalyst slices. Keep the + // framework-only Apple Vision implementation enabled for + // macNative while leaving language/inference on their safe + // unsupported native stubs. + boolean includeApplePackageDependencies = + !macNativeBuilder.isEnabled() + || entry.iosDependenciesSupportMacCatalyst(); boolean handledViaSpm = false; - if (projectPrefersSpm && !entry.iosSpmSpecs().isEmpty()) { - for (AiDependencyTable.IosSpm spm : entry.iosSpmSpecs()) { + if (includeApplePackageDependencies + && projectPrefersSpm + && !entry.iosSpmSpecs().isEmpty()) { + for (PlatformFeatureCatalog.IosSpm spm : entry.iosSpmSpecs()) { if (spmPackages.length() > 0) spmPackages.append(';'); spmPackages.append(spm.identity).append('|') .append(spm.url).append('|') @@ -1040,7 +1065,7 @@ public void usesClassMethod(String cls, String method) { } handledViaSpm = true; } - if (!handledViaSpm) { + if (includeApplePackageDependencies && !handledViaSpm) { for (String pod : entry.iosPods()) { if (iosPods.length() > 0) iosPods += ","; iosPods += pod; @@ -2401,7 +2426,7 @@ public void usesClassMethod(String cls, String method) { // First-class Bluetooth: weak-link CoreBluetooth and compile in // the CN1Bluetooth natives only when the app references // com.codename1.bluetooth.*. The NSBluetooth* privacy strings - // are defaulted (only-if-unset) by the AiDependencyTable entry + // are defaulted (only-if-unset) by the PlatformFeatureCatalog entry // through the standard plist application above. Background // operation is opt-in through the ios.bluetooth.background hint // ("central", "peripheral" or "central,peripheral"), merged into @@ -2452,7 +2477,7 @@ public void usesClassMethod(String cls, String method) { // INCLUDE_CAMERA_USAGE (the old modal Capture API): the new // AVFoundation natives are only built when the app actually // references com.codename1.camera.*, matching the AVFoundation - // framework injection driven by the same scan via AiDependencyTable. + // framework injection driven by the same scan via PlatformFeatureCatalog. if (usesCn1Camera) { try { replaceInFile(new File(buildinRes, @@ -2468,7 +2493,7 @@ public void usesClassMethod(String cls, String method) { // Augmented reality: uncomment INCLUDE_CN1_AR so the CN1AR // natives (ARKit + ARSCNView) compile in, and link ARKit / // SceneKit explicitly -- neither is default-linked and the - // AiDependencyTable iosFrameworks field is documentation-only. + // PlatformFeatureCatalog iosFrameworks field is documentation-only. // Apps that never reference com.codename1.ar leave the define // commented out so no ARKit symbol is referenced, which keeps // Apple's API-usage scan quiet and tvOS/watchOS slices clean. @@ -2490,6 +2515,56 @@ public void usesClassMethod(String cls, String method) { } } + if (usesCn1Vision) { + try { + replaceInFile(new File(buildinRes, + "CodenameOne_GLViewController.h"), + "//#define INCLUDE_CN1_VISION", + "#define INCLUDE_CN1_VISION"); + } catch (IOException ex) { + throw new BuildException( + "Failed to enable INCLUDE_CN1_VISION", ex); + } + String visionLibs = + "Vision.framework;CoreImage.framework;CoreVideo.framework"; + if (addLibs == null || addLibs.length() == 0) { + addLibs = visionLibs; + } else if (!addLibs.toLowerCase().contains("vision.framework")) { + addLibs = addLibs + ";" + visionLibs; + } + } + + if (usesCn1Language) { + try { + replaceInFile(new File(buildinRes, + "CodenameOne_GLViewController.h"), + "//#define INCLUDE_CN1_LANGUAGE", + "#define INCLUDE_CN1_LANGUAGE"); + } catch (IOException ex) { + throw new BuildException( + "Failed to enable INCLUDE_CN1_LANGUAGE", ex); + } + } + + if (usesCn1Inference) { + try { + replaceInFile(new File(buildinRes, + "CodenameOne_GLViewController.h"), + "//#define INCLUDE_CN1_INFERENCE", + "#define INCLUDE_CN1_INFERENCE"); + } catch (IOException ex) { + throw new BuildException( + "Failed to enable INCLUDE_CN1_INFERENCE", ex); + } + String inferenceLibs = + "CoreML.framework;Metal.framework;Accelerate.framework"; + if (addLibs == null || addLibs.length() == 0) { + addLibs = inferenceLibs; + } else if (!addLibs.toLowerCase().contains("coreml.framework")) { + addLibs = addLibs + ";" + inferenceLibs; + } + } + // CarPlay: link CarPlay.framework (+ MediaPlayer for the now-playing template) and // inject the per-category carplay entitlement. The CarPlay entitlements are granted by // Apple per app category, so we only inject the ones the project opts into via the @@ -5482,4 +5557,20 @@ private static String join(String[] strs, String sep) { return out.toString(); } + private static boolean isVisionAnalyzerClass(String cls) { + return "com/codename1/ai/vision/TextRecognizer".equals(cls) + || "com/codename1/ai/vision/BarcodeScanner".equals(cls) + || "com/codename1/ai/vision/FaceDetector".equals(cls) + || "com/codename1/ai/vision/ImageLabeler".equals(cls) + || "com/codename1/ai/vision/PoseDetector".equals(cls) + || "com/codename1/ai/vision/SelfieSegmenter".equals(cls) + || "com/codename1/ai/vision/DocumentScanner".equals(cls); + } + + private static boolean isLanguageFeatureClass(String cls) { + return "com/codename1/ai/language/LanguageIdentifier".equals(cls) + || "com/codename1/ai/language/Translator".equals(cls) + || "com/codename1/ai/language/SmartReply".equals(cls); + } + } diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacNativeBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacNativeBuilder.java index e0e522494b5..78f906090a9 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacNativeBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacNativeBuilder.java @@ -334,7 +334,7 @@ private void writeEntitlementsFile(BuildRequest request, File appSrcDir, // explicit device entitlements; without them the OS refuses to // open the AVCaptureSession even when the Info.plist usage // descriptions are present. We piggyback on the iOS plist hints - // already populated by AiDependencyTable -- when the build + // already populated by PlatformFeatureCatalog -- when the build // pipeline detected com.codename1.camera.* (or any other API // that triggers NSCameraUsageDescription / NSMicrophoneUsageDescription) // we mirror that into the Mac entitlements. Developers may diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/CN1BuildMojo.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/CN1BuildMojo.java index d4f55d37b00..12700c0df49 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/CN1BuildMojo.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/CN1BuildMojo.java @@ -1466,6 +1466,13 @@ private void doIOSLocalBuild(File tmpProjectDir, Properties props, File distJar) : getGeneratedIOSProjectSourceDirectory(); output.getParentFile().mkdirs(); try { + // This directory is a generated target. Replacing it is + // required when class scanning removes a dependency: + // copyDirectory() alone leaves stale Podfiles, workspaces, + // Pods and optional native sources from the prior build. + if (output.exists()) { + FileUtils.deleteDirectory(output); + } getLog().info("Copying Xcode Project to "+output); FileUtils.copyDirectory(xcodeProject, output); } catch (IOException ex) { diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java new file mode 100644 index 00000000000..c6909a3f87d --- /dev/null +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java @@ -0,0 +1,51 @@ +package com.codename1.builders; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +class AndroidAiSourceSelectionTest { + @Test + void selectsOneSourcePerVisionFeature() { + assertEquals("AndroidTextRecognitionAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/TextRecognizer")); + assertEquals("AndroidBarcodeScanningAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/BarcodeScanner")); + assertEquals("AndroidFaceDetectionAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/FaceDetector")); + assertEquals("AndroidImageLabelingAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/ImageLabeler")); + assertEquals("AndroidPoseDetectionAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/PoseDetector")); + assertEquals("AndroidSelfieSegmentationAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/SelfieSegmenter")); + } + + @Test + void selectsOneSourcePerLanguageFeature() { + assertEquals("AndroidLanguageIdAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/language/LanguageIdentifier")); + assertEquals("AndroidTranslationAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/language/Translator")); + assertEquals("AndroidSmartReplyAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/language/SmartReply")); + } + + @Test + void ignoresSharedApiAndUnrelatedClasses() { + assertNull(AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/VisionPipeline")); + assertNull(AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ui/Form")); + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java new file mode 100644 index 00000000000..466765fa678 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.inference; + +import com.codename1.impl.InferenceImpl; +import com.codename1.junit.UITestBase; +import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; + +class InferenceApiTest extends UITestBase { + @Test + void tensorsAreImmutableAndValidateShape() { + float[] source = new float[] {1, 2}; + Tensor tensor = Tensor.floats("input", new int[] {1, 2}, source); + source[0] = 9; + assertArrayEquals(new float[] {1, 2}, (float[]) tensor.getData()); + float[] returned = (float[]) tensor.getData(); + returned[1] = 9; + assertArrayEquals(new float[] {1, 2}, (float[]) tensor.getData()); + assertThrows(IllegalArgumentException.class, () -> + Tensor.floats("bad", new int[] {3}, new float[] {1, 2})); + } + + @Test + void sessionLifecycleForwardsToBackend() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + implementation.setInferenceImpl(backend); + InferenceSession session = await(InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions().threads(2))); + assertEquals(2, session.getInputs()[0].getShape()[1]); + Tensor[] output = await(session.run(new Tensor[] { + Tensor.floats("input", new int[] {1, 2}, new float[] {1, 2}) + })); + assertArrayEquals(new float[] {3}, (float[]) output[0].getData()); + session.resizeInput("input", new int[] {1, 4}); + assertEquals("input", backend.resizedName); + session.close(); + session.close(); + assertEquals(1, backend.closeCount); + assertThrows(IllegalStateException.class, session::getInputs); + } + + private T await(AsyncResource resource) { + final AtomicReference value = new AtomicReference(); + resource.ready(new SuccessCallback() { + public void onSucess(T result) { + value.set(result); + } + }); + flushSerialCalls(); + assertTrue(resource.isDone()); + assertNotNull(value.get()); + return resource.get(); + } + + private static final class RecordingInferenceImpl extends InferenceImpl { + final Object handle = new Object(); + String resizedName; + int closeCount; + + public boolean isSupported() { + return true; + } + + public AsyncResource open(ModelSource source, InferenceOptions options) { + AsyncResource result = new AsyncResource(); + result.complete(handle); + return result; + } + + public TensorInfo[] getInputs(Object value) { + assertSame(handle, value); + return new TensorInfo[] { + new TensorInfo("input", TensorType.FLOAT32, new int[] {1, 2}, 0) + }; + } + + public TensorInfo[] getOutputs(Object value) { + return new TensorInfo[] { + new TensorInfo("output", TensorType.FLOAT32, new int[] {1}, 0) + }; + } + + public AsyncResource run(Object value, Tensor[] inputs) { + AsyncResource result = new AsyncResource(); + result.complete(new Tensor[] { + Tensor.floats("output", new int[] {1}, new float[] {3}) + }); + return result; + } + + public void resizeInput(Object value, String name, int[] shape) { + resizedName = name; + } + + public void close(Object value) { + closeCount++; + } + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java new file mode 100644 index 00000000000..a4564ab35a0 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.language; + +import com.codename1.impl.LanguageImpl; +import com.codename1.junit.UITestBase; +import com.codename1.util.AsyncResource; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class LanguageApiTest extends UITestBase { + @Test + void languageOperationsForwardBackendAndOptions() { + RecordingLanguageImpl backend = new RecordingLanguageImpl(); + implementation.setLanguageImpl(backend); + LanguageOptions options = new LanguageOptions() + .backend(LanguageBackends.mlKit()) + .minimumConfidence(.4f); + + assertTrue(LanguageIdentifier.isSupported(options)); + assertTrue(Translator.isSupported(options)); + assertTrue(SmartReply.isSupported(options)); + + LanguageCandidate[] candidates = + LanguageIdentifier.identify("bonjour", options).get(); + assertEquals("fr", candidates[0].getLanguageTag()); + assertEquals("language-id", backend.feature); + assertEquals("ml-kit", backend.backend); + assertEquals(.4f, backend.options.getMinimumConfidence()); + + assertEquals("hello", + Translator.translate("bonjour", "fr", "en", options).get()); + assertEquals("translation", backend.feature); + + String[] replies = SmartReply.suggest(new SmartReplyMessage[] { + new SmartReplyMessage("Are you coming?", "remote", false, 1) + }, options).get(); + assertEquals("Yes", replies[0]); + assertEquals("smart-reply", backend.feature); + } + + @Test + void missingBackendReportsUnsupported() { + implementation.setLanguageImpl(null); + assertFalse(LanguageIdentifier.isSupported()); + AsyncResource result = + LanguageIdentifier.identify("hello", null); + assertTrue(result.isDone()); + assertThrows(AsyncResource.AsyncExecutionException.class, result::get); + } + + private static final class RecordingLanguageImpl extends LanguageImpl { + String feature; + String backend; + LanguageOptions options; + + public boolean isSupported(String value, String backendId) { + feature = value; + backend = backendId; + return true; + } + + public AsyncResource identify( + String text, String backendId, LanguageOptions value) { + backend = backendId; + options = value; + AsyncResource result = + new AsyncResource(); + result.complete(new LanguageCandidate[] { + new LanguageCandidate("fr", .9f) + }); + return result; + } + + public AsyncResource translate( + String text, String sourceLanguage, String targetLanguage, + String backendId, LanguageOptions value) { + feature = "translation"; + AsyncResource result = new AsyncResource(); + result.complete("hello"); + return result; + } + + public AsyncResource suggestReplies( + SmartReplyMessage[] conversation, String backendId, + LanguageOptions value) { + feature = "smart-reply"; + AsyncResource result = new AsyncResource(); + result.complete(new String[] {"Yes"}); + return result; + } + + public void close() { + } + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java new file mode 100644 index 00000000000..6548c5cb298 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.vision; + +import com.codename1.camera.FrameFormat; +import com.codename1.impl.VisionImpl; +import com.codename1.junit.UITestBase; +import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; + +class VisionApiTest extends UITestBase { + @Test + void visionImageDefensivelyCopiesInputAndOutput() { + byte[] source = new byte[] {1, 2, 3}; + VisionImage image = VisionImage.encoded(source); + source[0] = 9; + assertEquals(1, image.getEncodedBytes()[0]); + byte[] returned = image.getEncodedBytes(); + returned[1] = 9; + assertEquals(2, image.getEncodedBytes()[1]); + } + + @Test + void pixelInputNormalizesRotation() { + VisionImage image = VisionImage.pixels(new byte[] {1, 2, 3, 4}, + 1, 1, FrameFormat.RGBA8888, -90); + assertEquals(270, image.getRotationDegrees()); + } + + @Test + void backendMetadataIsOptionalAndImmutable() { + Map values = new HashMap(); + values.put("nativeType", "qr"); + VisionMetadata metadata = new VisionMetadata("ml-kit", values); + values.put("nativeType", "changed"); + assertEquals("ml-kit", metadata.getBackendId()); + assertEquals("qr", metadata.get("nativeType")); + assertThrows(UnsupportedOperationException.class, + () -> metadata.getValues().put("x", "y")); + Barcode barcode = new Barcode("value", "QR_CODE", null, + VisionRect.EMPTY, null, metadata); + assertSame(metadata, barcode.getMetadata()); + } + + @Test + void analyzerForwardsFeatureBackendAndOptions() { + RecordingVisionImpl backend = new RecordingVisionImpl(); + implementation.setVisionImpl(backend); + VisionOptions options = new VisionOptions() + .backend(VisionBackends.mlKit()).minimumConfidence(.6f); + TextRecognizer recognizer = new TextRecognizer(options); + TextRecognitionResult result = await(recognizer.process( + VisionImage.encoded(new byte[] {1}))); + assertEquals("hello", result.getText()); + assertEquals(VisionFeature.TEXT_RECOGNITION, backend.feature); + assertEquals("ml-kit", backend.backend); + assertSame(options, backend.options); + recognizer.close(); + assertEquals(1, backend.closeCount); + } + + @Test + void unsupportedAnalyzerReturnsFailedResource() { + implementation.setVisionImpl(null); + AsyncResource result = + new TextRecognizer().process(VisionImage.encoded(new byte[] {1})); + assertTrue(result.isDone()); + assertThrows(AsyncResource.AsyncExecutionException.class, result::get); + } + + private T await(AsyncResource resource) { + final AtomicReference value = new AtomicReference(); + resource.ready(new SuccessCallback() { + public void onSucess(T result) { + value.set(result); + } + }); + flushSerialCalls(); + assertTrue(resource.isDone()); + assertNotNull(value.get()); + return resource.get(); + } + + private static final class RecordingVisionImpl extends VisionImpl { + VisionFeature feature; + String backend; + VisionOptions options; + int closeCount; + + public boolean isSupported(VisionFeature feature, String backendId) { + return true; + } + + @SuppressWarnings("unchecked") + public AsyncResource analyze(VisionFeature value, String backendId, + VisionImage image, VisionOptions opts) { + feature = value; + backend = backendId; + options = opts; + AsyncResource result = new AsyncResource(); + result.complete((T) new TextRecognitionResult("hello", null)); + return result; + } + + public void close() { + closeCount++; + } + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java b/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java index d9d71e2e30d..0ee88888071 100644 --- a/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java +++ b/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java @@ -936,6 +936,37 @@ public com.codename1.impl.ARImpl createARImpl() { return arImpl; } + private com.codename1.impl.VisionImpl visionImpl; + private com.codename1.impl.InferenceImpl inferenceImpl; + private com.codename1.impl.LanguageImpl languageImpl; + + public void setVisionImpl(com.codename1.impl.VisionImpl visionImpl) { + this.visionImpl = visionImpl; + } + + @Override + public com.codename1.impl.VisionImpl createVisionImpl() { + return visionImpl; + } + + public void setInferenceImpl(com.codename1.impl.InferenceImpl inferenceImpl) { + this.inferenceImpl = inferenceImpl; + } + + @Override + public com.codename1.impl.InferenceImpl createInferenceImpl() { + return inferenceImpl; + } + + public void setLanguageImpl(com.codename1.impl.LanguageImpl languageImpl) { + this.languageImpl = languageImpl; + } + + @Override + public com.codename1.impl.LanguageImpl createLanguageImpl() { + return languageImpl; + } + private com.codename1.sensors.MotionSensorManager motionSensorManager; /** diff --git a/maven/platform-feature-catalog/pom.xml b/maven/platform-feature-catalog/pom.xml new file mode 100644 index 00000000000..5a94c125518 --- /dev/null +++ b/maven/platform-feature-catalog/pom.xml @@ -0,0 +1,31 @@ + + + 4.0.0 + + com.codenameone + codenameone + 8.0-SNAPSHOT + + codenameone-platform-feature-catalog + Codename One Platform Feature Catalog + Shared builder registry for optional platform APIs and native dependencies + + 1.8 + 1.8 + UTF-8 + + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AiDependencyTable.java b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java similarity index 68% rename from maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AiDependencyTable.java rename to maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java index 589034db84d..f082b4c91dd 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AiDependencyTable.java +++ b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java @@ -20,7 +20,7 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.builders; +package com.codename1.build.shared; import java.util.ArrayList; import java.util.Arrays; @@ -35,9 +35,9 @@ * in {@code com.codename1.media}) to the native dependencies and * permissions each one requires. * - *

The build server's class scanners ({@link IPhoneBuilder} and - * {@link AndroidGradleBuilder}) call into this table from inside their - * existing {@link Executor.ClassScanner#usesClass(String)} blocks; the + *

The build server's class scanners ({@code IPhoneBuilder} and + * {@code AndroidGradleBuilder}) call into this table from inside their + * existing {@code Executor.ClassScanner.usesClass(String)} blocks; the * resulting set of {@link Entry} records is then applied just before * iOS pods / SPM are resolved and just before the Android Gradle * dependencies / manifest fragments are written.

@@ -46,7 +46,7 @@ * needs change (different pod version, additional plist entry) should * be edited here, not in the builder hot loop.

*/ -public final class AiDependencyTable { +public final class PlatformFeatureCatalog { private static final List ENTRIES; @@ -79,78 +79,100 @@ public final class AiDependencyTable { .iosFrameworks("AVFAudio") .description("Text-to-speech")); - // ML Kit feature submodules. Class prefix matches the - // (forward-referenced) cn1libs' package layout. - e.add(new Entry("com/codename1/ai/mlkit/text/") - .iosPod("GoogleMLKit/TextRecognition") + // Built-in vision APIs. Android uses ML Kit by default. iOS uses + // Apple Vision/VisionKit unless VisionBackends.mlKit() is selected. + // The compound entries require both the feature and selector method, + // so iOS only bundles ML Kit pods for features actually used. + e.add(new Entry("com/codename1/ai/vision/TextRecognizer") + .iosFrameworks("Vision", "CoreImage") .androidGradle("com.google.mlkit:text-recognition:16.0.0") - .iosPlist("NSCameraUsageDescription", - "Used to recognise text from your camera.") - .description("ML Kit Text Recognition")); + .description("Text recognition")); + e.add(new Entry("com/codename1/ai/vision/TextRecognizer") + .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") + .iosPod("GoogleMLKit/TextRecognition") + .iosDependenciesUnsupportedOnMacCatalyst() + .description("ML Kit iOS text-recognition backend")); - e.add(new Entry("com/codename1/ai/mlkit/barcode/") - .iosPod("GoogleMLKit/BarcodeScanning") + e.add(new Entry("com/codename1/ai/vision/BarcodeScanner") + .iosFrameworks("Vision", "CoreImage") .androidGradle("com.google.mlkit:barcode-scanning:17.2.0") - .iosPlist("NSCameraUsageDescription", - "Used to scan barcodes with your camera.") - .androidFeatures("android.hardware.camera") - .description("ML Kit Barcode Scanning")); + .description("Barcode scanning")); + e.add(new Entry("com/codename1/ai/vision/BarcodeScanner") + .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") + .iosPod("GoogleMLKit/BarcodeScanning") + .iosDependenciesUnsupportedOnMacCatalyst() + .description("ML Kit iOS barcode backend")); - e.add(new Entry("com/codename1/ai/mlkit/face/") - .iosPod("GoogleMLKit/FaceDetection") + e.add(new Entry("com/codename1/ai/vision/FaceDetector") + .iosFrameworks("Vision", "CoreImage") .androidGradle("com.google.mlkit:face-detection:16.1.5") - .iosPlist("NSCameraUsageDescription", - "Used to detect faces in images.") - .description("ML Kit Face Detection")); + .description("Face detection")); + e.add(new Entry("com/codename1/ai/vision/FaceDetector") + .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") + .iosPod("GoogleMLKit/FaceDetection") + .iosDependenciesUnsupportedOnMacCatalyst() + .description("ML Kit iOS face-detection backend")); - e.add(new Entry("com/codename1/ai/mlkit/labeling/") - .iosPod("GoogleMLKit/ImageLabeling") + e.add(new Entry("com/codename1/ai/vision/ImageLabeler") + .iosFrameworks("Vision", "CoreML") .androidGradle("com.google.mlkit:image-labeling:17.0.7") - .description("ML Kit Image Labeling")); - - e.add(new Entry("com/codename1/ai/mlkit/translate/") - .iosPod("GoogleMLKit/Translate") - .androidGradle("com.google.mlkit:translate:17.0.1") - .description("ML Kit Translation")); - - e.add(new Entry("com/codename1/ai/mlkit/smartreply/") - .iosPod("GoogleMLKit/SmartReply") - .androidGradle("com.google.mlkit:smart-reply:17.0.2") - .description("ML Kit Smart Reply")); - - e.add(new Entry("com/codename1/ai/mlkit/langid/") - .iosPod("GoogleMLKit/LanguageID") - .androidGradle("com.google.mlkit:language-id:17.0.4") - .description("ML Kit Language ID")); + .description("Image labeling")); + e.add(new Entry("com/codename1/ai/vision/ImageLabeler") + .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") + .iosPod("GoogleMLKit/ImageLabeling") + .iosDependenciesUnsupportedOnMacCatalyst() + .description("ML Kit iOS image-labeling backend")); - e.add(new Entry("com/codename1/ai/mlkit/pose/") - .iosPod("GoogleMLKit/PoseDetection") + e.add(new Entry("com/codename1/ai/vision/PoseDetector") + .iosFrameworks("Vision", "CoreML") .androidGradle("com.google.mlkit:pose-detection:18.0.0-beta3") - .description("ML Kit Pose Detection")); - - e.add(new Entry("com/codename1/ai/mlkit/segmentation/") + .description("Pose detection")); + e.add(new Entry("com/codename1/ai/vision/PoseDetector") + .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") + .iosPod("GoogleMLKit/PoseDetection") + .iosDependenciesUnsupportedOnMacCatalyst() + .description("ML Kit iOS pose-detection backend")); + + e.add(new Entry("com/codename1/ai/vision/SelfieSegmenter") + .iosFrameworks("Vision", "CoreML") + .androidGradle( + "com.google.mlkit:segmentation-selfie:16.0.0-beta5") + .description("Selfie segmentation")); + e.add(new Entry("com/codename1/ai/vision/SelfieSegmenter") + .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") .iosPod("GoogleMLKit/SegmentationSelfie") - .androidGradle("com.google.mlkit:segmentation-selfie:16.0.0-beta4") - .description("ML Kit Selfie Segmentation")); - - e.add(new Entry("com/codename1/ai/mlkit/docscan/") - .iosPod("GoogleMLKit/DocumentScanner") - .iosFrameworks("VisionKit") - .androidGradle("com.google.android.gms:play-services-mlkit-document-scanner:16.0.0-beta1") - .description("ML Kit Document Scanner")); - - // TFLite has both Pods and SPM publishers. We register both - // so IOSDependencyManager.resolve() can route to whichever - // the project uses. - e.add(new Entry("com/codename1/ai/tflite/") - .iosPod("TensorFlowLiteSwift") - .iosSpm("TensorFlowLiteSwift", - "https://github.com/tensorflow/tensorflow.git", - "from:2.13.0", - "TensorFlowLite") - .androidGradle("org.tensorflow:tensorflow-lite:2.13.0") - .androidGradle("org.tensorflow:tensorflow-lite-support:0.4.4") - .description("TensorFlow Lite interpreter")); + .iosDependenciesUnsupportedOnMacCatalyst() + .description("ML Kit iOS selfie-segmentation backend")); + + e.add(new Entry("com/codename1/ai/vision/DocumentScanner") + .iosFrameworks("VisionKit", "Vision", "CoreImage") + .description("Still-image document correction (Apple platforms)")); + + // The common inference API is backed by LiteRT/TensorFlow Lite. + // iOS may select the Core ML delegate without changing + // model formats. + e.add(new Entry("com/codename1/ai/inference/InferenceSession") + .iosPod("TensorFlowLiteObjC/CoreML") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosFrameworks("CoreML", "Metal", "Accelerate") + .androidGradle("com.google.ai.edge.litert:litert:1.0.1") + .description("LiteRT inference")); + + e.add(new Entry("com/codename1/ai/language/LanguageIdentifier") + .iosPod("GoogleMLKit/LanguageID") + .iosDependenciesUnsupportedOnMacCatalyst() + .androidGradle("com.google.mlkit:language-id:17.0.6") + .description("On-device language identification")); + e.add(new Entry("com/codename1/ai/language/Translator") + .iosPod("GoogleMLKit/Translate") + .iosDependenciesUnsupportedOnMacCatalyst() + .androidGradle("com.google.mlkit:translate:17.0.3") + .description("On-device translation")); + e.add(new Entry("com/codename1/ai/language/SmartReply") + .iosPod("GoogleMLKit/SmartReply") + .iosDependenciesUnsupportedOnMacCatalyst() + .androidGradle("com.google.mlkit:smart-reply:17.0.4") + .description("On-device smart reply")); e.add(new Entry("com/codename1/ai/whisper/") .iosFrameworks("Accelerate") @@ -229,7 +251,7 @@ public final class AiDependencyTable { ENTRIES = Collections.unmodifiableList(e); } - private AiDependencyTable() { + private PlatformFeatureCatalog() { } /** All registered entries. Mostly useful for tests and tooling. */ @@ -249,7 +271,7 @@ public static List matchesFor(String internalClassName) { } List out = new ArrayList(); for (Entry e : ENTRIES) { - if (e.matches(internalClassName)) { + if (e.matchesClass(internalClassName) && !e.hasMethodRequirement()) { out.add(e); } } @@ -257,24 +279,37 @@ public static List matchesFor(String internalClassName) { } /** - * Builder/scanner output: the de-duplicated union of every entry - * fired by a class scan. Use {@link Accumulator#consume(String)} - * from inside an {@link Executor.ClassScanner#usesClass(String)} - * implementation. + * Builder/scanner output: the de-duplicated union of entries whose class + * and optional method requirements were observed. */ public static final class Accumulator { - private final Set hits = new LinkedHashSet(); + private final Set classes = new LinkedHashSet(); + private final Set methods = new LinkedHashSet(); public void consume(String internalClassName) { - hits.addAll(matchesFor(internalClassName)); + if (internalClassName != null) { + classes.add(internalClassName); + } + } + + public void consumeMethod(String internalClassName, String methodName) { + if (internalClassName != null && methodName != null) { + methods.add(internalClassName + "#" + methodName); + } } public Set hits() { + Set hits = new LinkedHashSet(); + for (Entry entry : ENTRIES) { + if (entry.requirementsMet(classes, methods)) { + hits.add(entry); + } + } return hits; } public boolean anyRequiresBigUpload() { - for (Entry e : hits) { + for (Entry e : hits()) { if (e.requiresBigUpload) { return true; } @@ -290,6 +325,8 @@ public boolean anyRequiresBigUpload() { */ public static final class Entry { private final String classPrefix; + private String methodOwner; + private String methodPrefix; private final List iosPods = new ArrayList(); private final List iosSpm = new ArrayList(); private final List iosFrameworks = new ArrayList(); @@ -298,6 +335,7 @@ public static final class Entry { private final List androidPermissions = new ArrayList(); private final List androidFeatures = new ArrayList(); private final List androidMetaData = new ArrayList(); + private boolean iosDependenciesSupportMacCatalyst = true; private boolean requiresBigUpload; private String description = ""; @@ -305,13 +343,45 @@ public static final class Entry { this.classPrefix = classPrefix; } - boolean matches(String internalClassName) { + boolean matchesClass(String internalClassName) { if (classPrefix.endsWith("/")) { return internalClassName.startsWith(classPrefix); } return internalClassName.equals(classPrefix); } + boolean hasMethodRequirement() { + return methodOwner != null; + } + + boolean requirementsMet(Set classes, Set methods) { + boolean classSeen = false; + for (String cls : classes) { + if (matchesClass(cls)) { + classSeen = true; + break; + } + } + if (!classSeen) { + return false; + } + if (!hasMethodRequirement()) { + return true; + } + for (String method : methods) { + if (method.startsWith(methodOwner + "#" + methodPrefix)) { + return true; + } + } + return false; + } + + Entry requiresMethod(String owner, String methodNamePrefix) { + this.methodOwner = owner; + this.methodPrefix = methodNamePrefix; + return this; + } + Entry iosPod(String pod) { iosPods.add(pod); return this; @@ -323,6 +393,11 @@ Entry iosSpm(String identity, String url, String requirement, String... products return this; } + Entry iosDependenciesUnsupportedOnMacCatalyst() { + iosDependenciesSupportMacCatalyst = false; + return this; + } + Entry iosFrameworks(String... fws) { for (String f : fws) { iosFrameworks.add(f); @@ -381,6 +456,14 @@ public List iosSpmSpecs() { return Collections.unmodifiableList(iosSpm); } + /** + * Whether this entry's CocoaPod/SPM payload has a Mac Catalyst slice. + * System frameworks are not affected by this flag. + */ + public boolean iosDependenciesSupportMacCatalyst() { + return iosDependenciesSupportMacCatalyst; + } + public List iosFrameworks() { return Collections.unmodifiableList(iosFrameworks); } diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AiDependencyTableTest.java b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java similarity index 59% rename from maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AiDependencyTableTest.java rename to maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java index f3d291de18b..f3b60f82584 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AiDependencyTableTest.java +++ b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java @@ -20,7 +20,7 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -package com.codename1.builders; +package com.codename1.build.shared; import org.junit.jupiter.api.Test; @@ -31,27 +31,41 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -class AiDependencyTableTest { +class PlatformFeatureCatalogTest { @Test - void mlkitTextRecognizerMapsToPodAndGradleDep() { - List hits = AiDependencyTable.matchesFor( - "com/codename1/ai/mlkit/text/TextRecognizer"); + void builtInTextRecognizerMapsToAppleVisionAndAndroidMlKit() { + List hits = PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/vision/TextRecognizer"); assertEquals(1, hits.size(), "expected one entry to fire"); - AiDependencyTable.Entry e = hits.get(0); - assertTrue(e.iosPods().contains("GoogleMLKit/TextRecognition")); + PlatformFeatureCatalog.Entry e = hits.get(0); + assertTrue(e.iosFrameworks().contains("Vision")); + assertTrue(e.iosPods().isEmpty()); assertTrue(e.androidGradleDeps().get(0).startsWith("com.google.mlkit:text-recognition")); - // Camera plist string is injected because text recognition - // is virtually always used with the camera. - assertNotNull(findPlistDefault(e, "NSCameraUsageDescription")); + assertTrue(e.iosPlistEntries().isEmpty(), + "Still-image analysis must not imply camera permission"); + } + + @Test + void explicitMlKitBackendAddsOnlyUsedFeaturePod() { + PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); + acc.consume("com/codename1/ai/vision/TextRecognizer"); + acc.consumeMethod("com/codename1/ai/vision/VisionBackends", "mlKit"); + boolean foundTextPod = false; + for (PlatformFeatureCatalog.Entry e : acc.hits()) { + foundTextPod |= e.iosPods().contains("GoogleMLKit/TextRecognition"); + assertFalse(e.iosPods().contains("GoogleMLKit/FaceDetection"), + "Unused vision features must not be bundled"); + } + assertTrue(foundTextPod); } @Test void speechRecognizerInjectsMicAndSpeechPlist() { - List hits = AiDependencyTable.matchesFor( + List hits = PlatformFeatureCatalog.matchesFor( "com/codename1/media/SpeechRecognizer"); assertEquals(1, hits.size()); - AiDependencyTable.Entry e = hits.get(0); + PlatformFeatureCatalog.Entry e = hits.get(0); assertTrue(e.iosFrameworks().contains("Speech")); assertNotNull(findPlistDefault(e, "NSMicrophoneUsageDescription")); assertNotNull(findPlistDefault(e, "NSSpeechRecognitionUsageDescription")); @@ -60,10 +74,10 @@ void speechRecognizerInjectsMicAndSpeechPlist() { @Test void textToSpeechInjectsNoPermissions() { - List hits = AiDependencyTable.matchesFor( + List hits = PlatformFeatureCatalog.matchesFor( "com/codename1/media/TextToSpeech"); assertEquals(1, hits.size()); - AiDependencyTable.Entry e = hits.get(0); + PlatformFeatureCatalog.Entry e = hits.get(0); assertTrue(e.iosFrameworks().contains("AVFAudio")); assertTrue(e.androidPermissions().isEmpty(), "TTS is built-in on every supported OS -- no permission needed"); @@ -77,10 +91,10 @@ void llmClientNeedsNothingExtra() { // means no plist string, no extra permission. They still // register so future diagnostics ("which AI APIs does this // app use?") can enumerate them. - List hits = AiDependencyTable.matchesFor( + List hits = PlatformFeatureCatalog.matchesFor( "com/codename1/ai/LlmClient"); assertEquals(1, hits.size()); - AiDependencyTable.Entry e = hits.get(0); + PlatformFeatureCatalog.Entry e = hits.get(0); assertTrue(e.iosPods().isEmpty()); assertTrue(e.androidGradleDeps().isEmpty()); assertTrue(e.androidPermissions().isEmpty()); @@ -88,17 +102,17 @@ void llmClientNeedsNothingExtra() { @Test void stableDiffusionFlagsBigUpload() { - AiDependencyTable.Accumulator acc = new AiDependencyTable.Accumulator(); + PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); acc.consume("com/codename1/ai/imagegen/StableDiffusion"); assertTrue(acc.anyRequiresBigUpload(), "On-device SD ships a 1-2 GB Core ML model -- cloud builds must abort with a friendly message"); } @Test - void mlkitDoesNotFlagBigUpload() { - AiDependencyTable.Accumulator acc = new AiDependencyTable.Accumulator(); - acc.consume("com/codename1/ai/mlkit/text/TextRecognizer"); - acc.consume("com/codename1/ai/mlkit/barcode/BarcodeScanner"); + void builtInVisionDoesNotFlagBigUpload() { + PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); + acc.consume("com/codename1/ai/vision/TextRecognizer"); + acc.consume("com/codename1/ai/vision/BarcodeScanner"); acc.consume("com/codename1/ai/whisper/WhisperRecognizer"); assertFalse(acc.anyRequiresBigUpload(), "ML Kit models stream lazily, Whisper bundles a small static lib -- neither exceeds the 2 GB cap"); @@ -109,9 +123,9 @@ void unrelatedClassesProduceNoHits() { // Sanity: we mustn't false-positive on classes outside the // AI namespace, because the scanner walks every class in // the user's app. - assertTrue(AiDependencyTable.matchesFor("com/codename1/ui/Form").isEmpty()); - assertTrue(AiDependencyTable.matchesFor("java/lang/Object").isEmpty()); - assertTrue(AiDependencyTable.matchesFor(null).isEmpty()); + assertTrue(PlatformFeatureCatalog.matchesFor("com/codename1/ui/Form").isEmpty()); + assertTrue(PlatformFeatureCatalog.matchesFor("java/lang/Object").isEmpty()); + assertTrue(PlatformFeatureCatalog.matchesFor(null).isEmpty()); } @Test @@ -120,10 +134,10 @@ void cameraEntryInjectsAvFoundationAndCameraXGradleDeps() { // the iOS frameworks, iOS plist usage descriptions, Android // permissions, and the four CameraX Gradle dependencies that the // AndroidCameraImpl reflection layer resolves at runtime. - List hits = AiDependencyTable.matchesFor( + List hits = PlatformFeatureCatalog.matchesFor( "com/codename1/camera/Camera"); assertEquals(1, hits.size(), "expected the camera entry to fire"); - AiDependencyTable.Entry e = hits.get(0); + PlatformFeatureCatalog.Entry e = hits.get(0); // iOS side assertTrue(e.iosFrameworks().contains("AVFoundation")); @@ -159,24 +173,71 @@ void cameraEntryInjectsAvFoundationAndCameraXGradleDeps() { void cameraEntryFiresOnAnySubpackageClass() { // The prefix matcher must hit any class inside com.codename1.camera, // not just the entry point. - assertEquals(1, AiDependencyTable.matchesFor("com/codename1/camera/CameraView").size()); - assertEquals(1, AiDependencyTable.matchesFor("com/codename1/camera/CameraSession").size()); - assertEquals(1, AiDependencyTable.matchesFor("com/codename1/camera/internal/Foo").size()); + assertEquals(1, PlatformFeatureCatalog.matchesFor("com/codename1/camera/CameraView").size()); + assertEquals(1, PlatformFeatureCatalog.matchesFor("com/codename1/camera/CameraSession").size()); + assertEquals(1, PlatformFeatureCatalog.matchesFor("com/codename1/camera/internal/Foo").size()); + } + + @Test + void inferenceUsesCoreMlEnabledObjectiveCPod() { + // The Objective-C native bridge uses the Core ML-enabled TFLite + // subspec. The Swift package exposes a different surface and must + // not be selected for this implementation. + List hits = PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/inference/InferenceSession"); + assertEquals(1, hits.size()); + PlatformFeatureCatalog.Entry e = hits.get(0); + assertTrue(e.iosPods().contains("TensorFlowLiteObjC/CoreML")); + assertTrue(e.iosSpmSpecs().isEmpty()); + assertFalse(e.iosDependenciesSupportMacCatalyst(), + "The official TensorFlow Lite Objective-C XCFramework has no Catalyst slice"); + } + + @Test + void thirdPartyAppleAiPackagesAreExcludedFromCatalyst() { + PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); + acc.consume("com/codename1/ai/vision/TextRecognizer"); + acc.consumeMethod("com/codename1/ai/vision/VisionBackends", "mlKit"); + acc.consume("com/codename1/ai/language/LanguageIdentifier"); + acc.consume("com/codename1/ai/inference/InferenceSession"); + + boolean foundSystemVision = false; + for (PlatformFeatureCatalog.Entry e : acc.hits()) { + if (e.iosPods().isEmpty()) { + foundSystemVision |= e.iosFrameworks().contains("Vision"); + assertTrue(e.iosDependenciesSupportMacCatalyst(), + "Apple system-framework entries should remain enabled"); + } else { + assertFalse(e.iosDependenciesSupportMacCatalyst(), + e.description() + " must not inject an iOS-only package into Catalyst"); + } + } + assertTrue(foundSystemVision); + } + + @Test + void androidAdaptersReceiveOnlyTheirFeatureDependency() { + List vision = PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/vision/TextRecognizer"); + assertEquals(1, vision.get(0).androidGradleDeps().size()); + assertTrue(vision.get(0).androidGradleDeps().get(0) + .startsWith("com.google.mlkit:text-recognition:")); + + List language = PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/language/LanguageIdentifier"); + assertEquals(1, language.get(0).androidGradleDeps().size()); + assertTrue(language.get(0).androidGradleDeps().get(0) + .startsWith("com.google.mlkit:language-id:")); } @Test - void tfliteHasBothPodAndSpmSpec() { - // TFLite is published as both a CocoaPod and a Swift Package. - // The table records both so projects can route the dep - // through whichever manager they prefer; the IPhoneBuilder - // applies whichever matches the project's current - // ios.dependencyManager setting. - List hits = AiDependencyTable.matchesFor( - "com/codename1/ai/tflite/Interpreter"); + void documentCorrectionDoesNotInjectUnusedAndroidScanner() { + List hits = PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/vision/DocumentScanner"); assertEquals(1, hits.size()); - AiDependencyTable.Entry e = hits.get(0); - assertFalse(e.iosPods().isEmpty(), "expected a CocoaPods spec"); - assertFalse(e.iosSpmSpecs().isEmpty(), "expected an SPM spec"); + assertTrue(hits.get(0).androidGradleDeps().isEmpty(), + "The interactive Google scanner cannot implement a still-image analyzer"); + assertTrue(hits.get(0).iosFrameworks().contains("VisionKit")); } @Test @@ -184,18 +245,18 @@ void accumulatorDeduplicates() { // Same class twice in the same scan shouldn't add the entry // twice -- otherwise we'd inject duplicate Gradle / pod // lines on the wire. - AiDependencyTable.Accumulator acc = new AiDependencyTable.Accumulator(); - acc.consume("com/codename1/ai/mlkit/text/TextRecognizer"); - acc.consume("com/codename1/ai/mlkit/text/OptionsBuilder"); + PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); + acc.consume("com/codename1/ai/vision/TextRecognizer"); + acc.consume("com/codename1/ai/vision/TextRecognizer"); assertEquals(1, acc.hits().size()); } @Test void arApiInjectsArKitCameraAndArCore() { - List hits = AiDependencyTable.matchesFor( + List hits = PlatformFeatureCatalog.matchesFor( "com/codename1/ar/AR"); assertEquals(1, hits.size(), "expected the AR entry to fire"); - AiDependencyTable.Entry e = hits.get(0); + PlatformFeatureCatalog.Entry e = hits.get(0); // iOS: ARKit + SceneKit (linked explicitly by IPhoneBuilder) and the // camera usage string, overridable via ios.NSCameraUsageDescription. assertTrue(e.iosFrameworks().contains("ARKit")); @@ -214,17 +275,17 @@ void arApiInjectsArKitCameraAndArCore() { @Test void arEntryMatchesTheWholePackageButNothingElse() { - assertEquals(1, AiDependencyTable.matchesFor("com/codename1/ar/ARSession").size()); - assertEquals(1, AiDependencyTable.matchesFor("com/codename1/ar/ARNode").size()); + assertEquals(1, PlatformFeatureCatalog.matchesFor("com/codename1/ar/ARSession").size()); + assertEquals(1, PlatformFeatureCatalog.matchesFor("com/codename1/ar/ARNode").size()); // The pure-core VR package must NOT pull the AR native dependencies. - assertTrue(AiDependencyTable.matchesFor("com/codename1/vr/VRView").isEmpty()); + assertTrue(PlatformFeatureCatalog.matchesFor("com/codename1/vr/VRView").isEmpty()); } @Test void nonArEntriesCarryNoMetaData() { // The meta-data field is new; make sure the existing entries did not // accidentally gain one. - for (AiDependencyTable.Entry e : AiDependencyTable.entries()) { + for (PlatformFeatureCatalog.Entry e : PlatformFeatureCatalog.entries()) { if (!e.classPrefix().startsWith("com/codename1/ar/")) { assertTrue(e.androidMetaDataEntries().isEmpty(), e.classPrefix() + " should carry no manifest meta-data"); @@ -242,9 +303,9 @@ void bluetoothEntryFiresForEverySubPackage() { "com/codename1/bluetooth/classic/RfcommConnection" }; for (String cls : classes) { - List hits = AiDependencyTable.matchesFor(cls); + List hits = PlatformFeatureCatalog.matchesFor(cls); assertEquals(1, hits.size(), "expected the bluetooth entry for " + cls); - AiDependencyTable.Entry e = hits.get(0); + PlatformFeatureCatalog.Entry e = hits.get(0); assertNotNull(findPlistDefault(e, "NSBluetoothAlwaysUsageDescription")); assertNotNull(findPlistDefault(e, "NSBluetoothPeripheralUsageDescription")); assertTrue(e.iosFrameworks().contains("CoreBluetooth")); @@ -258,13 +319,13 @@ void bluetoothEntryFiresForEverySubPackage() { @Test void bluetoothEntryDoesNotFireForUnrelatedClasses() { - assertTrue(AiDependencyTable.matchesFor("com/codename1/ui/Form").isEmpty()); + assertTrue(PlatformFeatureCatalog.matchesFor("com/codename1/ui/Form").isEmpty()); // "bluetoothle" cn1lib package must NOT trigger the core entry - assertTrue(AiDependencyTable.matchesFor( + assertTrue(PlatformFeatureCatalog.matchesFor( "com/codename1/bluetoothle/Bluetooth").isEmpty()); } - private static String findPlistDefault(AiDependencyTable.Entry e, String key) { + private static String findPlistDefault(PlatformFeatureCatalog.Entry e, String key) { for (String[] entry : e.iosPlistEntries()) { if (key.equals(entry[0])) { return entry[1]; diff --git a/maven/pom.xml b/maven/pom.xml index 2a683e8df92..86a4dfa3640 100644 --- a/maven/pom.xml +++ b/maven/pom.xml @@ -61,6 +61,7 @@ cn1-binaries + platform-feature-catalog java-runtime core factory @@ -80,17 +81,6 @@ cn1app-archetype cn1lib-archetype cn1-debug-proxy - cn1-ai-mlkit-text - cn1-ai-mlkit-barcode - cn1-ai-mlkit-face - cn1-ai-mlkit-labeling - cn1-ai-mlkit-translate - cn1-ai-mlkit-smartreply - cn1-ai-mlkit-langid - cn1-ai-mlkit-pose - cn1-ai-mlkit-segmentation - cn1-ai-mlkit-docscan - cn1-ai-tflite cn1-ai-whisper cn1-ai-stablediffusion cn1-admob diff --git a/scripts/gen-ai-cn1libs.py b/scripts/gen-ai-cn1libs.py index b10fb73f5f9..83496618f7c 100644 --- a/scripts/gen-ai-cn1libs.py +++ b/scripts/gen-ai-cn1libs.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 """ -Regenerates every AI cn1lib (cn1-ai-*) under maven/ as a proper multi-module -Maven project: root + common + ios + android + javase + lib. Each module's -sources are emitted by this script -- there are NO empty stubs. +Regenerates the deliberately external AI cn1libs listed in ``LIBS``. -Run with `python3 scripts/gen-ai-cn1libs.py` from any working directory. -Existing cn1-ai-* directories are wiped and rewritten. +Small platform AI features (vision, language services, and LiteRT/Core ML +inference) are built into Codename One and selected by the platform builders. +Only features with exceptionally large native runtimes or model payloads, +currently Whisper and Stable Diffusion, remain cn1libs. + +Run with ``python3 scripts/gen-ai-cn1libs.py`` from any working directory. +Only the cn1lib directories listed in ``LIBS`` are wiped and rewritten. """ from __future__ import annotations @@ -2606,17 +2609,6 @@ def lib_stablediffusion() -> Lib: LIBS: list[Lib] = [ - lib_mlkit_text(), - lib_mlkit_barcode(), - lib_mlkit_face(), - lib_mlkit_labeling(), - lib_mlkit_translate(), - lib_mlkit_smartreply(), - lib_mlkit_langid(), - lib_mlkit_pose(), - lib_mlkit_segmentation(), - lib_mlkit_docscan(), - lib_tflite(), lib_whisper(), lib_stablediffusion(), ] diff --git a/scripts/hellocodenameone/README.adoc b/scripts/hellocodenameone/README.adoc index caaea701d8a..c50f15c6bdd 100644 --- a/scripts/hellocodenameone/README.adoc +++ b/scripts/hellocodenameone/README.adoc @@ -124,6 +124,15 @@ When adding a conformance test: . Run the validator before pushing. The `Validate port status contract` workflow enforces the same rules in CI. +The on-device AI rows are permanent assertion tests rather than screenshots. +They validate immutable image, camera-frame, tensor, model-source, and options +contracts on every translated runtime; query every native analyzer or service; +and verify deterministic failed-resource behavior when a port has no backend. +They do not open camera hardware, download language models, or bundle a test +inference model. Registering each concrete entry point in the shared application +also exercises the builders' granular native source and dependency selection on +every source-build job. + CN1SS normalizes each run to `pass`, `fail`, `skip`, or `not-run` per test. A public feature checkbox is checked only when every mapped test passes. Skips, incomplete runs, stale reports, and failures remain distinct states; none is diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java index 30dd761a081..42730804048 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java @@ -366,6 +366,16 @@ private static int testTimeoutMs(BaseTest testClass) { // prompts). Self-skips on iOS / Android / JS where the open // call would surface an OS dialog. new CameraApiTest(), + // Built-in on-device AI coverage. These are assertion-only tests: + // portable value/lifecycle contracts run everywhere, while + // capability queries exercise each port's native bridge without + // requiring a camera, a downloaded language model, or a bundled + // inference model. Referencing the individual entry points also + // keeps the builders' granular dependency selection under the + // permanent cross-platform source-build suite. + new VisionOnDeviceApiTest(), + new LanguageOnDeviceApiTest(), + new InferenceOnDeviceApiTest(), // Exercises com.codename1.ar end-to-end: the unsupported // contract on the CI platforms (none has an AR runtime) and a // full session round trip when a backend is present. diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/InferenceOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/InferenceOnDeviceApiTest.java new file mode 100644 index 00000000000..b97199225e2 --- /dev/null +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/InferenceOnDeviceApiTest.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codenameone.examples.hellocodenameone.tests; + +import com.codename1.ai.inference.InferenceOptions; +import com.codename1.ai.inference.InferenceSession; +import com.codename1.ai.inference.ModelSource; +import com.codename1.ai.inference.Tensor; +import com.codename1.ai.inference.TensorInfo; +import com.codename1.ai.inference.TensorType; +import com.codename1.io.Log; +import com.codename1.util.AsyncResource; + +/** + * Cross-port, non-visual contract coverage for LiteRT model inference. + * + *

The portable tensor and model-source contracts run everywhere. The test + * also queries the native runtime so the builders select it where supported; + * an actual model is deliberately not bundled into the conformance app.

+ */ +public class InferenceOnDeviceApiTest extends BaseTest { + @Override + public boolean shouldTakeScreenshot() { + return false; + } + + @Override + public boolean runTest() { + try { + checkTensorContracts(); + checkModelAndOptions(); + checkCapability(); + done(); + return true; + } catch (Throwable t) { + fail("On-device inference API test failed: " + t); + return false; + } + } + + private void checkTensorContracts() { + int[] shape = new int[] {1, 2}; + float[] values = new float[] {1f, 2f}; + Tensor tensor = Tensor.floats("input", shape, values); + shape[1] = 9; + values[0] = 9f; + checkEqual(2, tensor.getShape()[1], + "Tensor must copy the input shape"); + checkEqual(1f, ((float[]) tensor.getData())[0], + "Tensor must copy the input data"); + int[] outputShape = tensor.getShape(); + float[] outputData = (float[]) tensor.getData(); + outputShape[1] = 8; + outputData[1] = 8f; + checkEqual(2, tensor.getShape()[1], + "Tensor must copy the output shape"); + checkEqual(2f, ((float[]) tensor.getData())[1], + "Tensor must copy the output data"); + check(tensor.getType() == TensorType.FLOAT32, + "Tensor FLOAT32 type"); + + TensorInfo info = new TensorInfo( + "output", TensorType.INT32, new int[] {1, 3}, 2); + int[] infoShape = info.getShape(); + infoShape[1] = 7; + checkEqual(3, info.getShape()[1], + "TensorInfo must copy the output shape"); + checkEqual(2, info.getIndex(), "TensorInfo index"); + + try { + Tensor.floats("bad", new int[] {3}, new float[] {1f, 2f}); + throw new IllegalStateException( + "Tensor accepted data that did not match its shape"); + } catch (IllegalArgumentException expected) { + // Shape/data validation is part of the public contract. + } + } + + private void checkModelAndOptions() { + byte[] bytes = new byte[] {1, 2, 3}; + ModelSource source = ModelSource.bytes(bytes); + bytes[0] = 9; + checkEqual(1, source.getBytes()[0], + "ModelSource must copy input bytes"); + byte[] returned = source.getBytes(); + returned[1] = 9; + checkEqual(2, source.getBytes()[1], + "ModelSource must copy output bytes"); + checkEqual(ModelSource.BYTES, source.getKind(), + "ModelSource byte kind"); + check("model.tflite".equals( + ModelSource.resource("model.tflite").getPath()), + "ModelSource resource path"); + + InferenceOptions options = new InferenceOptions() + .accelerator(InferenceOptions.Accelerator.CORE_ML) + .threads(-1) + .allowFallback(false); + check(options.getAccelerator() + == InferenceOptions.Accelerator.CORE_ML, + "inference accelerator"); + checkEqual(0, options.getThreads(), "thread lower clamp"); + check(!options.isFallbackAllowed(), "inference fallback option"); + } + + private void checkCapability() { + boolean supported = InferenceSession.isSupported(); + Log.p("InferenceOnDeviceApiTest: supported=" + supported); + if (!supported) { + AsyncResource result = InferenceSession.open( + ModelSource.bytes(new byte[] {1}), null); + check(result.isDone(), + "unsupported inference must complete immediately"); + try { + result.get(); + throw new IllegalStateException( + "unsupported inference unexpectedly opened a session"); + } catch (AsyncResource.AsyncExecutionException expected) { + // The documented unsupported-resource contract. + } + } + } + + private void check(boolean value, String label) { + if (!value) { + throw new IllegalStateException(label); + } + } + + private void checkEqual(int expected, int actual, String label) { + if (expected != actual) { + throw new IllegalStateException(label + ": expected " + + expected + " got " + actual); + } + } + + private void checkEqual(float expected, float actual, String label) { + if (Math.abs(expected - actual) > 0.0001f) { + throw new IllegalStateException(label + ": expected " + + expected + " got " + actual); + } + } +} diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java new file mode 100644 index 00000000000..143dd40b05d --- /dev/null +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codenameone.examples.hellocodenameone.tests; + +import com.codename1.ai.language.LanguageBackends; +import com.codename1.ai.language.LanguageCandidate; +import com.codename1.ai.language.LanguageIdentifier; +import com.codename1.ai.language.LanguageOptions; +import com.codename1.ai.language.SmartReply; +import com.codename1.ai.language.SmartReplyMessage; +import com.codename1.ai.language.Translator; +import com.codename1.io.Log; +import com.codename1.util.AsyncResource; + +/** + * Cross-port, non-visual contract coverage for on-device language services. + * + *

Capability checks are safe on every port and make the three independent + * services visible to the dependency scanner. Unsupported ports additionally + * exercise their immediate failed-resource fallback. Supported ports do not + * download mutable language models during the unattended suite.

+ */ +public class LanguageOnDeviceApiTest extends BaseTest { + @Override + public boolean shouldTakeScreenshot() { + return false; + } + + @Override + public boolean runTest() { + try { + checkValuesAndOptions(); + checkCapabilities(); + done(); + return true; + } catch (Throwable t) { + fail("On-device language API test failed: " + t); + return false; + } + } + + private void checkValuesAndOptions() { + LanguageOptions options = new LanguageOptions() + .minimumConfidence(2f); + checkEqual(1f, options.getMinimumConfidence(), + "confidence upper clamp"); + check("auto".equals(options.getBackend().getId()), + "default language backend"); + options.backend(null).minimumConfidence(-1f); + check(options.getBackend() == LanguageBackends.auto(), + "null language backend must restore auto"); + checkEqual(0f, options.getMinimumConfidence(), + "confidence lower clamp"); + + LanguageCandidate candidate = new LanguageCandidate("fr", .75f); + check("fr".equals(candidate.getLanguageTag()), + "language candidate tag"); + checkEqual(.75f, candidate.getConfidence(), + "language candidate confidence"); + + SmartReplyMessage message = + new SmartReplyMessage(null, "remote", false, 42L); + check("".equals(message.getText()), + "null smart-reply text must normalize to empty"); + check("remote".equals(message.getParticipantId()), + "smart-reply participant"); + check(!message.isLocalUser(), "smart-reply remote participant"); + checkEqual(42L, message.getTimestampMillis(), + "smart-reply timestamp"); + } + + private void checkCapabilities() { + boolean languageId = LanguageIdentifier.isSupported(); + boolean translation = Translator.isSupported(); + boolean smartReply = SmartReply.isSupported(); + Log.p("LanguageOnDeviceApiTest: languageId=" + languageId + + " translation=" + translation + + " smartReply=" + smartReply); + + if (!languageId) { + assertImmediateFailure(LanguageIdentifier.identify("hello", null), + "language identification"); + } + if (!translation) { + assertImmediateFailure(Translator.translate( + "bonjour", "fr", "en", null), "translation"); + } + if (!smartReply) { + assertImmediateFailure(SmartReply.suggest( + new SmartReplyMessage[] { + new SmartReplyMessage("Hello", "remote", false, 1) + }, null), "smart reply"); + } + } + + private void assertImmediateFailure(AsyncResource resource, + String label) { + check(resource.isDone(), label + + " unsupported fallback must complete immediately"); + try { + resource.get(); + throw new IllegalStateException(label + + " unsupported fallback unexpectedly succeeded"); + } catch (AsyncResource.AsyncExecutionException expected) { + // The documented unsupported-resource contract. + } + } + + private void check(boolean value, String label) { + if (!value) { + throw new IllegalStateException(label); + } + } + + private void checkEqual(long expected, long actual, String label) { + if (expected != actual) { + throw new IllegalStateException(label + ": expected " + + expected + " got " + actual); + } + } + + private void checkEqual(float expected, float actual, String label) { + if (Math.abs(expected - actual) > 0.0001f) { + throw new IllegalStateException(label + ": expected " + + expected + " got " + actual); + } + } +} diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java new file mode 100644 index 00000000000..69ea148c2e9 --- /dev/null +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codenameone.examples.hellocodenameone.tests; + +import com.codename1.ai.vision.BarcodeScanner; +import com.codename1.ai.vision.DocumentScanner; +import com.codename1.ai.vision.FaceDetector; +import com.codename1.ai.vision.ImageLabeler; +import com.codename1.ai.vision.PoseDetector; +import com.codename1.ai.vision.SelfieSegmenter; +import com.codename1.ai.vision.TextRecognizer; +import com.codename1.ai.vision.VisionAnalyzer; +import com.codename1.ai.vision.VisionBackends; +import com.codename1.ai.vision.VisionImage; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.camera.CameraFrame; +import com.codename1.camera.FrameFormat; +import com.codename1.io.Log; + +/** + * Cross-port, non-visual contract coverage for the built-in vision API. + * + *

The test intentionally avoids running an analyzer against camera hardware + * or a vendor model. It verifies the portable image/camera-frame ownership + * contract and asks every native analyzer for its capability. This keeps the + * suite deterministic while making every analyzer visible to the builders' + * granular dependency scanner.

+ */ +public class VisionOnDeviceApiTest extends BaseTest { + @Override + public boolean shouldTakeScreenshot() { + return false; + } + + @Override + public boolean runTest() { + try { + checkImageOwnership(); + checkOptions(); + checkAnalyzerCapabilitiesAndLifecycle(); + done(); + return true; + } catch (Throwable t) { + fail("On-device vision API test failed: " + t); + return false; + } + } + + private void checkImageOwnership() { + byte[] encodedSource = new byte[] {1, 2, 3}; + VisionImage encoded = VisionImage.encoded(encodedSource); + encodedSource[0] = 9; + checkEqual(1, encoded.getEncodedBytes()[0], + "VisionImage must copy encoded input"); + byte[] encodedOutput = encoded.getEncodedBytes(); + encodedOutput[1] = 9; + checkEqual(2, encoded.getEncodedBytes()[1], + "VisionImage must copy encoded output"); + + byte[] jpeg = new byte[] {4, 5}; + byte[] pixels = new byte[] {10, 20, 30, 40}; + CameraFrame frame = new CameraFrame(jpeg, pixels, 1, 1, -90, + 123456789L, FrameFormat.RGBA8888); + VisionImage cameraImage = VisionImage.fromCameraFrame(frame); + jpeg[0] = 99; + pixels[0] = 99; + checkEqual(4, cameraImage.getEncodedBytes()[0], + "VisionImage must own camera JPEG bytes"); + checkEqual(10, cameraImage.getPixels()[0], + "VisionImage must own camera pixel bytes"); + checkEqual(1, cameraImage.getWidth(), "camera image width"); + checkEqual(1, cameraImage.getHeight(), "camera image height"); + checkEqual(270, cameraImage.getRotationDegrees(), + "camera image rotation normalization"); + checkEqual(123456789L, cameraImage.getTimestampNanos(), + "camera image timestamp"); + check(cameraImage.getFormat() == FrameFormat.RGBA8888, + "camera image format"); + } + + private void checkOptions() { + VisionOptions options = new VisionOptions() + .backend(VisionBackends.appleVision()) + .minimumConfidence(2f) + .maximumResults(-1); + check("apple-vision".equals(options.getBackend().getId()), + "explicit Apple Vision backend id"); + checkEqual(1f, options.getMinimumConfidence(), + "confidence upper clamp"); + checkEqual(0, options.getMaximumResults(), + "maximum result lower clamp"); + options.backend(null).minimumConfidence(-1f); + check("auto".equals(options.getBackend().getId()), + "null backend must restore auto"); + checkEqual(0f, options.getMinimumConfidence(), + "confidence lower clamp"); + } + + private void checkAnalyzerCapabilitiesAndLifecycle() { + VisionAnalyzer[] analyzers = new VisionAnalyzer[] { + new TextRecognizer(), + new BarcodeScanner(), + new FaceDetector(), + new ImageLabeler(), + new PoseDetector(), + new SelfieSegmenter(), + new DocumentScanner() + }; + String[] names = new String[] { + "text", "barcode", "face", "label", "pose", "segmentation", + "document" + }; + VisionImage input = VisionImage.encoded(new byte[] {1}); + for (int i = 0; i < analyzers.length; i++) { + VisionAnalyzer analyzer = analyzers[i]; + boolean supported = analyzer.isSupported(); + Log.p("VisionOnDeviceApiTest: " + names[i] + + " supported=" + supported); + analyzer.close(); + check(!analyzer.isSupported(), + names[i] + " analyzer must report unsupported after close"); + boolean rejected = false; + try { + analyzer.process(input); + } catch (IllegalStateException expected) { + rejected = true; + } + check(rejected, names[i] + " analyzer accepted input after close"); + } + } + + private void check(boolean value, String label) { + if (!value) { + throw new IllegalStateException(label); + } + } + + private void checkEqual(int expected, int actual, String label) { + if (expected != actual) { + throw new IllegalStateException(label + ": expected " + + expected + " got " + actual); + } + } + + private void checkEqual(long expected, long actual, String label) { + if (expected != actual) { + throw new IllegalStateException(label + ": expected " + + expected + " got " + actual); + } + } + + private void checkEqual(float expected, float actual, String label) { + if (Math.abs(expected - actual) > 0.0001f) { + throw new IllegalStateException(label + ": expected " + + expected + " got " + actual); + } + } +} diff --git a/scripts/hellocodenameone/conformance/port_status.py b/scripts/hellocodenameone/conformance/port_status.py index 20f34f9f748..8dc15f6abcf 100755 --- a/scripts/hellocodenameone/conformance/port_status.py +++ b/scripts/hellocodenameone/conformance/port_status.py @@ -211,6 +211,19 @@ def validate(manifest: dict) -> dict: problems.append(f"Stored report {report_path} has an invalid result for {test}") elif result.get("status") == "skip": skipped_tests.add(test) + actual_summary = Counter( + result.get("status") + for result in report_tests.values() + if isinstance(result, dict) + ) + expected_summary = { + key: actual_summary.get(key, 0) + for key in ("pass", "fail", "skip", "not-run") + } + if report.get("summary") != expected_summary: + problems.append( + f"Stored report {report_path} summary does not match its test results" + ) manual_feature_count = 0 try: diff --git a/scripts/hellocodenameone/conformance/test_port_status.py b/scripts/hellocodenameone/conformance/test_port_status.py index 9b788e906c5..1c0d0cc1cb6 100755 --- a/scripts/hellocodenameone/conformance/test_port_status.py +++ b/scripts/hellocodenameone/conformance/test_port_status.py @@ -16,9 +16,9 @@ def setUpClass(cls): def test_contract_covers_registered_tests_and_goldens(self): counts = port_status.validate(self.manifest) - self.assertEqual(167, counts["tests"]) + self.assertEqual(170, counts["tests"]) self.assertEqual(1, counts["performance_tests"]) - self.assertGreaterEqual(counts["features"], 51) + self.assertGreaterEqual(counts["features"], 54) self.assertEqual(11, counts["ports"]) self.assertEqual(20, counts["manual_features"]) self.assertEqual(8, counts["deployment_platforms"]) @@ -27,6 +27,9 @@ def test_contract_covers_registered_tests_and_goldens(self): features = {feature["id"]: feature["tests"] for feature in self.manifest["features"]} self.assertEqual(["ARApiTest", "MotionSensorDeviceTest"], features["ar-motion-sensors"]) self.assertEqual(["CameraApiTest"], features["camera-access"]) + self.assertEqual(["VisionOnDeviceApiTest"], features["on-device-vision"]) + self.assertEqual(["LanguageOnDeviceApiTest"], features["on-device-language"]) + self.assertEqual(["InferenceOnDeviceApiTest"], features["on-device-inference"]) self.assertEqual(["CalendarApiTest"], features["calendar-integration"]) self.assertEqual(["VideoIODecodedFramesScreenshotTest"], features["video-decoding"]) self.assertEqual(["VideoIORoundTripTest"], features["video-round-trip"]) @@ -183,6 +186,34 @@ def test_cli_strict_gate_writes_report_before_returning_failure(self): self.assertFalse(report["suite_finished"]) self.assertGreater(report["summary"]["not-run"], 0) + def test_validate_rejects_inconsistent_stored_report_summary(self): + original_directory = self.manifest["report_directory"] + with tempfile.TemporaryDirectory(dir=port_status.REPO_ROOT) as tmp: + report_root = Path(tmp) + for port in self.manifest["ports"]: + source = port_status.REPO_ROOT / original_directory / ( + port["id"] + ".json" + ) + (report_root / source.name).write_text( + source.read_text(encoding="utf-8"), encoding="utf-8" + ) + android_path = report_root / "android.json" + android = port_status.read_json(android_path) + android["summary"]["not-run"] += 1 + android_path.write_text( + json.dumps(android, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + manifest = dict(self.manifest) + manifest["report_directory"] = str( + report_root.relative_to(port_status.REPO_ROOT) + ) + with self.assertRaisesRegex( + port_status.ContractError, + "summary does not match its test results", + ): + port_status.validate(manifest) + if __name__ == "__main__": unittest.main() diff --git a/scripts/initializr/common/src/main/resources/skill/SKILL.md b/scripts/initializr/common/src/main/resources/skill/SKILL.md index 1beefb158d7..e3d88dbd20b 100644 --- a/scripts/initializr/common/src/main/resources/skill/SKILL.md +++ b/scripts/initializr/common/src/main/resources/skill/SKILL.md @@ -37,7 +37,7 @@ This skill teaches you how to write code for a Codename One (CN1) cross-platform - `references/mobile-adaptability.md` — Density-independent units (mm), `convertToPixels`, `LayeredLayout` for responsive design, `Display.isTablet()`, font scaling. - `references/native-interfaces.md` — Authoring native interfaces for iOS/Android/JavaScript/Desktop with `cn1:generate-native-interfaces` and platform callbacks. - `references/cn1libs.md` — Creating, packaging, and consuming Codename One libraries (Maven and legacy `.cn1lib`). -- `references/ai-and-speech.md` — LLM client (`com.codename1.ai`), `ChatView`, `SpeechRecognizer`, `TextToSpeech`, non-prompting `SecureStorage` overloads, the ML Kit cn1libs, and the simulator's offline Ollama redirect. Read this when the user asks for chat, voice, embeddings, image generation, barcode/document/face detection, or wants to store an LLM API key. +- `references/ai-and-speech.md` — LLM client (`com.codename1.ai`), built-in vision/LiteRT/language APIs, `ChatView`, `SpeechRecognizer`, `TextToSpeech`, non-prompting `SecureStorage` overloads, and the simulator's offline Ollama redirect. Read this when the user asks for chat, voice, embeddings, image generation, barcode/document/face detection, or wants to store an LLM API key. - `references/printing.md` — Cross-platform printing (`com.codename1.printing`): `Printer.printPDF` / `printImage` / `print`, the `PrintResult` outcome, and per-platform caveats (iOS AirPrint, Android, desktop, native Windows, web). Read this when the user wants to print a document, report, image, or the current screen. - `references/games.md` — Game development (`com.codename1.gaming`): the `GameView` update loop, `Sprite` / `AnimatedSprite` / `SpriteSheet`, pollable `GameInput`, `TouchControls`, `SoundPool`, and 2D `com.codename1.gaming.physics` (Box2D). Read this for arcade/casual/scroller/board games or any real-time animated canvas. - `references/3d-graphics.md` — Portable GPU 3D (`com.codename1.gpu`): the `RenderView` + `Renderer` loop, declarative `Material` / `VertexFormat` (engine-generated shaders — no GLSL), `Primitives`, `GltfLoader` for glTF models, `Camera` / `Light` / `Matrix4`, and platform backends. Read this for product viewers, 3D scenes, or custom GPU rendering. diff --git a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md index 330d62b3b55..a2fbf0ba02b 100644 --- a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md +++ b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md @@ -1,6 +1,6 @@ # AI, Chat UI, and Speech Reference -Codename One ships a portable LLM client, a streaming chat component, speech recognition, text-to-speech, and ML Kit cn1lib bridges. All of it sits in the cross-platform `common/` module — the same call site runs on iOS, Android, JavaSE, and (where the backend supports it) JavaScript. +Codename One ships a portable LLM client, a streaming chat component, speech recognition, text-to-speech, and built-in on-device vision, language, and LiteRT inference APIs. **Read this reference when** the user asks to integrate an LLM, build a chat UI, voice input, voice output, image generation, embeddings, on-device barcode/face/document scanning, or wants to store an API key. @@ -13,11 +13,11 @@ Codename One ships a portable LLM client, a streaming chat component, speech rec | Speech-to-text | `com.codename1.media.SpeechRecognizer` | core (built-in) | | Text-to-speech | `com.codename1.media.TextToSpeech` | core (built-in) | | Silent secret storage (LLM API keys, etc.) | Single-arg overloads on `com.codename1.security.SecureStorage` | core (built-in) | -| Barcode scanning | `com.codename1.ai.mlkit.barcode.BarcodeScanner` | cn1lib `cn1-ai-mlkit-barcode` | -| Document scanning | `com.codename1.ai.mlkit.docscan.DocumentScanner` | cn1lib `cn1-ai-mlkit-docscan` | -| Face detection | `com.codename1.ai.mlkit.face.FaceDetector` | cn1lib `cn1-ai-mlkit-face` | +| Vision analysis | `com.codename1.ai.vision.*` | core (built-in) | +| LiteRT inference | `com.codename1.ai.inference.*` | core (built-in) | +| Language services | `com.codename1.ai.language.*` | core (built-in) | -The build-time scanner in the Codename One Maven plugin (`AiDependencyTable`) picks up references to any `com.codename1.ai.*` or `com.codename1.media.{Speech,Tts}*` class and automatically wires Pods (iOS), Swift Packages (iOS SPM), Gradle dependencies (Android), `Info.plist` usage strings, and Android permissions. You don't edit `codenameone_settings.properties` build hints for these classes. +The build-time scanner uses `PlatformFeatureCatalog` to add the required Apple frameworks, CocoaPods, Android dependencies, and permissions. Applications don't add AI cn1libs or build hints. ## LlmClient — chat, embeddings, image generation @@ -307,61 +307,46 @@ LlmClient client = LlmClient.openai(key); Base class returns `null` / `false` on platforms without an implementation, so you can wire this in without a platform check. -## ML Kit cn1libs - -Three cn1libs, each backed by ML Kit on iOS and Android. Drop the `pom` dependency in `common/pom.xml` and the build-time scanner takes care of Pods, Gradle deps, permissions, and usage strings. - -```xml - - com.codenameone - cn1-ai-mlkit-barcode-lib - ${cn1.version} - pom - -``` - -### Barcode scanner - -```java -import com.codename1.ai.mlkit.barcode.BarcodeScanner; - -Capture.capturePhoto(evt -> { - String path = (String) evt.getSource(); - byte[] bytes = readAllBytes(path); - BarcodeScanner.scan(bytes).ready(values -> { - for (String v : values) Log.p("Detected: " + v); - }); -}); -``` - -Decodes QR, EAN, Code 128, and the other ML Kit-supported formats. - -### Document scanner +## Built-in on-device ML ```java -import com.codename1.ai.mlkit.docscan.DocumentScanner; - -DocumentScanner.scanToFile(jpegBytes).ready(filePath -> { - // filePath points to the cropped, corrected document image -}); -``` +new TextRecognizer() + .process(VisionImage.encoded(jpegBytes)) + .ready(result -> Log.p(result.getText())); -iOS uses `VisionKit`. Android uses the Google Play Services document scanner — on devices without Play Services, the call resolves through `AsyncResource.except(...)`. +InferenceSession.open( + ModelSource.resource("/model.tflite"), + new InferenceOptions()) + .ready(session -> session.run(new Tensor[] {input})); -### Face detector - -```java -import com.codename1.ai.mlkit.face.FaceDetector; - -FaceDetector.detect(jpegBytes).ready(rects -> { - for (int i = 0; i < rects.length; i += 4) { - int x = rects[i], y = rects[i + 1], w = rects[i + 2], h = rects[i + 3]; - // draw rectangle, crop, etc. - } -}); +Translator.translate("Bonjour", "fr", "en", new LanguageOptions()) + .ready(result -> Log.p(result)); ``` -Returns a packed `int[]` — four ints per face. +`VisionImage` accepts JPEG, PNG, NV21, and RGBA8888. Use +`VisionPipeline` to connect a reusable analyzer to camera frames; it +copies callback-owned data and drops stale pending frames under load. +Use `ModelCache.fetch(httpsUrl, cacheKey, sha256)` for models too large +to bundle. iOS and Mac native default to Apple Vision; iOS also supports +explicit ML Kit selection. Android uses ML Kit. + +Dependency selection is per entry point. Referencing `TextRecognizer` +retains only the Android text adapter/artifact; it does not pull +barcode, face, labeling, pose, or segmentation. `LanguageIdentifier`, +`Translator`, and `SmartReply` likewise select independent artifacts. +On iOS, an ML Kit vision pod is selected only when both its analyzer +and `VisionBackends.mlKit()` are referenced. LiteRT is selected only by +`InferenceSession`. + +Call `isSupported()` before exposing a feature. Vision has native backends +on Android, iOS, and Mac native. Language services and LiteRT inference +have native backends on Android and iOS. JavaSE, JavaScript, native +Windows/Linux, watchOS, and tvOS return unsupported; Mac native returns +unsupported for language and LiteRT. An opt-in runtime can add another +backend, and the built-in fallback never uploads input to a cloud service. + +Do not add the retired `cn1-ai-mlkit-*` or `cn1-ai-tflite` dependencies. +There are no compatibility aliases for `com.codename1.ai.mlkit.*`. ## Common patterns @@ -419,5 +404,5 @@ view.setOnVoice(e -> SpeechRecognizer.recognize( - **Don't store an API key in source.** Use `SecureStorage.get("openai.key")` (single-arg overload) or pull it from a server-side proxy. Hard-coded keys leak through reverse-engineered binaries. - **Don't call `chatStream` from a tight UI loop.** A streaming call holds an HTTP connection until the response completes; one per user turn is correct, one per keystroke is a bug. - **Don't mutate `ChatView` on a non-EDT thread without going through the documented mutators.** `addMessage`, `appendToLastMessage`, and `setTypingIndicatorVisible` are thread-safe; arbitrary `view.add(...)` calls are not. -- **Don't assume the document scanner works on every Android device.** It requires Google Play Services. Wrap the call and fall back to `Capture.capturePhoto(...)` if the scanner returns an error. +- **Don't assume `DocumentScanner` is available on Android.** The built-in API corrects an existing image, while Google's Android scanner is an interactive camera flow. Check `isSupported()` and use your capture flow when it is false. - **Don't ship a project that bundles `cn1-ai-stablediffusion` to the cloud build server without checking.** The cn1lib carries multi-GB native blobs; the build will reject the upload with a `cn1.ai.requiresBigUpload` hint. Build locally for those projects. diff --git a/scripts/settings/common/src/main/java/com/codename1/settings/extensions/MavenCentralSearch.java b/scripts/settings/common/src/main/java/com/codename1/settings/extensions/MavenCentralSearch.java index 03b9ce67866..6cf22b2f905 100644 --- a/scripts/settings/common/src/main/java/com/codename1/settings/extensions/MavenCentralSearch.java +++ b/scripts/settings/common/src/main/java/com/codename1/settings/extensions/MavenCentralSearch.java @@ -20,14 +20,6 @@ public static List curated() { ArrayList out = new ArrayList(); out.add(maven("Google Maps", "Native Google Maps integration for Codename One apps.", "com.codenameone", "googlemaps-lib", "1.0.1", "pom")); - out.add(maven("ML Kit Barcode", "Decode QR, EAN, Code 128, and other ML Kit barcode formats.", - "com.codenameone", "cn1-ai-mlkit-barcode-lib", "LATEST", "pom")); - out.add(maven("ML Kit Document Scanner", "Capture and crop document photos with native VisionKit and Google Play Services support.", - "com.codenameone", "cn1-ai-mlkit-docscan-lib", "LATEST", "pom")); - out.add(maven("ML Kit Face Detection", "Detect faces and bounding rectangles using ML Kit-backed native providers.", - "com.codenameone", "cn1-ai-mlkit-face-lib", "LATEST", "pom")); - out.add(maven("TensorFlow Lite", "TensorFlow Lite inference bridge packaged as a Codename One cn1lib.", - "com.codenameone", "cn1-ai-tflite-lib", "LATEST", "pom")); out.add(maven("Whisper", "Speech-to-text support through the Codename One AI Whisper cn1lib.", "com.codenameone", "cn1-ai-whisper-lib", "LATEST", "pom")); out.add(legacy("Bouncy Castle SDK", "Legacy cn1lib catalog entry for Bouncy Castle cryptography support.")); From 33b0861a4b031d86a7bfa117637b2657096307c1 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 25 Jul 2026 14:46:27 +0300 Subject: [PATCH 02/80] Fix AI source and guide compliance --- .../ai/inference/InferenceException.java | 20 ++++- .../ai/inference/InferenceOptions.java | 20 ++++- .../ai/inference/InferenceSession.java | 20 ++++- .../codename1/ai/inference/ModelCache.java | 40 ++++++--- .../codename1/ai/inference/ModelSource.java | 20 ++++- .../com/codename1/ai/inference/Tensor.java | 20 ++++- .../codename1/ai/inference/TensorInfo.java | 20 ++++- .../codename1/ai/inference/TensorType.java | 20 ++++- .../codename1/ai/inference/package-info.java | 32 +++++-- .../ai/language/LanguageBackend.java | 20 ++++- .../ai/language/LanguageBackends.java | 20 ++++- .../ai/language/LanguageCandidate.java | 20 ++++- .../ai/language/LanguageIdentifier.java | 20 ++++- .../ai/language/LanguageOptions.java | 20 ++++- .../com/codename1/ai/language/SmartReply.java | 20 ++++- .../ai/language/SmartReplyMessage.java | 20 ++++- .../com/codename1/ai/language/Translator.java | 20 ++++- .../codename1/ai/language/package-info.java | 36 ++++++-- .../ai/vision/AbstractVisionAnalyzer.java | 18 ++++ .../src/com/codename1/ai/vision/Barcode.java | 20 ++++- .../codename1/ai/vision/BarcodeScanner.java | 20 ++++- .../ai/vision/DocumentScanResult.java | 20 ++++- .../codename1/ai/vision/DocumentScanner.java | 20 ++++- .../src/com/codename1/ai/vision/Face.java | 30 +++++-- .../com/codename1/ai/vision/FaceDetector.java | 20 ++++- .../com/codename1/ai/vision/ImageLabel.java | 20 ++++- .../com/codename1/ai/vision/ImageLabeler.java | 20 ++++- .../src/com/codename1/ai/vision/Pose.java | 20 ++++- .../com/codename1/ai/vision/PoseDetector.java | 20 ++++- .../codename1/ai/vision/SegmentationMask.java | 20 ++++- .../codename1/ai/vision/SelfieSegmenter.java | 20 ++++- .../ai/vision/TextRecognitionResult.java | 20 ++++- .../codename1/ai/vision/TextRecognizer.java | 20 ++++- .../codename1/ai/vision/VisionAnalyzer.java | 20 ++++- .../codename1/ai/vision/VisionBackend.java | 28 ++++++- .../codename1/ai/vision/VisionBackends.java | 28 ++++++- .../codename1/ai/vision/VisionException.java | 20 ++++- .../codename1/ai/vision/VisionFeature.java | 20 ++++- .../com/codename1/ai/vision/VisionImage.java | 24 +++++- .../codename1/ai/vision/VisionMetadata.java | 24 +++++- .../codename1/ai/vision/VisionOptions.java | 20 ++++- .../codename1/ai/vision/VisionPipeline.java | 24 +++++- .../ai/vision/VisionPipelineListener.java | 20 ++++- .../com/codename1/ai/vision/VisionPoint.java | 20 ++++- .../com/codename1/ai/vision/VisionRect.java | 20 ++++- .../com/codename1/ai/vision/package-info.java | 40 ++++++--- .../src/com/codename1/impl/InferenceImpl.java | 20 ++++- .../src/com/codename1/impl/LanguageImpl.java | 20 ++++- .../src/com/codename1/impl/VisionImpl.java | 20 ++++- .../ai/AndroidBarcodeScanningAdapter.java | 18 ++++ .../ai/AndroidFaceDetectionAdapter.java | 18 ++++ .../ai/AndroidImageLabelingAdapter.java | 18 ++++ .../impl/android/ai/AndroidInferenceImpl.java | 18 ++++ .../android/ai/AndroidLanguageAdapter.java | 18 ++++ .../android/ai/AndroidLanguageIdAdapter.java | 18 ++++ .../impl/android/ai/AndroidLanguageImpl.java | 18 ++++ .../ai/AndroidPoseDetectionAdapter.java | 18 ++++ .../ai/AndroidSelfieSegmentationAdapter.java | 18 ++++ .../android/ai/AndroidSmartReplyAdapter.java | 18 ++++ .../ai/AndroidTextRecognitionAdapter.java | 18 ++++ .../android/ai/AndroidTranslationAdapter.java | 18 ++++ .../impl/android/ai/AndroidVisionAdapter.java | 18 ++++ .../impl/android/ai/AndroidVisionImpl.java | 18 ++++ Ports/iOSPort/nativeSources/CN1Inference.m | 18 ++++ Ports/iOSPort/nativeSources/CN1Language.m | 18 ++++ Ports/iOSPort/nativeSources/CN1Vision.m | 18 ++++ .../codename1/impl/ios/IOSInferenceImpl.java | 18 ++++ .../codename1/impl/ios/IOSLanguageImpl.java | 18 ++++ .../com/codename1/impl/ios/IOSVisionImpl.java | 18 ++++ .../generated/AiAndSpeechOnDeviceSnippet.java | 83 +++++++++++++++++++ docs/developer-guide/Ai-And-Speech.asciidoc | 28 +------ .../AndroidAiSourceSelectionTest.java | 22 +++++ .../ai/inference/InferenceApiTest.java | 22 +++++ .../ai/language/LanguageApiTest.java | 22 +++++ .../codename1/ai/vision/VisionApiTest.java | 22 +++++ .../tests/InferenceOnDeviceApiTest.java | 22 +++++ .../tests/LanguageOnDeviceApiTest.java | 22 +++++ .../tests/VisionOnDeviceApiTest.java | 22 +++++ .../extensions/MavenCentralSearch.java | 22 +++++ 79 files changed, 1604 insertions(+), 127 deletions(-) create mode 100644 docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceException.java b/CodenameOne/src/com/codename1/ai/inference/InferenceException.java index 45c953e64ec..f4f127715e6 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceException.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceException.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.inference; -/** Failure while loading or executing a LiteRT model. */ +/// Failure while loading or executing a LiteRT model. public class InferenceException extends RuntimeException { public InferenceException(String message) { super(message); diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java index 38b5bfb7e17..68758838366 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.inference; -/** LiteRT session configuration. */ +/// LiteRT session configuration. public final class InferenceOptions { public enum Accelerator { AUTO, CPU, GPU, NPU, CORE_ML diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java index 8f9fc027ed3..becdc72fdec 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.inference; @@ -9,7 +27,7 @@ import com.codename1.util.AsyncResource; import com.codename1.util.SuccessCallback; -/** Reusable LiteRT model session. */ +/// Reusable LiteRT model session. public final class InferenceSession implements AutoCloseable { private final InferenceImpl implementation; private final Object handle; diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java index c23a5f6654b..82f6c9da0ea 100644 --- a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java +++ b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.inference; @@ -16,23 +34,19 @@ import java.io.IOException; import java.io.InputStream; -/** - * Downloads large LiteRT models once and exposes the cached file as a - * {@link ModelSource}. Small models can instead be packaged directly with - * {@link ModelSource#resource(String)}. - */ +/// Downloads large LiteRT models once and exposes the cached file as a +/// {@link ModelSource}. Small models can instead be packaged directly with +/// {@link ModelSource#resource(String)}. public final class ModelCache { private ModelCache() { } - /** - * Fetches a model into the app-private {@code ai-models} directory. - * - * @param url HTTPS URL of the model - * @param cacheKey stable cache name, independent of the URL - * @param sha256 optional lowercase or uppercase SHA-256 hex digest - * @return asynchronous cached model source - */ + /// Fetches a model into the app-private {@code ai-models} directory. + /// + /// @param url HTTPS URL of the model + /// @param cacheKey stable cache name, independent of the URL + /// @param sha256 optional lowercase or uppercase SHA-256 hex digest + /// @return asynchronous cached model source public static AsyncResource fetch( final String url, final String cacheKey, final String sha256) { if (url == null || !url.startsWith("https://")) { diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelSource.java b/CodenameOne/src/com/codename1/ai/inference/ModelSource.java index c7bf7ba28e3..f597a3f1fd5 100644 --- a/CodenameOne/src/com/codename1/ai/inference/ModelSource.java +++ b/CodenameOne/src/com/codename1/ai/inference/ModelSource.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.inference; -/** A `.tflite` model supplied as bytes, a filesystem path, or an app resource. */ +/// A `.tflite` model supplied as bytes, a filesystem path, or an app resource. public final class ModelSource { public static final int BYTES = 1; public static final int FILE = 2; diff --git a/CodenameOne/src/com/codename1/ai/inference/Tensor.java b/CodenameOne/src/com/codename1/ai/inference/Tensor.java index 67836a93146..4d0353dacba 100644 --- a/CodenameOne/src/com/codename1/ai/inference/Tensor.java +++ b/CodenameOne/src/com/codename1/ai/inference/Tensor.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.inference; -/** Named tensor value passed to or returned from an inference session. */ +/// Named tensor value passed to or returned from an inference session. public final class Tensor { private final String name; private final TensorType type; diff --git a/CodenameOne/src/com/codename1/ai/inference/TensorInfo.java b/CodenameOne/src/com/codename1/ai/inference/TensorInfo.java index 49774d930ea..b7b18d49603 100644 --- a/CodenameOne/src/com/codename1/ai/inference/TensorInfo.java +++ b/CodenameOne/src/com/codename1/ai/inference/TensorInfo.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.inference; -/** Immutable model input/output metadata. */ +/// Immutable model input/output metadata. public final class TensorInfo { private final String name; private final TensorType type; diff --git a/CodenameOne/src/com/codename1/ai/inference/TensorType.java b/CodenameOne/src/com/codename1/ai/inference/TensorType.java index 04f22652ea3..962af3d6b39 100644 --- a/CodenameOne/src/com/codename1/ai/inference/TensorType.java +++ b/CodenameOne/src/com/codename1/ai/inference/TensorType.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.inference; -/** Tensor element types supported by the portable LiteRT contract. */ +/// Tensor element types supported by the portable LiteRT contract. public enum TensorType { FLOAT32, INT32, diff --git a/CodenameOne/src/com/codename1/ai/inference/package-info.java b/CodenameOne/src/com/codename1/ai/inference/package-info.java index b62037c5e30..1718b6140fd 100644 --- a/CodenameOne/src/com/codename1/ai/inference/package-info.java +++ b/CodenameOne/src/com/codename1/ai/inference/package-info.java @@ -1,9 +1,29 @@ -/** - * Reusable on-device inference sessions for {@code .tflite} models. +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. * - *

Android uses LiteRT. iOS uses TensorFlow Lite Objective-C and may select - * the Core ML delegate. The builder links the runtime only when an application - * references {@link com.codename1.ai.inference.InferenceSession}. Other - * targets expose the same API with an explicit unsupported fallback.

+ * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ +/// Reusable on-device inference sessions for {@code .tflite} models. +/// +///

Android uses LiteRT. iOS uses TensorFlow Lite Objective-C and may select +/// the Core ML delegate. The builder links the runtime only when an application +/// references {@link com.codename1.ai.inference.InferenceSession}. Other +/// targets expose the same API with an explicit unsupported fallback.

package com.codename1.ai.inference; diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageBackend.java b/CodenameOne/src/com/codename1/ai/language/LanguageBackend.java index 854de1d0dc8..4c4bee2327a 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageBackend.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageBackend.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.language; -/** Identifies an on-device language implementation. */ +/// Identifies an on-device language implementation. public interface LanguageBackend { String getId(); } diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java b/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java index 0f3b8707ad8..10311cb940b 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.language; -/** Backend selectors for language identification, translation, and smart reply. */ +/// Backend selectors for language identification, translation, and smart reply. public final class LanguageBackends { private static final LanguageBackend AUTO = new Named("auto"); private static final LanguageBackend ML_KIT = new Named("ml-kit"); diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageCandidate.java b/CodenameOne/src/com/codename1/ai/language/LanguageCandidate.java index 31fe5176bba..e69d3897a80 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageCandidate.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageCandidate.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.language; -/** Language identification candidate. */ +/// Language identification candidate. public final class LanguageCandidate { private final String languageTag; private final float confidence; diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java index 438842a19d9..61a820ac4f0 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.language; @@ -8,7 +26,7 @@ import com.codename1.ui.Display; import com.codename1.util.AsyncResource; -/** On-device language identification. */ +/// On-device language identification. public final class LanguageIdentifier { private LanguageIdentifier() { } diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java b/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java index afeb1299ffb..4cec3d4a971 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.language; -/** Common options for on-device language services. */ +/// Common options for on-device language services. public final class LanguageOptions { private LanguageBackend backend = LanguageBackends.auto(); private float minimumConfidence; diff --git a/CodenameOne/src/com/codename1/ai/language/SmartReply.java b/CodenameOne/src/com/codename1/ai/language/SmartReply.java index 613e55aa7a4..2be20c971ad 100644 --- a/CodenameOne/src/com/codename1/ai/language/SmartReply.java +++ b/CodenameOne/src/com/codename1/ai/language/SmartReply.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.language; @@ -8,7 +26,7 @@ import com.codename1.ui.Display; import com.codename1.util.AsyncResource; -/** On-device short reply suggestions for a conversation. */ +/// On-device short reply suggestions for a conversation. public final class SmartReply { private SmartReply() { } diff --git a/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java b/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java index 1bda62dcea1..209f1dc90b9 100644 --- a/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java +++ b/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.language; -/** One message supplied to the on-device smart-reply model. */ +/// One message supplied to the on-device smart-reply model. public final class SmartReplyMessage { private final String text; private final String participantId; diff --git a/CodenameOne/src/com/codename1/ai/language/Translator.java b/CodenameOne/src/com/codename1/ai/language/Translator.java index 85d8a7b9c92..6045e8eba16 100644 --- a/CodenameOne/src/com/codename1/ai/language/Translator.java +++ b/CodenameOne/src/com/codename1/ai/language/Translator.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.language; @@ -8,7 +26,7 @@ import com.codename1.ui.Display; import com.codename1.util.AsyncResource; -/** On-device translation with lazily installed language-pair models. */ +/// On-device translation with lazily installed language-pair models. public final class Translator { private Translator() { } diff --git a/CodenameOne/src/com/codename1/ai/language/package-info.java b/CodenameOne/src/com/codename1/ai/language/package-info.java index 2b711adb17e..8debbe97ee3 100644 --- a/CodenameOne/src/com/codename1/ai/language/package-info.java +++ b/CodenameOne/src/com/codename1/ai/language/package-info.java @@ -1,11 +1,31 @@ -/** - * Vendor-neutral on-device language identification, translation, and smart - * reply. +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. * - *

Android and iOS builds use feature-scoped ML Kit components. Each - * public entry point is scanned independently, so language identification - * does not bundle translation or Smart Reply. Translation model payloads are - * installed lazily by ML Kit. Other targets expose the same API with an - * explicit unsupported fallback.

+ * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ +/// Vendor-neutral on-device language identification, translation, and smart +/// reply. +/// +///

Android and iOS builds use feature-scoped ML Kit components. Each +/// public entry point is scanned independently, so language identification +/// does not bundle translation or Smart Reply. Translation model payloads are +/// installed lazily by ML Kit. Other targets expose the same API with an +/// explicit unsupported fallback.

package com.codename1.ai.language; diff --git a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java index 48896135a22..083ae63633d 100644 --- a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java +++ b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; diff --git a/CodenameOne/src/com/codename1/ai/vision/Barcode.java b/CodenameOne/src/com/codename1/ai/vision/Barcode.java index da2625cb9e9..5081c421271 100644 --- a/CodenameOne/src/com/codename1/ai/vision/Barcode.java +++ b/CodenameOne/src/com/codename1/ai/vision/Barcode.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Portable barcode observation. */ +/// Portable barcode observation. public final class Barcode { private final String value; private final String format; diff --git a/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java b/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java index 53644a89e1b..90c70a4f552 100644 --- a/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java +++ b/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Creates reusable still-image or live-frame barcode analyzers. */ +/// Creates reusable still-image or live-frame barcode analyzers. public final class BarcodeScanner extends AbstractVisionAnalyzer { public BarcodeScanner() { this(null); diff --git a/CodenameOne/src/com/codename1/ai/vision/DocumentScanResult.java b/CodenameOne/src/com/codename1/ai/vision/DocumentScanResult.java index 85f4ce6acbb..2607ea7ab0e 100644 --- a/CodenameOne/src/com/codename1/ai/vision/DocumentScanResult.java +++ b/CodenameOne/src/com/codename1/ai/vision/DocumentScanResult.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Corrected document pages returned as encoded images. */ +/// Corrected document pages returned as encoded images. public final class DocumentScanResult { private final byte[][] pages; private final VisionMetadata metadata; diff --git a/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java b/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java index 63c1cb3f808..d6d80a6d056 100644 --- a/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java +++ b/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Creates reusable document-boundary and perspective-correction analyzers. */ +/// Creates reusable document-boundary and perspective-correction analyzers. public final class DocumentScanner extends AbstractVisionAnalyzer { public DocumentScanner() { this(null); diff --git a/CodenameOne/src/com/codename1/ai/vision/Face.java b/CodenameOne/src/com/codename1/ai/vision/Face.java index b6387d4fae0..56cbec325ed 100644 --- a/CodenameOne/src/com/codename1/ai/vision/Face.java +++ b/CodenameOne/src/com/codename1/ai/vision/Face.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; @@ -8,13 +26,11 @@ import java.util.HashMap; import java.util.Map; -/** - * Portable face observation. - * - *

Euler angles are expressed in degrees. Bounds and landmark points use - * the normalized, top-left-origin coordinate space defined by - * {@link VisionRect} and {@link VisionPoint}.

- */ +/// Portable face observation. +/// +///

Euler angles are expressed in degrees. Bounds and landmark points use +/// the normalized, top-left-origin coordinate space defined by +/// {@link VisionRect} and {@link VisionPoint}.

public final class Face { private final VisionRect bounds; private final Map landmarks; diff --git a/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java b/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java index a7f5c98f7c9..6afc9537d8f 100644 --- a/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java +++ b/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Creates reusable on-device face analyzers. */ +/// Creates reusable on-device face analyzers. public final class FaceDetector extends AbstractVisionAnalyzer { public FaceDetector() { this(null); diff --git a/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java b/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java index ac91a89de65..fd6256de7bb 100644 --- a/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java +++ b/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Portable image classification label. */ +/// Portable image classification label. public final class ImageLabel { private final String text; private final float confidence; diff --git a/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java b/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java index 598249da9eb..d23cf1c11fb 100644 --- a/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java +++ b/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Creates reusable on-device image classifiers. */ +/// Creates reusable on-device image classifiers. public final class ImageLabeler extends AbstractVisionAnalyzer { public ImageLabeler() { this(null); diff --git a/CodenameOne/src/com/codename1/ai/vision/Pose.java b/CodenameOne/src/com/codename1/ai/vision/Pose.java index 32b1b352270..9e772fa886d 100644 --- a/CodenameOne/src/com/codename1/ai/vision/Pose.java +++ b/CodenameOne/src/com/codename1/ai/vision/Pose.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Portable body-pose result. */ +/// Portable body-pose result. public final class Pose { private final Landmark[] landmarks; private final VisionMetadata metadata; diff --git a/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java b/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java index cd07c52970c..861fd48ae9c 100644 --- a/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java +++ b/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Creates reusable body-pose analyzers. */ +/// Creates reusable body-pose analyzers. public final class PoseDetector extends AbstractVisionAnalyzer { public PoseDetector() { this(null); diff --git a/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java b/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java index cd1e4b0791f..48332afcce4 100644 --- a/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java +++ b/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Per-pixel foreground confidence mask. */ +/// Per-pixel foreground confidence mask. public final class SegmentationMask { private final int width; private final int height; diff --git a/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java b/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java index 9e4254bfddc..e28928e24ad 100644 --- a/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java +++ b/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Creates reusable foreground/person segmentation analyzers. */ +/// Creates reusable foreground/person segmentation analyzers. public final class SelfieSegmenter extends AbstractVisionAnalyzer { public SelfieSegmenter() { this(null); diff --git a/CodenameOne/src/com/codename1/ai/vision/TextRecognitionResult.java b/CodenameOne/src/com/codename1/ai/vision/TextRecognitionResult.java index c3df7d0dad9..1f61ed83000 100644 --- a/CodenameOne/src/com/codename1/ai/vision/TextRecognitionResult.java +++ b/CodenameOne/src/com/codename1/ai/vision/TextRecognitionResult.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** OCR output with a portable block-level structure. */ +/// OCR output with a portable block-level structure. public final class TextRecognitionResult { public static final TextRecognitionResult EMPTY = new TextRecognitionResult("", new TextBlock[0]); diff --git a/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java b/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java index 244303fda23..e90b8b90c34 100644 --- a/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java +++ b/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Creates reusable on-device OCR analyzers. */ +/// Creates reusable on-device OCR analyzers. public final class TextRecognizer extends AbstractVisionAnalyzer { public TextRecognizer() { this(null); diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java b/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java index 135c779a224..c479a091667 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java @@ -1,12 +1,30 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; import com.codename1.util.AsyncResource; -/** Reusable, closable analyzer for still images or camera frames. */ +/// Reusable, closable analyzer for still images or camera frames. public interface VisionAnalyzer extends AutoCloseable { boolean isSupported(); AsyncResource process(VisionImage image); diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionBackend.java b/CodenameOne/src/com/codename1/ai/vision/VisionBackend.java index 0e0a7d44149..efd978f8121 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionBackend.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionBackend.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -9,10 +31,8 @@ */ package com.codename1.ai.vision; -/** - * Identifies a vision implementation. Use {@link VisionBackends} to obtain - * instances so builders can detect explicit optional-backend use. - */ +/// Identifies a vision implementation. Use {@link VisionBackends} to obtain +/// instances so builders can detect explicit optional-backend use. public interface VisionBackend { String getId(); } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java b/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java index 7a820d04306..191c18415ab 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -9,10 +31,8 @@ */ package com.codename1.ai.vision; -/** - * Vision backend selectors. {@code auto()} chooses Apple Vision on iOS and - * ML Kit on Android. Unsupported ports report that through the analyzer. - */ +/// Vision backend selectors. {@code auto()} chooses Apple Vision on iOS and +/// ML Kit on Android. Unsupported ports report that through the analyzer. public final class VisionBackends { private static final VisionBackend AUTO = new NamedBackend("auto"); private static final VisionBackend APPLE = new NamedBackend("apple-vision"); diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionException.java b/CodenameOne/src/com/codename1/ai/vision/VisionException.java index 8fcaab2acc0..48d1ab9c0fc 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionException.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionException.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Failure reported by an on-device vision backend. */ +/// Failure reported by an on-device vision backend. public class VisionException extends RuntimeException { public static final int UNSUPPORTED = 1; public static final int INVALID_IMAGE = 2; diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionFeature.java b/CodenameOne/src/com/codename1/ai/vision/VisionFeature.java index d9460fcfded..d66d0ced60f 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionFeature.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionFeature.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Features understood by the shared vision backend. */ +/// Features understood by the shared vision backend. public enum VisionFeature { TEXT_RECOGNITION, BARCODE_SCANNING, diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java index 9d82a8877e7..db745e92fbb 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java @@ -1,16 +1,32 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; import com.codename1.camera.CameraFrame; import com.codename1.camera.FrameFormat; -/** - * Immutable vision input. Camera-frame factories copy the callback-owned bytes - * so asynchronous analyzers never retain recycled camera buffers. - */ +/// Immutable vision input. Camera-frame factories copy the callback-owned bytes +/// so asynchronous analyzers never retain recycled camera buffers. public final class VisionImage { private final byte[] encodedBytes; private final byte[] pixels; diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionMetadata.java b/CodenameOne/src/com/codename1/ai/vision/VisionMetadata.java index 8c0ee8c6e2a..b3c718f172e 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionMetadata.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionMetadata.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; @@ -8,10 +26,8 @@ import java.util.HashMap; import java.util.Map; -/** - * Optional backend identity and backend-specific string values attached to a - * vision result. Portable code should use the typed result fields first. - */ +/// Optional backend identity and backend-specific string values attached to a +/// vision result. Portable code should use the typed result fields first. public final class VisionMetadata { private final String backendId; private final Map values; diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java b/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java index 778f0263495..6905ae54cda 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Common analyzer options; feature-specific options may extend this class. */ +/// Common analyzer options; feature-specific options may extend this class. public class VisionOptions { private VisionBackend backend = VisionBackends.auto(); private float minimumConfidence; diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java index 08f9731f427..da80f49817d 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; @@ -10,10 +28,8 @@ import com.codename1.ui.Display; import com.codename1.util.SuccessCallback; -/** - * Connects a camera frame stream to an analyzer with keep-only-latest - * backpressure. The pipeline owns the analyzer and closes it on close. - */ +/// Connects a camera frame stream to an analyzer with keep-only-latest +/// backpressure. The pipeline owns the analyzer and closes it on close. public final class VisionPipeline implements AutoCloseable { private final CameraSession session; private final VisionAnalyzer analyzer; diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionPipelineListener.java b/CodenameOne/src/com/codename1/ai/vision/VisionPipelineListener.java index 332e018db3b..c2f41f95ce3 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionPipelineListener.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionPipelineListener.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Receives live vision results and recoverable analysis failures on the EDT. */ +/// Receives live vision results and recoverable analysis failures on the EDT. public interface VisionPipelineListener { void result(T value, VisionImage source); void error(Throwable error); diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionPoint.java b/CodenameOne/src/com/codename1/ai/vision/VisionPoint.java index 581f19e56ef..20d9441ae2b 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionPoint.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionPoint.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Immutable point normalized to a top-left-origin 0..1 coordinate space. */ +/// Immutable point normalized to a top-left-origin 0..1 coordinate space. public final class VisionPoint { private final float x; private final float y; diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionRect.java b/CodenameOne/src/com/codename1/ai/vision/VisionRect.java index b708081a95a..75609b578d5 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionRect.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionRect.java @@ -1,10 +1,28 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.ai.vision; -/** Immutable normalized rectangle using a top-left origin. */ +/// Immutable normalized rectangle using a top-left origin. public final class VisionRect { public static final VisionRect EMPTY = new VisionRect(0, 0, 0, 0); diff --git a/CodenameOne/src/com/codename1/ai/vision/package-info.java b/CodenameOne/src/com/codename1/ai/vision/package-info.java index 131830dc56c..e9726740490 100644 --- a/CodenameOne/src/com/codename1/ai/vision/package-info.java +++ b/CodenameOne/src/com/codename1/ai/vision/package-info.java @@ -1,14 +1,34 @@ -/** - * Vendor-neutral on-device vision APIs for still images and live camera frames. +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. * - *

The automatic backend uses Apple Vision/Core Image on iOS and Mac - * Catalyst and ML Kit on Android. Unsupported ports report that through - * {@code isSupported()}. Optional backends are selected with - * {@link com.codename1.ai.vision.VisionBackends}.

+ * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). * - *

Each analyzer is a separate build-time feature. Referencing one analyzer - * causes the builder to retain only its platform adapter and native - * dependency. {@link com.codename1.ai.vision.VisionPipeline} safely copies - * callback-owned camera frames and drops stale pending frames under load.

+ * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ +/// Vendor-neutral on-device vision APIs for still images and live camera frames. +/// +///

The automatic backend uses Apple Vision/Core Image on iOS and Mac +/// Catalyst and ML Kit on Android. Unsupported ports report that through +/// {@code isSupported()}. Optional backends are selected with +/// {@link com.codename1.ai.vision.VisionBackends}.

+/// +///

Each analyzer is a separate build-time feature. Referencing one analyzer +/// causes the builder to retain only its platform adapter and native +/// dependency. {@link com.codename1.ai.vision.VisionPipeline} safely copies +/// callback-owned camera frames and drops stale pending frames under load.

package com.codename1.ai.vision; diff --git a/CodenameOne/src/com/codename1/impl/InferenceImpl.java b/CodenameOne/src/com/codename1/impl/InferenceImpl.java index 25936de936a..c79963e5ef2 100644 --- a/CodenameOne/src/com/codename1/impl/InferenceImpl.java +++ b/CodenameOne/src/com/codename1/impl/InferenceImpl.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl; @@ -10,7 +28,7 @@ import com.codename1.ai.inference.TensorInfo; import com.codename1.util.AsyncResource; -/** Port contract behind the built-in LiteRT inference API. @hidden */ +/// Port contract behind the built-in LiteRT inference API. @hidden public abstract class InferenceImpl { public abstract boolean isSupported(); public abstract AsyncResource open(ModelSource source, InferenceOptions options); diff --git a/CodenameOne/src/com/codename1/impl/LanguageImpl.java b/CodenameOne/src/com/codename1/impl/LanguageImpl.java index 9eab6f3919d..8bd9fe9995c 100644 --- a/CodenameOne/src/com/codename1/impl/LanguageImpl.java +++ b/CodenameOne/src/com/codename1/impl/LanguageImpl.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl; @@ -9,7 +27,7 @@ import com.codename1.ai.language.SmartReplyMessage; import com.codename1.util.AsyncResource; -/** Port contract behind the built-in on-device language APIs. @hidden */ +/// Port contract behind the built-in on-device language APIs. @hidden public abstract class LanguageImpl { public abstract boolean isSupported(String feature, String backendId); public abstract AsyncResource identify( diff --git a/CodenameOne/src/com/codename1/impl/VisionImpl.java b/CodenameOne/src/com/codename1/impl/VisionImpl.java index 007803dac7b..cb80164a25d 100644 --- a/CodenameOne/src/com/codename1/impl/VisionImpl.java +++ b/CodenameOne/src/com/codename1/impl/VisionImpl.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl; @@ -9,7 +27,7 @@ import com.codename1.ai.vision.VisionOptions; import com.codename1.util.AsyncResource; -/** Port contract behind {@code com.codename1.ai.vision}. @hidden */ +/// Port contract behind {@code com.codename1.ai.vision}. @hidden public abstract class VisionImpl { public abstract boolean isSupported(VisionFeature feature, String backendId); public abstract AsyncResource analyze(VisionFeature feature, String backendId, diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java index e02b4aa9ca6..961040e3031 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.android.ai; diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java index 99654d5a3ab..c3e9baa6627 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.android.ai; diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java index 91a134a1f1d..6829ae794b5 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.android.ai; diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java index c8f8ca6594a..c914020f2f2 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.android.ai; diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageAdapter.java index e08029fe522..ca96cb8c558 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageAdapter.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.android.ai; diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageIdAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageIdAdapter.java index 1a6ed6ad163..bca3039ec33 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageIdAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageIdAdapter.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.android.ai; diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageImpl.java index 02486ceeb2f..bab9094f966 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageImpl.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.android.ai; diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java index 3cd1de05ed6..e0ace3e4169 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.android.ai; diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidSelfieSegmentationAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSelfieSegmentationAdapter.java index decfc4265e1..aa425f0b04a 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidSelfieSegmentationAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSelfieSegmentationAdapter.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.android.ai; diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java index 728236c0a0b..ce57e48f41d 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.android.ai; diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java index ebfc61fdce3..26ec2db9073 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.android.ai; diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java index ec3f74818f3..0adc687356c 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.android.ai; diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionAdapter.java index 8d2fc880523..3ef9e9b3262 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionAdapter.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.android.ai; diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java index 4b41fb1078a..66aaac53b78 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.android.ai; diff --git a/Ports/iOSPort/nativeSources/CN1Inference.m b/Ports/iOSPort/nativeSources/CN1Inference.m index c33d7baf1c0..f5c9c84cf23 100644 --- a/Ports/iOSPort/nativeSources/CN1Inference.m +++ b/Ports/iOSPort/nativeSources/CN1Inference.m @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ #import "CodenameOne_GLViewController.h" #import "xmlvm.h" diff --git a/Ports/iOSPort/nativeSources/CN1Language.m b/Ports/iOSPort/nativeSources/CN1Language.m index 7bd07f87b90..71bcd97f52f 100644 --- a/Ports/iOSPort/nativeSources/CN1Language.m +++ b/Ports/iOSPort/nativeSources/CN1Language.m @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ #import "CodenameOne_GLViewController.h" #import "xmlvm.h" diff --git a/Ports/iOSPort/nativeSources/CN1Vision.m b/Ports/iOSPort/nativeSources/CN1Vision.m index a3d73dedbaf..f6483f34279 100644 --- a/Ports/iOSPort/nativeSources/CN1Vision.m +++ b/Ports/iOSPort/nativeSources/CN1Vision.m @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ #import "CodenameOne_GLViewController.h" #import "xmlvm.h" diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java index 3462dda4453..8a59c89c065 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.ios; diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java index 53177239793..da61099ec37 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.ios; diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java index 3198515df56..d362177f707 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java @@ -1,6 +1,24 @@ /* * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.ios; diff --git a/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java b/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java new file mode 100644 index 00000000000..b69bd8d8674 --- /dev/null +++ b/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.snippets.generated; + +import com.codename1.ai.inference.InferenceOptions; +import com.codename1.ai.inference.InferenceSession; +import com.codename1.ai.inference.ModelSource; +import com.codename1.ai.inference.Tensor; +import com.codename1.ai.language.LanguageOptions; +import com.codename1.ai.language.Translator; +import com.codename1.ai.vision.TextRecognizer; +import com.codename1.ai.vision.VisionBackends; +import com.codename1.ai.vision.VisionImage; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.io.Log; + +class AiAndSpeechOnDeviceSnippet { + private byte[] jpegBytes; + private float[] pixels; + + void vision() { + // tag::ai-and-speech-on-device-vision[] + VisionOptions options = new VisionOptions() + .backend(VisionBackends.appleVision()) + .minimumConfidence(0.5f); + + new TextRecognizer(options) + .process(VisionImage.encoded(jpegBytes)) + .ready(result -> Log.p(result.getText())) + .except(error -> Log.e(error)); + // end::ai-and-speech-on-device-vision[] + } + + void inference() { + // tag::ai-and-speech-on-device-inference[] + InferenceSession.open( + ModelSource.resource("/models/classifier.tflite"), + new InferenceOptions().accelerator( + InferenceOptions.Accelerator.AUTO)) + .ready(session -> { + Tensor input = Tensor.floats("serving_default_input", + new int[] {1, 224, 224, 3}, pixels); + session.run(new Tensor[] {input}) + .ready(outputs -> consume(outputs[0])) + .except(error -> Log.e(error)); + }); + // end::ai-and-speech-on-device-inference[] + } + + void language() { + // tag::ai-and-speech-on-device-language[] + if (Translator.isSupported()) { + Translator.translate("Where is the station?", "en", "fr", + new LanguageOptions()) + .ready(value -> Log.p(value)) + .except(error -> Log.e(error)); + } + // end::ai-and-speech-on-device-language[] + } + + private void consume(Tensor output) { + } +} diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index 080fb65a821..ea1d60f931f 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -285,14 +285,7 @@ and Core Image. iOS can select ML Kit explicitly: [source,java] ---- -VisionOptions options = new VisionOptions() - .backend(VisionBackends.appleVision()) - .minimumConfidence(0.5f); - -new TextRecognizer(options) - .process(VisionImage.encoded(jpegBytes)) - .ready(result -> Log.p(result.getText())) - .except(error -> Log.e(error)); +include::../demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java[tag=ai-and-speech-on-device-vision,indent=0] ---- Create and reuse an analyzer when processing a sequence. Call `close()` @@ -365,17 +358,7 @@ a native LiteRT runtime, including Mac native, return `false` from [source,java] ---- -InferenceSession.open( - ModelSource.resource("/models/classifier.tflite"), - new InferenceOptions().accelerator( - InferenceOptions.Accelerator.AUTO)) - .ready(session -> { - Tensor input = Tensor.floats("serving_default_input", - new int[] {1, 224, 224, 3}, pixels); - session.run(new Tensor[] {input}) - .ready(outputs -> consume(outputs[0])) - .except(error -> Log.e(error)); - }); +include::../demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java[tag=ai-and-speech-on-device-inference,indent=0] ---- Open is asynchronous because model initialization can allocate native @@ -400,12 +383,7 @@ SDKs. [source,java] ---- -if (Translator.isSupported()) { - Translator.translate("Where is the station?", "en", "fr", - new LanguageOptions()) - .ready(value -> Log.p(value)) - .except(error -> Log.e(error)); -} +include::../demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java[tag=ai-and-speech-on-device-language,indent=0] ---- `LanguageBackends.auto()` selects ML Kit on the supported Android and diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java index c6909a3f87d..617a2f2ee33 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ package com.codename1.builders; import org.junit.jupiter.api.Test; diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index 466765fa678..80a73e50f30 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ /* * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java index a4564ab35a0..997dedab097 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ /* * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java index 6548c5cb298..e69d83c7bf2 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ /* * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/InferenceOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/InferenceOnDeviceApiTest.java index b97199225e2..ae11d70a0a7 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/InferenceOnDeviceApiTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/InferenceOnDeviceApiTest.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ /* * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java index 143dd40b05d..b42582294e5 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ /* * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java index 69ea148c2e9..4d633af9a0d 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ /* * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. diff --git a/scripts/settings/common/src/main/java/com/codename1/settings/extensions/MavenCentralSearch.java b/scripts/settings/common/src/main/java/com/codename1/settings/extensions/MavenCentralSearch.java index 6cf22b2f905..c7b8663b85d 100644 --- a/scripts/settings/common/src/main/java/com/codename1/settings/extensions/MavenCentralSearch.java +++ b/scripts/settings/common/src/main/java/com/codename1/settings/extensions/MavenCentralSearch.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ package com.codename1.settings.extensions; import com.codename1.io.ConnectionRequest; From f576c5ce84e7027b7f8a0c71f8122721104d380c Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 25 Jul 2026 15:01:33 +0300 Subject: [PATCH 03/80] Fix AI guide prose lint --- docs/developer-guide/Ai-And-Speech.asciidoc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index ea1d60f931f..5833477bb81 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -295,13 +295,13 @@ linked backend, and OS version. `VisionPipeline` attaches an analyzer to a `CameraSession` frame stream and keeps only the newest pending frame. Live OCR and pose detection -therefore cannot build an unbounded queue when inference is slower than +therefore can't build an unbounded queue when inference is slower than the camera. Results and errors are delivered on the Codename One EDT. `DocumentScanner` currently performs still-image rectangle correction on Apple platforms. Google's Android document scanner is an interactive camera flow rather than an analyzer for an existing `VisionImage`, so the -Android backend reports it as unsupported instead of silently launching UI. +Android backend reports it as unsupported rather than launching a UI flow. ==== Vision feature and dependency selection @@ -344,7 +344,7 @@ each class the application actually uses: The optional iOS ML Kit pod is added only when both the analyzer and `VisionBackends.mlKit()` are referenced. Referencing one Android -analyzer does not compile or package the other five adapters. +analyzer doesn't compile or package the other five adapters. === Portable LiteRT inference @@ -370,7 +370,7 @@ Bundle small models as resources. For larger models use `ModelCache.fetch(url, cacheKey, sha256)`, which downloads over HTTPS, verifies the optional SHA-256 digest, and reuses the app-private cached file. The URL must use HTTPS. Treat the digest as required when the -model is not controlled by the same deployment pipeline as the app. +model isn't controlled by the same deployment pipeline as the app. === On-device language services @@ -378,7 +378,7 @@ model is not controlled by the same deployment pipeline as the app. `LanguageOptions`. Translation models are downloaded and cached by ML Kit when a language pair is first used. Android and iOS builders select `language-id`, `translate`, and `smart-reply` independently, so -using language identification alone does not package the other two +using language identification alone doesn't package the other two SDKs. [source,java] @@ -401,7 +401,7 @@ targets and Android uses ML Kit. Language services and LiteRT inference have native implementations on Android and iOS. JavaSE, JavaScript, native Windows, native Linux, watchOS, and tvOS report unsupported for these new APIs; Mac native reports unsupported for language and LiteRT. -No target silently sends data to a cloud inference service. Large +No target sends data to a cloud inference service. Large portable runtimes or model payloads for the remaining targets remain appropriate opt-in libraries. @@ -413,7 +413,7 @@ dependency decisions. === Migrating from the AI cn1libs The old `com.codename1.ai.mlkit.*` and `com.codename1.ai.tflite` -packages are not compatibility aliases. Remove those dependencies and +packages aren't compatibility aliases. Remove those dependencies and move directly to `com.codename1.ai.vision`, `com.codename1.ai.language`, or `com.codename1.ai.inference`. Result coordinates are normalized floats rather than packed @@ -423,8 +423,8 @@ and models are intentionally large. Remove the retired cn1lib dependency before adding the built-in class. Keeping both packages can leave duplicate ML Kit versions in an -Android or CocoaPods dependency graph. The builders deliberately do -not ship compatibility aliases because those aliases would keep the +Android or CocoaPods dependency graph. The builders don't ship +compatibility aliases because those aliases would keep the old native-interface and packaging contracts alive. ==== Putting it all together From c89cb9ecf0baf7949eaab49806b2feb45a9c044f Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 25 Jul 2026 15:20:30 +0300 Subject: [PATCH 04/80] Fix iOS inference file loading on Catalyst --- .../iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java index 8a59c89c065..807ad9ccfeb 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java @@ -29,13 +29,13 @@ import com.codename1.ai.inference.TensorInfo; import com.codename1.ai.inference.TensorType; import com.codename1.impl.InferenceImpl; +import com.codename1.io.FileSystemStorage; import com.codename1.io.JSONParser; import com.codename1.ui.Display; import com.codename1.util.AsyncResource; import com.codename1.util.Base64; import java.io.ByteArrayOutputStream; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.StringReader; @@ -363,7 +363,7 @@ private static byte[] loadModel(ModelSource source) throws IOException { } InputStream input; if (source.getKind() == ModelSource.FILE) { - input = new FileInputStream(source.getPath()); + input = FileSystemStorage.getInstance().openInputStream(source.getPath()); } else { input = Display.getInstance().getResourceAsStream( IOSInferenceImpl.class, source.getPath()); From 987f8fef3b67fdc6e86ba351ef9e8251c0319737 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 25 Jul 2026 15:40:25 +0300 Subject: [PATCH 05/80] Avoid iOS adapter outer captures --- .../codename1/impl/ios/IOSInferenceImpl.java | 16 ++++++++++++++-- .../com/codename1/impl/ios/IOSLanguageImpl.java | 17 ++++++++++++++++- .../com/codename1/impl/ios/IOSVisionImpl.java | 17 ++++++++++++++--- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java index 807ad9ccfeb..435eaf91005 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java @@ -63,6 +63,13 @@ public boolean isSupported() { public AsyncResource open(final ModelSource source, final InferenceOptions options) { final AsyncResource out = new AsyncResource(); + openInBackground(out, source, options); + return out; + } + + private static void openInBackground(final AsyncResource out, + final ModelSource source, + final InferenceOptions options) { Display.getInstance().scheduleBackgroundTask(new Runnable() { public void run() { try { @@ -81,7 +88,6 @@ public void run() { } } }); - return out; } @Override @@ -98,6 +104,13 @@ public TensorInfo[] getOutputs(Object handle) { public AsyncResource run(Object handle, final Tensor[] inputs) { final AsyncResource out = new AsyncResource(); final Handle checked = checked(handle); + runInBackground(out, checked, inputs); + return out; + } + + private static void runInBackground(final AsyncResource out, + final Handle checked, + final Tensor[] inputs) { final int id = checked.id; Display.getInstance().scheduleBackgroundTask(new Runnable() { public void run() { @@ -126,7 +139,6 @@ public void run() { } } }); - return out; } @Override diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java index da61099ec37..a75034f52e1 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java @@ -56,6 +56,11 @@ public boolean isSupported(String feature, String backendId) { @Override public AsyncResource identify( final String text, String backendId, final LanguageOptions options) { + return identifyInBackground(text, options); + } + + private static AsyncResource identifyInBackground( + final String text, final LanguageOptions options) { final AsyncResource out = new AsyncResource(); run(out, new NativeCall() { @@ -82,6 +87,12 @@ public AsyncResource translate( final String text, final String sourceLanguage, final String targetLanguage, String backendId, LanguageOptions options) { + return translateInBackground(text, sourceLanguage, targetLanguage); + } + + private static AsyncResource translateInBackground( + final String text, final String sourceLanguage, + final String targetLanguage) { final AsyncResource out = new AsyncResource(); run(out, new NativeCall() { public String call() throws Exception { @@ -97,8 +108,12 @@ public String call() throws Exception { public AsyncResource suggestReplies( SmartReplyMessage[] conversation, String backendId, LanguageOptions options) { + return suggestRepliesInBackground(conversationJson(conversation)); + } + + private static AsyncResource suggestRepliesInBackground( + final String json) { final AsyncResource out = new AsyncResource(); - final String json = conversationJson(conversation); run(out, new NativeCall() { public String[] call() throws Exception { Map root = parse(IOSImplementation.nativeInstance diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java index d362177f707..3ce6676297d 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java @@ -85,13 +85,25 @@ public AsyncResource analyze(final VisionFeature feature, "Apple Vision requires encoded, NV21, or RGBA8888 input")); return out; } + analyzeInBackground(out, feature, mlKit, input, + image.getRotationDegrees(), width, height, frameFormat); + return out; + } + + private static void analyzeInBackground(final AsyncResource out, + final VisionFeature feature, + final boolean mlKit, + final byte[] input, + final int rotation, + final int width, + final int height, + final int frameFormat) { Display.getInstance().scheduleBackgroundTask(new Runnable() { public void run() { try { String json = IOSImplementation.nativeInstance.cn1VisionAnalyze( input, feature.ordinal(), mlKit, - image.getRotationDegrees(), width, height, - frameFormat); + rotation, width, height, frameFormat); if (json == null || json.length() == 0) { throw new VisionException(VisionException.BACKEND_ERROR, "Apple Vision returned no result"); @@ -114,7 +126,6 @@ public void run() { } } }); - return out; } @SuppressWarnings({"rawtypes", "unchecked"}) From 65a724e0fef2580c459a4c2fd6f494bec3186eb5 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 25 Jul 2026 16:27:10 +0300 Subject: [PATCH 06/80] Satisfy core AI static analysis --- .../ai/inference/InferenceSession.java | 3 ++ .../codename1/ai/inference/ModelCache.java | 7 +++ .../codename1/ai/inference/ModelSource.java | 4 +- .../com/codename1/ai/inference/Tensor.java | 45 +++++++++++++------ .../ai/language/LanguageBackends.java | 1 + .../ai/vision/AbstractVisionAnalyzer.java | 3 ++ .../codename1/ai/vision/VisionAnalyzer.java | 1 + .../codename1/ai/vision/VisionBackends.java | 2 + .../codename1/ai/vision/VisionPipeline.java | 7 +++ 9 files changed, 59 insertions(+), 14 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java index becdc72fdec..d2b57ce88a0 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -56,10 +56,12 @@ public static AsyncResource open(ModelSource source, } impl.open(source, options == null ? new InferenceOptions() : options) .ready(new SuccessCallback() { + @Override public void onSucess(Object value) { out.complete(new InferenceSession(impl, value)); } }).except(new SuccessCallback() { + @Override public void onSucess(Throwable error) { out.error(error); } @@ -87,6 +89,7 @@ public void resizeInput(String name, int[] shape) { implementation.resizeInput(handle, name, shape); } + @Override public void close() { if (!closed) { closed = true; diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java index 82f6c9da0ea..b916fee8370 100644 --- a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java +++ b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java @@ -61,6 +61,7 @@ public static AsyncResource fetch( final AsyncResource out = new AsyncResource(); Display.getInstance().scheduleBackgroundTask(new Runnable() { + @Override public void run() { final FileSystemStorage fs = FileSystemStorage.getInstance(); final String directory = fs.getAppHomePath() + "ai-models/"; @@ -80,6 +81,7 @@ public void run() { return; } Display.getInstance().callSerially(new Runnable() { + @Override public void run() { download(out, url, sha256, target, fileName); } @@ -109,8 +111,10 @@ private static void download(final AsyncResource out, request.setUrl(url); request.setDestinationFile(temporary); request.addResponseListener(new ActionListener() { + @Override public void actionPerformed(NetworkEvent event) { Display.getInstance().scheduleBackgroundTask(new Runnable() { + @Override public void run() { try { if (!verify(temporary, sha256)) { @@ -130,6 +134,7 @@ public void run() { } }); ActionListener failure = new ActionListener() { + @Override public void actionPerformed(NetworkEvent event) { if (fs.exists(temporary)) { fs.delete(temporary); @@ -178,6 +183,7 @@ private static String safeName(String value) { private static void complete(final AsyncResource out, final T value) { Display.getInstance().callSerially(new Runnable() { + @Override public void run() { out.complete(value); } @@ -186,6 +192,7 @@ public void run() { private static void fail(final AsyncResource out, final Throwable error) { Display.getInstance().callSerially(new Runnable() { + @Override public void run() { out.error(new InferenceException("Could not cache LiteRT model", error)); } diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelSource.java b/CodenameOne/src/com/codename1/ai/inference/ModelSource.java index f597a3f1fd5..e5c6e6d0c14 100644 --- a/CodenameOne/src/com/codename1/ai/inference/ModelSource.java +++ b/CodenameOne/src/com/codename1/ai/inference/ModelSource.java @@ -67,7 +67,9 @@ public int getKind() { } public byte[] getBytes() { - if (bytes == null) return null; + if (bytes == null) { + return null; + } byte[] out = new byte[bytes.length]; System.arraycopy(bytes, 0, out, 0, bytes.length); return out; diff --git a/CodenameOne/src/com/codename1/ai/inference/Tensor.java b/CodenameOne/src/com/codename1/ai/inference/Tensor.java index 4d0353dacba..a02bec4a623 100644 --- a/CodenameOne/src/com/codename1/ai/inference/Tensor.java +++ b/CodenameOne/src/com/codename1/ai/inference/Tensor.java @@ -75,13 +75,22 @@ public Object getData() { private static void validateData(TensorType type, Object data) { boolean valid; switch (type) { - case FLOAT32: valid = data instanceof float[]; break; - case INT32: valid = data instanceof int[]; break; - case INT64: valid = data instanceof long[]; break; + case FLOAT32: + valid = data instanceof float[]; + break; + case INT32: + valid = data instanceof int[]; + break; + case INT64: + valid = data instanceof long[]; + break; case UINT8: case INT8: - case BOOL: valid = data instanceof byte[]; break; - default: valid = false; + case BOOL: + valid = data instanceof byte[]; + break; + default: + valid = false; } if (!valid) { throw new IllegalArgumentException("Data array does not match " + type); @@ -93,25 +102,35 @@ private static int elementCount(int[] shape) { return 1; } int out = 1; - for (int i = 0; i < shape.length; i++) { - if (shape[i] < 0) { + for (int dimension : shape) { + if (dimension < 0) { return -1; } - out *= shape[i]; + out *= dimension; } return out; } private static int dataLength(Object value) { - if (value instanceof float[]) return ((float[]) value).length; - if (value instanceof int[]) return ((int[]) value).length; - if (value instanceof long[]) return ((long[]) value).length; - if (value instanceof byte[]) return ((byte[]) value).length; + if (value instanceof float[]) { + return ((float[]) value).length; + } + if (value instanceof int[]) { + return ((int[]) value).length; + } + if (value instanceof long[]) { + return ((long[]) value).length; + } + if (value instanceof byte[]) { + return ((byte[]) value).length; + } throw new IllegalArgumentException("Unsupported tensor data array"); } private static int[] copy(int[] value) { - if (value == null) return new int[0]; + if (value == null) { + return new int[0]; + } int[] out = new int[value.length]; System.arraycopy(value, 0, out, 0, value.length); return out; diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java b/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java index 10311cb940b..fd83ab009f3 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java @@ -50,6 +50,7 @@ private Named(String id) { this.id = id; } + @Override public String getId() { return id; } diff --git a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java index 083ae63633d..53aa6cd9a4f 100644 --- a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java +++ b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java @@ -37,11 +37,13 @@ abstract class AbstractVisionAnalyzer implements VisionAnalyzer { this.options = options == null ? new VisionOptions() : options; } + @Override public final boolean isSupported() { VisionImpl impl = implementation(); return impl != null && impl.isSupported(feature, options.getBackend().getId()); } + @Override public final AsyncResource process(VisionImage image) { if (image == null) { throw new NullPointerException("image"); @@ -59,6 +61,7 @@ public final AsyncResource process(VisionImage image) { return impl.analyze(feature, options.getBackend().getId(), image, options); } + @Override public final void close() { closed = true; if (implementation != null) { diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java b/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java index c479a091667..f34c66d8710 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java @@ -28,5 +28,6 @@ public interface VisionAnalyzer extends AutoCloseable { boolean isSupported(); AsyncResource process(VisionImage image); + @Override void close(); } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java b/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java index 191c18415ab..892ba20d904 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java @@ -60,10 +60,12 @@ private NamedBackend(String id) { this.id = id; } + @Override public String getId() { return id; } + @Override public String toString() { return id; } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java index da80f49817d..50b9067b3a6 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java @@ -48,6 +48,7 @@ public VisionPipeline(CameraSession session, VisionAnalyzer analyzer, this.analyzer = analyzer; this.listener = listener; frameListener = new FrameListener() { + @Override public void onFrame(CameraFrame frame) { accept(VisionImage.fromCameraFrame(frame)); } @@ -71,16 +72,20 @@ private void accept(VisionImage image) { private void process(final VisionImage image) { analyzer.process(image).ready(new SuccessCallback() { + @Override public void onSucess(final T value) { onFinished(new Runnable() { + @Override public void run() { listener.result(value, image); } }); } }).except(new SuccessCallback() { + @Override public void onSucess(final Throwable error) { onFinished(new Runnable() { + @Override public void run() { listener.error(error); } @@ -91,6 +96,7 @@ public void run() { private void onFinished(final Runnable notification) { Display.getInstance().callSerially(new Runnable() { + @Override public void run() { VisionImage next; synchronized (VisionPipeline.this) { @@ -117,6 +123,7 @@ public boolean isBusy() { } } + @Override public void close() { synchronized (this) { if (closed) { From 7016de938716074b64414eb7f68e86ca4ab13bd9 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 25 Jul 2026 16:56:07 +0300 Subject: [PATCH 07/80] Exclude optional Android AI sources from port build --- Ports/Android/build.xml | 10 +++++----- Ports/Android/nbproject/project.properties | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Ports/Android/build.xml b/Ports/Android/build.xml index 3ef7fedd75f..d9a9730f8ee 100644 --- a/Ports/Android/build.xml +++ b/Ports/Android/build.xml @@ -110,14 +110,14 @@ - + excludes="com/codename1/impl/android/ar/**,com/codename1/impl/android/ai/**"> diff --git a/Ports/Android/nbproject/project.properties b/Ports/Android/nbproject/project.properties index ae84c3cc82f..a8e7118aaa4 100644 --- a/Ports/Android/nbproject/project.properties +++ b/Ports/Android/nbproject/project.properties @@ -26,11 +26,11 @@ dist.dir=dist dist.jar=${dist.dir}/Android.jar dist.javadoc.dir=${dist.dir}/javadoc endorsed.classpath= -# The ARCore-backed AR impl compiles against com.google.ar.core which is not -# in cn1-binaries; it is compiled inside user app builds where the Android -# builder adds the ARCore gradle dependency (and deletes the package for -# non-AR apps). Mirrors the maven-compiler exclude in maven/android/pom.xml. -excludes=com/codename1/impl/android/ar/** +# Optional AR and AI implementations compile against dependencies which are +# not in cn1-binaries. They are compiled inside user app builds where the +# Android builder adds only the dependencies and sources used by the app. +# Mirrors the maven-compiler excludes in maven/android/pom.xml. +excludes=com/codename1/impl/android/ar/**,com/codename1/impl/android/ai/** file.reference.android-billing-4.0.0.jar=../../../cn1-binaries/android/android-billing-4.0.0.jar file.reference.android-support-v7-appcompat.jar=../../../cn1-binaries/android/android-support-v7-appcompat.jar file.reference.android-support-v7-cardview.jar=../../../cn1-binaries/android/android-support-v7-cardview.jar From 6ed61e0bb032e7e35ab80126d23748d7e0a6439a Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 25 Jul 2026 18:21:42 +0300 Subject: [PATCH 08/80] Fix ML Kit builds on Apple Silicon simulators --- docs/ai-on-device-architecture.md | 10 +++++++ .../com/codename1/builders/IPhoneBuilder.java | 28 +++++++++++++++---- .../build/shared/PlatformFeatureCatalog.java | 24 ++++++++++++++++ .../shared/PlatformFeatureCatalogTest.java | 6 ++++ scripts/run-ios-native-tests.sh | 13 +++++++++ scripts/run-ios-ui-tests.sh | 18 ++++++++++-- 6 files changed, 91 insertions(+), 8 deletions(-) diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md index c24c58871b2..d603139533f 100644 --- a/docs/ai-on-device-architecture.md +++ b/docs/ai-on-device-architecture.md @@ -107,6 +107,16 @@ Catalyst. The builders omit those package dependencies from a Mac-native build; language and inference consequently use their explicit unsupported stubs there. +Google ML Kit's iOS binary frameworks contain device `arm64` and simulator +`x86_64` slices, but no `arm64` simulator slice. Catalog entries declare that +constraint independently from Catalyst support. When one of those entries is +selected, both builders set +`EXCLUDED_ARCHS[sdk=iphonesimulator*]=arm64` on the generated application and +Pods projects. The repository's iOS UI and native-test runners also select +`ARCHS=x86_64`, so Apple Silicon hosts use the supported simulator slice +without requiring an application build hint. TensorFlow Lite's XCFramework +does include an `arm64` simulator slice and does not trigger this fallback. + ## Image and camera contract `VisionImage` owns defensive copies of its input. It accepts encoded JPEG or diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java index 940ec0ad3e5..ea2182654dc 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java @@ -799,6 +799,7 @@ public boolean build(File sourceZip, BuildRequest request) throws BuildException // SPM specs, plist defaults and Android perms -- so the user // doesn't have to declare them by hand. final PlatformFeatureCatalog.Accumulator aiAcc = new PlatformFeatureCatalog.Accumulator(); + boolean excludeArm64Simulator = false; try { scanClassesForPermissions(classesDir, new Executor.ClassScanner() { @@ -1034,14 +1035,19 @@ public void usesClassMethod(String cls, String method) { boolean projectPrefersSpm = dependencyConfig.usesSwiftPackages() && !dependencyConfig.usesCocoaPods(); StringBuilder spmPackages = new StringBuilder(request.getArg("ios.spm.packages", "")); for (PlatformFeatureCatalog.Entry entry : aiAcc.hits()) { - // Google ML Kit and TensorFlowLiteObjC publish iOS/device and - // simulator slices, but not Mac Catalyst slices. Keep the - // framework-only Apple Vision implementation enabled for - // macNative while leaving language/inference on their safe - // unsupported native stubs. + // Third-party AI packages may omit Catalyst or arm64 + // simulator slices. Keep framework-only Apple Vision enabled + // for macNative and record the simulator constraint for the + // generated Xcode and Pods projects. boolean includeApplePackageDependencies = !macNativeBuilder.isEnabled() || entry.iosDependenciesSupportMacCatalyst(); + if (includeApplePackageDependencies + && !entry.iosDependenciesSupportArm64Simulator() + && (!entry.iosPods().isEmpty() + || !entry.iosSpmSpecs().isEmpty())) { + excludeArm64Simulator = true; + } boolean handledViaSpm = false; if (includeApplePackageDependencies && projectPrefersSpm @@ -3117,6 +3123,9 @@ public void usesClassMethod(String cls, String method) { targetStr = "8.0"; } addMinDeploymentTarget(targetStr); + String simulatorArchitectureSettings = excludeArm64Simulator + ? " config.build_settings['EXCLUDED_ARCHS[sdk=iphonesimulator*]'] = 'arm64'\n" + : ""; deploymentTargetStr = "begin\n" + " xcproj.targets.find{|e|e.name=='" + request.getMainClass() + "'}.build_configurations.each{|config| \n" + " config.build_settings['PRODUCT_BUNDLE_IDENTIFIER']='"+request.getPackageName()+"'\n" @@ -3133,6 +3142,7 @@ public void usesClassMethod(String cls, String method) { + " next if target.respond_to?(:product_type) && target.product_type == 'com.apple.product-type.app-extension'\n" + " target.build_configurations.each do |config|\n" + " config.build_settings['IPHONEOS_DEPLOYMENT_TARGET'] = '" + getDeploymentTarget(request) + "'\n" + + simulatorArchitectureSettings + " end\n" + " end\n" + " xcproj.save\n" @@ -3484,6 +3494,14 @@ public void usesClassMethod(String cls, String method) { if (useMetal) { buildSettings += " config.build_settings['CLANG_ENABLE_MODULES'] = \"YES\"\n"; } + if (excludeArm64Simulator) { + // Google ML Kit's binary frameworks contain device + // arm64 and simulator x86_64 slices. Apply the same + // exclusion to every pod target so CocoaPods doesn't + // drop its conflicting per-pod setting while merging + // the aggregate xcconfig. + buildSettings += " config.build_settings['EXCLUDED_ARCHS[sdk=iphonesimulator*]'] = \"arm64\"\n"; + } podFileContents += "\n\npost_install do |installer|\n" + diff --git a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java index f082b4c91dd..18765acb17f 100644 --- a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java +++ b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java @@ -91,6 +91,7 @@ public final class PlatformFeatureCatalog { .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") .iosPod("GoogleMLKit/TextRecognition") .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() .description("ML Kit iOS text-recognition backend")); e.add(new Entry("com/codename1/ai/vision/BarcodeScanner") @@ -101,6 +102,7 @@ public final class PlatformFeatureCatalog { .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") .iosPod("GoogleMLKit/BarcodeScanning") .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() .description("ML Kit iOS barcode backend")); e.add(new Entry("com/codename1/ai/vision/FaceDetector") @@ -111,6 +113,7 @@ public final class PlatformFeatureCatalog { .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") .iosPod("GoogleMLKit/FaceDetection") .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() .description("ML Kit iOS face-detection backend")); e.add(new Entry("com/codename1/ai/vision/ImageLabeler") @@ -121,6 +124,7 @@ public final class PlatformFeatureCatalog { .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") .iosPod("GoogleMLKit/ImageLabeling") .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() .description("ML Kit iOS image-labeling backend")); e.add(new Entry("com/codename1/ai/vision/PoseDetector") @@ -131,6 +135,7 @@ public final class PlatformFeatureCatalog { .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") .iosPod("GoogleMLKit/PoseDetection") .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() .description("ML Kit iOS pose-detection backend")); e.add(new Entry("com/codename1/ai/vision/SelfieSegmenter") @@ -142,6 +147,7 @@ public final class PlatformFeatureCatalog { .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") .iosPod("GoogleMLKit/SegmentationSelfie") .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() .description("ML Kit iOS selfie-segmentation backend")); e.add(new Entry("com/codename1/ai/vision/DocumentScanner") @@ -161,16 +167,19 @@ public final class PlatformFeatureCatalog { e.add(new Entry("com/codename1/ai/language/LanguageIdentifier") .iosPod("GoogleMLKit/LanguageID") .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() .androidGradle("com.google.mlkit:language-id:17.0.6") .description("On-device language identification")); e.add(new Entry("com/codename1/ai/language/Translator") .iosPod("GoogleMLKit/Translate") .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() .androidGradle("com.google.mlkit:translate:17.0.3") .description("On-device translation")); e.add(new Entry("com/codename1/ai/language/SmartReply") .iosPod("GoogleMLKit/SmartReply") .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() .androidGradle("com.google.mlkit:smart-reply:17.0.4") .description("On-device smart reply")); @@ -336,6 +345,7 @@ public static final class Entry { private final List androidFeatures = new ArrayList(); private final List androidMetaData = new ArrayList(); private boolean iosDependenciesSupportMacCatalyst = true; + private boolean iosDependenciesSupportArm64Simulator = true; private boolean requiresBigUpload; private String description = ""; @@ -398,6 +408,11 @@ Entry iosDependenciesUnsupportedOnMacCatalyst() { return this; } + Entry iosDependenciesUnsupportedOnArm64Simulator() { + iosDependenciesSupportArm64Simulator = false; + return this; + } + Entry iosFrameworks(String... fws) { for (String f : fws) { iosFrameworks.add(f); @@ -464,6 +479,15 @@ public boolean iosDependenciesSupportMacCatalyst() { return iosDependenciesSupportMacCatalyst; } + /** + * Whether this entry's CocoaPod/SPM payload has an arm64 iOS + * Simulator slice. Dependencies that return {@code false} still + * support the x86_64 simulator. + */ + public boolean iosDependenciesSupportArm64Simulator() { + return iosDependenciesSupportArm64Simulator; + } + public List iosFrameworks() { return Collections.unmodifiableList(iosFrameworks); } diff --git a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java index f3b60f82584..511ea3a58ba 100644 --- a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java +++ b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java @@ -56,6 +56,10 @@ void explicitMlKitBackendAddsOnlyUsedFeaturePod() { foundTextPod |= e.iosPods().contains("GoogleMLKit/TextRecognition"); assertFalse(e.iosPods().contains("GoogleMLKit/FaceDetection"), "Unused vision features must not be bundled"); + if (!e.iosPods().isEmpty()) { + assertFalse(e.iosDependenciesSupportArm64Simulator(), + "Google ML Kit binaries require the x86_64 iOS simulator"); + } } assertTrue(foundTextPod); } @@ -191,6 +195,8 @@ void inferenceUsesCoreMlEnabledObjectiveCPod() { assertTrue(e.iosSpmSpecs().isEmpty()); assertFalse(e.iosDependenciesSupportMacCatalyst(), "The official TensorFlow Lite Objective-C XCFramework has no Catalyst slice"); + assertTrue(e.iosDependenciesSupportArm64Simulator(), + "TensorFlow Lite's XCFramework includes an arm64 simulator slice"); } @Test diff --git a/scripts/run-ios-native-tests.sh b/scripts/run-ios-native-tests.sh index a194c170dd8..f1238f94fe7 100755 --- a/scripts/run-ios-native-tests.sh +++ b/scripts/run-ios-native-tests.sh @@ -209,6 +209,18 @@ fi if [ -n "${CN1_TEST_OPT_LEVEL:-}" ]; then DERIVED_ARGS+=("GCC_OPTIMIZATION_LEVEL=$CN1_TEST_OPT_LEVEL") fi +SIMULATOR_ARCH_ARGS=() +PODFILE_LOCK="$PROJECT_DIR/Podfile.lock" +if [ -f "$PODFILE_LOCK" ] && grep -q "GoogleMLKit/" "$PODFILE_LOCK"; then + # Match the UI build and the generated project: Google ML Kit currently + # supplies an x86_64 simulator slice, not an arm64 simulator slice. + SIMULATOR_ARCH_ARGS=( + "ARCHS=x86_64" + "ONLY_ACTIVE_ARCH=YES" + "EXCLUDED_ARCHS=arm64 armv7 armv7s" + ) + ri_log "Google ML Kit detected; selecting its supported x86_64 simulator slice" +fi ri_log "Running xcodebuild test (scheme=$TEST_SCHEME, destination=$DESTINATION)" set +e xcodebuild \ @@ -216,6 +228,7 @@ xcodebuild \ -scheme "$TEST_SCHEME" \ -destination "$DESTINATION" \ ${DERIVED_ARGS[@]+"${DERIVED_ARGS[@]}"} \ + ${SIMULATOR_ARCH_ARGS[@]+"${SIMULATOR_ARCH_ARGS[@]}"} \ test | tee "$TEST_LOG" RC=${PIPESTATUS[0]} set -e diff --git a/scripts/run-ios-ui-tests.sh b/scripts/run-ios-ui-tests.sh index 4dec3acb7f2..350224c7890 100755 --- a/scripts/run-ios-ui-tests.sh +++ b/scripts/run-ios-ui-tests.sh @@ -625,6 +625,18 @@ case "$HOST_ARCH" in arm64|x86_64) BUILD_ARCH="$HOST_ARCH" ;; *) BUILD_ARCH="arm64" ;; esac +SIMULATOR_EXCLUDED_ARCHS="armv7 armv7s" +FORCE_SIMULATOR_ARCH="false" +PODFILE_LOCK="$(dirname "$WORKSPACE_PATH")/Podfile.lock" +if [ -f "$PODFILE_LOCK" ] && grep -q "GoogleMLKit/" "$PODFILE_LOCK"; then + # Google ML Kit's iOS binaries contain a device arm64 slice and an + # x86_64 simulator slice, but no arm64 simulator slice. Xcode otherwise + # selects arm64 on Apple Silicon and the linker rejects the device object. + BUILD_ARCH="x86_64" + SIMULATOR_EXCLUDED_ARCHS="arm64 armv7 armv7s" + FORCE_SIMULATOR_ARCH="true" + ri_log "Google ML Kit detected; selecting its supported x86_64 simulator slice" +fi # A shared/stable derived-data dir can be supplied via CN1_IOS_DERIVED_DATA so the compiled # app + core are reused by a later step in the same job (e.g. the native test build) and across @@ -652,12 +664,12 @@ XCODE_BUILD_CMD=( -destination-timeout 120 -derivedDataPath "$DERIVED_DATA_DIR" ) -if [ "$USE_GENERIC_BUILD_DESTINATION" = "true" ]; then - ri_log "Forcing simulator ARCHS=$BUILD_ARCH for generic build destination" +if [ "$USE_GENERIC_BUILD_DESTINATION" = "true" ] || [ "$FORCE_SIMULATOR_ARCH" = "true" ]; then + ri_log "Forcing simulator ARCHS=$BUILD_ARCH" XCODE_BUILD_CMD+=( "ARCHS=$BUILD_ARCH" "ONLY_ACTIVE_ARCH=YES" - "EXCLUDED_ARCHS=armv7 armv7s" + "EXCLUDED_ARCHS=$SIMULATOR_EXCLUDED_ARCHS" ) fi # Optimize the translated C (Xcode's Debug config defaults to -O0). With -O0 the From a4484891e61c90d9161f4ebfa09e096496e77c18 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 25 Jul 2026 20:27:59 +0300 Subject: [PATCH 09/80] Support SIMD fallback on x86 iOS simulators --- Ports/iOSPort/nativeSources/IOSSimd.m | 127 ++++++++++++++++++ .../src/com/codename1/impl/ios/IOSSimd.java | 6 +- docs/ai-on-device-architecture.md | 3 + 3 files changed, 134 insertions(+), 2 deletions(-) diff --git a/Ports/iOSPort/nativeSources/IOSSimd.m b/Ports/iOSPort/nativeSources/IOSSimd.m index a287ec9a562..7b99ca83cf6 100644 --- a/Ports/iOSPort/nativeSources/IOSSimd.m +++ b/Ports/iOSPort/nativeSources/IOSSimd.m @@ -2,8 +2,15 @@ #include #include #include + +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#define CN1_IOS_NEON 1 #include +#else +#define CN1_IOS_NEON 0 +#endif +#if CN1_IOS_NEON static JAVA_ARRAY_BYTE cn1_saturating_byte(int value) { if (value > 127) { return 127; @@ -37,6 +44,7 @@ static uint8x16_t cn1_load_ints_to_u8x16(JAVA_ARRAY_INT* s, int offset) { int8x16_t out = vcombine_s8(vmovn_s16(lo16), vmovn_s16(hi16)); return vreinterpretq_u8_s8(out); } +#endif JAVA_OBJECT com_codename1_impl_ios_IOSSimd_allocByteNative___int_R_byte_1ARRAY(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT size) { return allocArrayAligned(threadStateData, size, &class_array1__JAVA_BYTE, sizeof(JAVA_ARRAY_BYTE), 1, 16); @@ -50,6 +58,11 @@ JAVA_OBJECT com_codename1_impl_ios_IOSSimd_allocFloatNative___int_R_float_1ARRAY return allocArrayAligned(threadStateData, size, &class_array1__JAVA_FLOAT, sizeof(JAVA_ARRAY_FLOAT), 1, 16); } +JAVA_BOOLEAN com_codename1_impl_ios_IOSSimd_isNativeSupported___R_boolean(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject) { + return CN1_IOS_NEON ? JAVA_TRUE : JAVA_FALSE; +} + +#if CN1_IOS_NEON JAVA_VOID com_codename1_impl_ios_IOSSimd_lookupBytes___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT table, JAVA_OBJECT indices, JAVA_OBJECT dst, JAVA_INT offset, JAVA_INT length) { JAVA_ARRAY_BYTE* t = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)table)->data; JAVA_ARRAY_BYTE* idx = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)indices)->data; @@ -1777,3 +1790,117 @@ JAVA_VOID com_codename1_impl_ios_IOSSimd_replaceTopByteFromUnsignedBytes___int_1 d[dstOffset + i] = (JAVA_INT)((r[rgbSrcOffset + i] & 0x00ffffff) | ((a[alphaSrcOffset + i] & 0xff) << 24)); } } +#else +/* + * Google ML Kit still requires an x86_64 simulator build on Apple Silicon. + * IOSSimd is NEON-specific, so expose the generic Simd implementations under + * the native iOS symbol names on non-NEON architectures. Mach-O indirect + * symbols preserve the exact generated native ABI without duplicating the + * scalar implementations or adding a second native source file. + */ +#define CN1_IOS_SIMD_ALIAS(name) \ + __asm__(".globl _com_codename1_impl_ios_IOSSimd_" #name "\n" \ + ".set _com_codename1_impl_ios_IOSSimd_" #name \ + ", _com_codename1_util_Simd_" #name); + +CN1_IOS_SIMD_ALIAS(lookupBytes___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(lookupBytes___byte_1ARRAY_byte_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(add___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(sub___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(mul___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(min___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(max___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(abs___byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(clamp___byte_1ARRAY_byte_1ARRAY_byte_byte_int_int) +CN1_IOS_SIMD_ALIAS(add___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(sub___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(mul___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(min___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(max___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(abs___int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(clamp___int_1ARRAY_int_1ARRAY_int_int_int_int) +CN1_IOS_SIMD_ALIAS(sum___int_1ARRAY_int_int_R_int) +CN1_IOS_SIMD_ALIAS(dot___int_1ARRAY_int_1ARRAY_int_int_R_int) +CN1_IOS_SIMD_ALIAS(add___float_1ARRAY_float_1ARRAY_float_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(sub___float_1ARRAY_float_1ARRAY_float_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(mul___float_1ARRAY_float_1ARRAY_float_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(min___float_1ARRAY_float_1ARRAY_float_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(max___float_1ARRAY_float_1ARRAY_float_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(abs___float_1ARRAY_float_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(clamp___float_1ARRAY_float_1ARRAY_float_float_int_int) +CN1_IOS_SIMD_ALIAS(sum___float_1ARRAY_int_int_R_float) +CN1_IOS_SIMD_ALIAS(dot___float_1ARRAY_float_1ARRAY_int_int_R_float) +CN1_IOS_SIMD_ALIAS(and___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(and___byte_1ARRAY_int_byte_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(or___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(or___byte_1ARRAY_int_byte_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(xor___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(not___byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpEq___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpEq___byte_1ARRAY_byte_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpLt___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpLt___byte_1ARRAY_byte_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpGt___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpRange___byte_1ARRAY_byte_byte_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(select___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(unpackUnsignedByteToInt___byte_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packIntToByteSaturating___int_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packIntToByteTruncate___int_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packIntToByteTruncate___int_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packIntToByteTruncateInterleaved4___int_1ARRAY_int_int_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packBytesInterleaved3___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packBytesInterleaved3___byte_1ARRAY_int_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packBytesInterleaved4___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packBytesInterleaved4___byte_1ARRAY_int_int_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(permuteBytes___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(and___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(and___int_1ARRAY_int_int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(or___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(or___int_1ARRAY_int_int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(xor___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(not___int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shl___int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shl___int_1ARRAY_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shrLogical___int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shrLogical___int_1ARRAY_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shrArithmetic___int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpEq___int_1ARRAY_int_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpLt___int_1ARRAY_int_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpGt___int_1ARRAY_int_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(select___byte_1ARRAY_int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shl___byte_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shl___byte_1ARRAY_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shrLogical___byte_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shrLogical___byte_1ARRAY_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(addWrapping___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(addWrapping___byte_1ARRAY_byte_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(subWrapping___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(subWrapping___byte_1ARRAY_byte_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(unpackUnsignedByteToInt___byte_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(unpackUnsignedByteToIntInterleaved3___byte_1ARRAY_int_int_1ARRAY_int_int_int_int) +CN1_IOS_SIMD_ALIAS(unpackBytesInterleaved3___byte_1ARRAY_int_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int) +CN1_IOS_SIMD_ALIAS(unpackBytesInterleaved3___byte_1ARRAY_int_byte_1ARRAY_int_int_int_int) +CN1_IOS_SIMD_ALIAS(unpackBytesInterleaved4___byte_1ARRAY_int_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int) +CN1_IOS_SIMD_ALIAS(unpackBytesInterleaved4___byte_1ARRAY_int_byte_1ARRAY_int_int_int_int_int) +CN1_IOS_SIMD_ALIAS(unpackLookupBytesInterleaved4___byte_1ARRAY_byte_1ARRAY_int_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_R_int) +CN1_IOS_SIMD_ALIAS(unpackLookupBytesInterleaved4___byte_1ARRAY_byte_1ARRAY_int_byte_1ARRAY_int_int_int_int_int_R_int) +CN1_IOS_SIMD_ALIAS(add___int_1ARRAY_int_int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpEq___int_1ARRAY_int_int_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpLt___int_1ARRAY_int_int_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(select___byte_1ARRAY_int_int_1ARRAY_int_int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(and___int_1ARRAY_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(or___int_1ARRAY_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(xor___int_1ARRAY_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpEq___int_1ARRAY_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpLt___int_1ARRAY_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpGt___int_1ARRAY_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(not___byte_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(select___byte_1ARRAY_int_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(select___byte_1ARRAY_int_int_1ARRAY_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(select___byte_1ARRAY_int_int_int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(blendByMaskTestNonzero___int_1ARRAY_int_int_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(blendByMaskTestNonzeroSubstituteOnKeepEq___int_1ARRAY_int_int_int_int_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(replaceTopByteFromUnsignedBytes___int_1ARRAY_int_byte_1ARRAY_int_int_1ARRAY_int_int) + +#undef CN1_IOS_SIMD_ALIAS +#endif diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSSimd.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSSimd.java index 759376a6a90..c7d454411df 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSSimd.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSSimd.java @@ -11,16 +11,18 @@ public class IOSSimd extends Simd { @Override public boolean isSupported() { - return true; + return isNativeSupported(); } /// NEON: the chained byte-shuffle codec (Base64) is ~75-83% faster than the /// autovectorized scalar even at -O2, so the codec should use the SIMD path. @Override public boolean isByteShuffleAccelerated() { - return true; + return isNativeSupported(); } + private native boolean isNativeSupported(); + @Override public byte[] allocByte(int size) { if (size < 16) { diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md index d603139533f..c9e73ef28b1 100644 --- a/docs/ai-on-device-architecture.md +++ b/docs/ai-on-device-architecture.md @@ -116,6 +116,9 @@ Pods projects. The repository's iOS UI and native-test runners also select `ARCHS=x86_64`, so Apple Silicon hosts use the supported simulator slice without requiring an application build hint. TensorFlow Lite's XCFramework does include an `arm64` simulator slice and does not trigger this fallback. +The iOS NEON implementation reports SIMD as unsupported in that x86_64 +configuration and aliases its native entry points to the generic scalar +implementation; device and arm64-simulator builds retain the NEON path. ## Image and camera contract From 6fe8457c0e480a1e68c568054d8bf98b596bf7aa Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sat, 25 Jul 2026 20:32:20 +0300 Subject: [PATCH 10/80] Add full headers to iOS SIMD sources --- Ports/iOSPort/nativeSources/IOSSimd.m | 23 +++++++++++++++++++ .../src/com/codename1/impl/ios/IOSSimd.java | 19 +++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/Ports/iOSPort/nativeSources/IOSSimd.m b/Ports/iOSPort/nativeSources/IOSSimd.m index 7b99ca83cf6..23eca3c9836 100644 --- a/Ports/iOSPort/nativeSources/IOSSimd.m +++ b/Ports/iOSPort/nativeSources/IOSSimd.m @@ -1,3 +1,26 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + #include "xmlvm.h" #include #include diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSSimd.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSSimd.java index c7d454411df..9884d1adc30 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSSimd.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSSimd.java @@ -1,5 +1,24 @@ /* * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.ios; From 7b9b250d82098ea1f5dc60113df376aae78b72fb Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 26 Jul 2026 02:27:03 +0300 Subject: [PATCH 11/80] Stabilize iOS FAB screenshot comparison --- .../screenshots/FloatingActionButtonTheme_light.tolerance | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 scripts/ios/screenshots/FloatingActionButtonTheme_light.tolerance diff --git a/scripts/ios/screenshots/FloatingActionButtonTheme_light.tolerance b/scripts/ios/screenshots/FloatingActionButtonTheme_light.tolerance new file mode 100644 index 00000000000..ea902126126 --- /dev/null +++ b/scripts/ios/screenshots/FloatingActionButtonTheme_light.tolerance @@ -0,0 +1,7 @@ +# The floating action button shadow has run-to-run anti-aliasing jitter on the +# iOS GL simulator. Three independent runs differed in at most 4 channel values +# across 0.279% of the frame while all geometry and the other 167 screenshots +# matched. Keep the pre-tightening gate for this screen only; a real layout, +# color, or missing-button regression still exceeds these narrow bounds. +maxChannelDelta=4 +maxMismatchPercent=0.30 From 6d709a54dd4445744c4ea91016ede1b6e9d0149f Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 26 Jul 2026 03:34:22 +0300 Subject: [PATCH 12/80] Allow slow Metal benchmark progress --- .github/workflows/scripts-ios.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/scripts-ios.yml b/.github/workflows/scripts-ios.yml index fee02d9c421..be9fd8bac5e 100644 --- a/.github/workflows/scripts-ios.yml +++ b/.github/workflows/scripts-ios.yml @@ -464,6 +464,14 @@ jobs: CN1SS_REPORT_TITLE: 'iOS Metal screenshot updates' CN1SS_SUCCESS_MESSAGE: '✅ Native iOS Metal screenshot tests passed.' CN1SS_COMMENT_LOG_PREFIX: '[run-ios-device-tests-metal]' + # The x86 simulator runs CommonWorkloadBenchmarkTest through + # ParparVM's non-NEON path. Its object-allocation block took 555s on + # an otherwise identical successful run, but exceeded the generic + # 720s idle gate on a contended runner while the hang sample showed + # active GC/allocation work. Allow that benchmark variance here; + # the suite's independent 35-minute absolute limit still catches a + # genuinely stuck app. + CN1SS_SUITE_IDLE_TIMEOUT_SECONDS: '1200' # Strict screenshot gating: fail on any pixel mismatch and on any # missing screenshot. The Metal backend must produce the full # baseline set; a test that hangs (e.g. the DialogTheme texture From c1f61e498256fb1430f976181b74553a38caa48f Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 26 Jul 2026 04:59:00 +0300 Subject: [PATCH 13/80] Allow slow GL simulator benchmark --- .github/workflows/scripts-ios.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/scripts-ios.yml b/.github/workflows/scripts-ios.yml index be9fd8bac5e..f3555202f5c 100644 --- a/.github/workflows/scripts-ios.yml +++ b/.github/workflows/scripts-ios.yml @@ -240,6 +240,13 @@ jobs: CN1SS_FAIL_ON_MISMATCH: '1' CN1SS_ALLOWED_MISSING: '0' CN1SS_PORT_ID: ios-gl + # Google ML Kit excludes arm64 simulators, so this job uses ParparVM's + # x86 path. On a contended runner the allocation benchmark remained + # active after the generic 720s idle limit, with less than three + # minutes left under the 35-minute absolute cap. Keep strict missing- + # screenshot gating, but allow the active benchmark to complete. + CN1SS_SUITE_IDLE_TIMEOUT_SECONDS: '1200' + CN1SS_SUITE_TIMEOUT_SECONDS: '3000' run: | set -euo pipefail mkdir -p "${ARTIFACTS_DIR}" @@ -251,9 +258,9 @@ jobs: "${{ steps.build-ios-app.outputs.workspace }}" \ "" \ "${{ steps.build-ios-app.outputs.scheme }}" - # The script enforces both a 35-minute absolute suite limit and a - # 12-minute no-progress limit. This outer cap also includes its - # native rebuild and cleanup, so it must leave those phases headroom. + # This job uses a 50-minute absolute suite limit and a 20-minute + # no-progress limit. The outer cap also includes its native rebuild + # and cleanup, so it leaves those phases headroom. timeout-minutes: 65 - name: Upload iOS OpenGL port status From e78d0c0ec9e998a79dfb5c20d5fca0d58e93d419 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 26 Jul 2026 17:40:56 +0300 Subject: [PATCH 14/80] Resolve on-device AI review feedback --- .../src/com/codename1/ai/ChatMessage.java | 27 + .../src/com/codename1/ai/ChatRequest.java | 58 + .../src/com/codename1/ai/ChatResponse.java | 12 + .../com/codename1/ai/ConversationStore.java | 11 + .../src/com/codename1/ai/Embedding.java | 8 + .../com/codename1/ai/EmbeddingRequest.java | 21 +- .../com/codename1/ai/EmbeddingResponse.java | 9 + .../codename1/ai/GenerateImageRequest.java | 20 + .../src/com/codename1/ai/ImageGenerator.java | 9 + .../src/com/codename1/ai/ImagePart.java | 11 + .../src/com/codename1/ai/LlmChatBinding.java | 6 + .../src/com/codename1/ai/LlmClient.java | 35 + .../src/com/codename1/ai/LlmException.java | 23 + .../src/com/codename1/ai/PromptTemplate.java | 13 + .../src/com/codename1/ai/RetryPolicy.java | 21 +- .../src/com/codename1/ai/SafetyFilter.java | 2 + .../com/codename1/ai/StreamingListener.java | 7 + .../src/com/codename1/ai/TextPart.java | 4 + CodenameOne/src/com/codename1/ai/Tool.java | 16 + .../src/com/codename1/ai/ToolCall.java | 12 + .../src/com/codename1/ai/ToolChoice.java | 3 + .../src/com/codename1/ai/ToolResultPart.java | 7 + CodenameOne/src/com/codename1/ai/Usage.java | 10 + .../ai/inference/InferenceException.java | 8 +- .../ai/inference/InferenceOptions.java | 25 +- .../ai/inference/InferenceSession.java | 34 +- .../codename1/ai/inference/ModelCache.java | 115 +- .../codename1/ai/inference/ModelSource.java | 17 +- .../com/codename1/ai/inference/Tensor.java | 39 +- .../codename1/ai/inference/TensorInfo.java | 13 +- .../ai/language/LanguageBackend.java | 5 +- .../ai/language/LanguageBackends.java | 27 +- .../ai/language/LanguageCandidate.java | 7 +- .../ai/language/LanguageIdentifier.java | 11 +- .../ai/language/LanguageOptions.java | 10 +- .../com/codename1/ai/language/SmartReply.java | 9 +- .../ai/language/SmartReplyMessage.java | 9 + .../com/codename1/ai/language/Translator.java | 11 +- .../codename1/ai/language/package-info.java | 10 +- .../ai/vision/AbstractVisionAnalyzer.java | 5 + .../src/com/codename1/ai/vision/Barcode.java | 25 +- .../codename1/ai/vision/BarcodeScanner.java | 4 + .../ai/vision/DocumentScanResult.java | 15 +- .../codename1/ai/vision/DocumentScanner.java | 4 + .../src/com/codename1/ai/vision/Face.java | 25 + .../com/codename1/ai/vision/FaceDetector.java | 4 + .../com/codename1/ai/vision/ImageLabel.java | 18 +- .../com/codename1/ai/vision/ImageLabeler.java | 4 + .../src/com/codename1/ai/vision/Pose.java | 20 +- .../com/codename1/ai/vision/PoseDetector.java | 6 +- .../codename1/ai/vision/SegmentationMask.java | 17 +- .../codename1/ai/vision/SelfieSegmenter.java | 4 + .../ai/vision/TextRecognitionResult.java | 25 +- .../codename1/ai/vision/TextRecognizer.java | 4 + .../codename1/ai/vision/VisionAnalyzer.java | 10 +- .../codename1/ai/vision/VisionBackend.java | 15 +- .../codename1/ai/vision/VisionBackends.java | 54 +- .../codename1/ai/vision/VisionException.java | 8 + .../com/codename1/ai/vision/VisionImage.java | 46 +- .../codename1/ai/vision/VisionMetadata.java | 14 +- .../codename1/ai/vision/VisionOptions.java | 15 +- .../codename1/ai/vision/VisionPipeline.java | 34 +- .../com/codename1/ai/vision/VisionPoint.java | 9 +- .../com/codename1/ai/vision/VisionRect.java | 13 +- .../impl/CodenameOneImplementation.java | 10 - .../ai/AndroidBarcodeScanningAdapter.java | 10 +- .../ai/AndroidFaceDetectionAdapter.java | 22 +- .../ai/AndroidImageLabelingAdapter.java | 20 +- .../impl/android/ai/AndroidInferenceImpl.java | 52 +- .../ai/AndroidPoseDetectionAdapter.java | 16 +- .../ai/AndroidSelfieSegmentationAdapter.java | 17 +- .../ai/AndroidTextRecognitionAdapter.java | 12 +- .../android/ai/AndroidTranslationAdapter.java | 8 +- .../impl/android/ai/AndroidVisionAdapter.java | 9 +- .../impl/android/ai/AndroidVisionImpl.java | 32 +- Ports/iOSPort/nativeSources/CN1Inference.m | 167 +- Ports/iOSPort/nativeSources/CN1Language.m | 46 +- Ports/iOSPort/nativeSources/CN1Vision.m | 123 +- .../codename1/impl/ios/IOSImplementation.java | 6 +- .../codename1/impl/ios/IOSInferenceImpl.java | 161 +- .../codename1/impl/ios/IOSLanguageImpl.java | 19 +- .../src/com/codename1/impl/ios/IOSNative.java | 11 +- .../com/codename1/impl/ios/IOSVisionImpl.java | 43 +- docs/ai-on-device-architecture.md | 23 +- docs/developer-guide/Ai-And-Speech.asciidoc | 26 +- .../com/codename1/builders/IPhoneBuilder.java | 47 +- .../IPhoneBuilderDependencyConfigTest.java | 24 + .../ai/inference/InferenceApiTest.java | 58 +- .../ai/language/LanguageApiTest.java | 2 +- .../codename1/ai/vision/VisionApiTest.java | 7 +- .../build/shared/PlatformFeatureCatalog.java | 40 +- .../shared/PlatformFeatureCatalogTest.java | 27 +- scripts/gen-ai-cn1libs.py | 1551 ----------------- .../tests/VisionOnDeviceApiTest.java | 35 +- .../skill/references/ai-and-speech.md | 8 +- 95 files changed, 1752 insertions(+), 1973 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/ChatMessage.java b/CodenameOne/src/com/codename1/ai/ChatMessage.java index 5ddc222b77a..98464eee911 100644 --- a/CodenameOne/src/com/codename1/ai/ChatMessage.java +++ b/CodenameOne/src/com/codename1/ai/ChatMessage.java @@ -39,10 +39,19 @@ public final class ChatMessage { private final String name; private final String toolCallId; + /// Creates a message with no assistant tool calls or provider name. + /// @param role speaker role; required + /// @param parts ordered multimodal content; {@code null} becomes empty public ChatMessage(Role role, List parts) { this(role, parts, null, null, null); } + /// Creates a fully specified normalized message. + /// @param role speaker role; required + /// @param parts ordered content parts; {@code null} becomes empty + /// @param toolCalls assistant tool calls; {@code null} becomes empty + /// @param name optional provider-visible participant name + /// @param toolCallId tool call answered by a {@link Role#TOOL} message public ChatMessage(Role role, List parts, List toolCalls, String name, String toolCallId) { if (role == null) { @@ -57,20 +66,29 @@ public ChatMessage(Role role, List parts, List toolCalls, this.toolCallId = toolCallId; } + /// @param text system instruction text + /// @return a single-part system message public static ChatMessage system(String text) { return single(Role.SYSTEM, new TextPart(text)); } + /// @param text user input text + /// @return a single-part user message public static ChatMessage user(String text) { return single(Role.USER, new TextPart(text)); } + /// @param text assistant output text + /// @return a single-part assistant message public static ChatMessage assistant(String text) { return single(Role.ASSISTANT, new TextPart(text)); } /// Builds a USER message containing both a text and image part -- /// the common multi-modal pattern. + /// @param text optional text accompanying the image + /// @param image image content sent to the model + /// @return multimodal user message public static ChatMessage userWithImage(String text, ImagePart image) { List parts = new ArrayList(2); if (text != null && text.length() > 0) { @@ -81,6 +99,9 @@ public static ChatMessage userWithImage(String text, ImagePart image) { } /// Builds a TOOL message wrapping the result of a previous tool call. + /// @param toolCallId provider id of the answered tool call + /// @param resultJson application result encoded as JSON + /// @return tool-role result message public static ChatMessage toolResult(String toolCallId, String resultJson) { return new ChatMessage(Role.TOOL, Arrays.asList(new ToolResultPart(toolCallId, resultJson)), @@ -93,22 +114,27 @@ private static ChatMessage single(Role r, MessagePart p) { return new ChatMessage(r, parts); } + /// @return speaker role public Role getRole() { return role; } + /// @return immutable content parts in message order public List getParts() { return parts; } + /// @return immutable assistant tool calls public List getToolCalls() { return toolCalls; } + /// @return optional participant name, or {@code null} public String getName() { return name; } + /// @return answered tool-call id for tool messages, or {@code null} public String getToolCallId() { return toolCallId; } @@ -116,6 +142,7 @@ public String getToolCallId() { /// Convenience: concatenates the text of every [TextPart]. Image /// and tool-result parts are skipped. Useful for `ChatView` /// rendering when you don't care about multi-modal content. + /// @return text parts joined with newlines public String getText() { StringBuilder sb = new StringBuilder(); for (MessagePart p : parts) { diff --git a/CodenameOne/src/com/codename1/ai/ChatRequest.java b/CodenameOne/src/com/codename1/ai/ChatRequest.java index d6458e5a323..b0cc8e357fa 100644 --- a/CodenameOne/src/com/codename1/ai/ChatRequest.java +++ b/CodenameOne/src/com/codename1/ai/ChatRequest.java @@ -67,54 +67,68 @@ private ChatRequest(Builder b) { this.safetyFilter = b.safetyFilter; } + /// Starts an empty request builder. At least one message is required. + /// @return a new builder public static Builder builder() { return new Builder(); } + /// @return requested provider model, or {@code null} for the client default public String getModel() { return model; } + /// @return immutable conversation messages in provider order public List getMessages() { return messages; } + /// @return sampling temperature, or {@code null} for the provider default public Float getTemperature() { return temperature; } + /// @return maximum generated tokens, or {@code null} for the provider default public Integer getMaxTokens() { return maxTokens; } + /// @return nucleus-sampling probability, or {@code null} when unspecified public Float getTopP() { return topP; } + /// @return immutable stop sequences; empty when none were requested public List getStopSequences() { return stopSequences; } + /// @return deterministic sampling seed, or {@code null} when unspecified public Long getSeed() { return seed; } + /// @return requested text/JSON response format, or {@code null} public ResponseFormat getResponseFormat() { return responseFormat; } + /// @return immutable tools offered to the model public List getTools() { return tools; } + /// @return tool-selection policy, or {@code null} for provider behavior public ToolChoice getToolChoice() { return toolChoice; } + /// @return immutable provider metadata attached to this request public Map getMetadata() { return metadata; } + /// @return client-side preflight filter, or {@code null} when disabled public SafetyFilter getSafetyFilter() { return safetyFilter; } @@ -138,6 +152,8 @@ public Builder toBuilder() { return b; } + /// Mutable fluent builder for {@link ChatRequest}. Collection arguments + /// are copied when the immutable request is built. public static final class Builder { private String model; private List messages = new ArrayList(); @@ -155,72 +171,114 @@ public static final class Builder { Builder() { } + /// Selects a provider model. + /// @param model provider model id; {@code null} uses the client default + /// @return this builder public Builder model(String model) { this.model = model; return this; } + /// Replaces the conversation history. + /// @param messages messages in chronological order; {@code null} clears them + /// @return this builder public Builder messages(List messages) { this.messages = messages == null ? new ArrayList() : new ArrayList(messages); return this; } + /// Appends one conversation message. + /// @param m message to append + /// @return this builder public Builder addMessage(ChatMessage m) { this.messages.add(m); return this; } + /// Sets sampling temperature. + /// @param t provider-supported temperature, or {@code null} to omit + /// @return this builder public Builder temperature(Float t) { this.temperature = t; return this; } + /// Limits response length. + /// @param n maximum generated token count, or {@code null} to omit + /// @return this builder public Builder maxTokens(Integer n) { this.maxTokens = n; return this; } + /// Sets nucleus sampling probability. + /// @param p provider-supported probability, or {@code null} to omit + /// @return this builder public Builder topP(Float p) { this.topP = p; return this; } + /// Sets sequences that terminate generation. + /// @param stops stop strings, or {@code null} for none + /// @return this builder public Builder stopSequences(List stops) { this.stopSequences = stops; return this; } + /// Requests deterministic sampling where supported. + /// @param seed provider sampling seed, or {@code null} to omit + /// @return this builder public Builder seed(Long seed) { this.seed = seed; return this; } + /// Requests text or structured JSON output. + /// @param f desired format, or {@code null} for provider default + /// @return this builder public Builder responseFormat(ResponseFormat f) { this.responseFormat = f; return this; } + /// Replaces the callable tools advertised to the model. + /// @param tools tool definitions, or {@code null} for none + /// @return this builder public Builder tools(List tools) { this.tools = tools; return this; } + /// Controls whether and which tool the model may call. + /// @param choice selection policy, or {@code null} for provider default + /// @return this builder public Builder toolChoice(ToolChoice choice) { this.toolChoice = choice; return this; } + /// Attaches provider-specific request metadata. + /// @param meta metadata copied into the request, or {@code null} + /// @return this builder public Builder metadata(Map meta) { this.metadata = meta; return this; } + /// Installs a client-side filter evaluated before network submission. + /// @param f safety policy, or {@code null} to disable it + /// @return this builder public Builder safetyFilter(SafetyFilter f) { this.safetyFilter = f; return this; } + /// Validates and creates an immutable request. + /// @return completed request + /// @throws IllegalStateException when no messages were supplied public ChatRequest build() { if (messages.isEmpty()) { throw new IllegalStateException("at least one message is required"); diff --git a/CodenameOne/src/com/codename1/ai/ChatResponse.java b/CodenameOne/src/com/codename1/ai/ChatResponse.java index 53f08263b0a..a60dd24817a 100644 --- a/CodenameOne/src/com/codename1/ai/ChatResponse.java +++ b/CodenameOne/src/com/codename1/ai/ChatResponse.java @@ -40,6 +40,12 @@ public final class ChatResponse { private final Usage usage; private final String modelUsed; + /// Creates a normalized terminal response. + /// @param assistantMessage aggregated assistant output, or {@code null} + /// @param toolCalls requested tool calls; {@code null} becomes empty + /// @param finishReason normalized provider finish reason + /// @param usage token accounting, or {@code null} + /// @param modelUsed provider-reported model id, or {@code null} public ChatResponse(ChatMessage assistantMessage, List toolCalls, String finishReason, Usage usage, String modelUsed) { this.assistantMessage = assistantMessage; @@ -50,28 +56,34 @@ public ChatResponse(ChatMessage assistantMessage, List toolCalls, this.modelUsed = modelUsed; } + /// @return normalized assistant message, including any multimodal parts public ChatMessage getAssistantMessage() { return assistantMessage; } + /// @return immutable tool calls requested by the model public List getToolCalls() { return toolCalls; } /// Convenience: the assembled assistant text. Equivalent to /// `getAssistantMessage().getText()` when there is one. + /// @return assistant text, or an empty string without a message public String getText() { return assistantMessage == null ? "" : assistantMessage.getText(); } + /// @return provider finish reason such as {@code stop} or {@code length} public String getFinishReason() { return finishReason; } + /// @return token accounting, or {@code null} when the provider omitted it public Usage getUsage() { return usage; } + /// @return provider-reported model id, or {@code null} when omitted public String getModelUsed() { return modelUsed; } diff --git a/CodenameOne/src/com/codename1/ai/ConversationStore.java b/CodenameOne/src/com/codename1/ai/ConversationStore.java index a3b72d7191b..3cf8d38baa0 100644 --- a/CodenameOne/src/com/codename1/ai/ConversationStore.java +++ b/CodenameOne/src/com/codename1/ai/ConversationStore.java @@ -45,6 +45,8 @@ public final class ConversationStore { private final String storageKey; + /// Creates a store using one app-private {@link Storage} key. + /// @param storageKey non-empty key dedicated to this conversation public ConversationStore(String storageKey) { if (storageKey == null || storageKey.length() == 0) { throw new IllegalArgumentException("storageKey is required"); @@ -52,6 +54,9 @@ public ConversationStore(String storageKey) { this.storageKey = storageKey; } + /// Replaces the persisted history. + /// @param messages chronological history; {@code null} stores an empty list + /// @throws IOException when JSON encoding or storage fails public void save(List messages) throws IOException { List serialized = new ArrayList(messages == null ? 0 : messages.size()); if (messages != null) { @@ -90,6 +95,10 @@ public void save(List messages) throws IOException { Storage.getInstance().writeObject(storageKey, payload); } + /// Loads persisted history. Missing or incompatible data is treated as an + /// empty conversation so upgrades do not prevent application startup. + /// @return mutable chronological message list + /// @throws IOException when stored JSON cannot be decoded public List load() throws IOException { Object raw = Storage.getInstance().readObject(storageKey); if (raw == null) { @@ -130,10 +139,12 @@ public List load() throws IOException { return out; } + /// Deletes the persisted conversation. public void clear() { Storage.getInstance().deleteStorageFile(storageKey); } + /// @return app-private storage key used by this store public String getStorageKey() { return storageKey; } diff --git a/CodenameOne/src/com/codename1/ai/Embedding.java b/CodenameOne/src/com/codename1/ai/Embedding.java index bb49091a2b9..b3f5d245f56 100644 --- a/CodenameOne/src/com/codename1/ai/Embedding.java +++ b/CodenameOne/src/com/codename1/ai/Embedding.java @@ -24,23 +24,31 @@ /// A single embedding vector. `index` matches the position of the /// corresponding input string in the original request. +/// One vector returned by an embedding provider. The vector is defensively +/// copied on construction and access so callers cannot mutate stored results. public final class Embedding { private final float[] vector; private final int index; + /// Creates an embedding result. + /// @param vector numeric embedding coordinates + /// @param index zero-based position of the corresponding request input public Embedding(float[] vector, int index) { this.vector = vector == null ? new float[0] : vector; this.index = index; } + /// @return a defensive copy of the embedding coordinates public float[] getVector() { return vector; } + /// @return zero-based position of this item in the request public int getIndex() { return index; } + /// @return number of coordinates in the embedding public int getDimensions() { return vector.length; } diff --git a/CodenameOne/src/com/codename1/ai/EmbeddingRequest.java b/CodenameOne/src/com/codename1/ai/EmbeddingRequest.java index d8cf2b7e0bf..97fcbdfe1cc 100644 --- a/CodenameOne/src/com/codename1/ai/EmbeddingRequest.java +++ b/CodenameOne/src/com/codename1/ai/EmbeddingRequest.java @@ -42,27 +42,35 @@ private EmbeddingRequest(Builder b) { this.dimensions = b.dimensions; } + /// @return a new empty embedding request builder public static Builder builder() { return new Builder(); } - /// Convenience for the single-input case. + /// Creates a single-input request. + /// @param model provider model id, or {@code null} for the client default + /// @param text text to embed + /// @return immutable embedding request public static EmbeddingRequest of(String model, String text) { return builder().model(model).inputs(Arrays.asList(text)).build(); } + /// @return requested model id, or {@code null} for the client default public String getModel() { return model; } + /// @return immutable input strings in request order public List getInputs() { return inputs; } + /// @return requested output dimensions, or {@code null} for model default public Integer getDimensions() { return dimensions; } + /// Mutable builder for an immutable {@link EmbeddingRequest}. public static final class Builder { private String model; private List inputs = new ArrayList(); @@ -71,27 +79,38 @@ public static final class Builder { Builder() { } + /// @param m provider model id, or {@code null} for client default + /// @return this builder public Builder model(String m) { this.model = m; return this; } + /// Replaces all input strings. + /// @param in strings to embed; {@code null} clears the list + /// @return this builder public Builder inputs(List in) { this.inputs = in == null ? new ArrayList() : new ArrayList(in); return this; } + /// @param s input string to append + /// @return this builder public Builder addInput(String s) { this.inputs.add(s); return this; } + /// @param d requested vector dimensions, or {@code null} for model default + /// @return this builder public Builder dimensions(Integer d) { this.dimensions = d; return this; } + /// @return validated immutable request + /// @throws IllegalStateException when no input strings were supplied public EmbeddingRequest build() { if (inputs.isEmpty()) { throw new IllegalStateException("at least one input is required"); diff --git a/CodenameOne/src/com/codename1/ai/EmbeddingResponse.java b/CodenameOne/src/com/codename1/ai/EmbeddingResponse.java index d2553a48e96..d47f0fa6108 100644 --- a/CodenameOne/src/com/codename1/ai/EmbeddingResponse.java +++ b/CodenameOne/src/com/codename1/ai/EmbeddingResponse.java @@ -26,11 +26,17 @@ import java.util.Collections; import java.util.List; +/// Immutable provider response containing embeddings, token accounting, and +/// the model name reported by the service. public final class EmbeddingResponse { private final List data; private final Usage usage; private final String modelUsed; + /// Creates a normalized embedding response. + /// @param data embedding items in provider response order; {@code null} becomes empty + /// @param usage provider token accounting, or {@code null} when omitted + /// @param modelUsed model identifier returned by the provider public EmbeddingResponse(List data, Usage usage, String modelUsed) { this.data = data == null ? Collections.emptyList() : Collections.unmodifiableList(new ArrayList(data)); @@ -38,14 +44,17 @@ public EmbeddingResponse(List data, Usage usage, String modelUsed) { this.modelUsed = modelUsed; } + /// @return immutable embedding items in request order public List getData() { return data; } + /// @return provider token accounting, or {@code null} when unavailable public Usage getUsage() { return usage; } + /// @return provider-reported model identifier, or {@code null} when omitted public String getModelUsed() { return modelUsed; } diff --git a/CodenameOne/src/com/codename1/ai/GenerateImageRequest.java b/CodenameOne/src/com/codename1/ai/GenerateImageRequest.java index 1f308e137e8..c22635c97ba 100644 --- a/CodenameOne/src/com/codename1/ai/GenerateImageRequest.java +++ b/CodenameOne/src/com/codename1/ai/GenerateImageRequest.java @@ -23,6 +23,9 @@ package com.codename1.ai; /// Request payload for [ImageGenerator#generate(GenerateImageRequest)]. +/// Mutable image-generation request shared by the supported providers. +/// Provider-specific values that are unsupported may be ignored or rejected +/// by the selected {@link ImageGenerator}. public final class GenerateImageRequest { private final String prompt; private String model; @@ -32,6 +35,8 @@ public final class GenerateImageRequest { private int count = 1; private Long seed; + /// Creates a request with required prompt text. + /// @param prompt description of the image to generate public GenerateImageRequest(String prompt) { if (prompt == null || prompt.length() == 0) { throw new IllegalArgumentException("prompt is required"); @@ -39,19 +44,24 @@ public GenerateImageRequest(String prompt) { this.prompt = prompt; } + /// @return image description sent to the provider public String getPrompt() { return prompt; } + /// @return requested model id, or {@code null} for generator default public String getModel() { return model; } + /// @param model provider model id, or {@code null} for generator default + /// @return this request public GenerateImageRequest setModel(String model) { this.model = model; return this; } + /// @return requested dimensions such as {@code 1024x1024} public String getSize() { return size; } @@ -63,24 +73,31 @@ public GenerateImageRequest setSize(String size) { return this; } + /// @return provider style value, or {@code null} when unspecified public String getStyle() { return style; } + /// @param style provider-specific rendering style, or {@code null} + /// @return this request public GenerateImageRequest setStyle(String style) { this.style = style; return this; } + /// @return provider quality tier, or {@code null} when unspecified public String getQuality() { return quality; } + /// @param quality provider-specific quality tier, or {@code null} + /// @return this request public GenerateImageRequest setQuality(String quality) { this.quality = quality; return this; } + /// @return requested image count public int getCount() { return count; } @@ -92,10 +109,13 @@ public GenerateImageRequest setCount(int count) { return this; } + /// @return deterministic seed, or {@code null} when unspecified public Long getSeed() { return seed; } + /// @param seed deterministic seed where supported, or {@code null} + /// @return this request public GenerateImageRequest setSeed(Long seed) { this.seed = seed; return this; diff --git a/CodenameOne/src/com/codename1/ai/ImageGenerator.java b/CodenameOne/src/com/codename1/ai/ImageGenerator.java index dc0e6fc8418..e78eceed084 100644 --- a/CodenameOne/src/com/codename1/ai/ImageGenerator.java +++ b/CodenameOne/src/com/codename1/ai/ImageGenerator.java @@ -50,6 +50,9 @@ /// ``` public abstract class ImageGenerator { + /// Creates an OpenAI image generator. + /// @param apiKey OpenAI API key + /// @return configured generator public static ImageGenerator openai(String apiKey) { return new OpenAiImageGenerator(apiKey); } @@ -57,6 +60,8 @@ public static ImageGenerator openai(String apiKey) { /// Replicate runs a wide catalog of third-party image models /// (SDXL, Flux, etc.) behind a uniform REST API. Pass the API /// token from `https://replicate.com/account`. + /// @param apiKey Replicate API token + /// @return configured generator public static ImageGenerator replicate(String apiKey) { return new ReplicateImageGenerator(apiKey); } @@ -65,6 +70,7 @@ public static ImageGenerator replicate(String apiKey) { /// `cn1-ai-stablediffusion`; without it this returns an /// `AsyncResource` that completes with /// `UnsupportedOperationException`. + /// @return optional on-device generator or an unsupported stub public static ImageGenerator onDevice() { // Lazy lookup so app code can compile even without the // cn1lib. The cn1lib registers an implementation via @@ -82,6 +88,9 @@ public AsyncResource generate(GenerateImageRequest req) { }; } + /// Generates an image without blocking the EDT. + /// @param req provider-neutral generation request + /// @return asynchronous generated image public abstract AsyncResource generate(GenerateImageRequest req); // --------------------- OpenAI --------------------- diff --git a/CodenameOne/src/com/codename1/ai/ImagePart.java b/CodenameOne/src/com/codename1/ai/ImagePart.java index df8a81e0426..18f8e546237 100644 --- a/CodenameOne/src/com/codename1/ai/ImagePart.java +++ b/CodenameOne/src/com/codename1/ai/ImagePart.java @@ -26,6 +26,8 @@ /// raw bytes (the provider encodes them as base64 inline data) or from /// a publicly-reachable URL -- both modes are accepted by OpenAI, /// Anthropic, and Gemini. +/// Image content within a multimodal {@link ChatMessage}. An image is either +/// inline encoded bytes with a MIME type or a provider-accessible URL. public final class ImagePart extends MessagePart { private final byte[] data; private final String mimeType; @@ -33,6 +35,9 @@ public final class ImagePart extends MessagePart { /// Inline image bytes. `mimeType` must be set (e.g. `"image/png"`, /// `"image/jpeg"`); the providers reject inline images without it. + /// Creates an inline image part. + /// @param data encoded image bytes, defensively copied + /// @param mimeType media type such as {@code image/png} public ImagePart(byte[] data, String mimeType) { if (data == null || mimeType == null) { throw new IllegalArgumentException("data and mimeType are required"); @@ -43,6 +48,8 @@ public ImagePart(byte[] data, String mimeType) { } /// Remote image by URL. Only HTTPS is portable across providers. + /// Creates a remotely addressed image part. + /// @param url provider-accessible HTTPS or data URL public ImagePart(String url) { if (url == null) { throw new IllegalArgumentException("url is required"); @@ -52,18 +59,22 @@ public ImagePart(String url) { this.url = url; } + /// @return a defensive copy of inline bytes, or {@code null} for a URL image public byte[] getData() { return data; } + /// @return MIME type of inline bytes, or {@code null} for a URL image public String getMimeType() { return mimeType; } + /// @return provider-accessible URL, or {@code null} for an inline image public String getUrl() { return url; } + /// @return {@code true} when this part references a URL instead of bytes public boolean isUrl() { return url != null; } diff --git a/CodenameOne/src/com/codename1/ai/LlmChatBinding.java b/CodenameOne/src/com/codename1/ai/LlmChatBinding.java index b4618cc219f..932219ba686 100644 --- a/CodenameOne/src/com/codename1/ai/LlmChatBinding.java +++ b/CodenameOne/src/com/codename1/ai/LlmChatBinding.java @@ -69,6 +69,12 @@ public final class LlmChatBinding { private LlmChatBinding() { } + /// Connects a chat view to a streaming client. Each submitted user message + /// replays the current view history while retaining the base request's + /// model, sampling, tool, metadata, and safety settings. + /// @param view chat UI to drive + /// @param client provider client used for every turn + /// @param baseRequest request template and optional initial messages public static void bind(final ChatView view, final LlmClient client, final ChatRequest baseRequest) { diff --git a/CodenameOne/src/com/codename1/ai/LlmClient.java b/CodenameOne/src/com/codename1/ai/LlmClient.java index fea905929a4..4bf5f1dec79 100644 --- a/CodenameOne/src/com/codename1/ai/LlmClient.java +++ b/CodenameOne/src/com/codename1/ai/LlmClient.java @@ -80,28 +80,44 @@ protected LlmClient(String baseUrl) { /// OpenAI / OpenAI-compatible (Together, Groq, Fireworks, vLLM, /// Ollama, etc.). Uses the public endpoint by default; override /// with [#setBaseUrl(String)]. + /// @param apiKey provider API key + /// @return configured client public static LlmClient openai(String apiKey) { return SimulatorRedirect.maybeWrap(new OpenAiClient(apiKey, DEFAULT_OPENAI_URL)); } + /// Creates a client for Anthropic's Messages API. + /// @param apiKey Anthropic API key + /// @return configured client public static LlmClient anthropic(String apiKey) { return SimulatorRedirect.maybeWrap(new AnthropicClient(apiKey, DEFAULT_ANTHROPIC_URL)); } + /// Creates a client for Gemini's OpenAI-compatible endpoint. + /// @param apiKey Google AI API key + /// @return configured client public static LlmClient gemini(String apiKey) { return SimulatorRedirect.maybeWrap(new GeminiClient(apiKey, DEFAULT_GEMINI_URL)); } /// Default Ollama install: `http://localhost:11434/v1`, model /// `llama3.2`. + /// @return local Ollama client public static LlmClient ollama() { return ollama("llama3.2"); } + /// Creates a client for the default local Ollama endpoint. + /// @param defaultModel model used when requests omit one + /// @return local Ollama client public static LlmClient ollama(String defaultModel) { return ollama(DEFAULT_OLLAMA_URL, defaultModel); } + /// Creates an Ollama client for a custom endpoint. + /// @param baseUrl OpenAI-compatible endpoint root + /// @param defaultModel model used when requests omit one + /// @return configured Ollama client public static LlmClient ollama(String baseUrl, String defaultModel) { OpenAiClient c = new OpenAiClient("ollama", baseUrl); c.setDefaultModel(defaultModel); @@ -111,6 +127,10 @@ public static LlmClient ollama(String baseUrl, String defaultModel) { /// Generic OpenAI-compatible endpoint (llama.cpp server, vLLM, /// LM Studio, a custom proxy). `apiKey` may be empty for local /// services that don't authenticate. + /// @param baseUrl OpenAI-compatible endpoint root + /// @param apiKey API key, or empty for an unauthenticated service + /// @param defaultModel model used when requests omit one + /// @return configured compatible client public static LlmClient localOpenAiCompatible(String baseUrl, String apiKey, String defaultModel) { OpenAiClient c = new OpenAiClient(apiKey == null ? "" : apiKey, baseUrl); c.setDefaultModel(defaultModel); @@ -120,33 +140,48 @@ public static LlmClient localOpenAiCompatible(String baseUrl, String apiKey, Str /// Non-streaming chat. Equivalent to `chatStream` with a no-op /// listener but optimized -- the provider skips the SSE response /// and returns a single JSON object. + /// @param req immutable chat request + /// @return asynchronous normalized provider response public abstract AsyncResource chat(ChatRequest req); /// Streaming chat. `listener` fires for every content delta / /// tool-call fragment on the EDT. The returned `AsyncResource` /// completes with the aggregated final response once the stream /// ends; cancel it to close the underlying socket. + /// @param req immutable chat request + /// @param listener incremental callbacks delivered on the EDT + /// @return asynchronous aggregated response public abstract AsyncResource chatStream(ChatRequest req, StreamingListener listener); + /// Creates embeddings for one or more strings. + /// @param req immutable embedding request + /// @return asynchronous normalized embedding response public abstract AsyncResource embed(EmbeddingRequest req); /// One of `"openai"`, `"anthropic"`, `"gemini"`, `"ollama"`, /// `"local"`. Used by `ChatView` and tests to vary behaviour by /// provider. + /// @return stable provider family id public abstract String getProvider(); + /// @return current provider endpoint root public String getBaseUrl() { return baseUrl; } + /// Overrides the provider endpoint for proxies or compatible services. + /// @param baseUrl endpoint root used by subsequent calls public void setBaseUrl(String baseUrl) { this.baseUrl = baseUrl; } + /// @return network timeout in milliseconds public int getHttpTimeoutMs() { return httpTimeoutMs; } + /// Sets the network timeout for subsequent calls. + /// @param httpTimeoutMs timeout in milliseconds public void setHttpTimeoutMs(int httpTimeoutMs) { this.httpTimeoutMs = httpTimeoutMs; } diff --git a/CodenameOne/src/com/codename1/ai/LlmException.java b/CodenameOne/src/com/codename1/ai/LlmException.java index 6caf452f400..c4f5994981c 100644 --- a/CodenameOne/src/com/codename1/ai/LlmException.java +++ b/CodenameOne/src/com/codename1/ai/LlmException.java @@ -100,19 +100,39 @@ public enum ErrorType { private final ErrorType type; private final int retryAfterSeconds; + /// Creates an unclassified provider failure. + /// @param message user-readable failure description public LlmException(String message) { this(message, -1, null, null, null, ErrorType.UNKNOWN, -1); } + /// Creates an unclassified failure with its underlying cause. + /// @param message user-readable failure description + /// @param cause network, parsing, or provider cause public LlmException(String message, Throwable cause) { this(message, -1, null, null, cause, ErrorType.UNKNOWN, -1); } + /// Creates a classified provider failure without retry timing. + /// @param message user-readable failure description + /// @param httpStatus HTTP status, or {@code -1} without a response + /// @param providerErrorCode provider-specific machine code + /// @param rawBody raw response body retained for diagnostics + /// @param cause originating error, or {@code null} + /// @param type portable failure classification public LlmException(String message, int httpStatus, String providerErrorCode, String rawBody, Throwable cause, ErrorType type) { this(message, httpStatus, providerErrorCode, rawBody, cause, type, -1); } + /// Creates a fully classified provider failure. + /// @param message user-readable failure description + /// @param httpStatus HTTP status, or {@code -1} without a response + /// @param providerErrorCode provider-specific machine code + /// @param rawBody raw response body retained for diagnostics + /// @param cause originating error, or {@code null} + /// @param type portable failure classification + /// @param retryAfterSeconds server-requested delay, or {@code -1} public LlmException(String message, int httpStatus, String providerErrorCode, String rawBody, Throwable cause, ErrorType type, int retryAfterSeconds) { @@ -134,14 +154,17 @@ public ErrorType getType() { return type; } + /// @return HTTP status, or {@code -1} when no response was received public int getHttpStatus() { return httpStatus; } + /// @return provider-specific error code, or {@code null} public String getProviderErrorCode() { return providerErrorCode; } + /// @return raw response body for diagnostics, or {@code null} public String getRawBody() { return rawBody; } diff --git a/CodenameOne/src/com/codename1/ai/PromptTemplate.java b/CodenameOne/src/com/codename1/ai/PromptTemplate.java index 7bfb090dfa6..6769d42f5fc 100644 --- a/CodenameOne/src/com/codename1/ai/PromptTemplate.java +++ b/CodenameOne/src/com/codename1/ai/PromptTemplate.java @@ -45,15 +45,25 @@ private PromptTemplate(String template) { this.template = template; } + /// Starts a template instance. + /// @param template text containing optional {@code {name}} placeholders + /// @return mutable template values bound to the supplied text public static PromptTemplate of(String template) { return new PromptTemplate(template); } + /// Binds one placeholder. A {@code null} value renders as an empty string. + /// @param key placeholder name without braces + /// @param value replacement value + /// @return this template public PromptTemplate put(String key, String value) { values.put(key, value == null ? "" : value); return this; } + /// Adds all supplied placeholder values. + /// @param map replacements keyed by placeholder name; {@code null} is ignored + /// @return this template public PromptTemplate putAll(Map map) { if (map != null) { values.putAll(map); @@ -64,6 +74,7 @@ public PromptTemplate putAll(Map map) { /// Renders the final string. Unknown placeholders are left /// intact (`{like_this}`) so they're easy to spot in test /// output -- silently dropping them tends to hide bugs. + /// @return rendered prompt text public String build() { StringBuilder out = new StringBuilder(template.length() + 32); int i = 0; @@ -88,11 +99,13 @@ public String build() { } /// Convenience: render and wrap as a [ChatMessage] with USER role. + /// @return rendered user message public ChatMessage asUser() { return ChatMessage.user(build()); } /// Convenience: render and wrap as a [ChatMessage] with SYSTEM role. + /// @return rendered system message public ChatMessage asSystem() { return ChatMessage.system(build()); } diff --git a/CodenameOne/src/com/codename1/ai/RetryPolicy.java b/CodenameOne/src/com/codename1/ai/RetryPolicy.java index 00beb3fd0ca..cbfef885cba 100644 --- a/CodenameOne/src/com/codename1/ai/RetryPolicy.java +++ b/CodenameOne/src/com/codename1/ai/RetryPolicy.java @@ -60,23 +60,34 @@ private RetryPolicy(int maxAttempts, long initialDelayMs, long maxDelayMs, /// 4 attempts, starting at 500 ms, doubling, capped at 30 s, with /// jitter. Good default for chat workloads. + /// @return recommended transient-failure policy public static RetryPolicy exponentialBackoff() { return new RetryPolicy(4, 500L, 30000L, 2.0, true); } /// No retries -- failures are returned to the caller as-is. + /// @return single-attempt policy public static RetryPolicy none() { return new RetryPolicy(1, 0L, 0L, 1.0, false); } + /// Creates a bounded exponential retry policy. + /// @param maxAttempts total attempts including the initial call + /// @param initialDelayMs delay before the first retry + /// @param maxDelayMs upper bound for computed delays + /// @param multiplier growth factor applied after each attempt + /// @param jitter whether to randomize each delay from zero to its bound + /// @return normalized retry policy public static RetryPolicy custom(int maxAttempts, long initialDelayMs, long maxDelayMs, double multiplier, boolean jitter) { return new RetryPolicy(maxAttempts, initialDelayMs, maxDelayMs, multiplier, jitter); } - /// Inspect a thrown exception and decide whether to retry. Apps - /// can override to add provider-specific rules (e.g. retry on a - /// custom 5xx code). + /// Inspects a thrown exception and decides whether this policy permits + /// another attempt. + /// @param t operation failure + /// @param attemptsSoFar attempts already made, including the failed one + /// @return {@code true} for a transient failure within the attempt limit public boolean shouldRetry(Throwable t, int attemptsSoFar) { if (attemptsSoFar >= maxAttempts) { return false; @@ -96,6 +107,9 @@ public boolean shouldRetry(Throwable t, int attemptsSoFar) { /// Returns the delay to wait before the next attempt, honouring /// `Retry-After` from rate-limit exceptions when present. + /// @param t failure that triggered the retry + /// @param attemptIndex zero-based retry index + /// @return delay in milliseconds public long computeDelayMs(Throwable t, int attemptIndex /* 0-based */) { if (t instanceof LlmException && ((LlmException) t).getType() == LlmException.ErrorType.RATE_LIMIT) { @@ -120,6 +134,7 @@ public long computeDelayMs(Throwable t, int attemptIndex /* 0-based */) { return (long) delay; } + /// @return total allowed attempts including the initial call public int getMaxAttempts() { return maxAttempts; } diff --git a/CodenameOne/src/com/codename1/ai/SafetyFilter.java b/CodenameOne/src/com/codename1/ai/SafetyFilter.java index 00ce689ec35..17404b682ca 100644 --- a/CodenameOne/src/com/codename1/ai/SafetyFilter.java +++ b/CodenameOne/src/com/codename1/ai/SafetyFilter.java @@ -31,6 +31,8 @@ public interface SafetyFilter { /// Returns `null` to allow the call, or a human-readable reason /// string to block it. + /// @param messages immutable request conversation + /// @return {@code null} to allow submission, otherwise the rejection reason String check(List messages); /// A built-in filter that allows everything. Useful as a default diff --git a/CodenameOne/src/com/codename1/ai/StreamingListener.java b/CodenameOne/src/com/codename1/ai/StreamingListener.java index 5bb48067cf3..bb26acef231 100644 --- a/CodenameOne/src/com/codename1/ai/StreamingListener.java +++ b/CodenameOne/src/com/codename1/ai/StreamingListener.java @@ -35,6 +35,7 @@ public interface StreamingListener { /// A chunk of assistant text. Append it to whatever text buffer /// you're rendering. + /// @param textDelta next ordered text fragment void onContentDelta(String textDelta); /// A tool-call fragment. `index` lets you correlate fragments @@ -44,16 +45,22 @@ public interface StreamingListener { /// JSON; concatenate fragments for the same `index` to reassemble. /// `id` is the provider's tool-call id, present on the first /// fragment. + /// @param index zero-based tool call within the response + /// @param id provider tool-call id, normally present on the first fragment + /// @param name requested tool name, normally present on the first fragment + /// @param argumentsFragment next JSON argument fragment void onToolCallDelta(int index, String id, String name, String argumentsFragment); /// Token-accounting update. Most providers send this once at the /// end; some send incremental counts. + /// @param usage latest provider token counts void onUsage(Usage usage); /// Mid-stream error (e.g. connection reset). The `AsyncResource` /// returned by `chatStream` will also complete with this same /// exception, so listeners can typically ignore this and react to /// the resource. Implemented for parity with other SDKs. + /// @param t stream failure also delivered by the returned resource void onError(Throwable t); /// No-op default implementation. Subclass and override only what diff --git a/CodenameOne/src/com/codename1/ai/TextPart.java b/CodenameOne/src/com/codename1/ai/TextPart.java index 45ee958d1f7..9eb9a4d1a57 100644 --- a/CodenameOne/src/com/codename1/ai/TextPart.java +++ b/CodenameOne/src/com/codename1/ai/TextPart.java @@ -23,13 +23,17 @@ package com.codename1.ai; /// A plain-text fragment of a [ChatMessage]. +/// Text content within a multimodal {@link ChatMessage}. public final class TextPart extends MessagePart { private final String text; + /// Creates a text part. + /// @param text text sent to or received from the model public TextPart(String text) { this.text = text == null ? "" : text; } + /// @return text carried by this part public String getText() { return text; } diff --git a/CodenameOne/src/com/codename1/ai/Tool.java b/CodenameOne/src/com/codename1/ai/Tool.java index 720aae3d669..7d003b367db 100644 --- a/CodenameOne/src/com/codename1/ai/Tool.java +++ b/CodenameOne/src/com/codename1/ai/Tool.java @@ -65,10 +65,19 @@ public final class Tool { private final String parametersJsonSchema; private final ToolHandler handler; + /// Creates a description-only tool for manual dispatch. + /// @param name provider-visible function name + /// @param description guidance that helps the model choose the tool + /// @param parametersJsonSchema JSON Schema for accepted arguments public Tool(String name, String description, String parametersJsonSchema) { this(name, description, parametersJsonSchema, null); } + /// Creates a tool definition with an application executor. + /// @param name provider-visible function name + /// @param description guidance that helps the model choose the tool + /// @param parametersJsonSchema JSON Schema for accepted arguments + /// @param handler executor called by {@link #invoke(String)}, or {@code null} public Tool(String name, String description, String parametersJsonSchema, ToolHandler handler) { if (name == null || name.length() == 0) { @@ -82,26 +91,33 @@ public Tool(String name, String description, String parametersJsonSchema, this.handler = handler; } + /// @return provider-visible function name public String getName() { return name; } + /// @return description used by the model to decide when to call the tool public String getDescription() { return description; } + /// @return JSON Schema describing accepted function arguments public String getParametersJsonSchema() { return parametersJsonSchema; } /// The optional executor wired up by the constructor. Returns /// `null` for description-only tools. + /// @return registered executor, or {@code null} public ToolHandler getHandler() { return handler; } /// Invokes the handler with the given arguments JSON. Throws /// `IllegalStateException` when no handler was registered. + /// @param argumentsJson model-generated arguments + /// @return handler result encoded as JSON + /// @throws Exception when the handler rejects or cannot execute the call public String invoke(String argumentsJson) throws Exception { if (handler == null) { throw new IllegalStateException( diff --git a/CodenameOne/src/com/codename1/ai/ToolCall.java b/CodenameOne/src/com/codename1/ai/ToolCall.java index 5fddfa4394d..87038a8af25 100644 --- a/CodenameOne/src/com/codename1/ai/ToolCall.java +++ b/CodenameOne/src/com/codename1/ai/ToolCall.java @@ -38,20 +38,27 @@ public final class ToolCall { private final String name; private final String argumentsJson; + /// Creates a normalized model tool call. + /// @param id provider id used to associate the result + /// @param name requested tool name + /// @param argumentsJson model-generated JSON arguments; {@code null} becomes {@code {}} public ToolCall(String id, String name, String argumentsJson) { this.id = id; this.name = name; this.argumentsJson = argumentsJson == null ? "{}" : argumentsJson; } + /// @return provider id used to associate the eventual tool result public String getId() { return id; } + /// @return requested tool name public String getName() { return name; } + /// @return model-generated arguments encoded as JSON public String getArgumentsJson() { return argumentsJson; } @@ -65,6 +72,9 @@ public String getArgumentsJson() { /// Throws `IllegalArgumentException` when no tool in `tools` /// has a matching name, or `IllegalStateException` when the /// matching tool has no handler registered. + /// @param tools definitions offered with the request + /// @return handler result encoded as JSON + /// @throws Exception when lookup or application execution fails public String execute(List tools) throws Exception { Tool match = findTool(tools); if (match == null) { @@ -79,6 +89,8 @@ public String execute(List tools) throws Exception { /// Looks up the matching [Tool] without invoking it. Useful when /// the caller wants to dispatch by hand but still benefit from /// the name-matching plumbing. + /// @param tools candidate definitions, or {@code null} + /// @return matching definition, or {@code null} public Tool findTool(List tools) { if (tools == null) { return null; diff --git a/CodenameOne/src/com/codename1/ai/ToolChoice.java b/CodenameOne/src/com/codename1/ai/ToolChoice.java index 018e17c4546..0b705e757c2 100644 --- a/CodenameOne/src/com/codename1/ai/ToolChoice.java +++ b/CodenameOne/src/com/codename1/ai/ToolChoice.java @@ -52,10 +52,13 @@ public static ToolChoice named(String toolName) { return new ToolChoice("named", toolName); } + /// @return portable choice mode: {@code auto}, {@code none}, + /// {@code required}, or {@code named} public String getMode() { return mode; } + /// @return forced tool name for a named choice, otherwise {@code null} public String getForcedToolName() { return forcedToolName; } diff --git a/CodenameOne/src/com/codename1/ai/ToolResultPart.java b/CodenameOne/src/com/codename1/ai/ToolResultPart.java index 0ea7ea02a73..5b4cfff94a5 100644 --- a/CodenameOne/src/com/codename1/ai/ToolResultPart.java +++ b/CodenameOne/src/com/codename1/ai/ToolResultPart.java @@ -25,6 +25,8 @@ /// The result of a tool invocation, sent back to the model so it can /// continue reasoning. Pairs with the originating [ToolCall] via the /// `toolCallId`. The carrying [ChatMessage] should use [Role#TOOL]. +/// Result content sent back to a model after executing a requested tool call. +/// The result is JSON text so providers receive the original structured value. public final class ToolResultPart extends MessagePart { private final String toolCallId; private final String resultJson; @@ -32,15 +34,20 @@ public final class ToolResultPart extends MessagePart { /// `resultJson` is the literal JSON string the tool produced. If /// the tool result isn't valid JSON, wrap it like /// `"{\"text\":\"...\"}"` -- the providers expect JSON-shaped values. + /// Creates a tool result part. + /// @param toolCallId id of the {@link ToolCall} being answered + /// @param resultJson JSON value returned by the application tool public ToolResultPart(String toolCallId, String resultJson) { this.toolCallId = toolCallId; this.resultJson = resultJson == null ? "" : resultJson; } + /// @return provider tool-call id this result answers public String getToolCallId() { return toolCallId; } + /// @return application result encoded as JSON public String getResultJson() { return resultJson; } diff --git a/CodenameOne/src/com/codename1/ai/Usage.java b/CodenameOne/src/com/codename1/ai/Usage.java index e7cf089ea1a..96a3f0802fb 100644 --- a/CodenameOne/src/com/codename1/ai/Usage.java +++ b/CodenameOne/src/com/codename1/ai/Usage.java @@ -24,25 +24,35 @@ /// Token accounting returned by the provider. Any field that the /// provider didn't return is `-1`. +/// Token counts reported for one provider operation. Providers may count +/// tokens differently, so use these values for billing and diagnostics rather +/// than comparing tokenizers across services. public final class Usage { private final int promptTokens; private final int completionTokens; private final int totalTokens; + /// Creates a usage record from provider counters. + /// @param promptTokens tokens consumed by request input + /// @param completionTokens tokens generated in the response + /// @param totalTokens provider-reported total token count public Usage(int promptTokens, int completionTokens, int totalTokens) { this.promptTokens = promptTokens; this.completionTokens = completionTokens; this.totalTokens = totalTokens; } + /// @return tokens consumed by request input public int getPromptTokens() { return promptTokens; } + /// @return tokens generated in the response public int getCompletionTokens() { return completionTokens; } + /// @return provider-reported total token count public int getTotalTokens() { return totalTokens; } diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceException.java b/CodenameOne/src/com/codename1/ai/inference/InferenceException.java index f4f127715e6..959059567ba 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceException.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceException.java @@ -22,12 +22,18 @@ */ package com.codename1.ai.inference; -/// Failure while loading or executing a LiteRT model. +/// Reports model validation, delegate selection, allocation, or invocation +/// failures from the portable inference API. public class InferenceException extends RuntimeException { + /// Creates an inference failure with a user-readable explanation. + /// @param message failure explanation public InferenceException(String message) { super(message); } + /// Creates an inference failure that preserves the native/port cause. + /// @param message failure explanation + /// @param cause originating error public InferenceException(String message, Throwable cause) { super(message, cause); } diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java index 68758838366..440a672b060 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java @@ -22,8 +22,14 @@ */ package com.codename1.ai.inference; -/// LiteRT session configuration. +/// Configures how a reusable {@link InferenceSession} is created. +/// +/// Accelerator requests are portable preferences rather than promises. +/// With fallback enabled, a backend may execute on CPU when the requested +/// delegate is unavailable. With fallback disabled, opening the session +/// fails instead of silently changing the execution target. public final class InferenceOptions { + /// Execution targets understood by the portable inference API. public enum Accelerator { AUTO, CPU, GPU, NPU, CORE_ML } @@ -32,29 +38,46 @@ public enum Accelerator { private int threads; private boolean allowFallback = true; + /// Requests an execution target for the model. + /// + /// @param value requested target; {@code null} restores {@link Accelerator#AUTO} + /// @return this options object public InferenceOptions accelerator(Accelerator value) { accelerator = value == null ? Accelerator.AUTO : value; return this; } + /// Sets the CPU worker count. Non-positive values let the native runtime + /// choose its default. + /// + /// @param value requested worker count + /// @return this options object public InferenceOptions threads(int value) { threads = Math.max(0, value); return this; } + /// Controls whether opening may fall back from an unavailable accelerator + /// to CPU execution. + /// + /// @param value {@code true} to permit CPU fallback + /// @return this options object public InferenceOptions allowFallback(boolean value) { allowFallback = value; return this; } + /// @return the requested accelerator, never {@code null} public Accelerator getAccelerator() { return accelerator; } + /// @return the requested CPU worker count, or a non-positive runtime default public int getThreads() { return threads; } + /// @return whether an unavailable accelerator may fall back to CPU public boolean isFallbackAllowed() { return allowFallback; } diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java index d2b57ce88a0..504b9c3dbe1 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -27,7 +27,12 @@ import com.codename1.util.AsyncResource; import com.codename1.util.SuccessCallback; -/// Reusable LiteRT model session. +/// Reusable, native on-device session for a TensorFlow Lite model. +/// +/// Opening and execution are asynchronous because model allocation and +/// delegates can be expensive. Metadata and resize operations are synchronous. +/// A session is not usable after {@link #close()}; applications should retain +/// and reuse one session instead of reopening the model for every input. public final class InferenceSession implements AutoCloseable { private final InferenceImpl implementation; private final Object handle; @@ -38,11 +43,20 @@ private InferenceSession(InferenceImpl implementation, Object handle) { this.handle = handle; } + /// Tests whether the current port includes a native LiteRT runtime. + /// + /// @return {@code true} when sessions can be opened on this target public static boolean isSupported() { InferenceImpl impl = Display.getInstance().getInferenceBackend(); return impl != null && impl.isSupported(); } + /// Opens and allocates a model session off the EDT. + /// + /// @param source bytes, resource, or file containing a `.tflite` model + /// @param options execution options; {@code null} uses defaults + /// @return an asynchronous session, failed with {@link InferenceException} + /// when the model or requested accelerator cannot be used public static AsyncResource open(ModelSource source, InferenceOptions options) { if (source == null) { @@ -69,27 +83,45 @@ public void onSucess(Throwable error) { return out; } + /// Returns the model's current input metadata. Shapes reflect the most + /// recent successful {@link #resizeInput(String, int[])} call. + /// + /// @return a defensive copy of the input metadata array public TensorInfo[] getInputs() { ensureOpen(); return implementation.getInputs(handle); } + /// @return a defensive copy of the model's current output metadata public TensorInfo[] getOutputs() { ensureOpen(); return implementation.getOutputs(handle); } + /// Copies input tensors to native memory, invokes the model, and returns + /// every output tensor. Named tensors are matched by name; unnamed tensors + /// are matched by position. + /// + /// @param inputs one tensor for each model input + /// @return asynchronous output tensors in model output order public AsyncResource run(Tensor[] inputs) { ensureOpen(); return implementation.run(handle, inputs == null ? new Tensor[0] : inputs); } + /// Resizes an input and reallocates native tensors before the next run. + /// + /// @param name model input name, or {@code null} for the first input + /// @param shape new non-negative dimensions public void resizeInput(String name, int[] shape) { ensureOpen(); implementation.resizeInput(handle, name, shape); } @Override + /// Releases the interpreter, delegates, and any temporary staged model. + /// The original file supplied by {@link ModelSource#file(String)} is never + /// deleted. Calling this method more than once has no effect. public void close() { if (!closed) { closed = true; diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java index b916fee8370..0fae5106a66 100644 --- a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java +++ b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java @@ -34,14 +34,22 @@ import java.io.IOException; import java.io.InputStream; -/// Downloads large LiteRT models once and exposes the cached file as a -/// {@link ModelSource}. Small models can instead be packaged directly with +/// Downloads a large model into app-private storage and exposes it as a +/// file-backed {@link ModelSource}. Downloads require HTTPS, use a temporary +/// file, and are promoted atomically only after optional digest verification. +/// +/// Supply a SHA-256 digest for third-party or remotely mutable models. Without +/// a digest HTTPS authenticates the connection but does not pin the executable +/// model payload. Small first-party models can instead be packaged with /// {@link ModelSource#resource(String)}. public final class ModelCache { private ModelCache() { } /// Fetches a model into the app-private {@code ai-models} directory. + /// A stale {@code .download} file is deleted and restarted rather than + /// resumed because the portable network layer cannot prove that a server's + /// partial response still represents the pinned model. /// /// @param url HTTPS URL of the model /// @param cacheKey stable cache name, independent of the URL @@ -55,11 +63,13 @@ public static AsyncResource fetch( if (cacheKey == null || cacheKey.length() == 0) { throw new IllegalArgumentException("cacheKey must not be empty"); } - if (sha256 != null && sha256.length() != 64) { + if (sha256 != null && !isSha256(sha256)) { throw new IllegalArgumentException("SHA-256 must contain 64 hex characters"); } final AsyncResource out = new AsyncResource(); + final Completion completion = + new Completion(out); Display.getInstance().scheduleBackgroundTask(new Runnable() { @Override public void run() { @@ -70,20 +80,20 @@ public void run() { final String target = directory + fileName; try { if (fs.exists(target) && verify(target, sha256)) { - complete(out, ModelSource.file(target)); + completion.complete(ModelSource.file(target)); return; } if (fs.exists(target)) { fs.delete(target); } } catch (IOException error) { - fail(out, error); + completion.fail(error); return; } Display.getInstance().callSerially(new Runnable() { @Override public void run() { - download(out, url, sha256, target, fileName); + download(completion, url, sha256, target, fileName); } }); } @@ -91,18 +101,22 @@ public void run() { return out; } + /// Fetches a model without content pinning. Prefer the three-argument + /// overload for any model that is not versioned by the app itself. + /// + /// @param url HTTPS model URL + /// @param cacheKey stable cache name + /// @return asynchronous cached file source public static AsyncResource fetch(String url, String cacheKey) { return fetch(url, cacheKey, null); } - private static void download(final AsyncResource out, + private static void download(final Completion completion, String url, final String sha256, final String target, final String fileName) { final FileSystemStorage fs = FileSystemStorage.getInstance(); final String temporary = target + ".download"; - if (fs.exists(temporary)) { - fs.delete(temporary); - } + prepareTemporary(fs, temporary); final ConnectionRequest request = new ConnectionRequest(); request.setPost(false); request.setFailSilently(true); @@ -117,17 +131,14 @@ public void actionPerformed(NetworkEvent event) { @Override public void run() { try { - if (!verify(temporary, sha256)) { - fs.delete(temporary); - throw new IOException("Downloaded model SHA-256 does not match"); - } + verifyDownloaded(fs, temporary, sha256); if (fs.exists(target)) { fs.delete(target); } fs.rename(temporary, fileName); - complete(out, ModelSource.file(target)); + completion.complete(ModelSource.file(target)); } catch (IOException error) { - fail(out, error); + completion.fail(error); } } }); @@ -140,7 +151,7 @@ public void actionPerformed(NetworkEvent event) { fs.delete(temporary); } Throwable error = event.getError(); - fail(out, error == null + completion.fail(error == null ? new IOException("Model download failed with HTTP " + event.getResponseCode()) : error); @@ -169,6 +180,20 @@ private static boolean verify(String path, String expected) throws IOException { } } + static void prepareTemporary(FileSystemStorage fs, String temporary) { + if (fs.exists(temporary)) { + fs.delete(temporary); + } + } + + static void verifyDownloaded(FileSystemStorage fs, String temporary, + String expected) throws IOException { + if (!verify(temporary, expected)) { + fs.delete(temporary); + throw new IOException("Downloaded model SHA-256 does not match"); + } + } + private static String safeName(String value) { StringBuilder out = new StringBuilder(); for (int i = 0; i < value.length(); i++) { @@ -181,21 +206,53 @@ private static String safeName(String value) { return out.toString(); } - private static void complete(final AsyncResource out, final T value) { - Display.getInstance().callSerially(new Runnable() { - @Override - public void run() { - out.complete(value); + private static boolean isSha256(String value) { + if (value.length() != 64) { + return false; + } + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') + || (c >= 'A' && c <= 'F'))) { + return false; } - }); + } + return true; } - private static void fail(final AsyncResource out, final Throwable error) { - Display.getInstance().callSerially(new Runnable() { - @Override - public void run() { - out.error(new InferenceException("Could not cache LiteRT model", error)); + static final class Completion { + private final AsyncResource resource; + private boolean done; + + Completion(AsyncResource resource) { + this.resource = resource; + } + + synchronized void complete(final T value) { + if (done) { + return; } - }); + done = true; + Display.getInstance().callSerially(new Runnable() { + @Override + public void run() { + resource.complete(value); + } + }); + } + + synchronized void fail(final Throwable error) { + if (done) { + return; + } + done = true; + Display.getInstance().callSerially(new Runnable() { + @Override + public void run() { + resource.error(new InferenceException( + "Could not cache LiteRT model", error)); + } + }); + } } } diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelSource.java b/CodenameOne/src/com/codename1/ai/inference/ModelSource.java index e5c6e6d0c14..610abfbf6aa 100644 --- a/CodenameOne/src/com/codename1/ai/inference/ModelSource.java +++ b/CodenameOne/src/com/codename1/ai/inference/ModelSource.java @@ -22,7 +22,10 @@ */ package com.codename1.ai.inference; -/// A `.tflite` model supplied as bytes, a filesystem path, or an app resource. +/// Describes where an {@link InferenceSession} should obtain a `.tflite` +/// model. Byte sources are defensively copied. File sources are especially +/// useful with {@link ModelCache} because native ports can open the cached +/// path without loading the full model into the Java heap. public final class ModelSource { public static final int BYTES = 1; public static final int FILE = 2; @@ -38,6 +41,9 @@ private ModelSource(int kind, byte[] bytes, String path) { this.path = path; } + /// Creates an in-memory model source. + /// @param value complete model bytes, copied by this method + /// @return a byte-backed source public static ModelSource bytes(byte[] value) { if (value == null || value.length == 0) { throw new IllegalArgumentException("Model bytes must not be empty"); @@ -47,10 +53,16 @@ public static ModelSource bytes(byte[] value) { return new ModelSource(BYTES, copy, null); } + /// Creates a source for an existing private filesystem path. + /// @param path path understood by {@code FileSystemStorage} + /// @return a file-backed source; the session never owns or deletes the file public static ModelSource file(String path) { return named(FILE, path); } + /// Creates a source for a model packaged in application resources. + /// @param path absolute classpath-style resource path + /// @return a resource-backed source public static ModelSource resource(String path) { return named(RESOURCE, path); } @@ -62,10 +74,12 @@ private static ModelSource named(int kind, String path) { return new ModelSource(kind, null, path); } + /// @return {@link #BYTES}, {@link #FILE}, or {@link #RESOURCE} public int getKind() { return kind; } + /// @return a defensive copy of model bytes, or {@code null} for non-byte sources public byte[] getBytes() { if (bytes == null) { return null; @@ -75,6 +89,7 @@ public byte[] getBytes() { return out; } + /// @return the file/resource path, or {@code null} for a byte source public String getPath() { return path; } diff --git a/CodenameOne/src/com/codename1/ai/inference/Tensor.java b/CodenameOne/src/com/codename1/ai/inference/Tensor.java index a02bec4a623..bfe2c7e72cf 100644 --- a/CodenameOne/src/com/codename1/ai/inference/Tensor.java +++ b/CodenameOne/src/com/codename1/ai/inference/Tensor.java @@ -22,13 +22,23 @@ */ package com.codename1.ai.inference; -/// Named tensor value passed to or returned from an inference session. +/// Immutable named tensor passed to or returned from an inference session. +/// The primitive data array and shape are copied on construction and by their +/// public getters, so callers can safely reuse or mutate their source arrays. public final class Tensor { private final String name; private final TensorType type; private final int[] shape; private final Object data; + /// Creates a tensor and validates that its type, shape, and primitive array + /// agree. Dynamic dimensions may be negative and skip the element-count + /// check; fixed shapes are overflow-checked. + /// + /// @param name model tensor name, or {@code null} for positional matching + /// @param type element type + /// @param shape tensor dimensions; an empty shape represents a scalar + /// @param data matching primitive array public Tensor(String name, TensorType type, int[] shape, Object data) { if (type == null || data == null) { throw new NullPointerException("type and data are required"); @@ -44,34 +54,53 @@ public Tensor(String name, TensorType type, int[] shape, Object data) { this.data = copyData(data); } + /// @return a FLOAT32 tensor with defensively copied data public static Tensor floats(String name, int[] shape, float[] data) { return new Tensor(name, TensorType.FLOAT32, shape, data); } + /// @return an INT32 tensor with defensively copied data public static Tensor ints(String name, int[] shape, int[] data) { return new Tensor(name, TensorType.INT32, shape, data); } + /// Creates a byte-array tensor for UINT8, INT8, or BOOL data. + /// @return a tensor with defensively copied data public static Tensor bytes(String name, TensorType type, int[] shape, byte[] data) { return new Tensor(name, type, shape, data); } + /// @return the model tensor name, or {@code null} for an unnamed tensor public String getName() { return name; } + /// @return the element type public TensorType getType() { return type; } + /// @return a defensive copy of tensor dimensions public int[] getShape() { return copy(shape); } + /// @return a defensive copy of the matching primitive data array public Object getData() { return copyData(data); } + /// Returns the tensor's immutable backing primitive array to a port + /// implementation, avoiding an additional full-tensor copy at the native + /// boundary. Application code must use {@link #getData()}. + /// + /// @return the backing {@code float[]}, {@code int[]}, {@code long[]}, or + /// {@code byte[]} matching {@link #getType()} + /// @hidden + public Object getDataUnsafe() { + return data; + } + private static void validateData(TensorType type, Object data) { boolean valid; switch (type) { @@ -101,14 +130,18 @@ private static int elementCount(int[] shape) { if (shape == null || shape.length == 0) { return 1; } - int out = 1; + long out = 1; for (int dimension : shape) { if (dimension < 0) { return -1; } out *= dimension; + if (out > Integer.MAX_VALUE) { + throw new IllegalArgumentException( + "Tensor shape contains more than 2147483647 elements"); + } } - return out; + return (int) out; } private static int dataLength(Object value) { diff --git a/CodenameOne/src/com/codename1/ai/inference/TensorInfo.java b/CodenameOne/src/com/codename1/ai/inference/TensorInfo.java index b7b18d49603..39e42c55005 100644 --- a/CodenameOne/src/com/codename1/ai/inference/TensorInfo.java +++ b/CodenameOne/src/com/codename1/ai/inference/TensorInfo.java @@ -22,13 +22,20 @@ */ package com.codename1.ai.inference; -/// Immutable model input/output metadata. +/// Immutable metadata for one model input or output. Shapes may contain a +/// negative dynamic dimension until the input has been resized and tensors +/// reallocated. public final class TensorInfo { private final String name; private final TensorType type; private final int[] shape; private final int index; + /// Creates tensor metadata. + /// @param name runtime tensor name, possibly empty + /// @param type element type + /// @param shape current tensor dimensions + /// @param index native model index public TensorInfo(String name, TensorType type, int[] shape, int index) { this.name = name; this.type = type; @@ -36,18 +43,22 @@ public TensorInfo(String name, TensorType type, int[] shape, int index) { this.index = index; } + /// @return the runtime tensor name public String getName() { return name; } + /// @return the tensor element type public TensorType getType() { return type; } + /// @return a defensive copy of current dimensions public int[] getShape() { return copy(shape); } + /// @return the zero-based model input or output index public int getIndex() { return index; } diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageBackend.java b/CodenameOne/src/com/codename1/ai/language/LanguageBackend.java index 4c4bee2327a..0801906c637 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageBackend.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageBackend.java @@ -22,7 +22,10 @@ */ package com.codename1.ai.language; -/// Identifies an on-device language implementation. +/// Identifies a native language-service implementation. Obtain instances from +/// {@link LanguageBackends}; builder dependency selection relies on calls to +/// those selector methods. public interface LanguageBackend { + /// @return stable backend identifier passed to the current platform port String getId(); } diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java b/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java index fd83ab009f3..3d3be29d9cc 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java @@ -22,25 +22,40 @@ */ package com.codename1.ai.language; -/// Backend selectors for language identification, translation, and smart reply. +/// Selects native backends for language identification, translation, and +/// Smart Reply. Automatic selection uses ML Kit on Android, Apple Natural +/// Language for iOS identification, and feature-scoped ML Kit components for +/// iOS translation and Smart Reply. public final class LanguageBackends { private static final LanguageBackend AUTO = new Named("auto"); private static final LanguageBackend ML_KIT = new Named("ml-kit"); - private static final LanguageBackend LITE_RT = new Named("litert"); + private static final LanguageBackend APPLE_NATURAL_LANGUAGE = + new Named("apple-natural-language"); private LanguageBackends() { } + /// @return the platform-recommended dependency-minimal backend public static LanguageBackend auto() { return AUTO; } - public static LanguageBackend mlKit() { - return ML_KIT; + /// Selects Apple's dependency-free Natural Language framework for + /// language identification. This backend is available on iOS 12 and + /// newer and is not available on Android. + /// + /// @return the Apple Natural Language backend selector + public static LanguageBackend appleNaturalLanguage() { + return APPLE_NATURAL_LANGUAGE; } - public static LanguageBackend liteRt() { - return LITE_RT; + /// Selects ML Kit specifically for language identification. Calling this + /// method lets the builder add the Language ID pod only when the + /// application opts out of the Apple-native iOS default. + /// + /// @return the ML Kit language-identification selector + public static LanguageBackend mlKitLanguageIdentification() { + return ML_KIT; } private static final class Named implements LanguageBackend { diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageCandidate.java b/CodenameOne/src/com/codename1/ai/language/LanguageCandidate.java index e69d3897a80..bdfdb5b7036 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageCandidate.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageCandidate.java @@ -22,20 +22,25 @@ */ package com.codename1.ai.language; -/// Language identification candidate. +/// One ranked BCP-47 language candidate returned by +/// {@link LanguageIdentifier#identify(String, LanguageOptions)}. public final class LanguageCandidate { private final String languageTag; private final float confidence; + /// @param languageTag BCP-47 tag, or {@code und} when undetermined + /// @param confidence backend confidence in the range 0..1 public LanguageCandidate(String languageTag, float confidence) { this.languageTag = languageTag; this.confidence = confidence; } + /// @return BCP-47 language tag public String getLanguageTag() { return languageTag; } + /// @return backend confidence in the range 0..1 public float getConfidence() { return confidence; } diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java index 61a820ac4f0..169d90a1d16 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java @@ -26,15 +26,20 @@ import com.codename1.ui.Display; import com.codename1.util.AsyncResource; -/// On-device language identification. +/// Identifies possible languages entirely on device. Results are ranked by +/// descending backend confidence and filtered by +/// {@link LanguageOptions#getMinimumConfidence()}. public final class LanguageIdentifier { private LanguageIdentifier() { } + /// @return whether automatic language identification is available public static boolean isSupported() { return isSupported(new LanguageOptions()); } + /// @param options backend selection, or {@code null} for defaults + /// @return whether the selected backend is available on this target public static boolean isSupported(LanguageOptions options) { LanguageImpl impl = Display.getInstance().getLanguageBackend(); LanguageOptions actual = options == null ? new LanguageOptions() : options; @@ -42,6 +47,10 @@ public static boolean isSupported(LanguageOptions options) { actual.getBackend().getId()); } + /// Identifies possible languages off the EDT without uploading text. + /// @param text non-null text to classify + /// @param options backend and confidence options, or {@code null} + /// @return asynchronous ranked candidates; may be empty for undetermined text public static AsyncResource identify( String text, LanguageOptions options) { LanguageOptions actual = options == null ? new LanguageOptions() : options; diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java b/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java index 4cec3d4a971..acb277c2e69 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java @@ -22,25 +22,33 @@ */ package com.codename1.ai.language; -/// Common options for on-device language services. +/// Reusable options shared by language identification, translation, and +/// Smart Reply operations. public final class LanguageOptions { private LanguageBackend backend = LanguageBackends.auto(); private float minimumConfidence; + /// @param value backend selector; {@code null} restores automatic selection + /// @return this options object public LanguageOptions backend(LanguageBackend value) { backend = value == null ? LanguageBackends.auto() : value; return this; } + /// Sets the minimum language-identification confidence, clamped to 0..1. + /// @param value requested threshold + /// @return this options object public LanguageOptions minimumConfidence(float value) { minimumConfidence = Math.max(0, Math.min(1, value)); return this; } + /// @return selected backend, never {@code null} public LanguageBackend getBackend() { return backend; } + /// @return language-identification threshold in the range 0..1 public float getMinimumConfidence() { return minimumConfidence; } diff --git a/CodenameOne/src/com/codename1/ai/language/SmartReply.java b/CodenameOne/src/com/codename1/ai/language/SmartReply.java index 2be20c971ad..e7a0ecb162c 100644 --- a/CodenameOne/src/com/codename1/ai/language/SmartReply.java +++ b/CodenameOne/src/com/codename1/ai/language/SmartReply.java @@ -26,15 +26,19 @@ import com.codename1.ui.Display; import com.codename1.util.AsyncResource; -/// On-device short reply suggestions for a conversation. +/// Produces short reply suggestions for a chronological conversation without +/// uploading its messages. ML Kit may return no suggestions when the language +/// or conversation context is unsupported. public final class SmartReply { private SmartReply() { } + /// @return whether automatic Smart Reply is available public static boolean isSupported() { return isSupported(new LanguageOptions()); } + /// @return whether the selected backend supports Smart Reply public static boolean isSupported(LanguageOptions options) { LanguageOptions actual = options == null ? new LanguageOptions() : options; @@ -43,6 +47,9 @@ public static boolean isSupported(LanguageOptions options) { "smart-reply", actual.getBackend().getId()); } + /// @param conversation chronological messages, oldest first + /// @param options backend options, or {@code null} + /// @return asynchronous suggestions, possibly an empty array public static AsyncResource suggest(SmartReplyMessage[] conversation, LanguageOptions options) { LanguageOptions actual = options == null ? new LanguageOptions() : options; diff --git a/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java b/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java index 209f1dc90b9..46383b56e58 100644 --- a/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java +++ b/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java @@ -29,6 +29,11 @@ public final class SmartReplyMessage { private final boolean localUser; private final long timestampMillis; + /// Creates one conversation turn used to generate reply suggestions. + /// @param text message body; {@code null} becomes an empty string + /// @param participantId stable speaker id used to group conversation turns + /// @param localUser whether this message was written by the current user + /// @param timestampMillis message time in Unix epoch milliseconds public SmartReplyMessage(String text, String participantId, boolean localUser, long timestampMillis) { this.text = text == null ? "" : text; @@ -37,18 +42,22 @@ public SmartReplyMessage(String text, String participantId, this.timestampMillis = timestampMillis; } + /// @return message text supplied to the local model public String getText() { return text; } + /// @return stable participant identifier used to group remote speakers public String getParticipantId() { return participantId; } + /// @return whether this message was authored by the device user public boolean isLocalUser() { return localUser; } + /// @return conversation timestamp in milliseconds since the epoch public long getTimestampMillis() { return timestampMillis; } diff --git a/CodenameOne/src/com/codename1/ai/language/Translator.java b/CodenameOne/src/com/codename1/ai/language/Translator.java index 6045e8eba16..243c15d57e6 100644 --- a/CodenameOne/src/com/codename1/ai/language/Translator.java +++ b/CodenameOne/src/com/codename1/ai/language/Translator.java @@ -26,15 +26,19 @@ import com.codename1.ui.Display; import com.codename1.util.AsyncResource; -/// On-device translation with lazily installed language-pair models. +/// Translates text on device with lazily installed language-pair models. +/// The first request for a pair may take longer while ML Kit downloads the +/// model; download failures are reported through the returned resource. public final class Translator { private Translator() { } + /// @return whether automatic on-device translation is available public static boolean isSupported() { return isSupported(new LanguageOptions()); } + /// @return whether the selected backend supports translation public static boolean isSupported(LanguageOptions options) { LanguageOptions actual = options == null ? new LanguageOptions() : options; LanguageImpl impl = Display.getInstance().getLanguageBackend(); @@ -42,6 +46,11 @@ public static boolean isSupported(LanguageOptions options) { actual.getBackend().getId()); } + /// @param text source text + /// @param sourceLanguage BCP-47/ML Kit source language tag + /// @param targetLanguage BCP-47/ML Kit target language tag + /// @param options backend options, or {@code null} + /// @return asynchronous translated text public static AsyncResource translate(String text, String sourceLanguage, String targetLanguage, LanguageOptions options) { diff --git a/CodenameOne/src/com/codename1/ai/language/package-info.java b/CodenameOne/src/com/codename1/ai/language/package-info.java index 8debbe97ee3..09e12f7f923 100644 --- a/CodenameOne/src/com/codename1/ai/language/package-info.java +++ b/CodenameOne/src/com/codename1/ai/language/package-info.java @@ -23,9 +23,13 @@ /// Vendor-neutral on-device language identification, translation, and smart /// reply. /// -///

Android and iOS builds use feature-scoped ML Kit components. Each -/// public entry point is scanned independently, so language identification -/// does not bundle translation or Smart Reply. Translation model payloads are +///

Android uses feature-scoped ML Kit components. On iOS, automatic +/// language identification uses Apple's Natural Language framework, while +/// translation and Smart Reply use their matching ML Kit components. An +/// application may opt into ML Kit language identification with +/// {@link com.codename1.ai.language.LanguageBackends#mlKitLanguageIdentification()}. +/// Each entry point and optional selector is scanned independently, so using +/// one feature does not bundle the others. Translation model payloads are /// installed lazily by ML Kit. Other targets expose the same API with an /// explicit unsupported fallback.

package com.codename1.ai.language; diff --git a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java index 53aa6cd9a4f..e230f511ed7 100644 --- a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java +++ b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java @@ -38,12 +38,16 @@ abstract class AbstractVisionAnalyzer implements VisionAnalyzer { } @Override + /// @return whether this analyzer's feature/backend is available and open public final boolean isSupported() { VisionImpl impl = implementation(); return impl != null && impl.isSupported(feature, options.getBackend().getId()); } @Override + /// Starts one analysis using the retained native backend. + /// @param image immutable encoded or raw input + /// @return asynchronous typed result public final AsyncResource process(VisionImage image) { if (image == null) { throw new NullPointerException("image"); @@ -62,6 +66,7 @@ public final AsyncResource process(VisionImage image) { } @Override + /// Idempotently releases the retained native backend. public final void close() { closed = true; if (implementation != null) { diff --git a/CodenameOne/src/com/codename1/ai/vision/Barcode.java b/CodenameOne/src/com/codename1/ai/vision/Barcode.java index 5081c421271..802501ed8d4 100644 --- a/CodenameOne/src/com/codename1/ai/vision/Barcode.java +++ b/CodenameOne/src/com/codename1/ai/vision/Barcode.java @@ -22,7 +22,11 @@ */ package com.codename1.ai.vision; -/// Portable barcode observation. +/// Portable barcode observation with normalized geometry. Format names are +/// backend-neutral constants such as {@code QR_CODE}, {@code DATA_MATRIX}, +/// {@code CODE_128}, and {@code EAN_13}; unknown symbologies use +/// {@code UNKNOWN}. Corner points are ordered around the code when supplied +/// by the backend and may be empty when geometry is unavailable. public final class Barcode { private final String value; private final String format; @@ -31,11 +35,24 @@ public final class Barcode { private final VisionPoint[] corners; private final VisionMetadata metadata; + /// Creates a barcode without backend metadata. + /// @param value decoded display value, or {@code null} + /// @param format normalized symbology name + /// @param rawBytes original payload bytes when available + /// @param bounds normalized top-left-origin bounds + /// @param corners normalized corner points public Barcode(String value, String format, byte[] rawBytes, VisionRect bounds, VisionPoint[] corners) { this(value, format, rawBytes, bounds, corners, null); } + /// Creates a complete barcode observation. + /// @param value decoded display value, or {@code null} + /// @param format normalized symbology name + /// @param rawBytes original payload bytes when available + /// @param bounds normalized top-left-origin bounds + /// @param corners normalized corner points + /// @param metadata backend identity and optional diagnostic values public Barcode(String value, String format, byte[] rawBytes, VisionRect bounds, VisionPoint[] corners, VisionMetadata metadata) { @@ -52,28 +69,34 @@ public Barcode(String value, String format, byte[] rawBytes, this.metadata = metadata; } + /// @return decoded display value, or {@code null} when decoding failed public String getValue() { return value; } + /// @return normalized symbology name, never a vendor numeric identifier public String getFormat() { return format; } + /// @return a defensive copy of payload bytes, or {@code null} if unavailable public byte[] getRawBytes() { return copy(rawBytes); } + /// @return normalized top-left-origin bounds public VisionRect getBounds() { return bounds; } + /// @return defensive copy of normalized corners; possibly empty public VisionPoint[] getCorners() { VisionPoint[] out = new VisionPoint[corners.length]; System.arraycopy(corners, 0, out, 0, corners.length); return out; } + /// @return backend metadata, or {@code null} for manually created results public VisionMetadata getMetadata() { return metadata; } diff --git a/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java b/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java index 90c70a4f552..577c4e98f7a 100644 --- a/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java +++ b/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java @@ -24,10 +24,14 @@ /// Creates reusable still-image or live-frame barcode analyzers. public final class BarcodeScanner extends AbstractVisionAnalyzer { + /// Creates an analyzer using the platform default backend and options. + /// VisionOptions public BarcodeScanner() { this(null); } + /// Creates a reusable analyzer with explicit backend and result options. + /// options configuration captured by this analyzer; null uses defaults public BarcodeScanner(VisionOptions options) { super(VisionFeature.BARCODE_SCANNING, options); } diff --git a/CodenameOne/src/com/codename1/ai/vision/DocumentScanResult.java b/CodenameOne/src/com/codename1/ai/vision/DocumentScanResult.java index 2607ea7ab0e..d74bbe3414e 100644 --- a/CodenameOne/src/com/codename1/ai/vision/DocumentScanResult.java +++ b/CodenameOne/src/com/codename1/ai/vision/DocumentScanResult.java @@ -22,15 +22,24 @@ */ package com.codename1.ai.vision; -/// Corrected document pages returned as encoded images. +/// Corrected document pages returned as encoded image data. Pages are ordered +/// as detected and both construction and access make defensive copies, so the +/// result can safely cross asynchronous boundaries. The still-image scanner is +/// currently Apple-only; Android's ML Kit document API owns an interactive +/// camera flow and does not implement this analyzer contract. public final class DocumentScanResult { private final byte[][] pages; private final VisionMetadata metadata; + /// Creates a corrected document scan without backend metadata. + /// @param pages encoded corrected page images, deeply defensively copied public DocumentScanResult(byte[][] pages) { this(pages, null); } + /// Creates a corrected document scan with backend diagnostics. + /// @param pages encoded corrected page images, deeply defensively copied + /// @param metadata backend details, or {@code null} public DocumentScanResult(byte[][] pages, VisionMetadata metadata) { if (pages == null) { this.pages = new byte[0][]; @@ -48,10 +57,13 @@ public DocumentScanResult(byte[][] pages, VisionMetadata metadata) { this.metadata = metadata; } + /// @return number of corrected pages public int getPageCount() { return pages.length; } + /// @param index zero-based page index + /// @return defensive copy of that page's encoded image public byte[] getPage(int index) { byte[] page = pages[index]; byte[] out = new byte[page.length]; @@ -59,6 +71,7 @@ public byte[] getPage(int index) { return out; } + /// @return backend metadata, or {@code null} for manually created results public VisionMetadata getMetadata() { return metadata; } diff --git a/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java b/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java index d6d80a6d056..9d9a38a3822 100644 --- a/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java +++ b/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java @@ -24,10 +24,14 @@ /// Creates reusable document-boundary and perspective-correction analyzers. public final class DocumentScanner extends AbstractVisionAnalyzer { + /// Creates an analyzer using the platform default backend and options. + /// VisionOptions public DocumentScanner() { this(null); } + /// Creates a reusable analyzer with explicit backend and result options. + /// options configuration captured by this analyzer; null uses defaults public DocumentScanner(VisionOptions options) { super(VisionFeature.DOCUMENT_SCANNING, options); } diff --git a/CodenameOne/src/com/codename1/ai/vision/Face.java b/CodenameOne/src/com/codename1/ai/vision/Face.java index 56cbec325ed..0d4ac975b2f 100644 --- a/CodenameOne/src/com/codename1/ai/vision/Face.java +++ b/CodenameOne/src/com/codename1/ai/vision/Face.java @@ -41,6 +41,14 @@ public final class Face { private final int trackingId; private final VisionMetadata metadata; + /// Creates a detected face without backend metadata. + /// @param bounds face bounds in the oriented image coordinate space + /// @param landmarks named feature points, defensively copied + /// @param yaw horizontal Euler angle in degrees + /// @param pitch vertical Euler angle in degrees + /// @param roll in-plane Euler angle in degrees + /// @param smilingProbability smile confidence, or a negative value when unavailable + /// @param trackingId stable streaming id, or a negative value when unavailable public Face(VisionRect bounds, Map landmarks, float yaw, float pitch, float roll, float smilingProbability, int trackingId) { @@ -48,6 +56,15 @@ public Face(VisionRect bounds, Map landmarks, trackingId, null); } + /// Creates a detected face with backend diagnostics. + /// @param bounds face bounds in the oriented image coordinate space + /// @param landmarks named feature points, defensively copied + /// @param yaw horizontal Euler angle in degrees + /// @param pitch vertical Euler angle in degrees + /// @param roll in-plane Euler angle in degrees + /// @param smilingProbability smile confidence, or a negative value when unavailable + /// @param trackingId stable streaming id, or a negative value when unavailable + /// @param metadata backend details, or {@code null} public Face(VisionRect bounds, Map landmarks, float yaw, float pitch, float roll, float smilingProbability, int trackingId, @@ -64,34 +81,42 @@ public Face(VisionRect bounds, Map landmarks, this.metadata = metadata; } + /// @return normalized top-left-origin face bounds public VisionRect getBounds() { return bounds; } + /// @return immutable named landmark map; possibly empty public Map getLandmarks() { return landmarks; } + /// @return left/right head rotation in degrees, or 0 if unavailable public float getYaw() { return yaw; } + /// @return up/down head rotation in degrees, or 0 if unavailable public float getPitch() { return pitch; } + /// @return in-plane head rotation in degrees, or 0 if unavailable public float getRoll() { return roll; } + /// @return smile probability in 0..1, or -1 when unavailable public float getSmilingProbability() { return smilingProbability; } + /// @return stream tracking identifier, or -1 when unavailable public int getTrackingId() { return trackingId; } + /// @return backend metadata, or {@code null} when manually constructed public VisionMetadata getMetadata() { return metadata; } diff --git a/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java b/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java index 6afc9537d8f..852b6ee18e0 100644 --- a/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java +++ b/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java @@ -24,10 +24,14 @@ /// Creates reusable on-device face analyzers. public final class FaceDetector extends AbstractVisionAnalyzer { + /// Creates an analyzer using the platform default backend and options. + /// VisionOptions public FaceDetector() { this(null); } + /// Creates a reusable analyzer with explicit backend and result options. + /// options configuration captured by this analyzer; null uses defaults public FaceDetector(VisionOptions options) { super(VisionFeature.FACE_DETECTION, options); } diff --git a/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java b/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java index fd6256de7bb..a3e63a7bc19 100644 --- a/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java +++ b/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java @@ -22,17 +22,29 @@ */ package com.codename1.ai.vision; -/// Portable image classification label. +/// Portable ranked image-classification label. Confidence is normalized to +/// 0..1. The numeric index is a backend/model class index and should not be +/// treated as stable across different backends; portable code should prefer +/// the text label. public final class ImageLabel { private final String text; private final float confidence; private final int index; private final VisionMetadata metadata; + /// Creates a label without backend metadata. + /// @param text normalized label text + /// @param confidence provider confidence, normally from zero to one + /// @param index provider label index, or a negative value when unavailable public ImageLabel(String text, float confidence, int index) { this(text, confidence, index, null); } + /// Creates a label with backend diagnostics. + /// @param text normalized label text + /// @param confidence provider confidence, normally from zero to one + /// @param index provider label index, or a negative value when unavailable + /// @param metadata backend details, or {@code null} public ImageLabel(String text, float confidence, int index, VisionMetadata metadata) { this.text = text == null ? "" : text; @@ -41,18 +53,22 @@ public ImageLabel(String text, float confidence, int index, this.metadata = metadata; } + /// @return human-readable class label public String getText() { return text; } + /// @return classification confidence in the range 0..1 public float getConfidence() { return confidence; } + /// @return backend/model class index; not portable across models public int getIndex() { return index; } + /// @return backend metadata, or {@code null} when manually constructed public VisionMetadata getMetadata() { return metadata; } diff --git a/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java b/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java index d23cf1c11fb..9bd368d250f 100644 --- a/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java +++ b/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java @@ -24,10 +24,14 @@ /// Creates reusable on-device image classifiers. public final class ImageLabeler extends AbstractVisionAnalyzer { + /// Creates an analyzer using the platform default backend and options. + /// VisionOptions public ImageLabeler() { this(null); } + /// Creates a reusable analyzer with explicit backend and result options. + /// options configuration captured by this analyzer; null uses defaults public ImageLabeler(VisionOptions options) { super(VisionFeature.IMAGE_LABELING, options); } diff --git a/CodenameOne/src/com/codename1/ai/vision/Pose.java b/CodenameOne/src/com/codename1/ai/vision/Pose.java index 9e772fa886d..93606870389 100644 --- a/CodenameOne/src/com/codename1/ai/vision/Pose.java +++ b/CodenameOne/src/com/codename1/ai/vision/Pose.java @@ -22,15 +22,23 @@ */ package com.codename1.ai.vision; -/// Portable body-pose result. +/// Portable body-pose result. Landmark coordinates use the normalized +/// top-left-origin image space defined by {@link VisionPoint}; confidence is +/// in the range 0..1. Landmark names are backend-neutral where the native +/// backend exposes a known joint name. public final class Pose { private final Landmark[] landmarks; private final VisionMetadata metadata; + /// Creates a pose without backend metadata. + /// @param landmarks detected named body landmarks, defensively copied public Pose(Landmark[] landmarks) { this(landmarks, null); } + /// Creates a pose with backend diagnostics. + /// @param landmarks detected named body landmarks, defensively copied + /// @param metadata backend details, or {@code null} public Pose(Landmark[] landmarks, VisionMetadata metadata) { if (landmarks == null) { this.landmarks = new Landmark[0]; @@ -41,35 +49,45 @@ public Pose(Landmark[] landmarks, VisionMetadata metadata) { this.metadata = metadata; } + /// @return defensive copy of detected body landmarks public Landmark[] getLandmarks() { Landmark[] out = new Landmark[landmarks.length]; System.arraycopy(landmarks, 0, out, 0, landmarks.length); return out; } + /// @return backend metadata, or {@code null} when manually constructed public VisionMetadata getMetadata() { return metadata; } + /// One named body joint with normalized position and confidence. public static final class Landmark { private final String name; private final VisionPoint position; private final float confidence; + /// Creates one detected body joint. + /// @param name joint name + /// @param position normalized joint position + /// @param confidence in-frame confidence in 0..1 public Landmark(String name, VisionPoint position, float confidence) { this.name = name; this.position = position; this.confidence = confidence; } + /// @return portable/native joint name public String getName() { return name; } + /// @return normalized top-left-origin joint position public VisionPoint getPosition() { return position; } + /// @return in-frame confidence in the range 0..1 public float getConfidence() { return confidence; } diff --git a/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java b/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java index 861fd48ae9c..4748505b7b1 100644 --- a/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java +++ b/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java @@ -23,11 +23,15 @@ package com.codename1.ai.vision; /// Creates reusable body-pose analyzers. -public final class PoseDetector extends AbstractVisionAnalyzer { +public final class PoseDetector extends AbstractVisionAnalyzer { + /// Creates an analyzer using the platform default backend and options. + /// VisionOptions public PoseDetector() { this(null); } + /// Creates a reusable analyzer with explicit backend and result options. + /// options configuration captured by this analyzer; null uses defaults public PoseDetector(VisionOptions options) { super(VisionFeature.POSE_DETECTION, options); } diff --git a/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java b/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java index 48332afcce4..cd6287e6b08 100644 --- a/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java +++ b/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java @@ -22,17 +22,28 @@ */ package com.codename1.ai.vision; -/// Per-pixel foreground confidence mask. +/// Dense per-pixel foreground confidence mask. The confidence array is +/// row-major with exactly {@code width * height} values in the range 0..1. +/// It is defensively copied on construction and access. public final class SegmentationMask { private final int width; private final int height; private final float[] confidence; private final VisionMetadata metadata; + /// Creates a row-major confidence mask without backend metadata. + /// @param width mask width in pixels + /// @param height mask height in pixels + /// @param confidence one foreground probability per pixel, defensively copied public SegmentationMask(int width, int height, float[] confidence) { this(width, height, confidence, null); } + /// Creates a row-major confidence mask with backend diagnostics. + /// @param width mask width in pixels + /// @param height mask height in pixels + /// @param confidence one foreground probability per pixel, defensively copied + /// @param metadata backend details, or {@code null} public SegmentationMask(int width, int height, float[] confidence, VisionMetadata metadata) { if (width < 0 || height < 0 @@ -46,20 +57,24 @@ public SegmentationMask(int width, int height, float[] confidence, this.metadata = metadata; } + /// @return mask width, which may differ from source image width public int getWidth() { return width; } + /// @return mask height, which may differ from source image height public int getHeight() { return height; } + /// @return defensive copy of row-major foreground confidences public float[] getConfidence() { float[] out = new float[confidence.length]; System.arraycopy(confidence, 0, out, 0, confidence.length); return out; } + /// @return backend metadata, or {@code null} when manually constructed public VisionMetadata getMetadata() { return metadata; } diff --git a/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java b/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java index e28928e24ad..8b6d8eebccf 100644 --- a/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java +++ b/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java @@ -24,10 +24,14 @@ /// Creates reusable foreground/person segmentation analyzers. public final class SelfieSegmenter extends AbstractVisionAnalyzer { + /// Creates an analyzer using the platform default backend and options. + /// VisionOptions public SelfieSegmenter() { this(null); } + /// Creates a reusable analyzer with explicit backend and result options. + /// options configuration captured by this analyzer; null uses defaults public SelfieSegmenter(VisionOptions options) { super(VisionFeature.SELFIE_SEGMENTATION, options); } diff --git a/CodenameOne/src/com/codename1/ai/vision/TextRecognitionResult.java b/CodenameOne/src/com/codename1/ai/vision/TextRecognitionResult.java index 1f61ed83000..244ec2e97dc 100644 --- a/CodenameOne/src/com/codename1/ai/vision/TextRecognitionResult.java +++ b/CodenameOne/src/com/codename1/ai/vision/TextRecognitionResult.java @@ -22,7 +22,10 @@ */ package com.codename1.ai.vision; -/// OCR output with a portable block-level structure. +/// OCR output containing the complete recognized text and portable +/// block-level geometry. Block bounds use normalized top-left-origin +/// coordinates; a language tag may be absent when the backend does not expose +/// per-block language identification. public final class TextRecognitionResult { public static final TextRecognitionResult EMPTY = new TextRecognitionResult("", new TextBlock[0]); @@ -31,10 +34,17 @@ public final class TextRecognitionResult { private final TextBlock[] blocks; private final VisionMetadata metadata; + /// Creates an OCR result without backend metadata. + /// @param text full recognized text in reading order + /// @param blocks structured text regions, defensively copied public TextRecognitionResult(String text, TextBlock[] blocks) { this(text, blocks, null); } + /// Creates an OCR result with backend diagnostics. + /// @param text full recognized text in reading order + /// @param blocks structured text regions, defensively copied + /// @param metadata backend details, or {@code null} public TextRecognitionResult(String text, TextBlock[] blocks, VisionMetadata metadata) { this.text = text == null ? "" : text; @@ -47,26 +57,35 @@ public TextRecognitionResult(String text, TextBlock[] blocks, this.metadata = metadata; } + /// @return complete recognized text in reading order public String getText() { return text; } + /// @return defensive copy of recognized blocks public TextBlock[] getBlocks() { TextBlock[] out = new TextBlock[blocks.length]; System.arraycopy(blocks, 0, out, 0, blocks.length); return out; } + /// @return backend metadata, or {@code null} when manually constructed public VisionMetadata getMetadata() { return metadata; } + /// One recognized text block with confidence and normalized bounds. public static final class TextBlock { private final String text; private final float confidence; private final VisionRect bounds; private final String languageTag; + /// Creates one OCR block. + /// @param text recognized text + /// @param confidence recognition confidence in 0..1 + /// @param bounds normalized block bounds + /// @param languageTag BCP-47 tag, or {@code null} public TextBlock(String text, float confidence, VisionRect bounds, String languageTag) { this.text = text == null ? "" : text; this.confidence = confidence; @@ -74,18 +93,22 @@ public TextBlock(String text, float confidence, VisionRect bounds, String langua this.languageTag = languageTag; } + /// @return recognized block text public String getText() { return text; } + /// @return recognition confidence in the range 0..1 public float getConfidence() { return confidence; } + /// @return normalized top-left-origin bounds public VisionRect getBounds() { return bounds; } + /// @return BCP-47 language tag, or {@code null} when unavailable public String getLanguageTag() { return languageTag; } diff --git a/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java b/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java index e90b8b90c34..7da6c711ef1 100644 --- a/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java +++ b/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java @@ -24,10 +24,14 @@ /// Creates reusable on-device OCR analyzers. public final class TextRecognizer extends AbstractVisionAnalyzer { + /// Creates an analyzer using the platform default backend and options. + /// VisionOptions public TextRecognizer() { this(null); } + /// Creates a reusable analyzer with explicit backend and result options. + /// options configuration captured by this analyzer; null uses defaults public TextRecognizer(VisionOptions options) { super(VisionFeature.TEXT_RECOGNITION, options); } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java b/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java index f34c66d8710..a5e126e7a4c 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java @@ -24,10 +24,18 @@ import com.codename1.util.AsyncResource; -/// Reusable, closable analyzer for still images or camera frames. +/// Reusable, closable on-device analyzer for still images or camera frames. +/// Implementations may retain native detectors and models between calls, so +/// create one analyzer per stream/workflow and close it when finished. public interface VisionAnalyzer extends AutoCloseable { + /// Tests the exact feature/backend pair configured for this analyzer. + /// @return {@code true} when the current target supports it boolean isSupported(); + /// Starts one asynchronous analysis without uploading the image. + /// @param image encoded or raw input + /// @return asynchronous typed result delivered on the EDT AsyncResource process(VisionImage image); + /// Releases native detector/model resources; further processing fails. @Override void close(); } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionBackend.java b/CodenameOne/src/com/codename1/ai/vision/VisionBackend.java index efd978f8121..00019a8de00 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionBackend.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionBackend.java @@ -20,19 +20,12 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - */ package com.codename1.ai.vision; -/// Identifies a vision implementation. Use {@link VisionBackends} to obtain -/// instances so builders can detect explicit optional-backend use. +/// Identifies a vision implementation. Use {@link VisionBackends} instead of +/// implementing this interface: builder scanning of those selector calls is +/// what adds an optional, feature-specific native dependency. public interface VisionBackend { + /// @return stable identifier passed to the current platform implementation String getId(); } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java b/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java index 892ba20d904..7534cb4cbad 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java @@ -20,19 +20,12 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - */ package com.codename1.ai.vision; -/// Vision backend selectors. {@code auto()} chooses Apple Vision on iOS and -/// ML Kit on Android. Unsupported ports report that through the analyzer. +/// Vision backend selectors. {@link #auto()} chooses Apple Vision on iOS and +/// ML Kit on Android. Feature-specific ML Kit methods are separate so one +/// selector never adds unrelated OCR, barcode, face, pose, labeling, or +/// segmentation pods. public final class VisionBackends { private static final VisionBackend AUTO = new NamedBackend("auto"); private static final VisionBackend APPLE = new NamedBackend("apple-vision"); @@ -41,15 +34,52 @@ public final class VisionBackends { private VisionBackends() { } + /// @return the dependency-minimal platform default public static VisionBackend auto() { return AUTO; } + /// Selects Apple Vision/Core Image without adding a third-party dependency. + /// @return Apple-native backend selector public static VisionBackend appleVision() { return APPLE; } - public static VisionBackend mlKit() { + /// Selects ML Kit for text recognition on iOS. Android already uses ML + /// Kit for the automatic backend. + /// + /// @return the ML Kit backend selector + public static VisionBackend mlKitTextRecognition() { + return ML_KIT; + } + + /// Selects ML Kit for barcode scanning on iOS. + /// @return the ML Kit backend selector + public static VisionBackend mlKitBarcodeScanning() { + return ML_KIT; + } + + /// Selects ML Kit for face detection on iOS. + /// @return the ML Kit backend selector + public static VisionBackend mlKitFaceDetection() { + return ML_KIT; + } + + /// Selects ML Kit for image labeling on iOS. + /// @return the ML Kit backend selector + public static VisionBackend mlKitImageLabeling() { + return ML_KIT; + } + + /// Selects ML Kit for pose detection on iOS. + /// @return the ML Kit backend selector + public static VisionBackend mlKitPoseDetection() { + return ML_KIT; + } + + /// Selects ML Kit for selfie segmentation on iOS. + /// @return the ML Kit backend selector + public static VisionBackend mlKitSelfieSegmentation() { return ML_KIT; } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionException.java b/CodenameOne/src/com/codename1/ai/vision/VisionException.java index 48d1ab9c0fc..570f0760d34 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionException.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionException.java @@ -32,16 +32,24 @@ public class VisionException extends RuntimeException { private final int code; + /// Creates a classified vision failure. + /// @param code portable failure code defined by this class + /// @param message user-readable failure description public VisionException(int code, String message) { super(message); this.code = code; } + /// Creates a classified vision failure with its native or port cause. + /// @param code portable failure code defined by this class + /// @param message user-readable failure description + /// @param cause originating detector or image-conversion error public VisionException(int code, String message, Throwable cause) { super(message, cause); this.code = code; } + /// @return one of this class's portable failure-code constants public int getCode() { return code; } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java index db745e92fbb..bfa7d774e56 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java @@ -25,8 +25,10 @@ import com.codename1.camera.CameraFrame; import com.codename1.camera.FrameFormat; -/// Immutable vision input. Camera-frame factories copy the callback-owned bytes -/// so asynchronous analyzers never retain recycled camera buffers. +/// Immutable input for encoded still images or raw camera pixels. Factory +/// methods defensively copy their arrays. In particular, +/// {@link #fromCameraFrame(CameraFrame)} detaches from callback-owned buffers +/// that a camera backend may recycle when the callback returns. public final class VisionImage { private final byte[] encodedBytes; private final byte[] pixels; @@ -47,6 +49,9 @@ private VisionImage(byte[] encodedBytes, byte[] pixels, int width, int height, this.format = format == null ? FrameFormat.JPEG : format; } + /// Creates an encoded JPEG or PNG input. + /// @param bytes complete encoded image, copied by this method + /// @return immutable image input public static VisionImage encoded(byte[] bytes) { if (bytes == null || bytes.length == 0) { throw new IllegalArgumentException("Image bytes must not be empty"); @@ -54,6 +59,13 @@ public static VisionImage encoded(byte[] bytes) { return new VisionImage(bytes, null, 0, 0, 0, 0, FrameFormat.JPEG); } + /// Creates raw NV21 or RGBA8888 input with display orientation metadata. + /// @param bytes pixel buffer in {@code format}, copied by this method + /// @param width unrotated pixel width + /// @param height unrotated pixel height + /// @param format supported raw frame format + /// @param rotationDegrees clockwise display rotation + /// @return immutable image input public static VisionImage pixels(byte[] bytes, int width, int height, FrameFormat format, int rotationDegrees) { if (bytes == null || bytes.length == 0 || width <= 0 || height <= 0) { @@ -62,6 +74,9 @@ public static VisionImage pixels(byte[] bytes, int width, int height, return new VisionImage(null, bytes, width, height, rotationDegrees, 0, format); } + /// Copies a camera frame, including timestamp, format, and orientation. + /// @param frame callback-owned frame + /// @return detached immutable input safe for asynchronous analysis public static VisionImage fromCameraFrame(CameraFrame frame) { if (frame == null) { throw new NullPointerException("frame"); @@ -71,30 +86,57 @@ public static VisionImage fromCameraFrame(CameraFrame frame) { frame.getTimestampNanos(), frame.getFormat()); } + /// @return a defensive copy of encoded bytes, or {@code null} for raw input public byte[] getEncodedBytes() { return copy(encodedBytes); } + /// Returns the immutable object's backing encoded buffer to a port + /// implementation. Application code must use {@link #getEncodedBytes()}, + /// which preserves the class's defensive-copy contract. + /// + /// @return the backing encoded buffer, or {@code null} for a raw image + /// @hidden + public byte[] getEncodedBytesUnsafe() { + return encodedBytes; + } + + /// @return a defensive copy of raw pixels, or {@code null} for encoded input public byte[] getPixels() { return copy(pixels); } + /// Returns the immutable object's backing pixel buffer to a port + /// implementation. The returned array must never be modified or retained + /// beyond the native call that consumes it. + /// + /// @return the backing pixel buffer, or {@code null} for an encoded image + /// @hidden + public byte[] getPixelsUnsafe() { + return pixels; + } + + /// @return raw pixel width, or zero for encoded input public int getWidth() { return width; } + /// @return raw pixel height, or zero for encoded input public int getHeight() { return height; } + /// @return normalized clockwise rotation in the range 0..359 public int getRotationDegrees() { return rotationDegrees; } + /// @return capture timestamp in nanoseconds, or zero for manual input public long getTimestampNanos() { return timestampNanos; } + /// @return encoded/raw frame format public FrameFormat getFormat() { return format; } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionMetadata.java b/CodenameOne/src/com/codename1/ai/vision/VisionMetadata.java index b3c718f172e..47446f2fb9c 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionMetadata.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionMetadata.java @@ -26,12 +26,17 @@ import java.util.HashMap; import java.util.Map; -/// Optional backend identity and backend-specific string values attached to a -/// vision result. Portable code should use the typed result fields first. +/// Optional backend identity and backend-specific diagnostic strings attached +/// to a vision result. Portable application logic should use typed result +/// fields first; metadata keys are intentionally not guaranteed across +/// backends. The values map is immutable. public final class VisionMetadata { private final String backendId; private final Map values; + /// Creates backend metadata with optional diagnostic values. + /// @param backendId stable portable backend id + /// @param values backend-specific string diagnostics, defensively copied public VisionMetadata(String backendId, Map values) { this.backendId = backendId; this.values = values == null || values.isEmpty() @@ -39,10 +44,13 @@ public VisionMetadata(String backendId, Map values) { : Collections.unmodifiableMap(new HashMap(values)); } + /// Creates metadata containing only the selected backend id. + /// @param backendId stable portable backend id public VisionMetadata(String backendId) { this(backendId, null); } + /// @return stable backend identifier such as {@code apple-vision} or {@code ml-kit} public String getBackendId() { return backendId; } @@ -51,6 +59,8 @@ public Map getValues() { return values; } + /// @param key backend-defined diagnostic key + /// @return associated value, or {@code null} public String get(String key) { return values.get(key); } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java b/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java index 6905ae54cda..88b83f395c0 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java @@ -22,35 +22,48 @@ */ package com.codename1.ai.vision; -/// Common analyzer options; feature-specific options may extend this class. +/// Common analyzer configuration. An analyzer captures the supplied options +/// when constructed; reuse it for frames needing the same backend, confidence +/// threshold, and result limit. public class VisionOptions { private VisionBackend backend = VisionBackends.auto(); private float minimumConfidence; private int maximumResults; + /// @param value selector, or {@code null} to restore automatic selection + /// @return this options object public VisionOptions backend(VisionBackend value) { backend = value == null ? VisionBackends.auto() : value; return this; } + /// Sets the confidence threshold, clamped to 0..1. + /// @param value requested threshold + /// @return this options object public VisionOptions minimumConfidence(float value) { minimumConfidence = Math.max(0, Math.min(1, value)); return this; } + /// Sets a non-negative result limit; zero means backend default/unlimited. + /// @param value requested limit + /// @return this options object public VisionOptions maximumResults(int value) { maximumResults = Math.max(0, value); return this; } + /// @return selected backend, never {@code null} public VisionBackend getBackend() { return backend; } + /// @return confidence threshold in the range 0..1 public float getMinimumConfidence() { return minimumConfidence; } + /// @return non-negative result limit; zero means backend default/unlimited public int getMaximumResults() { return maximumResults; } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java index 50b9067b3a6..b4341887c01 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java @@ -28,8 +28,11 @@ import com.codename1.ui.Display; import com.codename1.util.SuccessCallback; -/// Connects a camera frame stream to an analyzer with keep-only-latest -/// backpressure. The pipeline owns the analyzer and closes it on close. +/// Connects a camera frame stream to a reusable analyzer with keep-only-latest +/// backpressure. At most one analysis and one pending frame are retained, so a +/// slow model cannot build an unbounded queue. Results and errors arrive on +/// the EDT. The pipeline owns and closes the analyzer but not the camera +/// session. public final class VisionPipeline implements AutoCloseable { private final CameraSession session; private final VisionAnalyzer analyzer; @@ -39,6 +42,10 @@ public final class VisionPipeline implements AutoCloseable { private boolean busy; private boolean closed; + /// Attaches immediately as the session's frame listener. + /// @param session active camera session whose frames should be analyzed + /// @param analyzer reusable analyzer owned by this pipeline + /// @param listener EDT result/error listener public VisionPipeline(CameraSession session, VisionAnalyzer analyzer, VisionPipelineListener listener) { if (session == null || analyzer == null || listener == null) { @@ -71,7 +78,22 @@ private void accept(VisionImage image) { } private void process(final VisionImage image) { - analyzer.process(image).ready(new SuccessCallback() { + final com.codename1.util.AsyncResource operation; + try { + operation = analyzer.process(image); + if (operation == null) { + throw new IllegalStateException("Vision analyzer returned no operation"); + } + } catch (final Throwable error) { + onFinished(new Runnable() { + @Override + public void run() { + listener.error(error); + } + }); + return; + } + operation.ready(new SuccessCallback() { @Override public void onSucess(final T value) { onFinished(new Runnable() { @@ -117,12 +139,18 @@ public void run() { }); } + /// Returns whether a frame is currently being analyzed. + /// + /// @return {@code true} while an analysis is in flight public boolean isBusy() { synchronized (this) { return busy; } } + /// Stops accepting frames, detaches from the camera session, discards the + /// pending frame, and closes the analyzer. Calling this method more than + /// once has no effect. @Override public void close() { synchronized (this) { diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionPoint.java b/CodenameOne/src/com/codename1/ai/vision/VisionPoint.java index 20d9441ae2b..2bb8f1f8d90 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionPoint.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionPoint.java @@ -22,20 +22,27 @@ */ package com.codename1.ai.vision; -/// Immutable point normalized to a top-left-origin 0..1 coordinate space. +/// Immutable point in a normalized, top-left-origin coordinate space. X grows +/// right and Y grows down; 0 and 1 correspond to image edges, independent of +/// source pixel dimensions. public final class VisionPoint { private final float x; private final float y; + /// Creates a point in the oriented input image's top-left coordinate space. + /// @param x horizontal coordinate + /// @param y vertical coordinate public VisionPoint(float x, float y) { this.x = x; this.y = y; } + /// @return horizontal coordinate normalized to the oriented input width public float getX() { return x; } + /// @return downward vertical coordinate normalized to input height public float getY() { return y; } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionRect.java b/CodenameOne/src/com/codename1/ai/vision/VisionRect.java index 75609b578d5..c343eda5b37 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionRect.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionRect.java @@ -22,7 +22,9 @@ */ package com.codename1.ai.vision; -/// Immutable normalized rectangle using a top-left origin. +/// Immutable normalized rectangle using a top-left origin. X/Y identify the +/// upper-left corner and width/height are fractions of the oriented input +/// dimensions. {@link #EMPTY} represents unavailable geometry. public final class VisionRect { public static final VisionRect EMPTY = new VisionRect(0, 0, 0, 0); @@ -31,6 +33,11 @@ public final class VisionRect { private final float width; private final float height; + /// Creates a rectangle in the oriented input image's top-left coordinate space. + /// @param x left coordinate + /// @param y top coordinate + /// @param width non-negative rectangle width + /// @param height non-negative rectangle height public VisionRect(float x, float y, float width, float height) { this.x = x; this.y = y; @@ -38,18 +45,22 @@ public VisionRect(float x, float y, float width, float height) { this.height = height; } + /// @return normalized left coordinate public float getX() { return x; } + /// @return normalized top coordinate public float getY() { return y; } + /// @return width as a fraction of oriented input width public float getWidth() { return width; } + /// @return height as a fraction of oriented input height public float getHeight() { return height; } diff --git a/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java b/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java index 3af7f9c7ad0..0b12a9d5e50 100644 --- a/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java +++ b/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java @@ -7385,8 +7385,6 @@ public void capturePhoto(ActionListener response) { /// Factory for the low-level `com.codename1.camera.Camera` API. Each call /// returns a fresh per-session backend, or `null` on platforms that do not /// implement the new API. Subclasses override to wire in their port. - /// - /// @hidden public CameraImpl createCameraImpl() { return null; } @@ -7394,29 +7392,21 @@ public CameraImpl createCameraImpl() { /// Factory for the `com.codename1.ar.AR` augmented reality API. Each call /// returns a fresh per-session backend, or `null` on platforms without AR /// support. Subclasses override to wire in their port. - /// - /// @hidden public ARImpl createARImpl() { return null; } /// Factory for the built-in on-device vision API. - /// - /// @hidden public VisionImpl createVisionImpl() { return null; } /// Factory for the built-in LiteRT inference API. - /// - /// @hidden public InferenceImpl createInferenceImpl() { return null; } /// Factory for built-in on-device language services. - /// - /// @hidden public LanguageImpl createLanguageImpl() { return null; } diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java index 961040e3031..f3136106a68 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java @@ -37,13 +37,13 @@ /** ML Kit barcode scanning; retained only for {@code BarcodeScanner} users. */ final class AndroidBarcodeScanningAdapter extends AndroidVisionAdapter { + private final BarcodeScanner client = BarcodeScanning.getClient(); @Override @SuppressWarnings("unchecked") void analyze(InputImage input, final int imageWidth, final int imageHeight, VisionOptions options, AsyncResource resource) { final AsyncResource out = (AsyncResource) resource; - final BarcodeScanner client = BarcodeScanning.getClient(); client.process(input).addOnSuccessListener( new OnSuccessListener>() { public void onSuccess( @@ -68,9 +68,13 @@ public void onSuccess( corners, METADATA); } complete(out, result); - client.close(); } - }).addOnFailureListener(failure(out, client)); + }).addOnFailureListener(failure(out)); + } + + @Override + void close() { + client.close(); } private static String barcodeFormat(int format) { diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java index c3e9baa6627..904523d589c 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java @@ -39,20 +39,18 @@ /** ML Kit face detection; retained only for {@code FaceDetector} users. */ final class AndroidFaceDetectionAdapter extends AndroidVisionAdapter { + private final FaceDetector client = FaceDetection.getClient( + new FaceDetectorOptions.Builder() + .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_ALL) + .setClassificationMode( + FaceDetectorOptions.CLASSIFICATION_MODE_ALL) + .enableTracking().build()); @Override @SuppressWarnings("unchecked") void analyze(InputImage input, final int imageWidth, final int imageHeight, VisionOptions options, AsyncResource resource) { final AsyncResource out = (AsyncResource) resource; - FaceDetectorOptions detectorOptions = - new FaceDetectorOptions.Builder() - .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_ALL) - .setClassificationMode( - FaceDetectorOptions.CLASSIFICATION_MODE_ALL) - .enableTracking().build(); - final FaceDetector client = - FaceDetection.getClient(detectorOptions); client.process(input).addOnSuccessListener( new OnSuccessListener>() { public void onSuccess( @@ -88,9 +86,13 @@ public void onSuccess( tracking == null ? -1 : tracking, METADATA); } complete(out, result); - client.close(); } - }).addOnFailureListener(failure(out, client)); + }).addOnFailureListener(failure(out)); + } + + @Override + void close() { + client.close(); } private static void addLandmark(Map out, String name, diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java index 6829ae794b5..7ddbfec5112 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java @@ -35,16 +35,18 @@ /** ML Kit image labeling; retained only for {@code ImageLabeler} users. */ final class AndroidImageLabelingAdapter extends AndroidVisionAdapter { + private ImageLabeler client; @Override @SuppressWarnings("unchecked") void analyze(InputImage input, int imageWidth, int imageHeight, VisionOptions options, AsyncResource resource) { final AsyncResource out = (AsyncResource) resource; - final ImageLabeler client = ImageLabeling.getClient( - new ImageLabelerOptions.Builder() - .setConfidenceThreshold(options.getMinimumConfidence()) - .build()); + if (client == null) { + client = ImageLabeling.getClient(new ImageLabelerOptions.Builder() + .setConfidenceThreshold(options.getMinimumConfidence()) + .build()); + } client.process(input).addOnSuccessListener( new OnSuccessListener>() { public void onSuccess( @@ -57,8 +59,14 @@ public void onSuccess( value.getConfidence(), value.getIndex(), METADATA); } complete(out, result); - client.close(); } - }).addOnFailureListener(failure(out, client)); + }).addOnFailureListener(failure(out)); + } + + @Override + void close() { + if (client != null) { + client.close(); + } } } diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java index c914020f2f2..f7a7aa0209c 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java @@ -41,6 +41,7 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.channels.FileChannel; import java.util.HashMap; import java.util.Map; @@ -66,15 +67,38 @@ public AsyncResource open(final ModelSource source, Display.getInstance().scheduleBackgroundTask(new Runnable() { public void run() { try { + InferenceOptions.Accelerator accelerator = + options.getAccelerator(); + if ((accelerator == InferenceOptions.Accelerator.GPU + || accelerator == InferenceOptions.Accelerator.CORE_ML) + && !options.isFallbackAllowed()) { + throw new InferenceException(accelerator + + " acceleration is unavailable on Android"); + } ByteBuffer model = loadModel(source); Interpreter.Options nativeOptions = new Interpreter.Options(); if (options.getThreads() > 0) { nativeOptions.setNumThreads(options.getThreads()); } - if (options.getAccelerator() == InferenceOptions.Accelerator.NPU) { + if (accelerator == InferenceOptions.Accelerator.NPU) { nativeOptions.setUseNNAPI(true); } - final Handle handle = new Handle(new Interpreter(model, nativeOptions)); + Interpreter interpreter; + try { + interpreter = new Interpreter(model, nativeOptions); + } catch (Throwable acceleratedFailure) { + if (accelerator != InferenceOptions.Accelerator.NPU + || !options.isFallbackAllowed()) { + throw acceleratedFailure; + } + Interpreter.Options cpuOptions = new Interpreter.Options(); + if (options.getThreads() > 0) { + cpuOptions.setNumThreads(options.getThreads()); + } + model.rewind(); + interpreter = new Interpreter(model, cpuOptions); + } + final Handle handle = new Handle(interpreter); Display.getInstance().callSerially(new Runnable() { public void run() { out.complete(handle); @@ -216,7 +240,7 @@ private static ByteBuffer toBuffer(Tensor value, DataType nativeType) { throw new IllegalArgumentException("Input " + value.getName() + " type does not match the model"); } - Object data = value.getData(); + Object data = value.getDataUnsafe(); int byteCount; if (data instanceof float[]) byteCount = ((float[]) data).length * 4; else if (data instanceof int[]) byteCount = ((int[]) data).length * 4; @@ -266,19 +290,23 @@ private static int elementCount(int[] shape) { } private static ByteBuffer loadModel(ModelSource source) throws IOException { + if (source.getKind() == ModelSource.FILE) { + FileInputStream file = new FileInputStream(source.getPath()); + try { + return file.getChannel().map(FileChannel.MapMode.READ_ONLY, + 0, file.getChannel().size()); + } finally { + file.close(); + } + } byte[] bytes; if (source.getKind() == ModelSource.BYTES) { bytes = source.getBytes(); } else { - InputStream input; - if (source.getKind() == ModelSource.FILE) { - input = new FileInputStream(source.getPath()); - } else { - input = Display.getInstance().getResourceAsStream( - AndroidInferenceImpl.class, source.getPath()); - if (input == null) { - throw new IOException("Model resource not found: " + source.getPath()); - } + InputStream input = Display.getInstance().getResourceAsStream( + AndroidInferenceImpl.class, source.getPath()); + if (input == null) { + throw new IOException("Model resource not found: " + source.getPath()); } try { ByteArrayOutputStream output = new ByteArrayOutputStream(); diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java index e0ace3e4169..1f89668b537 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java @@ -37,16 +37,16 @@ /** ML Kit pose detection; retained only for {@code PoseDetector} users. */ final class AndroidPoseDetectionAdapter extends AndroidVisionAdapter { + private final PoseDetector client = PoseDetection.getClient( + new PoseDetectorOptions.Builder() + .setDetectorMode(PoseDetectorOptions.STREAM_MODE) + .build()); @Override @SuppressWarnings("unchecked") void analyze(InputImage input, final int imageWidth, final int imageHeight, VisionOptions options, AsyncResource resource) { final AsyncResource out = (AsyncResource) resource; - final PoseDetector client = PoseDetection.getClient( - new PoseDetectorOptions.Builder() - .setDetectorMode(PoseDetectorOptions.SINGLE_IMAGE_MODE) - .build()); client.process(input).addOnSuccessListener( new OnSuccessListener() { public void onSuccess(com.google.mlkit.vision.pose.Pose value) { @@ -62,8 +62,12 @@ public void onSuccess(com.google.mlkit.vision.pose.Pose value) { point.getInFrameLikelihood()); } complete(out, new Pose(landmarks, METADATA)); - client.close(); } - }).addOnFailureListener(failure(out, client)); + }).addOnFailureListener(failure(out)); + } + + @Override + void close() { + client.close(); } } diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidSelfieSegmentationAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSelfieSegmentationAdapter.java index aa425f0b04a..a80c61f7100 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidSelfieSegmentationAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSelfieSegmentationAdapter.java @@ -37,17 +37,16 @@ * ML Kit selfie segmentation; retained only for {@code SelfieSegmenter} users. */ final class AndroidSelfieSegmentationAdapter extends AndroidVisionAdapter { + private final Segmenter client = Segmentation.getClient( + new SelfieSegmenterOptions.Builder() + .setDetectorMode(SelfieSegmenterOptions.STREAM_MODE) + .build()); @Override @SuppressWarnings("unchecked") void analyze(InputImage input, int imageWidth, int imageHeight, VisionOptions options, AsyncResource resource) { final AsyncResource out = (AsyncResource) resource; - final Segmenter client = Segmentation.getClient( - new SelfieSegmenterOptions.Builder() - .setDetectorMode( - SelfieSegmenterOptions.SINGLE_IMAGE_MODE) - .build()); client.process(input).addOnSuccessListener( new OnSuccessListener< com.google.mlkit.vision.segmentation.SegmentationMask>() { @@ -61,8 +60,12 @@ public void onSuccess( complete(out, new SegmentationMask( value.getWidth(), value.getHeight(), confidence, METADATA)); - client.close(); } - }).addOnFailureListener(failure(out, client)); + }).addOnFailureListener(failure(out)); + } + + @Override + void close() { + client.close(); } } diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java index 26ec2db9073..80f92d4208e 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java @@ -36,6 +36,8 @@ /** ML Kit text recognition; retained only for {@code TextRecognizer} users. */ final class AndroidTextRecognitionAdapter extends AndroidVisionAdapter { + private final TextRecognizer client = TextRecognition.getClient( + TextRecognizerOptions.DEFAULT_OPTIONS); @Override @SuppressWarnings("unchecked") void analyze(InputImage input, final int imageWidth, @@ -43,8 +45,6 @@ void analyze(InputImage input, final int imageWidth, AsyncResource resource) { final AsyncResource out = (AsyncResource) resource; - final TextRecognizer client = TextRecognition.getClient( - TextRecognizerOptions.DEFAULT_OPTIONS); client.process(input).addOnSuccessListener(new OnSuccessListener() { public void onSuccess(Text text) { List source = text.getTextBlocks(); @@ -59,8 +59,12 @@ public void onSuccess(Text text) { } complete(out, new TextRecognitionResult( text.getText(), blocks, METADATA)); - client.close(); } - }).addOnFailureListener(failure(out, client)); + }).addOnFailureListener(failure(out)); + } + + @Override + void close() { + client.close(); } } diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java index 0adc687356c..5d76ae49c4b 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java @@ -24,8 +24,8 @@ import com.codename1.ai.language.LanguageOptions; import com.codename1.util.AsyncResource; -import com.google.android.gms.tasks.Continuation; import com.google.android.gms.tasks.OnSuccessListener; +import com.google.android.gms.tasks.SuccessContinuation; import com.google.android.gms.tasks.Task; import com.google.mlkit.nl.translate.Translation; import com.google.mlkit.nl.translate.Translator; @@ -44,9 +44,9 @@ AsyncResource translate( .setTargetLanguage(targetLanguage).build(); final Translator client = Translation.getClient(translatorOptions); - client.downloadModelIfNeeded().continueWithTask( - new Continuation>() { - public Task then(Task ignored) { + client.downloadModelIfNeeded().onSuccessTask( + new SuccessContinuation() { + public Task then(Void ignored) { return client.translate(text); } }).addOnSuccessListener(new OnSuccessListener() { diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionAdapter.java index 3ef9e9b3262..546e22156cd 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionAdapter.java @@ -40,6 +40,8 @@ abstract class AndroidVisionAdapter { abstract void analyze(InputImage input, int imageWidth, int imageHeight, VisionOptions options, AsyncResource out); + abstract void close(); + static VisionRect normalized(Rect rect, int imageWidth, int imageHeight) { if (rect == null) { return VisionRect.EMPTY; @@ -58,8 +60,7 @@ public void run() { }); } - static OnFailureListener failure(final AsyncResource out, - final AutoCloseable client) { + static OnFailureListener failure(final AsyncResource out) { return new OnFailureListener() { public void onFailure(final Exception error) { Display.getInstance().callSerially(new Runnable() { @@ -69,10 +70,6 @@ public void run() { error.getMessage(), error)); } }); - try { - client.close(); - } catch (Exception ignored) { - } } }; } diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java index 66aaac53b78..aa911a8c13f 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java @@ -41,6 +41,8 @@ */ public final class AndroidVisionImpl extends VisionImpl { private volatile boolean closed; + private AndroidVisionAdapter adapter; + private VisionFeature adapterFeature; @Override public boolean isSupported(VisionFeature feature, String backendId) { @@ -67,9 +69,7 @@ public AsyncResource analyze(VisionFeature feature, String backendId, return out; } try { - AndroidVisionAdapter adapter = (AndroidVisionAdapter) - Class.forName(adapterClass(feature)).newInstance(); - adapter.analyze(decoded.input, decoded.width, decoded.height, + adapter(feature).analyze(decoded.input, decoded.width, decoded.height, options, (AsyncResource) out); } catch (Throwable error) { out.error(new VisionException(VisionException.BACKEND_ERROR, @@ -79,8 +79,24 @@ public AsyncResource analyze(VisionFeature feature, String backendId, return out; } + private synchronized AndroidVisionAdapter adapter(VisionFeature feature) + throws Exception { + if (closed) { + throw new IllegalStateException("Vision backend is closed"); + } + if (adapter == null) { + adapter = (AndroidVisionAdapter) Class.forName( + adapterClass(feature)).newInstance(); + adapterFeature = feature; + } else if (adapterFeature != feature) { + throw new IllegalStateException( + "A vision backend instance cannot analyze multiple features"); + } + return adapter; + } + private static DecodedInput decode(VisionImage image) { - byte[] encoded = image.getEncodedBytes(); + byte[] encoded = image.getEncodedBytesUnsafe(); if (encoded != null) { Bitmap bitmap = BitmapFactory.decodeByteArray( encoded, 0, encoded.length); @@ -88,7 +104,7 @@ private static DecodedInput decode(VisionImage image) { InputImage.fromBitmap(bitmap, image.getRotationDegrees()), bitmap.getWidth(), bitmap.getHeight()); } - byte[] pixels = image.getPixels(); + byte[] pixels = image.getPixelsUnsafe(); int width = image.getWidth(); int height = image.getHeight(); if (pixels == null || width <= 0 || height <= 0) { @@ -171,7 +187,11 @@ private static String adapterClass(VisionFeature feature) { } @Override - public void close() { + public synchronized void close() { closed = true; + if (adapter != null) { + adapter.close(); + adapter = null; + } } } diff --git a/Ports/iOSPort/nativeSources/CN1Inference.m b/Ports/iOSPort/nativeSources/CN1Inference.m index f5c9c84cf23..8c60f5378bf 100644 --- a/Ports/iOSPort/nativeSources/CN1Inference.m +++ b/Ports/iOSPort/nativeSources/CN1Inference.m @@ -37,6 +37,7 @@ @interface CN1InferenceHandle : NSObject @property(nonatomic, strong) TFLInterpreter *interpreter; @property(nonatomic, strong) TFLDelegate *delegate; @property(nonatomic, copy) NSString *modelPath; +@property(nonatomic) BOOL deleteModelOnClose; @end @implementation CN1InferenceHandle @@ -88,15 +89,8 @@ static void cn1InferenceEnsureHandles(void) { } } -static NSString *cn1InferenceOpen(NSData *model, int threads, int accelerator, - BOOL allowFallback) { - NSString *path = [NSTemporaryDirectory() stringByAppendingPathComponent: - [NSString stringWithFormat:@"cn1-litert-%@.tflite", - NSUUID.UUID.UUIDString]]; - if (![model writeToFile:path atomically:YES]) { - return cn1InferenceJSON(@{@"error": @"Could not stage LiteRT model"}); - } - +static NSString *cn1InferenceOpenPath(NSString *path, BOOL deleteModelOnClose, + int threads, int accelerator, BOOL allowFallback) { TFLInterpreterOptions *options = [[TFLInterpreterOptions alloc] init]; if (threads > 0) options.numberOfThreads = (NSUInteger)threads; NSMutableArray *delegates = [NSMutableArray array]; @@ -110,13 +104,17 @@ static void cn1InferenceEnsureHandles(void) { if (delegate != nil) [delegates addObject:delegate]; #endif if (delegate == nil && !allowFallback && accelerator != 0) { - [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + if (deleteModelOnClose) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + } return cn1InferenceJSON(@{ @"error": @"Core ML delegate is unavailable on this device" }); } } else if (accelerator == 2 && !allowFallback) { - [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + if (deleteModelOnClose) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + } return cn1InferenceJSON(@{ @"error": @"The iOS backend does not provide a GPU delegate" }); @@ -126,11 +124,15 @@ static void cn1InferenceEnsureHandles(void) { TFLInterpreter *interpreter = [[TFLInterpreter alloc] initWithModelPath:path options:options delegates:delegates error:&error]; if (interpreter == nil || error != nil) { - [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + if (deleteModelOnClose) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + } return cn1InferenceError(error, @"Could not create LiteRT interpreter"); } if (![interpreter allocateTensorsWithError:&error]) { - [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + if (deleteModelOnClose) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + } return cn1InferenceError(error, @"Could not allocate LiteRT tensors"); } @@ -138,6 +140,7 @@ static void cn1InferenceEnsureHandles(void) { value.interpreter = interpreter; value.delegate = delegate; value.modelPath = path; + value.deleteModelOnClose = deleteModelOnClose; cn1InferenceEnsureHandles(); int handle; @synchronized (cn1InferenceHandles) { @@ -147,6 +150,17 @@ static void cn1InferenceEnsureHandles(void) { return cn1InferenceJSON(@{@"handle": @(handle)}); } +static NSString *cn1InferenceOpen(NSData *model, int threads, int accelerator, + BOOL allowFallback) { + NSString *path = [NSTemporaryDirectory() stringByAppendingPathComponent: + [NSString stringWithFormat:@"cn1-litert-%@.tflite", + NSUUID.UUID.UUIDString]]; + if (![model writeToFile:path atomically:YES]) { + return cn1InferenceJSON(@{@"error": @"Could not stage LiteRT model"}); + } + return cn1InferenceOpenPath(path, YES, threads, accelerator, allowFallback); +} + static NSString *cn1InferenceMetadata(int handle, BOOL outputs) { CN1InferenceHandle *value = cn1InferenceHandle(handle); if (value == nil) { @@ -186,65 +200,47 @@ static void cn1InferenceEnsureHandles(void) { return cn1InferenceJSON(@{@"items": items}); } -static NSString *cn1InferenceRun(int handle, NSString *inputsJSON) { +static NSString *cn1InferenceCopyInput(int handle, int index, NSData *data) { CN1InferenceHandle *value = cn1InferenceHandle(handle); if (value == nil) { return cn1InferenceJSON(@{@"error": @"Unknown LiteRT handle"}); } NSError *error = nil; - NSDictionary *root = [NSJSONSerialization JSONObjectWithData: - [inputsJSON dataUsingEncoding:NSUTF8StringEncoding] - options:0 error:&error]; - if (root == nil || error != nil) { - return cn1InferenceError(error, @"Invalid LiteRT input JSON"); - } TFLInterpreter *interpreter = value.interpreter; - for (NSDictionary *item in root[@"items"] ?: @[]) { - NSUInteger index = [item[@"index"] unsignedIntegerValue]; - if (index >= interpreter.inputTensorCount) { - return cn1InferenceJSON(@{@"error": @"Invalid LiteRT input index"}); - } - TFLTensor *tensor = [interpreter inputTensorAtIndex:index error:&error]; - if (tensor == nil || error != nil) { - return cn1InferenceError(error, @"Could not read LiteRT input"); - } - NSString *expectedType = cn1InferenceType(tensor.dataType); - if (![expectedType isEqualToString:item[@"type"]]) { - return cn1InferenceJSON(@{@"error": @"LiteRT input type mismatch"}); - } - NSData *data = [[NSData alloc] initWithBase64EncodedString:item[@"data"] - options:0]; - if (data == nil || ![tensor copyData:data error:&error]) { - return cn1InferenceError(error, @"Could not copy LiteRT input"); - } + if (index < 0 || (NSUInteger)index >= interpreter.inputTensorCount) { + return cn1InferenceJSON(@{@"error": @"Invalid LiteRT input index"}); + } + TFLTensor *tensor = [interpreter inputTensorAtIndex:(NSUInteger)index + error:&error]; + if (tensor == nil || error != nil || ![tensor copyData:data error:&error]) { + return cn1InferenceError(error, @"Could not copy LiteRT input"); } + return cn1InferenceJSON(@{@"ok": @(YES)}); +} + +static NSString *cn1InferenceInvoke(int handle) { + CN1InferenceHandle *value = cn1InferenceHandle(handle); + if (value == nil) { + return cn1InferenceJSON(@{@"error": @"Unknown LiteRT handle"}); + } + NSError *error = nil; + TFLInterpreter *interpreter = value.interpreter; if (![interpreter invokeWithError:&error]) { return cn1InferenceError(error, @"LiteRT invocation failed"); } - NSMutableArray *items = [NSMutableArray array]; - for (NSUInteger i = 0; i < interpreter.outputTensorCount; i++) { - TFLTensor *tensor = [interpreter outputTensorAtIndex:i error:&error]; - if (tensor == nil || error != nil) { - return cn1InferenceError(error, @"Could not read LiteRT output"); - } - NSString *type = cn1InferenceType(tensor.dataType); - if (type == nil) { - return cn1InferenceJSON(@{@"error": @"Unsupported output tensor type"}); - } - NSArray *shape = [tensor shapeWithError:&error]; - NSData *data = [tensor dataWithError:&error]; - if (shape == nil || data == nil || error != nil) { - return cn1InferenceError(error, @"Could not copy LiteRT output"); - } - [items addObject:@{ - @"index": @(i), - @"name": tensor.name ?: @"", - @"type": type, - @"shape": shape, - @"data": [data base64EncodedStringWithOptions:0] - }]; + return cn1InferenceJSON(@{@"ok": @(YES)}); +} + +static NSData *cn1InferenceOutputData(int handle, int index) { + CN1InferenceHandle *value = cn1InferenceHandle(handle); + if (value == nil || index < 0 + || (NSUInteger)index >= value.interpreter.outputTensorCount) { + return nil; } - return cn1InferenceJSON(@{@"items": items}); + NSError *error = nil; + TFLTensor *tensor = [value.interpreter outputTensorAtIndex:(NSUInteger)index + error:&error]; + return tensor == nil || error != nil ? nil : [tensor dataWithError:&error]; } static NSString *cn1InferenceResize(int handle, int index, @@ -304,18 +300,57 @@ JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceMetadata___int_boolean_ #endif } -JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceRun___int_java_lang_String_R_java_lang_String( +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceOpenFile___java_lang_String_int_int_boolean_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT path, + JAVA_INT threads, JAVA_INT accelerator, JAVA_BOOLEAN allowFallback) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1InferenceOpenPath( + toNSString(CN1_THREAD_GET_STATE_PASS_ARG path), NO, + threads, accelerator, allowFallback)); +#else + return JAVA_NULL; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceCopyInput___int_int_byte_1ARRAY_R_java_lang_String( CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT handle, - JAVA_OBJECT inputsJSON) { + JAVA_INT index, JAVA_OBJECT input) { #if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + if (input == JAVA_NULL) { + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + @"{\"error\":\"Input data is null\"}"); + } + JAVA_ARRAY bytes = (JAVA_ARRAY)input; + NSData *data = [NSData dataWithBytes:bytes->data + length:(NSUInteger)bytes->length]; return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG - cn1InferenceRun(handle, - toNSString(CN1_THREAD_GET_STATE_PASS_ARG inputsJSON))); + cn1InferenceCopyInput(handle, index, data)); #else return JAVA_NULL; #endif } +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceInvoke___int_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT handle) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG cn1InferenceInvoke(handle)); +#else + return JAVA_NULL; +#endif +} + +JAVA_LONG com_codename1_impl_ios_IOSNative_cn1InferenceOutputData___int_int_R_long( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT handle, + JAVA_INT index) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + NSData *data = cn1InferenceOutputData(handle, index); + return data == nil ? 0 : (JAVA_LONG)CFBridgingRetain(data); +#else + return 0; +#endif +} + JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceResize___int_int_int_1ARRAY_R_java_lang_String( CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT handle, JAVA_INT index, JAVA_OBJECT shape) { @@ -347,7 +382,7 @@ void com_codename1_impl_ios_IOSNative_cn1InferenceClose___int( value = cn1InferenceHandles[@(handle)]; [cn1InferenceHandles removeObjectForKey:@(handle)]; } - if (value.modelPath != nil) { + if (value.deleteModelOnClose && value.modelPath != nil) { [[NSFileManager defaultManager] removeItemAtPath:value.modelPath error:nil]; } #endif diff --git a/Ports/iOSPort/nativeSources/CN1Language.m b/Ports/iOSPort/nativeSources/CN1Language.m index 71bcd97f52f..221ed376e99 100644 --- a/Ports/iOSPort/nativeSources/CN1Language.m +++ b/Ports/iOSPort/nativeSources/CN1Language.m @@ -25,6 +25,7 @@ #if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV #import +#import #import "java_lang_String.h" #if __has_include() @@ -58,7 +59,35 @@ }); } -static NSString *cn1IdentifyLanguage(NSString *text, float threshold) { +static NSString *cn1IdentifyLanguage(NSString *text, float threshold, + BOOL useMLKit) { + if (!useMLKit) { + if (@available(iOS 12.0, *)) { + NLLanguageRecognizer *recognizer = + [[NLLanguageRecognizer alloc] init]; + [recognizer processString:text ?: @""]; + NSDictionary *hypotheses = + [recognizer languageHypothesesWithMaximum:20]; + NSMutableArray *items = [NSMutableArray array]; + [hypotheses enumerateKeysAndObjectsUsingBlock: + ^(NLLanguage language, NSNumber *confidence, BOOL *stop) { + if (confidence.floatValue >= threshold) { + [items addObject:@{ + @"language": language ?: @"und", + @"confidence": confidence + }]; + } + }]; + [items sortUsingComparator:^NSComparisonResult( + NSDictionary *left, NSDictionary *right) { + return [right[@"confidence"] compare:left[@"confidence"]]; + }]; + return cn1LanguageJSON(@{@"items": items}); + } + return cn1LanguageJSON(@{ + @"error": @"Apple Natural Language requires iOS 12 or newer" + }); + } #if defined(CN1_HAS_MLKIT_LANGUAGE_ID) MLKLanguageIdentificationOptions *options = [[MLKLanguageIdentificationOptions alloc] @@ -179,11 +208,16 @@ } #endif -JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_cn1LanguageIsSupported___int_R_boolean( - CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT feature) { +JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_cn1LanguageIsSupported___int_boolean_R_boolean( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT feature, + JAVA_BOOLEAN useMLKit) { #if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV switch (feature) { case 0: + if (!useMLKit) { + if (@available(iOS 12.0, *)) return 1; + return 0; + } #if defined(CN1_HAS_MLKIT_LANGUAGE_ID) return 1; #else @@ -209,14 +243,14 @@ JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_cn1LanguageIsSupported___int_R_boo #endif } -JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1LanguageIdentify___java_lang_String_float_R_java_lang_String( +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1LanguageIdentify___java_lang_String_float_boolean_R_java_lang_String( CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT text, - JAVA_FLOAT minimumConfidence) { + JAVA_FLOAT minimumConfidence, JAVA_BOOLEAN useMLKit) { #if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG cn1IdentifyLanguage( toNSString(CN1_THREAD_GET_STATE_PASS_ARG text), - minimumConfidence)); + minimumConfidence, useMLKit)); #else return JAVA_NULL; #endif diff --git a/Ports/iOSPort/nativeSources/CN1Vision.m b/Ports/iOSPort/nativeSources/CN1Vision.m index f6483f34279..23ec13f1884 100644 --- a/Ports/iOSPort/nativeSources/CN1Vision.m +++ b/Ports/iOSPort/nativeSources/CN1Vision.m @@ -120,9 +120,32 @@ static UIImageOrientation cn1UIImageOrientation(int rotation) { } #endif -static NSString *cn1MLKitVisionPerform(NSData *data, int feature, int rotation) { +#if defined(CN1_HAS_MLKIT_BARCODE) +static NSString *cn1MLKitBarcodeFormat(MLKBarcodeFormat format) { + switch (format) { + case MLKBarcodeFormatAztec: return @"AZTEC"; + case MLKBarcodeFormatCodaBar: return @"CODABAR"; + case MLKBarcodeFormatCode39: return @"CODE_39"; + case MLKBarcodeFormatCode93: return @"CODE_93"; + case MLKBarcodeFormatCode128: return @"CODE_128"; + case MLKBarcodeFormatDataMatrix: return @"DATA_MATRIX"; + case MLKBarcodeFormatEAN8: return @"EAN_8"; + case MLKBarcodeFormatEAN13: return @"EAN_13"; + case MLKBarcodeFormatITF: return @"ITF"; + case MLKBarcodeFormatPDF417: return @"PDF417"; + case MLKBarcodeFormatQRCode: return @"QR_CODE"; + case MLKBarcodeFormatUPCA: return @"UPC_A"; + case MLKBarcodeFormatUPCE: return @"UPC_E"; + default: return @"UNKNOWN"; + } +} +#endif + +static NSString *cn1MLKitVisionPerform(NSData *data, CGImageRef rawImage, + int feature, int rotation) { #if defined(CN1_HAS_MLKIT_VISION) - UIImage *image = [UIImage imageWithData:data]; + UIImage *image = rawImage == NULL ? [UIImage imageWithData:data] + : [UIImage imageWithCGImage:rawImage]; if (image == nil) { return cn1VisionJSON(@{@"error": @"Could not decode ML Kit image"}); } @@ -174,8 +197,16 @@ static UIImageOrientation cn1UIImageOrientation(int rotation) { NSMutableDictionary *item = [NSMutableDictionary dictionaryWithDictionary:cn1MLKitRect(barcode.frame, image.size)]; item[@"value"] = barcode.rawValue ?: [NSNull null]; - item[@"format"] = [NSString stringWithFormat:@"%ld", - (long)barcode.format]; + item[@"format"] = cn1MLKitBarcodeFormat(barcode.format); + NSMutableArray *corners = [NSMutableArray array]; + for (NSValue *pointValue in barcode.cornerPoints ?: @[]) { + CGPoint point = pointValue.CGPointValue; + [corners addObject:@{ + @"x": @(point.x / image.size.width), + @"y": @(point.y / image.size.height) + }]; + } + item[@"corners"] = corners; [items addObject:item]; } return cn1VisionJSON(@{@"items": items}); @@ -293,7 +324,7 @@ static UIImageOrientation cn1UIImageOrientation(int rotation) { return cn1VisionJSON(@{ @"width": @(width), @"height": @(height), - @"data": [packed base64EncodedStringWithOptions:0] + @"dataPeer": @((uint64_t)CFBridgingRetain(packed)) }); #endif } @@ -314,10 +345,29 @@ static CGImagePropertyOrientation cn1CGOrientation(int rotation) { } } -static NSString *cn1VisionPerform(NSData *data, int feature, int rotation) { +static NSString *cn1AppleBarcodeFormat(NSString *symbology) { + NSString *value = symbology.uppercaseString; + if ([value containsString:@"QR"]) return @"QR_CODE"; + if ([value containsString:@"DATAMATRIX"]) return @"DATA_MATRIX"; + if ([value containsString:@"PDF417"]) return @"PDF417"; + if ([value containsString:@"CODE128"]) return @"CODE_128"; + if ([value containsString:@"CODE93"]) return @"CODE_93"; + if ([value containsString:@"CODE39"]) return @"CODE_39"; + if ([value containsString:@"EAN13"]) return @"EAN_13"; + if ([value containsString:@"EAN8"]) return @"EAN_8"; + if ([value containsString:@"UPCE"]) return @"UPC_E"; + if ([value containsString:@"ITF"]) return @"ITF"; + if ([value containsString:@"AZTEC"]) return @"AZTEC"; + return @"UNKNOWN"; +} + +static NSString *cn1VisionPerform(NSData *data, CGImageRef rawImage, + int feature, int rotation) { NSError *error = nil; - VNImageRequestHandler *handler = - [[VNImageRequestHandler alloc] initWithData:data + VNImageRequestHandler *handler = rawImage == NULL + ? [[VNImageRequestHandler alloc] initWithData:data + orientation:cn1CGOrientation(rotation) options:@{}] + : [[VNImageRequestHandler alloc] initWithCGImage:rawImage orientation:cn1CGOrientation(rotation) options:@{}]; if (feature == 0) { @@ -356,7 +406,17 @@ static CGImagePropertyOrientation cn1CGOrientation(int rotation) { [NSMutableDictionary dictionaryWithDictionary: cn1VisionRect(observation.boundingBox)]; item[@"value"] = observation.payloadStringValue ?: [NSNull null]; - item[@"format"] = observation.symbology ?: @"UNKNOWN"; + item[@"format"] = cn1AppleBarcodeFormat(observation.symbology); + item[@"corners"] = @[ + @{@"x": @(observation.topLeft.x), + @"y": @(1.0 - observation.topLeft.y)}, + @{@"x": @(observation.topRight.x), + @"y": @(1.0 - observation.topRight.y)}, + @{@"x": @(observation.bottomRight.x), + @"y": @(1.0 - observation.bottomRight.y)}, + @{@"x": @(observation.bottomLeft.x), + @"y": @(1.0 - observation.bottomLeft.y)} + ]; [items addObject:item]; } return cn1VisionJSON(@{@"items": items}); @@ -448,11 +508,12 @@ static CGImagePropertyOrientation cn1CGOrientation(int rotation) { return cn1VisionJSON(@{ @"width": @(width), @"height": @(height), - @"data": [packed base64EncodedStringWithOptions:0] + @"dataPeer": @((uint64_t)CFBridgingRetain(packed)) }); } } else if (feature == 6) { - UIImage *image = [UIImage imageWithData:data]; + UIImage *image = rawImage == NULL ? [UIImage imageWithData:data] + : [UIImage imageWithCGImage:rawImage]; if (image == nil) { return cn1VisionJSON(@{@"error": @"Could not decode document image"}); } @@ -481,7 +542,7 @@ static CGImagePropertyOrientation cn1CGOrientation(int rotation) { CGImageRelease(outputImage); NSData *jpeg = UIImageJPEGRepresentation(output, 0.92); return cn1VisionJSON(@{ - @"data": [jpeg base64EncodedStringWithOptions:0] + @"dataPeer": @((uint64_t)CFBridgingRetain(jpeg)) }); } return cn1VisionJSON(@{@"error": @"Vision feature is unavailable on this OS"}); @@ -559,17 +620,17 @@ static uint8_t cn1VisionClamp(int value) { } /* - * Converts the two raw CameraFrame formats to JPEG so both Apple Vision and - * the optional ML Kit path consume identical, orientation-neutral data. + * Converts the two raw CameraFrame formats to an uncompressed CGImage so + * Vision and ML Kit avoid a lossy JPEG encode/decode cycle on every frame. * format mirrors FrameFormat: 0=JPEG/encoded, 1=NV21, 2=RGBA8888. */ -static NSData *cn1VisionEncodeInput(NSData *data, int width, int height, - int format) { +static CGImageRef cn1VisionCreateRawImage(NSData *data, int width, int height, + int format) { if (format == 0) { - return data; + return NULL; } if (width <= 0 || height <= 0) { - return nil; + return NULL; } NSUInteger pixelCount = (NSUInteger)width * (NSUInteger)height; const uint8_t *source = data.bytes; @@ -577,12 +638,12 @@ static uint8_t cn1VisionClamp(int value) { uint8_t *dest = rgba.mutableBytes; if (format == 2) { if (data.length < pixelCount * 4) { - return nil; + return NULL; } memcpy(dest, source, pixelCount * 4); } else if (format == 1) { if (data.length < pixelCount + pixelCount / 2) { - return nil; + return NULL; } for (int y = 0; y < height; y++) { int uvRow = width * height + (y >> 1) * width; @@ -603,7 +664,7 @@ static uint8_t cn1VisionClamp(int value) { } } } else { - return nil; + return NULL; } CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); CGDataProviderRef provider = CGDataProviderCreateWithCFData( @@ -611,14 +672,9 @@ static uint8_t cn1VisionClamp(int value) { CGImageRef cgImage = CGImageCreate(width, height, 8, 32, width * 4, colorSpace, kCGBitmapByteOrderDefault | kCGImageAlphaLast, provider, NULL, false, kCGRenderingIntentDefault); - UIImage *image = cgImage == NULL ? nil - : [UIImage imageWithCGImage:cgImage]; - NSData *encoded = image == nil ? nil - : UIImageJPEGRepresentation(image, 0.95); - if (cgImage != NULL) CGImageRelease(cgImage); CGDataProviderRelease(provider); CGColorSpaceRelease(colorSpace); - return encoded; + return cgImage; } #endif @@ -634,14 +690,17 @@ JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1VisionAnalyze___byte_1ARRAY_int_ } JAVA_ARRAY bytes = (JAVA_ARRAY) encodedImage; NSData *data = [NSData dataWithBytes:bytes->data length:(NSUInteger) bytes->length]; - data = cn1VisionEncodeInput(data, width, height, frameFormat); - if (data == nil) { + CGImageRef rawImage = cn1VisionCreateRawImage( + data, width, height, frameFormat); + if (frameFormat != 0 && rawImage == NULL) { return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG @"{\"error\":\"Invalid raw vision image\"}"); } - return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG - mlKit ? cn1MLKitVisionPerform(data, feature, rotation) - : cn1VisionPerform(data, feature, rotation)); + NSString *result = mlKit + ? cn1MLKitVisionPerform(data, rawImage, feature, rotation) + : cn1VisionPerform(data, rawImage, feature, rotation); + if (rawImage != NULL) CGImageRelease(rawImage); + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG result); #else return JAVA_NULL; #endif diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java index 222d03ade86..35bda7d7edf 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java @@ -4777,9 +4777,9 @@ public com.codename1.impl.VisionImpl createVisionImpl() { @Override public com.codename1.impl.LanguageImpl createLanguageImpl() { - return nativeInstance.cn1LanguageIsSupported(0) - || nativeInstance.cn1LanguageIsSupported(1) - || nativeInstance.cn1LanguageIsSupported(2) + return nativeInstance.cn1LanguageIsSupported(0, false) + || nativeInstance.cn1LanguageIsSupported(1, false) + || nativeInstance.cn1LanguageIsSupported(2, false) ? new IOSLanguageImpl() : null; } diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java index 435eaf91005..879c158f31d 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java @@ -29,18 +29,14 @@ import com.codename1.ai.inference.TensorInfo; import com.codename1.ai.inference.TensorType; import com.codename1.impl.InferenceImpl; -import com.codename1.io.FileSystemStorage; import com.codename1.io.JSONParser; import com.codename1.ui.Display; import com.codename1.util.AsyncResource; -import com.codename1.util.Base64; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.StringReader; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -48,9 +44,13 @@ public final class IOSInferenceImpl extends InferenceImpl { private static final class Handle { final int id; + TensorInfo[] inputs; + TensorInfo[] outputs; - Handle(int id) { + Handle(int id, TensorInfo[] inputs, TensorInfo[] outputs) { this.id = id; + this.inputs = inputs; + this.outputs = outputs; } } @@ -73,11 +73,25 @@ private static void openInBackground(final AsyncResource out, Display.getInstance().scheduleBackgroundTask(new Runnable() { public void run() { try { - Map root = parse(IOSImplementation.nativeInstance.cn1InferenceOpen( - loadModel(source), options.getThreads(), - options.getAccelerator().ordinal(), - options.isFallbackAllowed())); - final Handle handle = new Handle(integer(root, "handle")); + String opened = source.getKind() == ModelSource.FILE + ? IOSImplementation.nativeInstance.cn1InferenceOpenFile( + source.getPath(), options.getThreads(), + options.getAccelerator().ordinal(), + options.isFallbackAllowed()) + : IOSImplementation.nativeInstance.cn1InferenceOpen( + loadModel(source), options.getThreads(), + options.getAccelerator().ordinal(), + options.isFallbackAllowed()); + Map root = parse(opened); + int id = integer(root, "handle"); + final Handle handle; + try { + handle = new Handle(id, + metadata(id, false), metadata(id, true)); + } catch (Throwable metadataError) { + IOSImplementation.nativeInstance.cn1InferenceClose(id); + throw metadataError; + } Display.getInstance().callSerially(new Runnable() { public void run() { out.complete(handle); @@ -92,12 +106,12 @@ public void run() { @Override public TensorInfo[] getInputs(Object handle) { - return metadata(checked(handle), false); + return copy(checked(handle).inputs); } @Override public TensorInfo[] getOutputs(Object handle) { - return metadata(checked(handle), true); + return copy(checked(handle).outputs); } @Override @@ -115,19 +129,52 @@ private static void runInBackground(final AsyncResource out, Display.getInstance().scheduleBackgroundTask(new Runnable() { public void run() { try { - TensorInfo[] metadata = metadata(checked, false); - Map root = parse(IOSImplementation.nativeInstance.cn1InferenceRun( - id, inputsJson(inputs, metadata))); - List items = list(root, "items"); - final Tensor[] result = new Tensor[items.size()]; + TensorInfo[] metadata = checked.inputs; + if (inputs.length != metadata.length) { + throw new IllegalArgumentException("Expected " + + metadata.length + " input tensors but received " + + inputs.length); + } + for (int i = 0; i < inputs.length; i++) { + Tensor tensor = inputs[i]; + int index = inputIndex(tensor, i, metadata); + TensorInfo expected = find(metadata, index); + if (tensor.getType() != expected.getType()) { + throw new IllegalArgumentException("Input " + + expected.getName() + " expects " + + expected.getType() + " but received " + + tensor.getType()); + } + parse(IOSImplementation.nativeInstance.cn1InferenceCopyInput( + id, index, encodeData(tensor.getType(), + tensor.getDataUnsafe()))); + } + parse(IOSImplementation.nativeInstance.cn1InferenceInvoke(id)); + TensorInfo[] outputs = checked.outputs; + final Tensor[] result = new Tensor[outputs.length]; for (int i = 0; i < result.length; i++) { - Map value = (Map) items.get(i); - TensorType type = TensorType.valueOf(string(value, "type")); - int[] shape = intArray(list(value, "shape")); - byte[] bytes = Base64.decode( - string(value, "data").getBytes("UTF-8")); - result[i] = new Tensor(string(value, "name"), type, shape, - decodeData(type, bytes, elementCount(shape))); + TensorInfo output = outputs[i]; + long data = IOSImplementation.nativeInstance + .cn1InferenceOutputData(id, output.getIndex()); + if (data == 0) { + throw new InferenceException( + "Could not read LiteRT output " + + output.getName()); + } + byte[] bytes; + try { + bytes = new byte[IOSImplementation.nativeInstance + .getNSDataSize(data)]; + IOSImplementation.nativeInstance.nsDataToByteArray( + data, bytes); + } finally { + IOSImplementation.nativeInstance.releasePeer(data); + } + int[] shape = output.getShape(); + result[i] = new Tensor(output.getName(), + output.getType(), shape, + decodeData(output.getType(), bytes, + elementCount(shape))); } Display.getInstance().callSerially(new Runnable() { public void run() { @@ -144,7 +191,7 @@ public void run() { @Override public void resizeInput(Object handle, String name, int[] shape) { Handle checked = checked(handle); - TensorInfo[] inputs = metadata(checked, false); + TensorInfo[] inputs = checked.inputs; int index = -1; for (int i = 0; i < inputs.length; i++) { if (name == null || name.equals(inputs[i].getName())) { @@ -158,6 +205,8 @@ public void resizeInput(Object handle, String name, int[] shape) { try { parse(IOSImplementation.nativeInstance.cn1InferenceResize( checked.id, index, shape)); + checked.inputs = metadata(checked.id, false); + checked.outputs = metadata(checked.id, true); } catch (Exception error) { throw new InferenceException("Could not resize input " + name, error); } @@ -168,10 +217,10 @@ public void close(Object handle) { IOSImplementation.nativeInstance.cn1InferenceClose(checked(handle).id); } - private static TensorInfo[] metadata(Handle handle, boolean outputs) { + private static TensorInfo[] metadata(int handle, boolean outputs) { try { Map root = parse(IOSImplementation.nativeInstance.cn1InferenceMetadata( - handle.id, outputs)); + handle, outputs)); List items = list(root, "items"); TensorInfo[] result = new TensorInfo[items.size()]; for (int i = 0; i < result.length; i++) { @@ -194,29 +243,6 @@ private static Handle checked(Object value) { return (Handle) value; } - private static String inputsJson(Tensor[] inputs, TensorInfo[] metadata) { - if (inputs.length != metadata.length) { - throw new IllegalArgumentException("Expected " + metadata.length - + " input tensors but received " + inputs.length); - } - List> items = new ArrayList>(); - for (int i = 0; i < inputs.length; i++) { - Tensor tensor = inputs[i]; - int index = inputIndex(tensor, i, metadata); - Map value = new HashMap(); - value.put("index", Integer.valueOf(index)); - value.put("name", tensor.getName()); - value.put("type", tensor.getType().name()); - value.put("shape", integerList(tensor.getShape())); - value.put("data", Base64.encode(encodeData( - tensor.getType(), tensor.getData()))); - items.add(value); - } - Map root = new HashMap(); - root.put("items", items); - return JSONParser.mapToJson(root); - } - private static int inputIndex(Tensor tensor, int fallback, TensorInfo[] metadata) { if (tensor.getName() != null) { @@ -231,6 +257,15 @@ private static int inputIndex(Tensor tensor, int fallback, return metadata[fallback].getIndex(); } + private static TensorInfo find(TensorInfo[] metadata, int index) { + for (int i = 0; i < metadata.length; i++) { + if (metadata[i].getIndex() == index) { + return metadata[i]; + } + } + throw new IllegalArgumentException("Unknown model input index " + index); + } + private static byte[] encodeData(TensorType type, Object value) { if (value instanceof byte[]) { return (byte[]) value; @@ -326,14 +361,6 @@ private static int elementCount(int[] shape) { return out; } - private static List integerList(int[] values) { - List out = new ArrayList(); - for (int i = 0; i < values.length; i++) { - out.add(Integer.valueOf(values[i])); - } - return out; - } - private static int[] intArray(List values) { int[] out = new int[values.size()]; for (int i = 0; i < out.length; i++) { @@ -374,14 +401,10 @@ private static byte[] loadModel(ModelSource source) throws IOException { return source.getBytes(); } InputStream input; - if (source.getKind() == ModelSource.FILE) { - input = FileSystemStorage.getInstance().openInputStream(source.getPath()); - } else { - input = Display.getInstance().getResourceAsStream( - IOSInferenceImpl.class, source.getPath()); - if (input == null) { - throw new IOException("Model resource not found: " + source.getPath()); - } + input = Display.getInstance().getResourceAsStream( + IOSInferenceImpl.class, source.getPath()); + if (input == null) { + throw new IOException("Model resource not found: " + source.getPath()); } try { ByteArrayOutputStream output = new ByteArrayOutputStream(); @@ -396,6 +419,12 @@ private static byte[] loadModel(ModelSource source) throws IOException { } } + private static TensorInfo[] copy(TensorInfo[] value) { + TensorInfo[] out = new TensorInfo[value.length]; + System.arraycopy(value, 0, out, 0, value.length); + return out; + } + private static void fail(final AsyncResource out, final String message, final Throwable cause) { Display.getInstance().callSerially(new Runnable() { diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java index a75034f52e1..6e53e4b3dd9 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java @@ -36,7 +36,7 @@ import java.util.List; import java.util.Map; -/** iOS ML Kit language services. */ +/** iOS Natural Language and feature-scoped ML Kit language services. */ public final class IOSLanguageImpl extends LanguageImpl { private static final int LANGUAGE_ID = 0; private static final int TRANSLATION = 1; @@ -46,27 +46,34 @@ public final class IOSLanguageImpl extends LanguageImpl { @Override public boolean isSupported(String feature, String backendId) { - if (closed || (!"auto".equals(backendId) && !"ml-kit".equals(backendId))) { + if (closed || (!"auto".equals(backendId) + && !"ml-kit".equals(backendId) + && !"apple-natural-language".equals(backendId))) { + return false; + } + if (!"language-id".equals(feature) + && "apple-natural-language".equals(backendId)) { return false; } return IOSImplementation.nativeInstance.cn1LanguageIsSupported( - featureId(feature)); + featureId(feature), "ml-kit".equals(backendId)); } @Override public AsyncResource identify( final String text, String backendId, final LanguageOptions options) { - return identifyInBackground(text, options); + return identifyInBackground(text, "ml-kit".equals(backendId), options); } private static AsyncResource identifyInBackground( - final String text, final LanguageOptions options) { + final String text, final boolean mlKit, + final LanguageOptions options) { final AsyncResource out = new AsyncResource(); run(out, new NativeCall() { public LanguageCandidate[] call() throws Exception { String json = IOSImplementation.nativeInstance.cn1LanguageIdentify( - text, options.getMinimumConfidence()); + text, options.getMinimumConfidence(), mlKit); Map root = parse(json); List values = list(root, "items"); LanguageCandidate[] result = new LanguageCandidate[values.size()]; diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java index 99880a36586..cd3f4b56dfb 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java @@ -543,16 +543,21 @@ native void fillGradient(int kind, int stopCount, float[] positions, float[] pre native String cn1VisionAnalyze(byte[] imageData, int feature, boolean mlKit, int rotationDegrees, int width, int height, int frameFormat); - native boolean cn1LanguageIsSupported(int feature); - native String cn1LanguageIdentify(String text, float minimumConfidence); + native boolean cn1LanguageIsSupported(int feature, boolean mlKit); + native String cn1LanguageIdentify(String text, float minimumConfidence, + boolean mlKit); native String cn1LanguageTranslate(String text, String sourceLanguage, String targetLanguage); native String cn1LanguageSmartReply(String conversationJson); native boolean cn1InferenceIsSupported(); native String cn1InferenceOpen(byte[] model, int threads, int accelerator, boolean allowFallback); + native String cn1InferenceOpenFile(String path, int threads, int accelerator, + boolean allowFallback); native String cn1InferenceMetadata(int handle, boolean outputs); - native String cn1InferenceRun(int handle, String inputsJson); + native String cn1InferenceCopyInput(int handle, int index, byte[] data); + native String cn1InferenceInvoke(int handle); + native long cn1InferenceOutputData(int handle, int index); native String cn1InferenceResize(int handle, int index, int[] shape); native void cn1InferenceClose(int handle); diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java index 3ce6676297d..57722cb2d11 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java @@ -74,8 +74,9 @@ public AsyncResource analyze(final VisionFeature feature, + (mlKit ? "ML Kit" : "Apple Vision"))); return out; } - byte[] encoded = image.getEncodedBytes(); - final byte[] input = encoded == null ? image.getPixels() : encoded; + byte[] encoded = image.getEncodedBytesUnsafe(); + final byte[] input = encoded == null + ? image.getPixelsUnsafe() : encoded; final int width = encoded == null ? image.getWidth() : 0; final int height = encoded == null ? image.getHeight() : 0; final int frameFormat = encoded == null @@ -158,9 +159,18 @@ private static T parse(VisionFeature feature, String json, Barcode[] values = new Barcode[items.size()]; for (int i = 0; i < values.length; i++) { Map value = (Map) items.get(i); + List points = value.get("corners") instanceof List + ? (List) value.get("corners") + : java.util.Collections.EMPTY_LIST; + VisionPoint[] corners = new VisionPoint[points.size()]; + for (int p = 0; p < corners.length; p++) { + Map point = (Map) points.get(p); + corners[p] = new VisionPoint(number(point, "x"), + number(point, "y")); + } values[i] = new Barcode(stringOrNull(value, "value"), string(value, "format"), null, rect(value), - new VisionPoint[0], metadata); + corners, metadata); } return (T) values; } @@ -195,8 +205,7 @@ private static T parse(VisionFeature feature, String json, return (T) new Pose(values, metadata); } case SELFIE_SEGMENTATION: { - String base64 = string(root, "data"); - byte[] bytes = Base64.decode(base64.getBytes("UTF-8")); + byte[] bytes = nativeData(root); float[] confidence = new float[bytes.length]; for (int i = 0; i < bytes.length; i++) { confidence[i] = (bytes[i] & 0xff) / 255f; @@ -205,9 +214,8 @@ private static T parse(VisionFeature feature, String json, integer(root, "height"), confidence, metadata); } case DOCUMENT_SCANNING: { - String base64 = string(root, "data"); return (T) new DocumentScanResult(new byte[][] { - Base64.decode(base64.getBytes("UTF-8")) + nativeData(root) }, metadata); } default: @@ -241,6 +249,27 @@ private static String stringOrNull(Map value, String key) { return out == null ? null : String.valueOf(out); } + private static byte[] nativeData(Map value) throws Exception { + Object peerValue = value.get("dataPeer"); + if (peerValue instanceof Number) { + long peer = ((Number) peerValue).longValue(); + if (peer == 0) { + throw new VisionException(VisionException.BACKEND_ERROR, + "Apple Vision returned no binary result"); + } + try { + byte[] out = new byte[IOSImplementation.nativeInstance + .getNSDataSize(peer)]; + IOSImplementation.nativeInstance.nsDataToByteArray(peer, out); + return out; + } finally { + IOSImplementation.nativeInstance.releasePeer(peer); + } + } + String base64 = string(value, "data"); + return Base64.decode(base64.getBytes("UTF-8")); + } + @Override public void close() { closed = true; diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md index c9e73ef28b1..75efc6d0261 100644 --- a/docs/ai-on-device-architecture.md +++ b/docs/ai-on-device-architecture.md @@ -22,7 +22,7 @@ Native backends are: | Target | Vision | Language | `.tflite` inference | | --- | --- | --- | --- | | Android | ML Kit | ML Kit | LiteRT | -| iOS | Apple Vision/Core Image; optional ML Kit | ML Kit | TensorFlow Lite Objective-C; optional Core ML delegate | +| iOS | Apple Vision/Core Image; optional ML Kit | Apple Natural Language for identification; ML Kit for translation, Smart Reply, and optional identification | TensorFlow Lite Objective-C; optional Core ML delegate | | Mac native | Apple Vision/Core Image through Mac Catalyst | Unsupported fallback | Unsupported fallback | | JavaSE, JavaScript, native Windows/Linux | Unsupported fallback | Unsupported fallback | Unsupported fallback | | watchOS, tvOS | Unsupported fallback | Unsupported fallback | Unsupported fallback | @@ -92,11 +92,13 @@ needed by a referenced analyzer. It adds a Google ML Kit vision pod only when it observes both: 1. the concrete analyzer class; and -2. a call to `VisionBackends.mlKit()`. +2. a call to the matching feature-specific selector, such as + `VisionBackends.mlKitTextRecognition()`. -On iOS, language entry points map directly to independent -`GoogleMLKit/LanguageID`, `GoogleMLKit/Translate`, and -`GoogleMLKit/SmartReply` pods. `InferenceSession` selects +On iOS, language identification defaults to the system Natural Language +framework. Calling `LanguageBackends.mlKitLanguageIdentification()` opts +into `GoogleMLKit/LanguageID`; translation and Smart Reply map independently +to `GoogleMLKit/Translate` and `GoogleMLKit/SmartReply`. `InferenceSession` selects `TensorFlowLiteObjC/CoreML`. The native implementation is disabled for watchOS and tvOS. Mac native uses @@ -124,8 +126,9 @@ implementation; device and arm64-simulator builds retain the NEON path. `VisionImage` owns defensive copies of its input. It accepts encoded JPEG or PNG, NV21, and RGBA8888. Android feeds raw NV21 directly to ML Kit and -converts RGBA to a bitmap. Apple converts the raw formats to an encoded image -inside `CN1Vision.m`. +converts RGBA to a bitmap. Apple creates a `CGImage` directly from NV21 or +RGBA memory and passes it to Vision or ML Kit without an intermediate JPEG +encode/decode. `VisionImage.fromCameraFrame()` is safe beyond the `FrameListener.onFrame()` callback because it copies the callback-owned @@ -141,8 +144,10 @@ actual backend id without exposing platform classes. `InferenceSession` supports named typed tensors, multiple inputs and outputs, input resizing, and reusable native sessions. Model sources are bytes, -resources, private files, or the HTTPS-only `ModelCache`. The cache can verify -a SHA-256 digest before publishing a downloaded model. +resources, private files, or the HTTPS-only `ModelCache`. File sources are +opened by path without copying the model through the Java heap. The cache can +verify a SHA-256 digest before publishing a downloaded model; callers should +always supply the digest for mutable or third-party downloads. All native sessions and analyzers must be closed. Expensive open, analysis, and inference work runs off the EDT; completion and error delivery return to diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index 5833477bb81..ead8e3caced 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -342,9 +342,11 @@ each class the application actually uses: | Core Image perspective correction |=== -The optional iOS ML Kit pod is added only when both the analyzer and -`VisionBackends.mlKit()` are referenced. Referencing one Android -analyzer doesn't compile or package the other five adapters. +The optional iOS ML Kit pod is added only when both the analyzer and its +feature-specific selector are referenced. For example, +`VisionBackends.mlKitBarcodeScanning()` adds only the barcode pod. +Referencing one Android analyzer doesn't compile or package the other +five adapters. === Portable LiteRT inference @@ -368,9 +370,12 @@ the port boundary. Always close the session. Bundle small models as resources. For larger models use `ModelCache.fetch(url, cacheKey, sha256)`, which downloads over HTTPS, -verifies the optional SHA-256 digest, and reuses the app-private cached -file. The URL must use HTTPS. Treat the digest as required when the -model isn't controlled by the same deployment pipeline as the app. +verifies a SHA-256 digest when supplied, and reuses the app-private cached +file. The URL must use HTTPS. Supply the digest for every third-party or +remotely mutable model: an omitted digest provides transport security but +does not pin the executable model payload. Interrupted downloads restart +through a temporary `.download` file and are never promoted to the cache +until verification succeeds. === On-device language services @@ -386,8 +391,13 @@ SDKs. include::../demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java[tag=ai-and-speech-on-device-language,indent=0] ---- -`LanguageBackends.auto()` selects ML Kit on the supported Android and -iOS targets. `LanguageBackends.mlKit()` makes that selection explicit. +`LanguageBackends.auto()` selects ML Kit on Android and Apple's Natural +Language framework for language identification on iOS. The Apple default +keeps arm64 simulator builds native and adds no third-party dependency. +`LanguageBackends.mlKitLanguageIdentification()` opts language +identification into the ML Kit iOS pod. Translation and Smart Reply use +their feature-scoped ML Kit components on both mobile platforms; those +two iOS components currently require the x86_64 simulator slice. Mac native and the other unsupported targets return `false` from the language service `isSupported()` checks. Completion and failure callbacks arrive on the EDT. diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java index ea2182654dc..1bb4eb14bd5 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java @@ -338,6 +338,24 @@ private static String append(String str, String separator, String append) { } return str + append; } + + private static String appendFrameworks(String libraries, + String... frameworks) { + String out = libraries == null ? "" : libraries; + for (String framework : frameworks) { + boolean present = false; + for (String item : out.split(";")) { + if (framework.equalsIgnoreCase(item.trim())) { + present = true; + break; + } + } + if (!present) { + out = out.length() == 0 ? framework : out + ";" + framework; + } + } + return out; + } private int getDeploymentTargetInt(BuildRequest request) { String target = getDeploymentTarget(request); @@ -2531,13 +2549,8 @@ public void usesClassMethod(String cls, String method) { throw new BuildException( "Failed to enable INCLUDE_CN1_VISION", ex); } - String visionLibs = - "Vision.framework;CoreImage.framework;CoreVideo.framework"; - if (addLibs == null || addLibs.length() == 0) { - addLibs = visionLibs; - } else if (!addLibs.toLowerCase().contains("vision.framework")) { - addLibs = addLibs + ";" + visionLibs; - } + addLibs = appendFrameworks(addLibs, "Vision.framework", + "CoreImage.framework", "CoreVideo.framework"); } if (usesCn1Language) { @@ -2562,13 +2575,8 @@ public void usesClassMethod(String cls, String method) { throw new BuildException( "Failed to enable INCLUDE_CN1_INFERENCE", ex); } - String inferenceLibs = - "CoreML.framework;Metal.framework;Accelerate.framework"; - if (addLibs == null || addLibs.length() == 0) { - addLibs = inferenceLibs; - } else if (!addLibs.toLowerCase().contains("coreml.framework")) { - addLibs = addLibs + ";" + inferenceLibs; - } + addLibs = appendFrameworks(addLibs, "CoreML.framework", + "Metal.framework", "Accelerate.framework"); } // CarPlay: link CarPlay.framework (+ MediaPlayer for the now-playing template) and @@ -3403,6 +3411,17 @@ public void usesClassMethod(String cls, String method) { + " end\n" + " raise \"Swift files/resources still present in Copy Bundle Resources: #{names.join(', ')}\"\n" + " end\n" + + " arc_sources = ['CN1Vision.m', 'CN1Language.m', 'CN1Inference.m']\n" + + " main_target.source_build_phase.files.each do |bf|\n" + + " ref = bf.file_ref\n" + + " name = ref && File.basename(ref.path || ref.name || '')\n" + + " next unless arc_sources.include?(name)\n" + + " settings = bf.settings || {}\n" + + " flags = settings['COMPILER_FLAGS'].to_s.split\n" + + " flags << '-fobjc-arc' unless flags.include?('-fobjc-arc')\n" + + " settings['COMPILER_FLAGS'] = flags.join(' ')\n" + + " bf.settings = settings\n" + + " end\n" + " end\n" + "rescue => e\n" + " puts \"Error while correcting Swift build phases: #{$!}\"\n" diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java index 3b4f48bae4e..e327db135b7 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java @@ -2,6 +2,7 @@ import org.junit.jupiter.api.Test; +import java.lang.reflect.Method; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -146,6 +147,29 @@ void rejectsUnknownDependencyManagerHint() { assertThrows(BuildException.class, () -> IOSDependencyManager.fromHint("gradle")); } + @Test + void appendsEachRequiredAiFrameworkIndependently() throws Exception { + Method method = IPhoneBuilder.class.getDeclaredMethod( + "appendFrameworks", String.class, String[].class); + method.setAccessible(true); + String value = (String) method.invoke(null, "Vision.framework", + new String[] {"Vision.framework", "CoreImage.framework", + "CoreVideo.framework"}); + assertEquals("Vision.framework;CoreImage.framework;CoreVideo.framework", + value); + + value = (String) method.invoke(null, + "CoreML.framework;Accelerate.framework", + new String[] {"CoreML.framework", "Metal.framework", + "Accelerate.framework"}); + assertEquals("CoreML.framework;Accelerate.framework;Metal.framework", + value); + + value = (String) method.invoke(null, "ThirdPartyVision.framework", + new String[] {"Vision.framework"}); + assertEquals("ThirdPartyVision.framework;Vision.framework", value); + } + private BuildRequest requestWithArgs(String... kvPairs) { BuildRequest out = new BuildRequest(); for (int i = 0; i < kvPairs.length; i += 2) { diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index 80a73e50f30..36f5ca33c1d 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -20,18 +20,16 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -/* - * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - */ package com.codename1.ai.inference; import com.codename1.impl.InferenceImpl; +import com.codename1.io.FileSystemStorage; import com.codename1.junit.UITestBase; import com.codename1.util.AsyncResource; import com.codename1.util.SuccessCallback; import org.junit.jupiter.api.Test; +import java.io.OutputStream; import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.*; @@ -48,6 +46,58 @@ void tensorsAreImmutableAndValidateShape() { assertArrayEquals(new float[] {1, 2}, (float[]) tensor.getData()); assertThrows(IllegalArgumentException.class, () -> Tensor.floats("bad", new int[] {3}, new float[] {1, 2})); + assertThrows(IllegalArgumentException.class, () -> + Tensor.bytes("overflow", TensorType.UINT8, + new int[] {Integer.MAX_VALUE, 2}, new byte[] {1})); + } + + @Test + void modelCacheCompletionIsSingleShot() { + AsyncResource resource = new AsyncResource(); + ModelCache.Completion completion = + new ModelCache.Completion(resource); + completion.complete("first"); + completion.fail(new RuntimeException("late failure")); + completion.complete("second"); + flushSerialCalls(); + assertEquals("first", resource.get()); + } + + @Test + void modelCacheRejectsInsecureAndMalformedRequests() { + assertThrows(IllegalArgumentException.class, + () -> ModelCache.fetch("http://example.com/model.tflite", "model")); + assertThrows(IllegalArgumentException.class, + () -> ModelCache.fetch("https://example.com/model.tflite", "")); + assertThrows(IllegalArgumentException.class, + () -> ModelCache.fetch("https://example.com/model.tflite", "model", "xyz")); + assertThrows(IllegalArgumentException.class, + () -> ModelCache.fetch("https://example.com/model.tflite", "model", + "gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg")); + } + + @Test + void modelCacheDiscardsPartialAndDigestMismatchFiles() throws Exception { + FileSystemStorage fs = FileSystemStorage.getInstance(); + String temporary = fs.getAppHomePath() + "model-cache-test.download"; + write(temporary, new byte[] {1, 2, 3}); + ModelCache.prepareTemporary(fs, temporary); + assertFalse(fs.exists(temporary), "a previous partial download must not be resumed"); + + write(temporary, new byte[] {4, 5, 6}); + assertThrows(java.io.IOException.class, () -> + ModelCache.verifyDownloaded(fs, temporary, + "0000000000000000000000000000000000000000000000000000000000000000")); + assertFalse(fs.exists(temporary), "a digest mismatch must delete executable data"); + } + + private static void write(String path, byte[] value) throws Exception { + OutputStream output = FileSystemStorage.getInstance().openOutputStream(path); + try { + output.write(value); + } finally { + output.close(); + } } @Test diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java index 997dedab097..d3302f70d7f 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java @@ -42,7 +42,7 @@ void languageOperationsForwardBackendAndOptions() { RecordingLanguageImpl backend = new RecordingLanguageImpl(); implementation.setLanguageImpl(backend); LanguageOptions options = new LanguageOptions() - .backend(LanguageBackends.mlKit()) + .backend(LanguageBackends.mlKitLanguageIdentification()) .minimumConfidence(.4f); assertTrue(LanguageIdentifier.isSupported(options)); diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java index e69d83c7bf2..2d3425f877d 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java @@ -20,10 +20,6 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -/* - * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - */ package com.codename1.ai.vision; import com.codename1.camera.FrameFormat; @@ -78,7 +74,8 @@ void analyzerForwardsFeatureBackendAndOptions() { RecordingVisionImpl backend = new RecordingVisionImpl(); implementation.setVisionImpl(backend); VisionOptions options = new VisionOptions() - .backend(VisionBackends.mlKit()).minimumConfidence(.6f); + .backend(VisionBackends.mlKitBarcodeScanning()) + .minimumConfidence(.6f); TextRecognizer recognizer = new TextRecognizer(options); TextRecognitionResult result = await(recognizer.process( VisionImage.encoded(new byte[] {1}))); diff --git a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java index 18765acb17f..0a17052cadc 100644 --- a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java +++ b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java @@ -88,7 +88,8 @@ public final class PlatformFeatureCatalog { .androidGradle("com.google.mlkit:text-recognition:16.0.0") .description("Text recognition")); e.add(new Entry("com/codename1/ai/vision/TextRecognizer") - .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") + .requiresMethod("com/codename1/ai/vision/VisionBackends", + "mlKitTextRecognition") .iosPod("GoogleMLKit/TextRecognition") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() @@ -99,7 +100,8 @@ public final class PlatformFeatureCatalog { .androidGradle("com.google.mlkit:barcode-scanning:17.2.0") .description("Barcode scanning")); e.add(new Entry("com/codename1/ai/vision/BarcodeScanner") - .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") + .requiresMethod("com/codename1/ai/vision/VisionBackends", + "mlKitBarcodeScanning") .iosPod("GoogleMLKit/BarcodeScanning") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() @@ -110,7 +112,8 @@ public final class PlatformFeatureCatalog { .androidGradle("com.google.mlkit:face-detection:16.1.5") .description("Face detection")); e.add(new Entry("com/codename1/ai/vision/FaceDetector") - .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") + .requiresMethod("com/codename1/ai/vision/VisionBackends", + "mlKitFaceDetection") .iosPod("GoogleMLKit/FaceDetection") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() @@ -121,7 +124,8 @@ public final class PlatformFeatureCatalog { .androidGradle("com.google.mlkit:image-labeling:17.0.7") .description("Image labeling")); e.add(new Entry("com/codename1/ai/vision/ImageLabeler") - .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") + .requiresMethod("com/codename1/ai/vision/VisionBackends", + "mlKitImageLabeling") .iosPod("GoogleMLKit/ImageLabeling") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() @@ -132,7 +136,8 @@ public final class PlatformFeatureCatalog { .androidGradle("com.google.mlkit:pose-detection:18.0.0-beta3") .description("Pose detection")); e.add(new Entry("com/codename1/ai/vision/PoseDetector") - .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") + .requiresMethod("com/codename1/ai/vision/VisionBackends", + "mlKitPoseDetection") .iosPod("GoogleMLKit/PoseDetection") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() @@ -144,7 +149,8 @@ public final class PlatformFeatureCatalog { "com.google.mlkit:segmentation-selfie:16.0.0-beta5") .description("Selfie segmentation")); e.add(new Entry("com/codename1/ai/vision/SelfieSegmenter") - .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKit") + .requiresMethod("com/codename1/ai/vision/VisionBackends", + "mlKitSelfieSegmentation") .iosPod("GoogleMLKit/SegmentationSelfie") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() @@ -165,19 +171,29 @@ public final class PlatformFeatureCatalog { .description("LiteRT inference")); e.add(new Entry("com/codename1/ai/language/LanguageIdentifier") + .iosFrameworks("NaturalLanguage") + .androidGradle("com.google.mlkit:language-id:17.0.6") + .description("On-device language identification")); + e.add(new Entry("com/codename1/ai/language/LanguageIdentifier") + .requiresMethod("com/codename1/ai/language/LanguageBackends", + "mlKitLanguageIdentification") .iosPod("GoogleMLKit/LanguageID") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() - .androidGradle("com.google.mlkit:language-id:17.0.6") - .description("On-device language identification")); + .description("ML Kit iOS language-identification backend")); + // CN1Language.m contains the dependency-free Apple identifier beside + // the ML Kit adapters, so every target that compiles this source must + // link the small system NaturalLanguage framework. e.add(new Entry("com/codename1/ai/language/Translator") .iosPod("GoogleMLKit/Translate") + .iosFrameworks("NaturalLanguage") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() .androidGradle("com.google.mlkit:translate:17.0.3") .description("On-device translation")); e.add(new Entry("com/codename1/ai/language/SmartReply") .iosPod("GoogleMLKit/SmartReply") + .iosFrameworks("NaturalLanguage") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() .androidGradle("com.google.mlkit:smart-reply:17.0.4") @@ -335,7 +351,7 @@ public boolean anyRequiresBigUpload() { public static final class Entry { private final String classPrefix; private String methodOwner; - private String methodPrefix; + private String methodName; private final List iosPods = new ArrayList(); private final List iosSpm = new ArrayList(); private final List iosFrameworks = new ArrayList(); @@ -379,16 +395,16 @@ boolean requirementsMet(Set classes, Set methods) { return true; } for (String method : methods) { - if (method.startsWith(methodOwner + "#" + methodPrefix)) { + if (method.equals(methodOwner + "#" + methodName)) { return true; } } return false; } - Entry requiresMethod(String owner, String methodNamePrefix) { + Entry requiresMethod(String owner, String methodName) { this.methodOwner = owner; - this.methodPrefix = methodNamePrefix; + this.methodName = methodName; return this; } diff --git a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java index 511ea3a58ba..f98891e8965 100644 --- a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java +++ b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java @@ -50,7 +50,8 @@ void builtInTextRecognizerMapsToAppleVisionAndAndroidMlKit() { void explicitMlKitBackendAddsOnlyUsedFeaturePod() { PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); acc.consume("com/codename1/ai/vision/TextRecognizer"); - acc.consumeMethod("com/codename1/ai/vision/VisionBackends", "mlKit"); + acc.consumeMethod("com/codename1/ai/vision/VisionBackends", + "mlKitTextRecognition"); boolean foundTextPod = false; for (PlatformFeatureCatalog.Entry e : acc.hits()) { foundTextPod |= e.iosPods().contains("GoogleMLKit/TextRecognition"); @@ -203,7 +204,8 @@ void inferenceUsesCoreMlEnabledObjectiveCPod() { void thirdPartyAppleAiPackagesAreExcludedFromCatalyst() { PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); acc.consume("com/codename1/ai/vision/TextRecognizer"); - acc.consumeMethod("com/codename1/ai/vision/VisionBackends", "mlKit"); + acc.consumeMethod("com/codename1/ai/vision/VisionBackends", + "mlKitTextRecognition"); acc.consume("com/codename1/ai/language/LanguageIdentifier"); acc.consume("com/codename1/ai/inference/InferenceSession"); @@ -221,6 +223,27 @@ void thirdPartyAppleAiPackagesAreExcludedFromCatalyst() { assertTrue(foundSystemVision); } + @Test + void selectorMethodsAreMatchedExactlyAndLanguageIdDefaultsToApple() { + PlatformFeatureCatalog.Accumulator acc = + new PlatformFeatureCatalog.Accumulator(); + acc.consume("com/codename1/ai/vision/TextRecognizer"); + acc.consumeMethod("com/codename1/ai/vision/VisionBackends", + "mlKitTextRecognitionFuture"); + for (PlatformFeatureCatalog.Entry entry : acc.hits()) { + assertTrue(entry.iosPods().isEmpty(), + "A method-name prefix must not select an optional pod"); + } + + List language = + PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/language/LanguageIdentifier"); + assertEquals(1, language.size()); + assertTrue(language.get(0).iosPods().isEmpty()); + assertTrue(language.get(0).iosFrameworks().contains("NaturalLanguage")); + assertTrue(language.get(0).iosDependenciesSupportArm64Simulator()); + } + @Test void androidAdaptersReceiveOnlyTheirFeatureDependency() { List vision = PlatformFeatureCatalog.matchesFor( diff --git a/scripts/gen-ai-cn1libs.py b/scripts/gen-ai-cn1libs.py index 83496618f7c..646ba1a5792 100644 --- a/scripts/gen-ai-cn1libs.py +++ b/scripts/gen-ai-cn1libs.py @@ -787,1557 +787,6 @@ def test_java(pkg: str, facade: str, mock_methods: str, test_methods: str) -> st # -------------------------------------------------------------------------- -def lib_mlkit_text() -> Lib: - facade_methods = textwrap.dedent("""\ - /// Runs OCR on the supplied image bytes (JPEG or PNG). Completes with - /// the recognised text. Empty image -> empty string. No text -> empty - /// string. Hard errors fire `AsyncResource.error(...)`. - public static AsyncResource recognize(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - if (imageBytes == null || imageBytes.length == 0) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(""); } - }); - return out; - } - final NativeTextRecognizer bridge = NativeLookup.create(NativeTextRecognizer.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException( - "TextRecognizer.recognize is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String result = bridge.recognize(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(result == null ? "" : result); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("TextRecognizer.recognize failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } - """) - ni_methods = "String recognize(byte[] imageBytes);\n" - - ios_h_decls = "-(NSString*)recognize:(NSData*)param;\n" - ios_impl = textwrap.dedent("""\ - -(NSString*)recognize:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return @""; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKTextRecognizerOptions *opts = [[MLKTextRecognizerOptions alloc] init]; - MLKTextRecognizer *recognizer = [MLKTextRecognizer textRecognizerWithOptions:opts]; - __block NSString *result = @""; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [recognizer processImage:vision completion:^(MLKText * _Nullable text, NSError * _Nullable err) { - if (text && !err) { - result = text.text ?: @""; - } else if (err) { - result = @""; - } - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; - } - """) - # ML Kit ships per-framework umbrella headers (named after the - # framework itself, e.g. MLKitTextRecognition.h). Importing the - # umbrella pulls every public class -- the per-class header paths - # we previously used aren't actually exported. - # MLKText (the result type) lives in MLKitTextRecognitionCommon, - # which the recognizer's main framework re-exports but clang's - # modules system still requires the explicit import. - ios_imports = textwrap.dedent("""\ - #import - #import - #import - """) - - android_impl = textwrap.dedent("""\ - public String recognize(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return ""; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.text.TextRecognizer rec = - com.google.mlkit.vision.text.TextRecognition.getClient( - com.google.mlkit.vision.text.latin.TextRecognizerOptions.DEFAULT_OPTIONS); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference(""); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - rec.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.vision.text.Text>() { - public void onSuccess(com.google.mlkit.vision.text.Text t) { - out.set(t.getText() == null ? "" : t.getText()); - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - """) - android_imports = "" - - javase_impl = textwrap.dedent("""\ - public String recognize(byte[] imageBytes) { - if (imageBytes == null || imageBytes.length == 0) return ""; - return "[mlkit-text simulator stub] " + imageBytes.length + " bytes"; - } - """) - - test_mock_methods = textwrap.dedent("""\ - String response = "hello"; - public String recognize(byte[] imageBytes) { - if (imageBytes == null) throw new NullPointerException(); - return response; - } - """) - test_methods = textwrap.dedent("""\ - @Test - void bridge_returns_canned_string() { - MockBridge b = new MockBridge(); - assertEquals("hello", b.recognize(new byte[]{1, 2, 3})); - } - - @Test - void bridge_reports_supported() { - MockBridge b = new MockBridge(); - assertTrue(b.isSupported()); - } - - @Test - void bridge_rejects_null_input() { - MockBridge b = new MockBridge(); - assertThrows(NullPointerException.class, () -> b.recognize(null)); - } - """) - - return Lib( - artifact="cn1-ai-mlkit-text", - pkg="mlkit.text", - facade="TextRecognizer", - short_desc="ML Kit Text Recognition (OCR)", - long_desc=( - "Extracts text strings from images entirely on-device via Google's ML Kit.\n" - "Bridges to `GoogleMLKit/TextRecognition` on iOS and\n" - "`com.google.mlkit:text-recognition` on Android." - ), - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls=ios_h_decls, - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports=android_imports, - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("recognize__byte_1ARRAY", 1),), - simulator_hints=( - ("ios.NSCameraUsageDescription", - "This app uses the camera to recognise text."), - ), - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/TextRecognition", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:text-recognition:16.0.0'", - }, - ) - - -# --- mlkit-barcode ------------------------------------------------------- - - -def lib_mlkit_barcode() -> Lib: - facade_methods = textwrap.dedent("""\ - public static AsyncResource scan(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - if (imageBytes == null || imageBytes.length == 0) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(new String[0]); } - }); - return out; - } - final NativeBarcodeScanner bridge = NativeLookup.create(NativeBarcodeScanner.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("BarcodeScanner.scan is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String[] r = bridge.scan(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new String[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("BarcodeScanner.scan failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } - """) - ni_methods = "String[] scan(byte[] imageBytes);\n" - - ios_impl = textwrap.dedent("""\ - -(NSData*)scan:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [self packStrings:@[]]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKBarcodeScannerOptions *opts = [[MLKBarcodeScannerOptions alloc] init]; - MLKBarcodeScanner *scanner = [MLKBarcodeScanner barcodeScannerWithOptions:opts]; - __block NSArray *values = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [scanner processImage:vision completion:^(NSArray * _Nullable barcodes, - NSError * _Nullable error) { - NSMutableArray *m = [NSMutableArray array]; - for (MLKBarcode *b in barcodes ?: @[]) { - if (b.rawValue) [m addObject:b.rawValue]; - } - values = m; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return [self packStrings:values]; - } - - -(NSData*)packStrings:(NSArray *)strings { - // Encode as length-prefixed UTF-8 (network byte order int + bytes). - NSMutableData *out = [NSMutableData data]; - uint32_t count = htonl((uint32_t)strings.count); - [out appendBytes:&count length:sizeof(count)]; - for (NSString *s in strings) { - NSData *u = [s dataUsingEncoding:NSUTF8StringEncoding]; - uint32_t len = htonl((uint32_t)u.length); - [out appendBytes:&len length:sizeof(len)]; - [out appendData:u]; - } - return out; - } - """) - ios_imports = textwrap.dedent("""\ - #import - #import - #import - """) - android_impl = textwrap.dedent("""\ - public String[] scan(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new String[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.barcode.BarcodeScannerOptions o = - new com.google.mlkit.vision.barcode.BarcodeScannerOptions.Builder().build(); - com.google.mlkit.vision.barcode.BarcodeScanner scanner = - com.google.mlkit.vision.barcode.BarcodeScanning.getClient(o); - final java.util.List out = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - scanner.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - java.util.List>() { - public void onSuccess(java.util.List rs) { - for (com.google.mlkit.vision.barcode.common.Barcode b : rs) { - String v = b.getRawValue(); - if (v != null) out.add(v); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.toArray(new String[0]); - } - """) - javase_impl = textwrap.dedent("""\ - public String[] scan(byte[] imageBytes) { - if (imageBytes == null || imageBytes.length == 0) return new String[0]; - // Deterministic stub for simulator runs. - return new String[]{"SIMULATOR_BARCODE_" + imageBytes.length}; - } - """) - test_mock_methods = textwrap.dedent("""\ - public String[] scan(byte[] imageBytes) { - return new String[]{"x", "y"}; - } - """) - test_methods = textwrap.dedent("""\ - @Test - void mock_bridge_returns_two_codes() { - MockBridge b = new MockBridge(); - String[] r = b.scan(new byte[]{1, 2, 3}); - assertEquals(2, r.length); - assertEquals("x", r[0]); - } - - @Test - void bridge_reports_supported() { - assertTrue(new MockBridge().isSupported()); - } - """) - return Lib( - artifact="cn1-ai-mlkit-barcode", - pkg="mlkit.barcode", - facade="BarcodeScanner", - short_desc="ML Kit Barcode Scanning", - long_desc=( - "Decodes barcodes (QR, EAN, UPC, Data Matrix, PDF417, etc.) from images.\n" - "Bridges to `MLKitBarcodeScanning` on iOS and\n" - "`com.google.mlkit:barcode-scanning` on Android." - ), - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSData*)scan:(NSData*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("scan__byte_1ARRAY", 1),), - simulator_hints=( - ("ios.NSCameraUsageDescription", - "This app uses the camera to scan barcodes."), - ), - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/BarcodeScanning", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:barcode-scanning:17.2.0'", - "codename1.arg.android.xpermissions": "", - }, - ) - - -# --- mlkit-face ---------------------------------------------------------- - - -def lib_mlkit_face() -> Lib: - facade_methods = textwrap.dedent("""\ - /// Returns an array of face bounding-box quadruples - /// (x, y, width, height) packed as `int[4 * n]`. - public static AsyncResource detect(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativeFaceDetector bridge = NativeLookup.create(NativeFaceDetector.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("FaceDetector.detect is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final int[] r = bridge.detect(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new int[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("FaceDetector.detect failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } - """) - ni_methods = "int[] detect(byte[] imageBytes);\n" - - ios_impl = textwrap.dedent("""\ - -(NSData*)detect:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [NSData data]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKFaceDetectorOptions *opts = [[MLKFaceDetectorOptions alloc] init]; - MLKFaceDetector *det = [MLKFaceDetector faceDetectorWithOptions:opts]; - __block NSArray *faces = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [det processImage:vision completion:^(NSArray * _Nullable f, NSError * _Nullable e) { - faces = f ?: @[]; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - NSMutableData *out = [NSMutableData data]; - for (MLKFace *face in faces) { - CGRect r = face.frame; - int32_t v[4] = { htonl((int32_t)r.origin.x), htonl((int32_t)r.origin.y), - htonl((int32_t)r.size.width), htonl((int32_t)r.size.height) }; - [out appendBytes:v length:sizeof(v)]; - } - return out; - } - """) - ios_imports = textwrap.dedent("""\ - #import - #import - #import - """) - android_impl = textwrap.dedent("""\ - public int[] detect(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new int[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.face.FaceDetector det = - com.google.mlkit.vision.face.FaceDetection.getClient( - new com.google.mlkit.vision.face.FaceDetectorOptions.Builder().build()); - final java.util.List rs = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - det.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - java.util.List>() { - public void onSuccess(java.util.List faces) { - for (com.google.mlkit.vision.face.Face f : faces) { - android.graphics.Rect r = f.getBoundingBox(); - rs.add(new int[]{r.left, r.top, r.width(), r.height()}); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - int[] flat = new int[rs.size() * 4]; - int i = 0; - for (int[] r : rs) { System.arraycopy(r, 0, flat, i, 4); i += 4; } - return flat; - } - """) - javase_impl = textwrap.dedent("""\ - public int[] detect(byte[] imageBytes) { - // Deterministic 1-face stub for simulator runs. - if (imageBytes == null || imageBytes.length == 0) return new int[0]; - return new int[]{10, 20, 100, 120}; - } - """) - test_mock_methods = textwrap.dedent("""\ - public int[] detect(byte[] imageBytes) { - return new int[]{1, 2, 3, 4, 5, 6, 7, 8}; - } - """) - test_methods = textwrap.dedent("""\ - @Test - void mock_bridge_returns_two_faces() { - MockBridge b = new MockBridge(); - int[] r = b.detect(new byte[]{1}); - assertEquals(8, r.length); - } - """) - return Lib( - artifact="cn1-ai-mlkit-face", - pkg="mlkit.face", - facade="FaceDetector", - short_desc="ML Kit Face Detection", - long_desc=( - "Detects faces in images and returns bounding boxes.\n" - "Bridges to `MLKitFaceDetection` on iOS and\n" - "`com.google.mlkit:face-detection` on Android." - ), - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSData*)detect:(NSData*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("detect__byte_1ARRAY", 1),), - simulator_hints=( - ("ios.NSCameraUsageDescription", - "This app uses the camera to detect faces."), - ), - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/FaceDetection", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:face-detection:16.1.5'", - }, - ) - - -# Below: shorter helpers for the remaining libs. Each follows the same -# pattern; the native bodies use the appropriate ML Kit / TFLite / whisper / -# CoreML API. - - -def _string_facade_methods(facade: str, ni: str, method: str, arg_kind: str = "bytes"): - """Build the AsyncResource facade body for a String-returning bridge.""" - arg_type = "byte[] imageBytes" if arg_kind == "bytes" else "String input" - arg_pass = "imageBytes" if arg_kind == "bytes" else "input" - return textwrap.dedent(f"""\ - public static AsyncResource {method}(final {arg_type}) {{ - final AsyncResource out = new AsyncResource(); - final {ni} bridge = NativeLookup.create({ni}.class); - if (bridge == null || !bridge.isSupported()) {{ - out.error(new LlmException("{facade}.{method} is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - }} - Display.getInstance().scheduleBackgroundTask(new Runnable() {{ - @Override public void run() {{ - try {{ - final String r = bridge.{method}({arg_pass}); - Display.getInstance().callSerially(new Runnable() {{ - @Override public void run() {{ out.complete(r == null ? "" : r); }} - }}); - }} catch (final Throwable t) {{ - Display.getInstance().callSerially(new Runnable() {{ - @Override public void run() {{ - out.error(new LlmException("{facade}.{method} failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - }} - }}); - }} - }} - }}); - return out; - }} - """) - - -def _strings_facade_methods(facade: str, ni: str, method: str, arg_kind: str = "bytes"): - arg_type = "byte[] imageBytes" if arg_kind == "bytes" else "String input" - arg_pass = "imageBytes" if arg_kind == "bytes" else "input" - return textwrap.dedent(f"""\ - public static AsyncResource {method}(final {arg_type}) {{ - final AsyncResource out = new AsyncResource(); - final {ni} bridge = NativeLookup.create({ni}.class); - if (bridge == null || !bridge.isSupported()) {{ - out.error(new LlmException("{facade}.{method} is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - }} - Display.getInstance().scheduleBackgroundTask(new Runnable() {{ - @Override public void run() {{ - try {{ - final String[] r = bridge.{method}({arg_pass}); - Display.getInstance().callSerially(new Runnable() {{ - @Override public void run() {{ out.complete(r == null ? new String[0] : r); }} - }}); - }} catch (final Throwable t) {{ - Display.getInstance().callSerially(new Runnable() {{ - @Override public void run() {{ - out.error(new LlmException("{facade}.{method} failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - }} - }}); - }} - }} - }}); - return out; - }} - """) - - -def lib_mlkit_labeling() -> Lib: - facade_methods = _strings_facade_methods("ImageLabeler", "NativeImageLabeler", "label") - ios_impl = textwrap.dedent("""\ - -(NSData*)label:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [self packStrings:@[]]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKImageLabelerOptions *opts = [[MLKImageLabelerOptions alloc] init]; - MLKImageLabeler *labeler = [MLKImageLabeler imageLabelerWithOptions:opts]; - __block NSArray *labels = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [labeler processImage:vision completion:^(NSArray * _Nullable r, NSError * _Nullable e) { - labels = r ?: @[]; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - NSMutableArray *m = [NSMutableArray array]; - for (MLKImageLabel *l in labels) { if (l.text) [m addObject:l.text]; } - return [self packStrings:m]; - } - - -(NSData*)packStrings:(NSArray *)strings { - NSMutableData *out = [NSMutableData data]; - uint32_t count = htonl((uint32_t)strings.count); - [out appendBytes:&count length:sizeof(count)]; - for (NSString *s in strings) { - NSData *u = [s dataUsingEncoding:NSUTF8StringEncoding]; - uint32_t len = htonl((uint32_t)u.length); - [out appendBytes:&len length:sizeof(len)]; - [out appendData:u]; - } - return out; - } - """) - ios_imports = textwrap.dedent("""\ - #import - #import - #import - #import - """) - android_impl = textwrap.dedent("""\ - public String[] label(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new String[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.label.ImageLabeler labeler = - com.google.mlkit.vision.label.ImageLabeling.getClient( - com.google.mlkit.vision.label.defaults.ImageLabelerOptions.DEFAULT_OPTIONS); - final java.util.List out = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - labeler.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - java.util.List>() { - public void onSuccess(java.util.List rs) { - for (com.google.mlkit.vision.label.ImageLabel l : rs) out.add(l.getText()); - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.toArray(new String[0]); - } - """) - javase_impl = textwrap.dedent("""\ - public String[] label(byte[] imageBytes) { - if (imageBytes == null || imageBytes.length == 0) return new String[0]; - return new String[]{"object", "stub", "simulator"}; - } - """) - test_mock_methods = textwrap.dedent("""\ - public String[] label(byte[] imageBytes) { return new String[]{"a", "b"}; } - """) - test_methods = textwrap.dedent("""\ - @Test - void mock_bridge_returns_labels() { - MockBridge b = new MockBridge(); - assertArrayEquals(new String[]{"a", "b"}, b.label(new byte[]{1})); - } - """) - return Lib( - artifact="cn1-ai-mlkit-labeling", - pkg="mlkit.labeling", - facade="ImageLabeler", - short_desc="ML Kit Image Labeling", - long_desc=( - "Returns descriptive labels for the contents of an image.\n" - "Bridges to `MLKitImageLabeling` on iOS and\n" - "`com.google.mlkit:image-labeling` on Android." - ), - facade_methods=facade_methods, - ni_methods="String[] label(byte[] imageBytes);\n", - ios_h_decls="-(NSData*)label:(NSData*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("label__byte_1ARRAY", 1),), - win_decls="public string[] label(byte[] param) { return null; }", - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/ImageLabeling", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:image-labeling:17.0.7'", - }, - ) - - -def lib_mlkit_translate() -> Lib: - facade_methods = textwrap.dedent("""\ - public static AsyncResource translate(final String text, - final String sourceLang, - final String targetLang) { - final AsyncResource out = new AsyncResource(); - final NativeTranslator bridge = NativeLookup.create(NativeTranslator.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("Translator.translate is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String r = bridge.translate(text, sourceLang, targetLang); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? "" : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("Translator.translate failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } - """) - ni_methods = "String translate(String text, String sourceLang, String targetLang);\n" - ios_impl = textwrap.dedent("""\ - -(NSString*)translate:(NSString*)param param1:(NSString*)param1 param2:(NSString*)param2 { - MLKTranslatorOptions *opts = [[MLKTranslatorOptions alloc] - initWithSourceLanguage:param1 - targetLanguage:param2]; - MLKTranslator *t = [MLKTranslator translatorWithOptions:opts]; - MLKModelDownloadConditions *cond = [[MLKModelDownloadConditions alloc] - initWithAllowsCellularAccess:YES - allowsBackgroundDownloading:YES]; - __block NSString *result = @""; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [t downloadModelIfNeededWithConditions:cond completion:^(NSError * _Nullable err) { - if (err) { dispatch_semaphore_signal(sem); return; } - [t translateText:param completion:^(NSString * _Nullable r, NSError * _Nullable e) { - if (r && !e) result = r; - dispatch_semaphore_signal(sem); - }]; - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; - } - """) - ios_imports = textwrap.dedent("""\ - #import - #import - """) - android_impl = textwrap.dedent("""\ - public String translate(String text, String sourceLang, String targetLang) { - com.google.mlkit.nl.translate.TranslatorOptions opts = - new com.google.mlkit.nl.translate.TranslatorOptions.Builder() - .setSourceLanguage(sourceLang) - .setTargetLanguage(targetLang) - .build(); - com.google.mlkit.nl.translate.Translator t = - com.google.mlkit.nl.translate.Translation.getClient(opts); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference(""); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - t.downloadModelIfNeeded() - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener() { - public void onSuccess(Void v) { - t.translate(text) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener() { - public void onSuccess(String r) { out.set(r); latch.countDown(); } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - """) - javase_impl = textwrap.dedent("""\ - public String translate(String text, String sourceLang, String targetLang) { - if (text == null) return ""; - return "[" + sourceLang + "->" + targetLang + "] " + text; - } - """) - test_mock_methods = textwrap.dedent("""\ - public String translate(String text, String sourceLang, String targetLang) { - return text + "@" + sourceLang + "->" + targetLang; - } - """) - test_methods = textwrap.dedent("""\ - @Test - void mock_translate_round_trip() { - MockBridge b = new MockBridge(); - assertEquals("hi@en->fr", b.translate("hi", "en", "fr")); - } - """) - return Lib( - artifact="cn1-ai-mlkit-translate", - pkg="mlkit.translate", - facade="Translator", - short_desc="ML Kit on-device Translation", - long_desc="Translates short text between language pairs entirely on-device.", - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSString*)translate:(NSString*)param param1:(NSString*)param1 param2:(NSString*)param2;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("translate__java_lang_String_java_lang_String_java_lang_String", 3),), - win_decls="public string translate(string param, string param1, string param2) { return null; }", - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/Translate", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:translate:17.0.3'", - }, - ) - - -def lib_mlkit_smartreply() -> Lib: - facade_methods = _strings_facade_methods("SmartReply", "NativeSmartReply", "suggest", arg_kind="text") - ni_methods = "String[] suggest(String conversationJson);\n" - ios_impl = textwrap.dedent("""\ - -(NSData*)suggest:(NSString*)param { - // param is a JSON array of {role,message,timestamp,userId}. - NSError *err = nil; - NSArray *items = [NSJSONSerialization JSONObjectWithData: - [param dataUsingEncoding:NSUTF8StringEncoding] - options:0 error:&err]; - NSMutableArray *messages = [NSMutableArray array]; - if ([items isKindOfClass:[NSArray class]]) { - for (NSDictionary *d in items) { - if (![d isKindOfClass:[NSDictionary class]]) continue; - NSString *role = d[@"role"] ?: @"user"; - BOOL isLocalUser = [role isEqualToString:@"user"]; - NSString *text = d[@"message"] ?: @""; - NSNumber *ts = d[@"timestamp"] ?: @0; - MLKTextMessage *m = [[MLKTextMessage alloc] - initWithText:text timestamp:[ts doubleValue] - userID:(d[@"userId"] ?: @"u") - isLocalUser:isLocalUser]; - [messages addObject:m]; - } - } - __block NSArray *out = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [[MLKSmartReply smartReply] suggestRepliesForMessages:messages - completion:^(MLKSmartReplySuggestionResult * _Nullable result, NSError * _Nullable e) { - NSMutableArray *m = [NSMutableArray array]; - for (MLKSmartReplySuggestion *s in result.suggestions ?: @[]) { - if (s.text) [m addObject:s.text]; - } - out = m; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return [self packStrings:out]; - } - - -(NSData*)packStrings:(NSArray *)strings { - NSMutableData *out = [NSMutableData data]; - uint32_t count = htonl((uint32_t)strings.count); - [out appendBytes:&count length:sizeof(count)]; - for (NSString *s in strings) { - NSData *u = [s dataUsingEncoding:NSUTF8StringEncoding]; - uint32_t len = htonl((uint32_t)u.length); - [out appendBytes:&len length:sizeof(len)]; - [out appendData:u]; - } - return out; - } - """) - ios_imports = textwrap.dedent("""\ - #import - #import - """) - android_impl = textwrap.dedent("""\ - public String[] suggest(String conversationJson) { - java.util.List msgs = - new java.util.ArrayList(); - try { - org.json.JSONArray a = new org.json.JSONArray(conversationJson); - for (int i = 0; i < a.length(); i++) { - org.json.JSONObject o = a.getJSONObject(i); - String role = o.optString("role", "user"); - long ts = o.optLong("timestamp", 0); - String text = o.optString("message", ""); - String userId = o.optString("userId", "u"); - if ("user".equals(role)) { - msgs.add(com.google.mlkit.nl.smartreply.TextMessage.createForLocalUser(text, ts)); - } else { - msgs.add(com.google.mlkit.nl.smartreply.TextMessage.createForRemoteUser(text, ts, userId)); - } - } - } catch (org.json.JSONException jex) { - return new String[0]; - } - final java.util.List out = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - com.google.mlkit.nl.smartreply.SmartReplyGenerator gen = - com.google.mlkit.nl.smartreply.SmartReply.getClient(); - gen.suggestReplies(msgs) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.nl.smartreply.SmartReplySuggestionResult>() { - public void onSuccess(com.google.mlkit.nl.smartreply.SmartReplySuggestionResult r) { - for (com.google.mlkit.nl.smartreply.SmartReplySuggestion s : r.getSuggestions()) { - out.add(s.getText()); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.toArray(new String[0]); - } - """) - javase_impl = textwrap.dedent("""\ - public String[] suggest(String conversationJson) { - return new String[]{"Sounds good", "Thanks!", "Got it"}; - } - """) - test_mock_methods = "public String[] suggest(String c) { return new String[]{\"ok\"}; }\n" - test_methods = textwrap.dedent("""\ - @Test - void mock_returns_single_suggestion() { - MockBridge b = new MockBridge(); - assertEquals(1, b.suggest("[]").length); - } - """) - return Lib( - artifact="cn1-ai-mlkit-smartreply", - pkg="mlkit.smartreply", - facade="SmartReply", - short_desc="ML Kit Smart Reply", - long_desc="Generates short reply suggestions for chat conversations on-device.", - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSData*)suggest:(NSString*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("suggest__java_lang_String", 1),), - win_decls="public string[] suggest(string param) { return null; }", - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/SmartReply", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:smart-reply:17.0.4'", - }, - ) - - -def lib_mlkit_langid() -> Lib: - facade_methods = _string_facade_methods("LanguageIdentifier", "NativeLanguageIdentifier", "identify", - arg_kind="text") - ni_methods = "String identify(String input);\n" - ios_impl = textwrap.dedent("""\ - -(NSString*)identify:(NSString*)param { - MLKLanguageIdentification *id = [MLKLanguageIdentification languageIdentification]; - __block NSString *result = @"und"; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [id identifyLanguageForText:param completion:^(NSString * _Nullable lang, NSError * _Nullable e) { - if (lang) result = lang; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; - } - """) - ios_imports = "#import \n" - android_impl = textwrap.dedent("""\ - public String identify(String input) { - com.google.mlkit.nl.languageid.LanguageIdentifier id = - com.google.mlkit.nl.languageid.LanguageIdentification.getClient(); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference("und"); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - id.identifyLanguage(input) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener() { - public void onSuccess(String s) { if (s != null) out.set(s); latch.countDown(); } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - """) - javase_impl = textwrap.dedent("""\ - public String identify(String input) { - // Crude language ID stub for simulator. - if (input == null || input.isEmpty()) return "und"; - return "en"; - } - """) - test_mock_methods = "public String identify(String input) { return \"en\"; }\n" - test_methods = textwrap.dedent("""\ - @Test - void mock_identifies_english() { - MockBridge b = new MockBridge(); - assertEquals("en", b.identify("hello")); - } - """) - return Lib( - artifact="cn1-ai-mlkit-langid", - pkg="mlkit.langid", - facade="LanguageIdentifier", - short_desc="ML Kit Language Identification", - long_desc="Identifies the language of a given text string.", - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSString*)identify:(NSString*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("identify__java_lang_String", 1),), - win_decls="public string identify(string param) { return null; }", - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/LanguageID", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:language-id:17.0.6'", - }, - ) - - -def lib_mlkit_pose() -> Lib: - facade_methods = textwrap.dedent("""\ - /// Returns 33 landmark triples (x,y,confidence) per detected pose - /// packed as `float[3 * 33]`. - public static AsyncResource detect(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativePoseDetector bridge = NativeLookup.create(NativePoseDetector.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("PoseDetector.detect is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final float[] r = bridge.detect(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new float[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("PoseDetector.detect failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } - """) - ni_methods = "float[] detect(byte[] imageBytes);\n" - ios_impl = textwrap.dedent("""\ - -(NSData*)detect:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [NSData data]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKPoseDetectorOptions *opts = [[MLKPoseDetectorOptions alloc] init]; - MLKPoseDetector *det = [MLKPoseDetector poseDetectorWithOptions:opts]; - __block MLKPose *pose = nil; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [det processImage:vision completion:^(NSArray * _Nullable r, NSError * _Nullable e) { - if (r.count > 0) pose = r[0]; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - float buf[99] = {0}; - if (pose) { - for (NSInteger i = 0; i < 33 && i < pose.landmarks.count; i++) { - MLKPoseLandmark *lm = pose.landmarks[i]; - buf[i * 3] = (float)lm.position.x; - buf[i * 3 + 1] = (float)lm.position.y; - buf[i * 3 + 2] = (float)lm.inFrameLikelihood; - } - } - // Pack as big-endian float bytes (matches JAVA_ARRAY_FLOAT on iOS port). - return [NSData dataWithBytes:buf length:sizeof(buf)]; - } - """) - ios_imports = textwrap.dedent("""\ - #import - #import - #import - """) - android_impl = textwrap.dedent("""\ - public float[] detect(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new float[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.pose.PoseDetector det = - com.google.mlkit.vision.pose.PoseDetection.getClient( - new com.google.mlkit.vision.pose.defaults.PoseDetectorOptions.Builder().build()); - final float[] out = new float[33 * 3]; - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - det.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.vision.pose.Pose>() { - public void onSuccess(com.google.mlkit.vision.pose.Pose p) { - java.util.List lms = p.getAllPoseLandmarks(); - for (int i = 0; i < 33 && i < lms.size(); i++) { - com.google.mlkit.vision.pose.PoseLandmark lm = lms.get(i); - out[i * 3] = lm.getPosition().x; - out[i * 3 + 1] = lm.getPosition().y; - out[i * 3 + 2] = lm.getInFrameLikelihood(); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out; - } - """) - javase_impl = textwrap.dedent("""\ - public float[] detect(byte[] imageBytes) { - float[] out = new float[99]; - for (int i = 0; i < 33; i++) { out[i * 3] = i; out[i * 3 + 2] = 0.5f; } - return out; - } - """) - test_mock_methods = "public float[] detect(byte[] imageBytes) { return new float[99]; }\n" - test_methods = textwrap.dedent("""\ - @Test - void mock_returns_33_landmarks() { - MockBridge b = new MockBridge(); - assertEquals(99, b.detect(new byte[]{1}).length); - } - """) - return Lib( - artifact="cn1-ai-mlkit-pose", - pkg="mlkit.pose", - facade="PoseDetector", - short_desc="ML Kit Pose Detection", - long_desc="Returns skeletal landmarks for human bodies detected in an image.", - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSData*)detect:(NSData*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("detect__byte_1ARRAY", 1),), - win_decls="public float[] detect(byte[] param) { return null; }", - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/PoseDetection", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:pose-detection:18.0.0-beta3'", - }, - ) - - -def lib_mlkit_segmentation() -> Lib: - facade_methods = textwrap.dedent("""\ - /// Returns a per-pixel mask separating foreground (person) from - /// background as `byte[width * height]` (0=background, 255=foreground). - public static AsyncResource segment(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativeSelfieSegmenter bridge = NativeLookup.create(NativeSelfieSegmenter.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("SelfieSegmenter.segment is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final byte[] r = bridge.segment(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new byte[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("SelfieSegmenter.segment failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } - """) - ni_methods = "byte[] segment(byte[] imageBytes);\n" - ios_impl = textwrap.dedent("""\ - -(NSData*)segment:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [NSData data]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKSelfieSegmenterOptions *opts = [[MLKSelfieSegmenterOptions alloc] init]; - opts.segmenterMode = MLKSegmenterModeSingleImage; - MLKSegmenter *seg = [MLKSegmenter segmenterWithOptions:opts]; - __block NSData *result = [NSData data]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [seg processImage:vision completion:^(MLKSegmentationMask * _Nullable mask, NSError * _Nullable e) { - if (mask) { - // MLKSegmentationMask exposes only `buffer` (a - // CVPixelBuffer); dimensions come from CVPixelBufferGet*. - CVPixelBufferRef buf = mask.buffer; - size_t w = CVPixelBufferGetWidth(buf); - size_t h = CVPixelBufferGetHeight(buf); - CVPixelBufferLockBaseAddress(buf, kCVPixelBufferLock_ReadOnly); - void *base = CVPixelBufferGetBaseAddress(buf); - NSMutableData *m = [NSMutableData dataWithLength:w * h]; - uint8_t *out = m.mutableBytes; - float *src = (float *)base; - for (size_t i = 0; i < w * h; i++) { - float v = src[i]; - out[i] = (uint8_t)(v * 255.0f); - } - CVPixelBufferUnlockBaseAddress(buf, kCVPixelBufferLock_ReadOnly); - result = m; - } - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; - } - """) - ios_imports = textwrap.dedent("""\ - #import - #import - #import - #import - """) - android_impl = textwrap.dedent("""\ - public byte[] segment(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new byte[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.segmentation.Segmenter seg = - com.google.mlkit.vision.segmentation.Segmentation.getClient( - new com.google.mlkit.vision.segmentation.SelfieSegmenterOptions.Builder() - .setDetectorMode( - com.google.mlkit.vision.segmentation.SelfieSegmenterOptions.SINGLE_IMAGE_MODE) - .build()); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference(new byte[0]); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - seg.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.vision.segmentation.SegmentationMask>() { - public void onSuccess(com.google.mlkit.vision.segmentation.SegmentationMask mask) { - int w = mask.getWidth(), h = mask.getHeight(); - java.nio.ByteBuffer buf = mask.getBuffer(); - buf.rewind(); - byte[] outb = new byte[w * h]; - for (int i = 0; i < w * h; i++) { - float v = buf.getFloat(); - outb[i] = (byte)(int)(v * 255); - } - out.set(outb); - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - """) - javase_impl = textwrap.dedent("""\ - public byte[] segment(byte[] imageBytes) { - // 8x8 checkerboard stub. - byte[] out = new byte[64]; - for (int i = 0; i < 64; i++) out[i] = (byte)(((i / 8) + (i % 8)) % 2 == 0 ? 255 : 0); - return out; - } - """) - test_mock_methods = "public byte[] segment(byte[] imageBytes) { return new byte[16]; }\n" - test_methods = textwrap.dedent("""\ - @Test - void mock_returns_mask_bytes() { - MockBridge b = new MockBridge(); - assertEquals(16, b.segment(new byte[]{1}).length); - } - """) - return Lib( - artifact="cn1-ai-mlkit-segmentation", - pkg="mlkit.segmentation", - facade="SelfieSegmenter", - short_desc="ML Kit Selfie Segmentation", - long_desc="Returns a per-pixel mask separating a person in the foreground from the background.", - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSData*)segment:(NSData*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("segment__byte_1ARRAY", 1),), - win_decls="public byte[] segment(byte[] param) { return null; }", - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/SegmentationSelfie", - "codename1.arg.android.gradleDep": - "implementation 'com.google.mlkit:segmentation-selfie:16.0.0-beta5'", - }, - ) - - -def lib_mlkit_docscan() -> Lib: - facade_methods = _string_facade_methods("DocumentScanner", "NativeDocumentScanner", "scanToFile", - arg_kind="bytes") - ni_methods = "String scanToFile(byte[] imageBytes);\n" - ios_impl = textwrap.dedent("""\ - // VisionKit-based fallback: Apple's VNDocumentCameraViewController is - // interactive; this bridge accepts a pre-captured image and returns its - // cropped JPEG path. On iOS 13+ VisionKit handles the live UI flow; the - // sample app drives that flow and feeds the bytes into the cn1lib. - -(NSString*)scanToFile:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return @""; - CIImage *ci = [CIImage imageWithCGImage:image.CGImage]; - CIContext *ctx = [CIContext context]; - CIDetector *det = [CIDetector detectorOfType:CIDetectorTypeRectangle context:ctx - options:@{CIDetectorAccuracy: CIDetectorAccuracyHigh}]; - NSArray *features = [det featuresInImage:ci]; - UIImage *cropped = image; - if (features.count > 0) { - CIRectangleFeature *rf = (CIRectangleFeature *)features.firstObject; - CIImage *flat = [ci imageByApplyingFilter:@"CIPerspectiveCorrection" withInputParameters:@{ - @"inputTopLeft": [CIVector vectorWithCGPoint:rf.topLeft], - @"inputTopRight": [CIVector vectorWithCGPoint:rf.topRight], - @"inputBottomLeft": [CIVector vectorWithCGPoint:rf.bottomLeft], - @"inputBottomRight": [CIVector vectorWithCGPoint:rf.bottomRight] - }]; - CGImageRef cg = [ctx createCGImage:flat fromRect:flat.extent]; - cropped = [UIImage imageWithCGImage:cg]; - CGImageRelease(cg); - } - NSString *path = [NSString stringWithFormat:@"%@/docscan-%@.jpg", - NSTemporaryDirectory(), [[NSUUID UUID] UUIDString]]; - [UIImageJPEGRepresentation(cropped, 0.92) writeToFile:path atomically:YES]; - return path; - } - """) - ios_imports = textwrap.dedent("""\ - #import - """) - android_impl = textwrap.dedent("""\ - public String scanToFile(byte[] imageBytes) { - try { - java.io.File f = java.io.File.createTempFile("docscan-", ".jpg"); - java.io.FileOutputStream fos = new java.io.FileOutputStream(f); - fos.write(imageBytes); - fos.close(); - return f.getAbsolutePath(); - } catch (java.io.IOException ioe) { - return ""; - } - } - """) - javase_impl = textwrap.dedent("""\ - public String scanToFile(byte[] imageBytes) { - try { - java.io.File f = java.io.File.createTempFile("docscan-stub-", ".jpg"); - java.nio.file.Files.write(f.toPath(), imageBytes); - return f.getAbsolutePath(); - } catch (java.io.IOException ioe) { - return ""; - } - } - """) - test_mock_methods = "public String scanToFile(byte[] imageBytes) { return \"/tmp/x.jpg\"; }\n" - test_methods = textwrap.dedent("""\ - @Test - void mock_returns_path() { - MockBridge b = new MockBridge(); - assertEquals("/tmp/x.jpg", b.scanToFile(new byte[]{1})); - } - """) - return Lib( - artifact="cn1-ai-mlkit-docscan", - pkg="mlkit.docscan", - facade="DocumentScanner", - short_desc="ML Kit / VisionKit Document Scanner", - long_desc=( - "Captures and crops document photos. On iOS uses Apple's VisionKit + Core Image " - "rectangle detection (no extra pod). On Android uses the Google Play services " - "document-scanner module." - ), - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSString*)scanToFile:(NSData*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("scanToFile__byte_1ARRAY", 1),), - win_decls="public string scanToFile(byte[] param) { return null; }", - build_hints={ - "codename1.arg.ios.add_frameworks": "VisionKit", - "codename1.arg.android.gradleDep": - "implementation 'com.google.android.gms:play-services-mlkit-document-scanner:16.0.0-beta1'", - }, - ) - - -def lib_tflite() -> Lib: - facade_methods = textwrap.dedent("""\ - /// Loads a TensorFlow Lite model from the supplied bytes and runs - /// inference against a float32 input tensor. Returns the output as - /// `float[]`. The model file is held in a native handle keyed by - /// the SHA-1 of the input bytes; repeated calls reuse the loaded - /// model. - public static AsyncResource run(final byte[] modelBytes, - final float[] input, - final int outputLength) { - final AsyncResource out = new AsyncResource(); - final NativeInterpreter bridge = NativeLookup.create(NativeInterpreter.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("Interpreter.run is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final float[] r = bridge.run(modelBytes, input, outputLength); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new float[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("Interpreter.run failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } - """) - ni_methods = "float[] run(byte[] modelBytes, float[] input, int outputLength);\n" - ios_impl = textwrap.dedent("""\ - -(NSData*)run:(NSData*)param param1:(NSData*)param1 param2:(int)param2 { - NSError *err = nil; - NSString *modelPath = [NSString stringWithFormat:@"%@/tflite-%@.tflite", - NSTemporaryDirectory(), [[NSUUID UUID] UUIDString]]; - [param writeToFile:modelPath atomically:YES]; - TFLInterpreter *interp = [[TFLInterpreter alloc] initWithModelPath:modelPath error:&err]; - if (err) return [NSData data]; - [interp allocateTensorsWithError:&err]; - if (err) return [NSData data]; - TFLTensor *in0 = [interp inputTensorAtIndex:0 error:&err]; - [in0 copyData:param1 error:&err]; - [interp invokeWithError:&err]; - TFLTensor *out0 = [interp outputTensorAtIndex:0 error:&err]; - NSData *outBytes = [out0 dataWithError:&err]; - return outBytes ?: [NSData data]; - } - """) - # TensorFlowLiteObjC pod installs the `TFLTensorFlowLite` framework - # with umbrella header `TFLTensorFlowLite.h`. The framework dir - # name (used as the angle-bracket prefix) is the module name - # `TFLTensorFlowLite`, not the pod name `TensorFlowLiteObjC`. - ios_imports = "#import \n" - android_impl = textwrap.dedent("""\ - public float[] run(byte[] modelBytes, float[] input, int outputLength) { - java.nio.ByteBuffer bb = java.nio.ByteBuffer.allocateDirect(modelBytes.length); - bb.order(java.nio.ByteOrder.nativeOrder()); - bb.put(modelBytes); - bb.rewind(); - org.tensorflow.lite.Interpreter interp = new org.tensorflow.lite.Interpreter(bb); - float[][] out = new float[1][outputLength]; - interp.run(new float[][]{input}, out); - interp.close(); - return out[0]; - } - """) - javase_impl = textwrap.dedent("""\ - public float[] run(byte[] modelBytes, float[] input, int outputLength) { - // Identity stub: returns first outputLength entries of input - // (or zero-padded if input shorter). Lets simulator test plumbing. - float[] out = new float[outputLength]; - int n = Math.min(input.length, outputLength); - System.arraycopy(input, 0, out, 0, n); - return out; - } - """) - test_mock_methods = textwrap.dedent("""\ - public float[] run(byte[] modelBytes, float[] input, int outputLength) { - float[] r = new float[outputLength]; - for (int i = 0; i < r.length; i++) r[i] = i; - return r; - } - """) - test_methods = textwrap.dedent("""\ - @Test - void mock_returns_increasing_vector() { - MockBridge b = new MockBridge(); - float[] r = b.run(new byte[0], new float[]{1f, 2f}, 4); - assertEquals(4, r.length); - assertEquals(3.0f, r[3], 1e-6); - } - """) - return Lib( - artifact="cn1-ai-tflite", - pkg="tflite", - facade="Interpreter", - short_desc="TensorFlow Lite on-device inference", - long_desc=( - "Loads a `.tflite` model and runs inference against `float[]` inputs.\n" - "Bridges to `TensorFlowLiteObjC` on iOS and `org.tensorflow:tensorflow-lite`\n" - "on Android." - ), - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSData*)run:(NSData*)param param1:(NSData*)param1 param2:(int)param2;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("run__byte_1ARRAY_float_1ARRAY_int", 3),), - win_decls="public float[] run(byte[] param, float[] param1, int param2) { return null; }", - build_hints={ - "codename1.arg.ios.pods": "TensorFlowLiteObjC", - "codename1.arg.android.gradleDep": "implementation 'org.tensorflow:tensorflow-lite:2.14.0'", - }, - ) - - def lib_whisper() -> Lib: facade_methods = textwrap.dedent("""\ /// Transcribes audio using a whisper.cpp model. `modelPath` is the diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java index 4d633af9a0d..c45b3ed4b60 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java @@ -27,6 +27,7 @@ package com.codenameone.examples.hellocodenameone.tests; import com.codename1.ai.vision.BarcodeScanner; +import com.codename1.ai.vision.Barcode; import com.codename1.ai.vision.DocumentScanner; import com.codename1.ai.vision.FaceDetector; import com.codename1.ai.vision.ImageLabeler; @@ -40,15 +41,15 @@ import com.codename1.camera.CameraFrame; import com.codename1.camera.FrameFormat; import com.codename1.io.Log; +import com.codename1.util.Base64; /** * Cross-port, non-visual contract coverage for the built-in vision API. * - *

The test intentionally avoids running an analyzer against camera hardware - * or a vendor model. It verifies the portable image/camera-frame ownership - * contract and asks every native analyzer for its capability. This keeps the - * suite deterministic while making every analyzer visible to the builders' - * granular dependency scanner.

+ *

In addition to portable ownership/lifecycle checks, supported mobile + * ports decode a bundled deterministic QR image. That assertion crosses the + * Java/native image boundary, native detector, JSON result mapping, format + * normalization, and geometry mapping without camera hardware.

*/ public class VisionOnDeviceApiTest extends BaseTest { @Override @@ -62,6 +63,7 @@ public boolean runTest() { checkImageOwnership(); checkOptions(); checkAnalyzerCapabilitiesAndLifecycle(); + checkNativeQrDecode(); done(); return true; } catch (Throwable t) { @@ -70,6 +72,29 @@ public boolean runTest() { } } + private void checkNativeQrDecode() throws Exception { + BarcodeScanner scanner = new BarcodeScanner(); + try { + if (!scanner.isSupported()) { + Log.p("VisionOnDeviceApiTest: native QR assertion skipped; " + + "barcode backend unsupported"); + return; + } + String png = "iVBORw0KGgoAAAANSUhEUgAAAMAAAADAAQAAAAB6p1GqAAAA5UlEQVR4Xu2UQQ7DIAwEnVOewU8L/JRncIJ6TaqkJT17pbBSImBy2HhtpP+R/B58tMCkBSY9AyRRBX1vFauNA2R9akh72Wo4tgygqtHQRKJiCVzA7LKBjAUT6IhWT/V9bAnAGIMM9j0GrmBIcTx3/kDjbHZlmNdy/Q9H0NBkJVZEm3kAxlJkR9u1a7SuAEUEQx31CxIA4VSxWecBuCnKS9B5mQPYAGi3aQVxa0QOAH82ALGijqddX2BGm5lOdCD0PrqNCCTrNgRMArpFW5Gu3GTuA8YYZKtjul4ZruBOC0xaYNJzwRtNmtsMgNUTWwAAAABJRU5ErkJggg=="; + Barcode[] values = scanner.process(VisionImage.encoded( + Base64.decode(png.getBytes("UTF-8")))).get(30000); + check(values.length > 0, "native QR detector returned no result"); + check("CN1-VISION-QR".equals(values[0].getValue()), + "native QR payload mismatch"); + check("QR_CODE".equals(values[0].getFormat()), + "native QR format was not normalized"); + check(values[0].getCorners().length == 4, + "native QR detector did not return four corners"); + } finally { + scanner.close(); + } + } + private void checkImageOwnership() { byte[] encodedSource = new byte[] {1, 2, 3}; VisionImage encoded = VisionImage.encoded(encodedSource); diff --git a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md index a2fbf0ba02b..b90c267a70b 100644 --- a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md +++ b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md @@ -335,10 +335,14 @@ retains only the Android text adapter/artifact; it does not pull barcode, face, labeling, pose, or segmentation. `LanguageIdentifier`, `Translator`, and `SmartReply` likewise select independent artifacts. On iOS, an ML Kit vision pod is selected only when both its analyzer -and `VisionBackends.mlKit()` are referenced. LiteRT is selected only by +and its matching feature-specific selector are referenced. For example, +`VisionBackends.mlKitBarcodeScanning()` selects only the barcode pod. +LiteRT is selected only by `InferenceSession`. -Call `isSupported()` before exposing a feature. Vision has native backends +Language identification defaults to Apple Natural Language on iOS and ML +Kit on Android. Translation and Smart Reply use feature-scoped ML Kit +components. Call `isSupported()` before exposing a feature. Vision has native backends on Android, iOS, and Mac native. Language services and LiteRT inference have native backends on Android and iOS. JavaSE, JavaScript, native Windows/Linux, watchOS, and tvOS return unsupported; Mac native returns From 5d0bcb145f640f59a155ff5a49a1f7d6d710a167 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 26 Jul 2026 18:05:35 +0300 Subject: [PATCH 15/80] Add required builder test copyright header --- .../IPhoneBuilderDependencyConfigTest.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java index e327db135b7..24986e64eae 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ package com.codename1.builders; import org.junit.jupiter.api.Test; From d6f3106bde6ef0088b822f4768987241f598526c Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 26 Jul 2026 18:22:44 +0300 Subject: [PATCH 16/80] Fix on-device AI guide prose checks --- docs/developer-guide/Ai-And-Speech.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index ead8e3caced..569e7f8bcd5 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -370,10 +370,10 @@ the port boundary. Always close the session. Bundle small models as resources. For larger models use `ModelCache.fetch(url, cacheKey, sha256)`, which downloads over HTTPS, -verifies a SHA-256 digest when supplied, and reuses the app-private cached +verifies the supplied SHA-256 digest, and reuses the app-private cached file. The URL must use HTTPS. Supply the digest for every third-party or remotely mutable model: an omitted digest provides transport security but -does not pin the executable model payload. Interrupted downloads restart +doesn't pin the executable model payload. Interrupted downloads restart through a temporary `.download` file and are never promoted to the cache until verification succeeds. From ea0fce9ac4a6c6d3675370b679450e14623007eb Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 26 Jul 2026 18:55:04 +0300 Subject: [PATCH 17/80] Harden builder archive extraction --- .../builders/AndroidGradleBuilder.java | 34 +++++---- .../java/com/codename1/builders/Executor.java | 58 ++++++++++++--- .../ExecutorArchiveExtractionTest.java | 74 +++++++++++++++++++ 3 files changed, 138 insertions(+), 28 deletions(-) create mode 100644 maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java index d5b4d87c097..a617300952b 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java @@ -5603,7 +5603,7 @@ public void extract(InputStream source, File dir, String sdkPath) throws IOExcep while ((entry = zis.getNextEntry()) != null) { debug("Extracting: " + entry); if (entry.isDirectory()) { - File d = new File(dir, entry.getName()); + File d = resolveArchiveEntry(dir, entry.getName()); d.mkdirs(); if (!addedSDKDir && sdkPath != null) { if (is_windows) { @@ -5628,7 +5628,7 @@ public void extract(InputStream source, File dir, String sdkPath) throws IOExcep String sdkPathProperties = "sdk.dir=" + sdkPath + "\n"; // write the files to the disk File destFile; - destFile = new File(dir, entry.getName()); + destFile = resolveArchiveEntry(dir, entry.getName()); destFile.getParentFile().mkdirs(); FileOutputStream fos = new FileOutputStream(destFile); fos.write(sdkPathProperties.getBytes(StandardCharsets.UTF_8)); @@ -5640,7 +5640,7 @@ public void extract(InputStream source, File dir, String sdkPath) throws IOExcep byte[] data = new byte[8192]; // write the files to the disk File destFile; - destFile = new File(dir, entry.getName()); + destFile = resolveArchiveEntry(dir, entry.getName()); destFile.getParentFile().mkdirs(); FileOutputStream fos = new FileOutputStream(destFile); dest = new BufferedOutputStream(fos, data.length); @@ -5741,18 +5741,18 @@ public void extractAAR(InputStream source, File dir, String sdkPath) throws IOEx } - protected File placeXMLFile(ZipEntry entry, File xmlDir, File resDir) { + protected File placeXMLFile(ZipEntry entry, File xmlDir, File resDir) throws IOException { String name = entry.getName(); if (name.endsWith("colors.xml")) { File parent = resDir.getParentFile(); resDir = new File(parent, "res"); File valsDir = new File(resDir, "values"); - return new File(valsDir, entry.getName()); + return resolveArchiveEntry(valsDir, entry.getName()); } else if (name.endsWith("layout.xml")) { File parent = resDir.getParentFile(); resDir = new File(parent, "res"); File layDir = new File(resDir, "layout"); - return new File(layDir, entry.getName()); + return resolveArchiveEntry(layDir, entry.getName()); } else { return super.placeXMLFile(entry, xmlDir, resDir); } @@ -5861,11 +5861,11 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD if (entry.isDirectory()) { if (!entryName.startsWith("raw")) { - File dir = new File(classesDir, entryName); + File dir = resolveArchiveEntry(classesDir, entryName); dir.mkdirs(); - dir = new File(resDir, entryName); + dir = resolveArchiveEntry(resDir, entryName); dir.mkdirs(); - dir = new File(sourceDir, entryName); + dir = resolveArchiveEntry(sourceDir, entryName); dir.mkdirs(); } continue; @@ -5877,24 +5877,26 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD // write the files to the disk File destFile; if (entryName.endsWith(".class")) { - destFile = new File(classesDir, entryName); + destFile = resolveArchiveEntry(classesDir, entryName); } else { if (entryName.endsWith(".java") || entryName.endsWith(".kt") || entryName.endsWith(".swift") || entryName.endsWith(".m") || entryName.endsWith(".h")) { - destFile = new File(sourceDir, entryName); + destFile = resolveArchiveEntry(sourceDir, entryName); } else { if (entryName.endsWith(".jar") || entryName.endsWith(".a") || entryName.endsWith(".dylib") || entryName.endsWith(".andlib") || entryName.endsWith(".aar")) { - destFile = new File(libsDir, entryName); + destFile = resolveArchiveEntry(libsDir, entryName); } else { if (useXMLDir() && entryName.endsWith(".xml")) { destFile = placeXMLFile(entry, xmlDir, resDir); } else if (entryName.startsWith("raw")) { - destFile = new File(xmlDir.getParentFile(), entryName.toLowerCase()); + destFile = resolveArchiveEntry(xmlDir.getParentFile(), + entryName.toLowerCase()); } else if (entryName.contains("notification_sound")) { - destFile = new File(xmlDir.getParentFile(), "raw/" + entryName.toLowerCase()); + destFile = resolveArchiveEntry(xmlDir.getParentFile(), + "raw/" + entryName.toLowerCase()); } else if ("google-services.json".equals(entryName)) { - destFile = new File(libsDir.getParentFile(), entryName); + destFile = resolveArchiveEntry(libsDir.getParentFile(), entryName); } else { - destFile = new File(resDir, entryName); + destFile = resolveArchiveEntry(resDir, entryName); } } } diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java index 8bc5d568271..5a2a8bbbb1e 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java @@ -1309,11 +1309,11 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD } if (entry.isDirectory()) { - File dir = new File(classesDir, entryName); + File dir = resolveArchiveEntry(classesDir, entryName); dir.mkdirs(); - dir = new File(resDir, entryName); + dir = resolveArchiveEntry(resDir, entryName); dir.mkdirs(); - dir = new File(sourceDir, entryName); + dir = resolveArchiveEntry(sourceDir, entryName); dir.mkdirs(); continue; } @@ -1328,22 +1328,22 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD log("!!!!Skipping "+entryName); continue; } else { - destFile = new File(classesDir, entryName); + destFile = resolveArchiveEntry(classesDir, entryName); } } else { if (entryName.endsWith(".java") || entryName.endsWith(".kt") || entryName.endsWith(".swift") || entryName.endsWith(".m") || entryName.endsWith(".h") || entryName.endsWith(".cs")) { - destFile = new File(sourceDir, entryName); + destFile = resolveArchiveEntry(sourceDir, entryName); } else { if (entryName.endsWith(".jar") || entryName.endsWith(".a") || entryName.endsWith(".dylib") || entryName.endsWith(".andlib") || entryName.endsWith(".aar") || entryName.endsWith(dll)) { - destFile = new File(libsDir, entryName); + destFile = resolveArchiveEntry(libsDir, entryName); } else { if (useXMLDir() && entryName.endsWith(".xml")) { destFile = placeXMLFile(entry, xmlDir, resDir); } else { if(entryName.equals("codenameone_settings.properties")) { - destFile = new File(sourceDir.getParentFile(), entryName); + destFile = resolveArchiveEntry(sourceDir.getParentFile(), entryName); } else { - destFile = new File(resDir, entryName); + destFile = resolveArchiveEntry(resDir, entryName); } } } @@ -1390,7 +1390,7 @@ public void extractZip(InputStream source, File destDir, ExtractionFilter filter if (entry.isDirectory()) { currentDir = entryName; - File dir = new File(destDir, entryName); + File dir = resolveArchiveEntry(destDir, entryName); dir.mkdirs(); continue; } @@ -1400,6 +1400,7 @@ public void extractZip(InputStream source, File destDir, ExtractionFilter filter // write the files to the disk File destFile = filter.destFile(currentDir, entryName); + destFile = requireArchiveDestination(destDir, destFile, entryName); destFile.getParentFile().mkdirs(); FileOutputStream fos = new FileOutputStream(destFile); dest = new BufferedOutputStream(fos, data.length); @@ -1416,7 +1417,7 @@ public void extractZip(InputStream source, File destDir, ExtractionFilter filter } } - protected File placeXMLFile(ZipEntry entry, File xmlDir, File resDir) { + protected File placeXMLFile(ZipEntry entry, File xmlDir, File resDir) throws IOException { boolean putInXMLDir = false; String name = entry.getName(); if (name.contains("/")) { @@ -1431,12 +1432,45 @@ protected File placeXMLFile(ZipEntry entry, File xmlDir, File resDir) { name = name.substring(name.lastIndexOf("/") + 1, name.length()); } if (putInXMLDir) { - return new File(xmlDir, name); + return resolveArchiveEntry(xmlDir, name); } else { - return new File(resDir, entry.getName()); + return resolveArchiveEntry(resDir, entry.getName()); } } + /** + * Resolves an archive entry beneath its intended extraction directory. + * Entries that are absolute or escape through {@code ..} components are + * rejected before any directory or file is created. + * + * @param destinationDirectory extraction root + * @param entryName untrusted name read from the archive + * @return canonical destination contained by {@code destinationDirectory} + * @throws IOException if the entry escapes the extraction root + */ + protected static File resolveArchiveEntry(File destinationDirectory, + String entryName) throws IOException { + if (new File(entryName).isAbsolute()) { + throw new IOException("Archive entry is absolute: " + entryName); + } + return requireArchiveDestination(destinationDirectory, + new File(destinationDirectory, entryName), entryName); + } + + private static File requireArchiveDestination(File destinationDirectory, + File destinationFile, String entryName) throws IOException { + File canonicalDirectory = destinationDirectory.getCanonicalFile(); + File canonicalDestination = destinationFile.getCanonicalFile(); + String directoryPath = canonicalDirectory.getPath(); + String directoryPrefix = directoryPath.endsWith(File.separator) + ? directoryPath : directoryPath + File.separator; + if (!canonicalDestination.getPath().startsWith(directoryPrefix)) { + throw new IOException("Archive entry escapes destination directory: " + + entryName); + } + return canonicalDestination; + } + public int executeProcess(ProcessBuilder pb) throws Exception { return executeProcess(pb, -1); } diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java new file mode 100644 index 00000000000..fd8da34c4ba --- /dev/null +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.builders; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class ExecutorArchiveExtractionTest { + @TempDir + Path temporaryDirectory; + + @Test + void resolvesNestedEntryInsideDestination() throws Exception { + File destination = temporaryDirectory.toFile(); + + File resolved = Executor.resolveArchiveEntry(destination, + "nested/model.tflite"); + + assertEquals(new File(destination, "nested/model.tflite") + .getCanonicalFile(), resolved); + } + + @Test + void rejectsParentTraversal() { + File destination = temporaryDirectory.resolve("archive").toFile(); + + assertThrows(IOException.class, () -> Executor.resolveArchiveEntry( + destination, "../escaped.txt")); + } + + @Test + void rejectsAbsoluteDestination() { + File destination = temporaryDirectory.resolve("archive").toFile(); + String absoluteEntry = temporaryDirectory.resolve("escaped.txt") + .toAbsolutePath().toString(); + + assertThrows(IOException.class, () -> Executor.resolveArchiveEntry( + destination, absoluteEntry)); + } + + @Test + void rejectsSiblingWithMatchingPrefix() { + File destination = temporaryDirectory.resolve("archive").toFile(); + + assertThrows(IOException.class, () -> Executor.resolveArchiveEntry( + destination, "../archive-escape/payload.bin")); + } +} From 2165939f6bcee8e8e3c90554928bcdad3b539ad4 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:47:44 +0300 Subject: [PATCH 18/80] Link Apple Natural Language framework --- .../src/main/java/com/codename1/builders/IPhoneBuilder.java | 2 ++ .../codename1/builders/IPhoneBuilderDependencyConfigTest.java | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java index 1bb4eb14bd5..b5759621a08 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java @@ -2563,6 +2563,8 @@ public void usesClassMethod(String cls, String method) { throw new BuildException( "Failed to enable INCLUDE_CN1_LANGUAGE", ex); } + addLibs = appendFrameworks(addLibs, + "NaturalLanguage.framework"); } if (usesCn1Inference) { diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java index 24986e64eae..fb388802ab8 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java @@ -187,6 +187,10 @@ void appendsEachRequiredAiFrameworkIndependently() throws Exception { assertEquals("CoreML.framework;Accelerate.framework;Metal.framework", value); + value = (String) method.invoke(null, "Vision.framework", + new String[] {"NaturalLanguage.framework"}); + assertEquals("Vision.framework;NaturalLanguage.framework", value); + value = (String) method.invoke(null, "ThirdPartyVision.framework", new String[] {"Vision.framework"}); assertEquals("ThirdPartyVision.framework;Vision.framework", value); From 52b7421f4738dca1edfa13d561b63abf195b5456 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 26 Jul 2026 23:02:30 +0300 Subject: [PATCH 19/80] Avoid blocking EDT in vision backend test --- .../tests/Cn1ssDeviceRunner.java | 7 ++ .../tests/VisionOnDeviceApiTest.java | 70 ++++++++++++------- 2 files changed, 51 insertions(+), 26 deletions(-) diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java index 42730804048..3b31c5bb407 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java @@ -121,6 +121,13 @@ private static int testTimeoutMs(BaseTest testClass) { && testClass instanceof ToastBarTopPositionScreenshotTest) { return 45000; } + if (!"HTML5".equals(Display.getInstance().getPlatformName()) + && testClass instanceof VisionOnDeviceApiTest) { + // The first Apple Vision request may compile its detector model on + // a cold simulator. Keep the EDT free and allow that one-time + // native initialization a bounded minute. + return TEST_TIMEOUT_MS_NATIVE * 2; + } return "HTML5".equals(Display.getInstance().getPlatformName()) ? TEST_TIMEOUT_MS_HTML5 : TEST_TIMEOUT_MS_NATIVE; diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java index c45b3ed4b60..1fdd5afd597 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java @@ -20,10 +20,6 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ -/* - * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - */ package com.codenameone.examples.hellocodenameone.tests; import com.codename1.ai.vision.BarcodeScanner; @@ -41,7 +37,9 @@ import com.codename1.camera.CameraFrame; import com.codename1.camera.FrameFormat; import com.codename1.io.Log; +import com.codename1.util.AsyncResource; import com.codename1.util.Base64; +import com.codename1.util.SuccessCallback; /** * Cross-port, non-visual contract coverage for the built-in vision API. @@ -63,36 +61,56 @@ public boolean runTest() { checkImageOwnership(); checkOptions(); checkAnalyzerCapabilitiesAndLifecycle(); - checkNativeQrDecode(); - done(); - return true; + return checkNativeQrDecode(); } catch (Throwable t) { fail("On-device vision API test failed: " + t); return false; } } - private void checkNativeQrDecode() throws Exception { - BarcodeScanner scanner = new BarcodeScanner(); - try { - if (!scanner.isSupported()) { - Log.p("VisionOnDeviceApiTest: native QR assertion skipped; " - + "barcode backend unsupported"); - return; - } - String png = "iVBORw0KGgoAAAANSUhEUgAAAMAAAADAAQAAAAB6p1GqAAAA5UlEQVR4Xu2UQQ7DIAwEnVOewU8L/JRncIJ6TaqkJT17pbBSImBy2HhtpP+R/B58tMCkBSY9AyRRBX1vFauNA2R9akh72Wo4tgygqtHQRKJiCVzA7LKBjAUT6IhWT/V9bAnAGIMM9j0GrmBIcTx3/kDjbHZlmNdy/Q9H0NBkJVZEm3kAxlJkR9u1a7SuAEUEQx31CxIA4VSxWecBuCnKS9B5mQPYAGi3aQVxa0QOAH82ALGijqddX2BGm5lOdCD0PrqNCCTrNgRMArpFW5Gu3GTuA8YYZKtjul4ZruBOC0xaYNJzwRtNmtsMgNUTWwAAAABJRU5ErkJggg=="; - Barcode[] values = scanner.process(VisionImage.encoded( - Base64.decode(png.getBytes("UTF-8")))).get(30000); - check(values.length > 0, "native QR detector returned no result"); - check("CN1-VISION-QR".equals(values[0].getValue()), - "native QR payload mismatch"); - check("QR_CODE".equals(values[0].getFormat()), - "native QR format was not normalized"); - check(values[0].getCorners().length == 4, - "native QR detector did not return four corners"); - } finally { + private boolean checkNativeQrDecode() throws Exception { + final BarcodeScanner scanner = new BarcodeScanner(); + if (!scanner.isSupported()) { + Log.p("VisionOnDeviceApiTest: native QR assertion skipped; " + + "barcode backend unsupported"); scanner.close(); + done(); + return true; } + String png = "iVBORw0KGgoAAAANSUhEUgAAAMAAAADAAQAAAAB6p1GqAAAA5UlEQVR4Xu2UQQ7DIAwEnVOewU8L/JRncIJ6TaqkJT17pbBSImBy2HhtpP+R/B58tMCkBSY9AyRRBX1vFauNA2R9akh72Wo4tgygqtHQRKJiCVzA7LKBjAUT6IhWT/V9bAnAGIMM9j0GrmBIcTx3/kDjbHZlmNdy/Q9H0NBkJVZEm3kAxlJkR9u1a7SuAEUEQx31CxIA4VSxWecBuCnKS9B5mQPYAGi3aQVxa0QOAH82ALGijqddX2BGm5lOdCD0PrqNCCTrNgRMArpFW5Gu3GTuA8YYZKtjul4ZruBOC0xaYNJzwRtNmtsMgNUTWwAAAABJRU5ErkJggg=="; + AsyncResource operation = scanner.process( + VisionImage.encoded(Base64.decode(png.getBytes("UTF-8")))); + operation.ready(new SuccessCallback() { + public void onSucess(Barcode[] values) { + try { + check(values.length > 0, + "native QR detector returned no result"); + check("CN1-VISION-QR".equals(values[0].getValue()), + "native QR payload mismatch"); + check("QR_CODE".equals(values[0].getFormat()), + "native QR format was not normalized"); + check(values[0].getCorners().length == 4, + "native QR detector did not return four corners"); + done(); + } catch (Throwable t) { + fail("On-device vision API test failed: " + t); + } finally { + scanner.close(); + } + } + }); + operation.except(new SuccessCallback() { + public void onSucess(Throwable error) { + try { + fail("On-device vision API test failed: " + error); + } finally { + scanner.close(); + } + } + }); + // Native completion is asynchronous. Returning lets the EDT service + // Apple Vision/ML Kit callbacks while the suite runner polls isDone(). + return true; } private void checkImageOwnership() { From f8d1552bd8bd51ab4a42099582308691d5a6fc6e Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 26 Jul 2026 23:25:04 +0300 Subject: [PATCH 20/80] Resolve AI backend review comments --- .../codename1/ai/inference/ModelCache.java | 14 +++++-- .../ai/AndroidBarcodeScanningAdapter.java | 5 ++- .../ai/AndroidFaceDetectionAdapter.java | 5 ++- .../ai/AndroidImageLabelingAdapter.java | 5 ++- .../impl/android/ai/AndroidInferenceImpl.java | 5 ++- .../ai/AndroidTextRecognitionAdapter.java | 5 ++- .../codename1/impl/ios/IOSInferenceImpl.java | 5 ++- .../com/codename1/impl/ios/IOSVisionImpl.java | 41 +++++++++++++++++-- .../ai/inference/InferenceApiTest.java | 10 +++++ 9 files changed, 82 insertions(+), 13 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java index 0fae5106a66..3b03e69e520 100644 --- a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java +++ b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java @@ -194,15 +194,23 @@ static void verifyDownloaded(FileSystemStorage fs, String temporary, } } - private static String safeName(String value) { + static String safeName(String value) { StringBuilder out = new StringBuilder(); for (int i = 0; i < value.length(); i++) { char c = value.charAt(i); - out.append((c >= 'a' && c <= 'z') + boolean safe = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') - || c == '-' || c == '_' ? c : '_'); + || c == '-' || c == '_'; + out.append(safe ? c : '_'); } + Hash hash = Hash.create(Hash.SHA256); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + hash.update((byte) (c >>> 8)); + hash.update((byte) c); + } + out.append('-').append(Hash.toHex(hash.digest()).substring(0, 32)); return out.toString(); } diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java index f3136106a68..58b0175c3e5 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java @@ -44,11 +44,14 @@ void analyze(InputImage input, final int imageWidth, final int imageHeight, VisionOptions options, AsyncResource resource) { final AsyncResource out = (AsyncResource) resource; + final int maximumResults = options.getMaximumResults(); client.process(input).addOnSuccessListener( new OnSuccessListener>() { public void onSuccess( List values) { - Barcode[] result = new Barcode[values.size()]; + int count = maximumResults == 0 ? values.size() + : Math.min(maximumResults, values.size()); + Barcode[] result = new Barcode[count]; for (int i = 0; i < result.length; i++) { com.google.mlkit.vision.barcode.common.Barcode value = values.get(i); diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java index 904523d589c..47ddb71dba5 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java @@ -51,11 +51,14 @@ void analyze(InputImage input, final int imageWidth, final int imageHeight, VisionOptions options, AsyncResource resource) { final AsyncResource out = (AsyncResource) resource; + final int maximumResults = options.getMaximumResults(); client.process(input).addOnSuccessListener( new OnSuccessListener>() { public void onSuccess( List values) { - Face[] result = new Face[values.size()]; + int count = maximumResults == 0 ? values.size() + : Math.min(maximumResults, values.size()); + Face[] result = new Face[count]; for (int i = 0; i < result.length; i++) { com.google.mlkit.vision.face.Face value = values.get(i); Map landmarks = diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java index 7ddbfec5112..3a23c1dc029 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java @@ -42,6 +42,7 @@ void analyze(InputImage input, int imageWidth, int imageHeight, VisionOptions options, AsyncResource resource) { final AsyncResource out = (AsyncResource) resource; + final int maximumResults = options.getMaximumResults(); if (client == null) { client = ImageLabeling.getClient(new ImageLabelerOptions.Builder() .setConfidenceThreshold(options.getMinimumConfidence()) @@ -51,7 +52,9 @@ void analyze(InputImage input, int imageWidth, int imageHeight, new OnSuccessListener>() { public void onSuccess( List values) { - ImageLabel[] result = new ImageLabel[values.size()]; + int count = maximumResults == 0 ? values.size() + : Math.min(maximumResults, values.size()); + ImageLabel[] result = new ImageLabel[count]; for (int i = 0; i < result.length; i++) { com.google.mlkit.vision.label.ImageLabel value = values.get(i); diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java index f7a7aa0209c..ec7cce647e6 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java @@ -29,6 +29,7 @@ import com.codename1.ai.inference.TensorInfo; import com.codename1.ai.inference.TensorType; import com.codename1.impl.InferenceImpl; +import com.codename1.io.FileSystemStorage; import com.codename1.ui.Display; import com.codename1.util.AsyncResource; @@ -291,7 +292,9 @@ private static int elementCount(int[] shape) { private static ByteBuffer loadModel(ModelSource source) throws IOException { if (source.getKind() == ModelSource.FILE) { - FileInputStream file = new FileInputStream(source.getPath()); + String nativePath = FileSystemStorage.getInstance().toNativePath( + source.getPath()); + FileInputStream file = new FileInputStream(nativePath); try { return file.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.getChannel().size()); diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java index 80f92d4208e..405e2bac4e8 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java @@ -45,11 +45,14 @@ void analyze(InputImage input, final int imageWidth, AsyncResource resource) { final AsyncResource out = (AsyncResource) resource; + final int maximumResults = options.getMaximumResults(); client.process(input).addOnSuccessListener(new OnSuccessListener() { public void onSuccess(Text text) { List source = text.getTextBlocks(); + int count = maximumResults == 0 ? source.size() + : Math.min(maximumResults, source.size()); TextRecognitionResult.TextBlock[] blocks = - new TextRecognitionResult.TextBlock[source.size()]; + new TextRecognitionResult.TextBlock[count]; for (int i = 0; i < blocks.length; i++) { Text.TextBlock block = source.get(i); blocks[i] = new TextRecognitionResult.TextBlock( diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java index 879c158f31d..5f81ef81da7 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java @@ -29,6 +29,7 @@ import com.codename1.ai.inference.TensorInfo; import com.codename1.ai.inference.TensorType; import com.codename1.impl.InferenceImpl; +import com.codename1.io.FileSystemStorage; import com.codename1.io.JSONParser; import com.codename1.ui.Display; import com.codename1.util.AsyncResource; @@ -75,7 +76,9 @@ public void run() { try { String opened = source.getKind() == ModelSource.FILE ? IOSImplementation.nativeInstance.cn1InferenceOpenFile( - source.getPath(), options.getThreads(), + FileSystemStorage.getInstance().toNativePath( + source.getPath()), + options.getThreads(), options.getAccelerator().ordinal(), options.isFallbackAllowed()) : IOSImplementation.nativeInstance.cn1InferenceOpen( diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java index 57722cb2d11..9b387e10bcb 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java @@ -43,6 +43,7 @@ import com.codename1.util.Base64; import java.io.StringReader; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -87,7 +88,8 @@ public AsyncResource analyze(final VisionFeature feature, return out; } analyzeInBackground(out, feature, mlKit, input, - image.getRotationDegrees(), width, height, frameFormat); + image.getRotationDegrees(), width, height, frameFormat, + options.getMinimumConfidence(), options.getMaximumResults()); return out; } @@ -98,7 +100,9 @@ private static void analyzeInBackground(final AsyncResource out, final int rotation, final int width, final int height, - final int frameFormat) { + final int frameFormat, + final float minimumConfidence, + final int maximumResults) { Display.getInstance().scheduleBackgroundTask(new Runnable() { public void run() { try { @@ -110,7 +114,8 @@ public void run() { "Apple Vision returned no result"); } final T value = parse(feature, json, - mlKit ? "ml-kit" : "apple-vision"); + mlKit ? "ml-kit" : "apple-vision", + minimumConfidence, maximumResults); Display.getInstance().callSerially(new Runnable() { public void run() { out.complete(value); @@ -131,7 +136,8 @@ public void run() { @SuppressWarnings({"rawtypes", "unchecked"}) private static T parse(VisionFeature feature, String json, - String backendId) throws Exception { + String backendId, float minimumConfidence, + int maximumResults) throws Exception { Map root = new JSONParser().parseJSON(new StringReader(json)); Object error = root.get("error"); if (error != null) { @@ -141,6 +147,7 @@ private static T parse(VisionFeature feature, String json, if (items == null) { items = java.util.Collections.EMPTY_LIST; } + items = filterItems(items, minimumConfidence, maximumResults); VisionMetadata metadata = new VisionMetadata(backendId); switch (feature) { case TEXT_RECOGNITION: { @@ -224,6 +231,32 @@ private static T parse(VisionFeature feature, String json, } } + @SuppressWarnings({"rawtypes", "unchecked"}) + private static List filterItems(List items, float minimumConfidence, + int maximumResults) { + if (items.isEmpty() + || (minimumConfidence <= 0 && maximumResults == 0)) { + return items; + } + List filtered = new ArrayList(); + for (int i = 0; i < items.size(); i++) { + Object item = items.get(i); + if (minimumConfidence > 0 && item instanceof Map) { + Object confidence = ((Map) item).get("confidence"); + if (confidence instanceof Number + && ((Number) confidence).floatValue() + < minimumConfidence) { + continue; + } + } + filtered.add(item); + if (maximumResults > 0 && filtered.size() >= maximumResults) { + break; + } + } + return filtered; + } + private static VisionRect rect(Map value) { return new VisionRect(number(value, "x"), number(value, "y"), number(value, "width"), number(value, "height")); diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index 36f5ca33c1d..c8632af7b89 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -76,6 +76,16 @@ void modelCacheRejectsInsecureAndMalformedRequests() { "gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg")); } + @Test + void modelCacheNamesDoNotAliasSanitizedKeys() { + assertNotEquals(ModelCache.safeName("model/v1"), + ModelCache.safeName("model?v1")); + assertNotEquals(ModelCache.safeName("model/v1"), + ModelCache.safeName("model_v1")); + assertTrue(ModelCache.safeName("model-v1_2") + .startsWith("model-v1_2-")); + } + @Test void modelCacheDiscardsPartialAndDigestMismatchFiles() throws Exception { FileSystemStorage fs = FileSystemStorage.getInstance(); From 6ed7db244e82121b2378f27bcbf292ec8ed9b1af Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Sun, 26 Jul 2026 23:42:56 +0300 Subject: [PATCH 21/80] Complete iOS AI fallback and face results --- Ports/iOSPort/nativeSources/CN1Inference.m | 17 +++- Ports/iOSPort/nativeSources/CN1Vision.m | 95 +++++++++++++++++++ .../com/codename1/impl/ios/IOSVisionImpl.java | 38 +++++++- 3 files changed, 144 insertions(+), 6 deletions(-) diff --git a/Ports/iOSPort/nativeSources/CN1Inference.m b/Ports/iOSPort/nativeSources/CN1Inference.m index 8c60f5378bf..8897696a184 100644 --- a/Ports/iOSPort/nativeSources/CN1Inference.m +++ b/Ports/iOSPort/nativeSources/CN1Inference.m @@ -123,13 +123,26 @@ static void cn1InferenceEnsureHandles(void) { NSError *error = nil; TFLInterpreter *interpreter = [[TFLInterpreter alloc] initWithModelPath:path options:options delegates:delegates error:&error]; - if (interpreter == nil || error != nil) { + BOOL interpreterCreated = interpreter != nil && error == nil; + BOOL tensorsAllocated = interpreterCreated + && [interpreter allocateTensorsWithError:&error]; + if ((!interpreterCreated || !tensorsAllocated) + && delegate != nil && allowFallback) { + delegate = nil; + error = nil; + interpreter = [[TFLInterpreter alloc] + initWithModelPath:path options:options delegates:@[] error:&error]; + interpreterCreated = interpreter != nil && error == nil; + tensorsAllocated = interpreterCreated + && [interpreter allocateTensorsWithError:&error]; + } + if (!interpreterCreated) { if (deleteModelOnClose) { [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; } return cn1InferenceError(error, @"Could not create LiteRT interpreter"); } - if (![interpreter allocateTensorsWithError:&error]) { + if (!tensorsAllocated) { if (deleteModelOnClose) { [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; } diff --git a/Ports/iOSPort/nativeSources/CN1Vision.m b/Ports/iOSPort/nativeSources/CN1Vision.m index 23ec13f1884..a808e591e3a 100644 --- a/Ports/iOSPort/nativeSources/CN1Vision.m +++ b/Ports/iOSPort/nativeSources/CN1Vision.m @@ -87,6 +87,68 @@ }; } +#if defined(CN1_HAS_MLKIT_FACE) +static void cn1MLKitAddFaceLandmark(NSMutableDictionary *landmarks, + NSString *name, MLKFace *face, MLKFaceLandmarkType type, + CGSize imageSize) { + MLKFaceLandmark *landmark = [face landmarkOfType:type]; + if (landmark != nil && imageSize.width > 0 && imageSize.height > 0) { + landmarks[name] = @{ + @"x": @(landmark.position.x / imageSize.width), + @"y": @(landmark.position.y / imageSize.height) + }; + } +} +#endif + +static NSDictionary *cn1VisionFaceLandmark( + VNFaceLandmarkRegion2D *region, CGRect faceBounds) { + if (region == nil || region.pointCount == 0 + || region.normalizedPoints == NULL) { + return nil; + } + double x = 0; + double y = 0; + for (NSUInteger i = 0; i < region.pointCount; i++) { + x += region.normalizedPoints[i].x; + y += region.normalizedPoints[i].y; + } + x = faceBounds.origin.x + + faceBounds.size.width * x / region.pointCount; + y = faceBounds.origin.y + + faceBounds.size.height * y / region.pointCount; + return @{@"x": @(x), @"y": @(1.0 - y)}; +} + +static NSDictionary *cn1VisionFaceLandmarkEdge( + VNFaceLandmarkRegion2D *region, CGRect faceBounds, BOOL left) { + if (region == nil || region.pointCount == 0 + || region.normalizedPoints == NULL) { + return nil; + } + CGPoint selected = region.normalizedPoints[0]; + for (NSUInteger i = 1; i < region.pointCount; i++) { + CGPoint candidate = region.normalizedPoints[i]; + if ((left && candidate.x < selected.x) + || (!left && candidate.x > selected.x)) { + selected = candidate; + } + } + double x = faceBounds.origin.x + + faceBounds.size.width * selected.x; + double y = faceBounds.origin.y + + faceBounds.size.height * selected.y; + return @{@"x": @(x), @"y": @(1.0 - y)}; +} + +static void cn1VisionAddFaceLandmark(NSMutableDictionary *landmarks, + NSString *name, VNFaceLandmarkRegion2D *region, CGRect faceBounds) { + NSDictionary *point = cn1VisionFaceLandmark(region, faceBounds); + if (point != nil) { + landmarks[name] = point; + } +} + static NSString *cn1VisionJSON(NSDictionary *value) { NSError *error = nil; NSData *data = [NSJSONSerialization dataWithJSONObject:value options:0 error:&error]; @@ -231,9 +293,25 @@ static UIImageOrientation cn1UIImageOrientation(int rotation) { for (MLKFace *face in result ?: @[]) { NSMutableDictionary *item = [NSMutableDictionary dictionaryWithDictionary:cn1MLKitRect(face.frame, image.size)]; + NSMutableDictionary *landmarks = [NSMutableDictionary dictionary]; + cn1MLKitAddFaceLandmark(landmarks, @"leftEye", face, + MLKFaceLandmarkTypeLeftEye, image.size); + cn1MLKitAddFaceLandmark(landmarks, @"rightEye", face, + MLKFaceLandmarkTypeRightEye, image.size); + cn1MLKitAddFaceLandmark(landmarks, @"noseBase", face, + MLKFaceLandmarkTypeNoseBase, image.size); + cn1MLKitAddFaceLandmark(landmarks, @"mouthLeft", face, + MLKFaceLandmarkTypeMouthLeft, image.size); + cn1MLKitAddFaceLandmark(landmarks, @"mouthRight", face, + MLKFaceLandmarkTypeMouthRight, image.size); + item[@"landmarks"] = landmarks; item[@"yaw"] = @(face.headEulerAngleY); item[@"pitch"] = @(face.headEulerAngleX); item[@"roll"] = @(face.headEulerAngleZ); + item[@"smilingProbability"] = face.hasSmilingProbability + ? @(face.smilingProbability) : @(-1); + item[@"trackingId"] = face.hasTrackingID + ? @(face.trackingID) : @(-1); [items addObject:item]; } return cn1VisionJSON(@{@"items": items}); @@ -430,11 +508,28 @@ static CGImagePropertyOrientation cn1CGOrientation(int rotation) { NSMutableDictionary *item = [NSMutableDictionary dictionaryWithDictionary: cn1VisionRect(observation.boundingBox)]; + NSMutableDictionary *landmarks = [NSMutableDictionary dictionary]; + VNFaceLandmarks2D *faceLandmarks = observation.landmarks; + cn1VisionAddFaceLandmark(landmarks, @"leftEye", + faceLandmarks.leftEye, observation.boundingBox); + cn1VisionAddFaceLandmark(landmarks, @"rightEye", + faceLandmarks.rightEye, observation.boundingBox); + cn1VisionAddFaceLandmark(landmarks, @"noseBase", + faceLandmarks.nose, observation.boundingBox); + NSDictionary *mouthLeft = cn1VisionFaceLandmarkEdge( + faceLandmarks.outerLips, observation.boundingBox, YES); + NSDictionary *mouthRight = cn1VisionFaceLandmarkEdge( + faceLandmarks.outerLips, observation.boundingBox, NO); + if (mouthLeft != nil) landmarks[@"mouthLeft"] = mouthLeft; + if (mouthRight != nil) landmarks[@"mouthRight"] = mouthRight; + item[@"landmarks"] = landmarks; item[@"yaw"] = @((observation.yaw ?: @0).doubleValue * 180.0 / M_PI); item[@"pitch"] = @0; item[@"roll"] = @((observation.roll ?: @0).doubleValue * 180.0 / M_PI); + item[@"smilingProbability"] = @(-1); + item[@"trackingId"] = @(-1); [items addObject:item]; } return cn1VisionJSON(@{@"items": items}); diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java index 9b387e10bcb..5b59ecdaefd 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java @@ -185,10 +185,14 @@ private static T parse(VisionFeature feature, String json, Face[] values = new Face[items.size()]; for (int i = 0; i < values.length; i++) { Map value = (Map) items.get(i); + Map landmarks = + parseLandmarks(value.get("landmarks")); values[i] = new Face(rect(value), - new HashMap(), + landmarks, number(value, "yaw"), number(value, "pitch"), - number(value, "roll"), -1, -1, metadata); + number(value, "roll"), + number(value, "smilingProbability", -1), + integer(value, "trackingId", -1), metadata); } return (T) values; } @@ -262,14 +266,40 @@ private static VisionRect rect(Map value) { number(value, "width"), number(value, "height")); } + @SuppressWarnings("rawtypes") + private static Map parseLandmarks(Object value) { + Map out = new HashMap(); + if (!(value instanceof Map)) { + return out; + } + Map source = (Map) value; + for (Object key : source.keySet()) { + Object point = source.get(key); + if (point instanceof Map) { + out.put(String.valueOf(key), new VisionPoint( + number((Map) point, "x"), + number((Map) point, "y"))); + } + } + return out; + } + private static float number(Map value, String key) { + return number(value, key, 0); + } + + private static float number(Map value, String key, float fallback) { Object out = value.get(key); - return out instanceof Number ? ((Number) out).floatValue() : 0; + return out instanceof Number ? ((Number) out).floatValue() : fallback; } private static int integer(Map value, String key) { + return integer(value, key, 0); + } + + private static int integer(Map value, String key, int fallback) { Object out = value.get(key); - return out instanceof Number ? ((Number) out).intValue() : 0; + return out instanceof Number ? ((Number) out).intValue() : fallback; } private static String string(Map value, String key) { From c5f07fbe95f29663cf54ca86482eb891c2f5c94b Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 00:01:17 +0300 Subject: [PATCH 22/80] Harden inference metadata and input mapping --- .../codename1/ai/inference/ModelCache.java | 5 +++ .../impl/android/ai/AndroidInferenceImpl.java | 42 ++++++++++--------- .../codename1/impl/ios/IOSInferenceImpl.java | 17 +++++++- .../ai/inference/InferenceApiTest.java | 6 +++ 4 files changed, 48 insertions(+), 22 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java index 3b03e69e520..1795033f22c 100644 --- a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java +++ b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java @@ -43,6 +43,8 @@ /// model payload. Small first-party models can instead be packaged with /// {@link ModelSource#resource(String)}. public final class ModelCache { + private static final int MAX_READABLE_CACHE_KEY_LENGTH = 160; + private ModelCache() { } @@ -204,6 +206,9 @@ static String safeName(String value) { || c == '-' || c == '_'; out.append(safe ? c : '_'); } + if (out.length() > MAX_READABLE_CACHE_KEY_LENGTH) { + out.setLength(MAX_READABLE_CACHE_KEY_LENGTH); + } Hash hash = Hash.create(Hash.SHA256); for (int i = 0; i < value.length(); i++) { char c = value.charAt(i); diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java index ec7cce647e6..a3a2e392c5f 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java @@ -142,11 +142,28 @@ public AsyncResource run(Object handle, final Tensor[] inputs) { Display.getInstance().scheduleBackgroundTask(new Runnable() { public void run() { try { - Object[] nativeInputs = new Object[interpreter.getInputTensorCount()]; - for (int i = 0; i < nativeInputs.length; i++) { - org.tensorflow.lite.Tensor metadata = interpreter.getInputTensor(i); - Tensor value = find(inputs, metadata.name(), i); - nativeInputs[i] = toBuffer(value, metadata.dataType()); + int inputCount = interpreter.getInputTensorCount(); + if (inputs == null || inputs.length != inputCount) { + throw new IllegalArgumentException("Expected " + + inputCount + " input tensors but received " + + (inputs == null ? 0 : inputs.length)); + } + Object[] nativeInputs = new Object[inputCount]; + boolean[] resolved = new boolean[inputCount]; + for (int i = 0; i < inputs.length; i++) { + Tensor value = inputs[i]; + int index = value.getName() == null ? i + : inputIndex(interpreter, value.getName()); + org.tensorflow.lite.Tensor metadata = + interpreter.getInputTensor(index); + if (resolved[index]) { + throw new IllegalArgumentException( + "Model input " + metadata.name() + + " was supplied more than once"); + } + resolved[index] = true; + nativeInputs[index] = + toBuffer(value, metadata.dataType()); } Map nativeOutputs = new HashMap(); ByteBuffer[] outputBuffers = @@ -198,21 +215,6 @@ private static Handle checked(Object value) { return (Handle) value; } - private static Tensor find(Tensor[] values, String name, int index) { - if (values == null) { - throw new IllegalArgumentException("No input tensors supplied"); - } - for (int i = 0; i < values.length; i++) { - if (name != null && name.equals(values[i].getName())) { - return values[i]; - } - } - if (index < values.length) { - return values[index]; - } - throw new IllegalArgumentException("Missing model input " + name); - } - private static int inputIndex(Interpreter interpreter, String name) { for (int i = 0; i < interpreter.getInputTensorCount(); i++) { if (name == null || name.equals(interpreter.getInputTensor(i).name())) { diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java index 5f81ef81da7..e2055d3f3c7 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java @@ -138,22 +138,35 @@ public void run() { + metadata.length + " input tensors but received " + inputs.length); } + int[] indices = new int[inputs.length]; for (int i = 0; i < inputs.length; i++) { Tensor tensor = inputs[i]; int index = inputIndex(tensor, i, metadata); TensorInfo expected = find(metadata, index); + for (int previous = 0; previous < i; previous++) { + if (indices[previous] == index) { + throw new IllegalArgumentException( + "Model input " + expected.getName() + + " was supplied more than once"); + } + } + indices[i] = index; if (tensor.getType() != expected.getType()) { throw new IllegalArgumentException("Input " + expected.getName() + " expects " + expected.getType() + " but received " + tensor.getType()); } + } + for (int i = 0; i < inputs.length; i++) { + Tensor tensor = inputs[i]; parse(IOSImplementation.nativeInstance.cn1InferenceCopyInput( - id, index, encodeData(tensor.getType(), + id, indices[i], encodeData(tensor.getType(), tensor.getDataUnsafe()))); } parse(IOSImplementation.nativeInstance.cn1InferenceInvoke(id)); - TensorInfo[] outputs = checked.outputs; + TensorInfo[] outputs = metadata(id, true); + checked.outputs = outputs; final Tensor[] result = new Tensor[outputs.length]; for (int i = 0; i < result.length; i++) { TensorInfo output = outputs[i]; diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index c8632af7b89..40c249d6421 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -84,6 +84,12 @@ void modelCacheNamesDoNotAliasSanitizedKeys() { ModelCache.safeName("model_v1")); assertTrue(ModelCache.safeName("model-v1_2") .startsWith("model-v1_2-")); + StringBuilder longKey = new StringBuilder(); + for (int i = 0; i < 512; i++) { + longKey.append('a'); + } + assertTrue((ModelCache.safeName(longKey.toString()) + + ".tflite.download").length() <= 255); } @Test From 26335fde25cdeac440160258883a2b64046ab496 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 00:13:46 +0300 Subject: [PATCH 23/80] Serialize inference lifecycle operations --- .../ai/inference/InferenceSession.java | 123 ++++++++++++++++-- .../ai/vision/AbstractVisionAnalyzer.java | 6 +- Ports/iOSPort/nativeSources/CN1Language.m | 2 +- Ports/iOSPort/nativeSources/CN1Vision.m | 4 + .../ai/inference/InferenceApiTest.java | 28 ++++ 5 files changed, 146 insertions(+), 17 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java index 504b9c3dbe1..7cfd3f6aff8 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -37,6 +37,8 @@ public final class InferenceSession implements AutoCloseable { private final InferenceImpl implementation; private final Object handle; private boolean closed; + private int activeRuns; + private boolean closePending; private InferenceSession(InferenceImpl implementation, Object handle) { this.implementation = implementation; @@ -88,43 +90,122 @@ public void onSucess(Throwable error) { /// /// @return a defensive copy of the input metadata array public TensorInfo[] getInputs() { - ensureOpen(); - return implementation.getInputs(handle); + synchronized (this) { + ensureOpen(); + return implementation.getInputs(handle); + } } - /// @return a defensive copy of the model's current output metadata + /// Returns the model's current output metadata. Backends refresh this + /// information after an invocation so models with dynamically resolved + /// output dimensions report the shape used to decode the returned tensor. + /// + /// @return a defensive copy of the output metadata array public TensorInfo[] getOutputs() { - ensureOpen(); - return implementation.getOutputs(handle); + synchronized (this) { + ensureOpen(); + return implementation.getOutputs(handle); + } } /// Copies input tensors to native memory, invokes the model, and returns /// every output tensor. Named tensors are matched by name; unnamed tensors - /// are matched by position. + /// are matched by position. Calling {@link #close()} while this operation + /// is pending prevents new work immediately but defers native release until + /// the returned resource succeeds or fails. /// /// @param inputs one tensor for each model input /// @return asynchronous output tensors in model output order public AsyncResource run(Tensor[] inputs) { - ensureOpen(); - return implementation.run(handle, inputs == null ? new Tensor[0] : inputs); + final AsyncResource result; + synchronized (this) { + ensureOpen(); + activeRuns++; + try { + result = implementation.run(handle, + inputs == null ? new Tensor[0] : inputs); + if (result == null) { + throw new InferenceException( + "Inference backend returned no asynchronous result"); + } + } catch (RuntimeException error) { + activeRuns--; + throw error; + } catch (Error error) { + activeRuns--; + throw error; + } + } + final PendingRun pending = new PendingRun(this); + result.ready(new SuccessCallback() { + @Override + public void onSucess(Tensor[] value) { + pending.finish(); + } + }).except(new SuccessCallback() { + @Override + public void onSucess(Throwable error) { + pending.finish(); + } + }); + return result; } /// Resizes an input and reallocates native tensors before the next run. /// + /// This method throws while an asynchronous {@link #run(Tensor[])} is + /// pending because native runtimes cannot safely reallocate tensors during + /// an invocation. + /// /// @param name model input name, or {@code null} for the first input /// @param shape new non-negative dimensions + /// @throws IllegalStateException if the session is closed or a run is pending public void resizeInput(String name, int[] shape) { - ensureOpen(); - implementation.resizeInput(handle, name, shape); + synchronized (this) { + ensureOpen(); + if (activeRuns > 0) { + throw new IllegalStateException( + "Cannot resize inputs while inference is running"); + } + implementation.resizeInput(handle, name, shape); + } } - @Override /// Releases the interpreter, delegates, and any temporary staged model. /// The original file supplied by {@link ModelSource#file(String)} is never - /// deleted. Calling this method more than once has no effect. + /// deleted. If a run is pending, release is deferred until that run + /// settles. Calling this method more than once has no effect. + @Override public void close() { - if (!closed) { + boolean release = false; + synchronized (this) { + if (closed) { + return; + } closed = true; + if (activeRuns == 0) { + release = true; + } else { + closePending = true; + } + } + if (release) { + implementation.close(handle); + } + } + + private void runFinished() { + boolean release = false; + synchronized (this) { + if (activeRuns > 0) { + activeRuns--; + } + if (activeRuns == 0 && closePending) { + closePending = false; + release = true; + } + } + if (release) { implementation.close(handle); } } @@ -134,4 +215,20 @@ private void ensureOpen() { throw new IllegalStateException("Inference session is closed"); } } + + private static final class PendingRun { + private final InferenceSession session; + private boolean finished; + + PendingRun(InferenceSession session) { + this.session = session; + } + + synchronized void finish() { + if (!finished) { + finished = true; + session.runFinished(); + } + } + } } diff --git a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java index e230f511ed7..c1b5cff62e2 100644 --- a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java +++ b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java @@ -37,17 +37,17 @@ abstract class AbstractVisionAnalyzer implements VisionAnalyzer { this.options = options == null ? new VisionOptions() : options; } - @Override /// @return whether this analyzer's feature/backend is available and open + @Override public final boolean isSupported() { VisionImpl impl = implementation(); return impl != null && impl.isSupported(feature, options.getBackend().getId()); } - @Override /// Starts one analysis using the retained native backend. /// @param image immutable encoded or raw input /// @return asynchronous typed result + @Override public final AsyncResource process(VisionImage image) { if (image == null) { throw new NullPointerException("image"); @@ -65,8 +65,8 @@ public final AsyncResource process(VisionImage image) { return impl.analyze(feature, options.getBackend().getId(), image, options); } - @Override /// Idempotently releases the retained native backend. + @Override public final void close() { closed = true; if (implementation != null) { diff --git a/Ports/iOSPort/nativeSources/CN1Language.m b/Ports/iOSPort/nativeSources/CN1Language.m index 221ed376e99..af27e9a3874 100644 --- a/Ports/iOSPort/nativeSources/CN1Language.m +++ b/Ports/iOSPort/nativeSources/CN1Language.m @@ -178,7 +178,7 @@ if (![value isKindOfClass:[NSDictionary class]]) continue; MLKTextMessage *message = [[MLKTextMessage alloc] initWithText:value[@"text"] ?: @"" - timestamp:[value[@"timestamp"] doubleValue] + timestamp:[value[@"timestamp"] doubleValue] / 1000.0 userID:value[@"participant"] ?: @"remote" isLocalUser:[value[@"local"] boolValue]]; [messages addObject:message]; diff --git a/Ports/iOSPort/nativeSources/CN1Vision.m b/Ports/iOSPort/nativeSources/CN1Vision.m index a808e591e3a..8a4b6f33732 100644 --- a/Ports/iOSPort/nativeSources/CN1Vision.m +++ b/Ports/iOSPort/nativeSources/CN1Vision.m @@ -613,6 +613,10 @@ static CGImagePropertyOrientation cn1CGOrientation(int rotation) { return cn1VisionJSON(@{@"error": @"Could not decode document image"}); } CIImage *source = [CIImage imageWithCGImage:image.CGImage]; + if (rotation != 0) { + source = [source imageByApplyingOrientation: + cn1CGOrientation(rotation)]; + } CIContext *context = [CIContext context]; CIDetector *detector = [CIDetector detectorOfType:CIDetectorTypeRectangle context:context diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index 40c249d6421..6fd5e562acc 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -135,6 +135,30 @@ void sessionLifecycleForwardsToBackend() { assertThrows(IllegalStateException.class, session::getInputs); } + @Test + void sessionDefersCloseUntilPendingRunSettles() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + backend.pendingRun = new AsyncResource(); + implementation.setInferenceImpl(backend); + InferenceSession session = await(InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions())); + AsyncResource run = session.run(new Tensor[] { + Tensor.floats("input", new int[] {1, 2}, + new float[] {1, 2}) + }); + assertThrows(IllegalStateException.class, + () -> session.resizeInput("input", new int[] {1, 4})); + session.close(); + assertEquals(0, backend.closeCount); + backend.pendingRun.complete(new Tensor[] { + Tensor.floats("output", new int[] {1}, new float[] {3}) + }); + flushSerialCalls(); + assertTrue(run.isDone()); + assertEquals(1, backend.closeCount); + assertThrows(IllegalStateException.class, session::getInputs); + } + private T await(AsyncResource resource) { final AtomicReference value = new AtomicReference(); resource.ready(new SuccessCallback() { @@ -152,6 +176,7 @@ private static final class RecordingInferenceImpl extends InferenceImpl { final Object handle = new Object(); String resizedName; int closeCount; + AsyncResource pendingRun; public boolean isSupported() { return true; @@ -177,6 +202,9 @@ public TensorInfo[] getOutputs(Object value) { } public AsyncResource run(Object value, Tensor[] inputs) { + if (pendingRun != null) { + return pendingRun; + } AsyncResource result = new AsyncResource(); result.complete(new Tensor[] { Tensor.floats("output", new int[] {1}, new float[] {3}) From 76d7daf2417c2d1f8963e77b4ef8a390aa13b945 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 00:24:32 +0300 Subject: [PATCH 24/80] Normalize pose results across backends --- .../ai/inference/InferenceSession.java | 8 ++- .../src/com/codename1/ai/vision/Pose.java | 15 ++++- .../ai/AndroidFaceDetectionAdapter.java | 3 +- .../ai/AndroidPoseDetectionAdapter.java | 62 +++++++++++++++++-- Ports/iOSPort/nativeSources/CN1Vision.m | 26 +++++++- .../ai/inference/InferenceApiTest.java | 3 + 6 files changed, 105 insertions(+), 12 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java index 7cfd3f6aff8..39014670317 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -112,14 +112,20 @@ public TensorInfo[] getOutputs() { /// every output tensor. Named tensors are matched by name; unnamed tensors /// are matched by position. Calling {@link #close()} while this operation /// is pending prevents new work immediately but defers native release until - /// the returned resource succeeds or fails. + /// the returned resource succeeds or fails. A session accepts one run at a + /// time because the underlying native interpreter is mutable. /// /// @param inputs one tensor for each model input /// @return asynchronous output tensors in model output order + /// @throws IllegalStateException if the session is closed or already running public AsyncResource run(Tensor[] inputs) { final AsyncResource result; synchronized (this) { ensureOpen(); + if (activeRuns > 0) { + throw new IllegalStateException( + "Inference session is already running"); + } activeRuns++; try { result = implementation.run(handle, diff --git a/CodenameOne/src/com/codename1/ai/vision/Pose.java b/CodenameOne/src/com/codename1/ai/vision/Pose.java index 93606870389..ae958ad0466 100644 --- a/CodenameOne/src/com/codename1/ai/vision/Pose.java +++ b/CodenameOne/src/com/codename1/ai/vision/Pose.java @@ -24,8 +24,13 @@ /// Portable body-pose result. Landmark coordinates use the normalized /// top-left-origin image space defined by {@link VisionPoint}; confidence is -/// in the range 0..1. Landmark names are backend-neutral where the native -/// backend exposes a known joint name. +/// in the range 0..1. +/// +/// Common backend-neutral names include `nose`, `leftEye`, `rightEye`, +/// `leftEar`, `rightEar`, and the `left`/`right` forms of `Shoulder`, `Elbow`, +/// `Wrist`, `Hip`, `Knee`, and `Ankle`. A backend can additionally report +/// finer eye, mouth, finger, heel, foot, `neck`, or `root` landmarks using the +/// same lower-camel-case convention. public final class Pose { private final Landmark[] landmarks; private final VisionMetadata metadata; @@ -77,7 +82,11 @@ public Landmark(String name, VisionPoint position, float confidence) { this.confidence = confidence; } - /// @return portable/native joint name + /// Returns the stable lower-camel-case joint name described by + /// {@link Pose}. Known native constants are normalized rather than + /// exposed as platform-specific numeric or symbolic identifiers. + /// + /// @return backend-neutral joint name, or `unknown` if unmapped public String getName() { return name; } diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java index 47ddb71dba5..bfa21a4c678 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java @@ -83,7 +83,8 @@ public void onSuccess( result[i] = new Face( normalized(value.getBoundingBox(), imageWidth, imageHeight), - landmarks, value.getHeadEulerAngleY(), 0, + landmarks, value.getHeadEulerAngleY(), + value.getHeadEulerAngleX(), value.getHeadEulerAngleZ(), smile == null ? -1 : smile, tracking == null ? -1 : tracking, METADATA); diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java index 1f89668b537..41f02218a6b 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java @@ -33,6 +33,7 @@ import com.google.mlkit.vision.pose.PoseLandmark; import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions; +import java.util.ArrayList; import java.util.List; /** ML Kit pose detection; retained only for {@code PoseDetector} users. */ @@ -47,21 +48,31 @@ void analyze(InputImage input, final int imageWidth, final int imageHeight, VisionOptions options, AsyncResource resource) { final AsyncResource out = (AsyncResource) resource; + final float minimumConfidence = options.getMinimumConfidence(); + final int maximumResults = options.getMaximumResults(); client.process(input).addOnSuccessListener( new OnSuccessListener() { public void onSuccess(com.google.mlkit.vision.pose.Pose value) { List source = value.getAllPoseLandmarks(); - Pose.Landmark[] landmarks = new Pose.Landmark[source.size()]; - for (int i = 0; i < landmarks.length; i++) { + List result = new ArrayList(); + for (int i = 0; i < source.size(); i++) { PoseLandmark point = source.get(i); - landmarks[i] = new Pose.Landmark( - String.valueOf(point.getLandmarkType()), + if (point.getInFrameLikelihood() < minimumConfidence) { + continue; + } + result.add(new Pose.Landmark( + landmarkName(point.getLandmarkType()), new VisionPoint( point.getPosition().x / imageWidth, point.getPosition().y / imageHeight), - point.getInFrameLikelihood()); + point.getInFrameLikelihood())); + if (maximumResults > 0 + && result.size() >= maximumResults) { + break; + } } - complete(out, new Pose(landmarks, METADATA)); + complete(out, new Pose(result.toArray( + new Pose.Landmark[result.size()]), METADATA)); } }).addOnFailureListener(failure(out)); } @@ -70,4 +81,43 @@ public void onSuccess(com.google.mlkit.vision.pose.Pose value) { void close() { client.close(); } + + private static String landmarkName(int type) { + switch (type) { + case PoseLandmark.NOSE: return "nose"; + case PoseLandmark.LEFT_EYE_INNER: return "leftEyeInner"; + case PoseLandmark.LEFT_EYE: return "leftEye"; + case PoseLandmark.LEFT_EYE_OUTER: return "leftEyeOuter"; + case PoseLandmark.RIGHT_EYE_INNER: return "rightEyeInner"; + case PoseLandmark.RIGHT_EYE: return "rightEye"; + case PoseLandmark.RIGHT_EYE_OUTER: return "rightEyeOuter"; + case PoseLandmark.LEFT_EAR: return "leftEar"; + case PoseLandmark.RIGHT_EAR: return "rightEar"; + case PoseLandmark.LEFT_MOUTH: return "leftMouth"; + case PoseLandmark.RIGHT_MOUTH: return "rightMouth"; + case PoseLandmark.LEFT_SHOULDER: return "leftShoulder"; + case PoseLandmark.RIGHT_SHOULDER: return "rightShoulder"; + case PoseLandmark.LEFT_ELBOW: return "leftElbow"; + case PoseLandmark.RIGHT_ELBOW: return "rightElbow"; + case PoseLandmark.LEFT_WRIST: return "leftWrist"; + case PoseLandmark.RIGHT_WRIST: return "rightWrist"; + case PoseLandmark.LEFT_PINKY: return "leftPinky"; + case PoseLandmark.RIGHT_PINKY: return "rightPinky"; + case PoseLandmark.LEFT_INDEX: return "leftIndex"; + case PoseLandmark.RIGHT_INDEX: return "rightIndex"; + case PoseLandmark.LEFT_THUMB: return "leftThumb"; + case PoseLandmark.RIGHT_THUMB: return "rightThumb"; + case PoseLandmark.LEFT_HIP: return "leftHip"; + case PoseLandmark.RIGHT_HIP: return "rightHip"; + case PoseLandmark.LEFT_KNEE: return "leftKnee"; + case PoseLandmark.RIGHT_KNEE: return "rightKnee"; + case PoseLandmark.LEFT_ANKLE: return "leftAnkle"; + case PoseLandmark.RIGHT_ANKLE: return "rightAnkle"; + case PoseLandmark.LEFT_HEEL: return "leftHeel"; + case PoseLandmark.RIGHT_HEEL: return "rightHeel"; + case PoseLandmark.LEFT_FOOT_INDEX: return "leftFootIndex"; + case PoseLandmark.RIGHT_FOOT_INDEX: return "rightFootIndex"; + default: return "unknown"; + } + } } diff --git a/Ports/iOSPort/nativeSources/CN1Vision.m b/Ports/iOSPort/nativeSources/CN1Vision.m index 8a4b6f33732..a1929051904 100644 --- a/Ports/iOSPort/nativeSources/CN1Vision.m +++ b/Ports/iOSPort/nativeSources/CN1Vision.m @@ -439,6 +439,30 @@ static CGImagePropertyOrientation cn1CGOrientation(int rotation) { return @"UNKNOWN"; } +static NSString *cn1ApplePoseLandmarkName( + VNHumanBodyPoseObservationJointName name) { + if ([name isEqual:VNHumanBodyPoseObservationJointNameNose]) return @"nose"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftEye]) return @"leftEye"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightEye]) return @"rightEye"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftEar]) return @"leftEar"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightEar]) return @"rightEar"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftShoulder]) return @"leftShoulder"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightShoulder]) return @"rightShoulder"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameNeck]) return @"neck"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftElbow]) return @"leftElbow"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightElbow]) return @"rightElbow"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftWrist]) return @"leftWrist"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightWrist]) return @"rightWrist"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftHip]) return @"leftHip"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightHip]) return @"rightHip"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRoot]) return @"root"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftKnee]) return @"leftKnee"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightKnee]) return @"rightKnee"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftAnkle]) return @"leftAnkle"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightAnkle]) return @"rightAnkle"; + return name ?: @"unknown"; +} + static NSString *cn1VisionPerform(NSData *data, CGImageRef rawImage, int feature, int rotation) { NSError *error = nil; @@ -567,7 +591,7 @@ static CGImagePropertyOrientation cn1CGOrientation(int rotation) { for (NSString *name in points) { VNRecognizedPoint *point = points[name]; [items addObject:@{ - @"name": name, + @"name": cn1ApplePoseLandmarkName(name), @"x": @(point.location.x), @"y": @(1.0 - point.location.y), @"confidence": @(point.confidence) diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index 6fd5e562acc..d6573f04063 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -146,6 +146,9 @@ void sessionDefersCloseUntilPendingRunSettles() { Tensor.floats("input", new int[] {1, 2}, new float[] {1, 2}) }); + assertThrows(IllegalStateException.class, () -> session.run( + new Tensor[] {Tensor.floats("input", new int[] {1, 2}, + new float[] {3, 4})})); assertThrows(IllegalStateException.class, () -> session.resizeInput("input", new int[] {1, 4})); session.close(); From 8f15981abe8e3f8148840bce03620b4d605aa740 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 00:29:35 +0300 Subject: [PATCH 25/80] Normalize iOS vision result details --- Ports/iOSPort/nativeSources/CN1Vision.m | 49 +++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/Ports/iOSPort/nativeSources/CN1Vision.m b/Ports/iOSPort/nativeSources/CN1Vision.m index a1929051904..cf4d9b51592 100644 --- a/Ports/iOSPort/nativeSources/CN1Vision.m +++ b/Ports/iOSPort/nativeSources/CN1Vision.m @@ -101,6 +101,45 @@ static void cn1MLKitAddFaceLandmark(NSMutableDictionary *landmarks, } #endif +#if defined(CN1_HAS_MLKIT_POSE) +static NSString *cn1MLKitPoseLandmarkName(MLKPoseLandmarkType type) { + if ([type isEqualToString:MLKPoseLandmarkTypeNose]) return @"nose"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftEyeInner]) return @"leftEyeInner"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftEye]) return @"leftEye"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftEyeOuter]) return @"leftEyeOuter"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightEyeInner]) return @"rightEyeInner"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightEye]) return @"rightEye"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightEyeOuter]) return @"rightEyeOuter"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftEar]) return @"leftEar"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightEar]) return @"rightEar"; + if ([type isEqualToString:MLKPoseLandmarkTypeMouthLeft]) return @"leftMouth"; + if ([type isEqualToString:MLKPoseLandmarkTypeMouthRight]) return @"rightMouth"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftShoulder]) return @"leftShoulder"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightShoulder]) return @"rightShoulder"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftElbow]) return @"leftElbow"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightElbow]) return @"rightElbow"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftWrist]) return @"leftWrist"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightWrist]) return @"rightWrist"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftPinkyFinger]) return @"leftPinky"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightPinkyFinger]) return @"rightPinky"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftIndexFinger]) return @"leftIndex"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightIndexFinger]) return @"rightIndex"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftThumb]) return @"leftThumb"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightThumb]) return @"rightThumb"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftHip]) return @"leftHip"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightHip]) return @"rightHip"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftKnee]) return @"leftKnee"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightKnee]) return @"rightKnee"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftAnkle]) return @"leftAnkle"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightAnkle]) return @"rightAnkle"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftHeel]) return @"leftHeel"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightHeel]) return @"rightHeel"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftToe]) return @"leftFootIndex"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightToe]) return @"rightFootIndex"; + return @"unknown"; +} +#endif + static NSDictionary *cn1VisionFaceLandmark( VNFaceLandmarkRegion2D *region, CGRect faceBounds) { if (region == nil || region.pointCount == 0 @@ -356,8 +395,7 @@ static UIImageOrientation cn1UIImageOrientation(int rotation) { MLKPose *pose = result.firstObject; for (MLKPoseLandmark *landmark in pose.landmarks ?: @[]) { [items addObject:@{ - @"name": [NSString stringWithFormat:@"%ld", - (long)landmark.type], + @"name": cn1MLKitPoseLandmarkName(landmark.type), @"x": @(landmark.position.x / image.size.width), @"y": @(landmark.position.y / image.size.height), @"confidence": @(landmark.inFrameLikelihood) @@ -549,7 +587,12 @@ static CGImagePropertyOrientation cn1CGOrientation(int rotation) { item[@"landmarks"] = landmarks; item[@"yaw"] = @((observation.yaw ?: @0).doubleValue * 180.0 / M_PI); - item[@"pitch"] = @0; + if (@available(iOS 15.0, *)) { + item[@"pitch"] = @((observation.pitch ?: @0).doubleValue + * 180.0 / M_PI); + } else { + item[@"pitch"] = @0; + } item[@"roll"] = @((observation.roll ?: @0).doubleValue * 180.0 / M_PI); item[@"smilingProbability"] = @(-1); From bff9439228cf7cc0a72984b53d75f16e3614a8a2 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 00:42:20 +0300 Subject: [PATCH 26/80] Fix vision threading and rotated geometry --- .../impl/android/ai/AndroidVisionImpl.java | 87 +++++++++++++------ Ports/iOSPort/nativeSources/CN1Vision.m | 29 ++++--- 2 files changed, 77 insertions(+), 39 deletions(-) diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java index aa911a8c13f..9037f9c8efc 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java @@ -31,6 +31,7 @@ import com.codename1.ai.vision.VisionOptions; import com.codename1.camera.FrameFormat; import com.codename1.impl.VisionImpl; +import com.codename1.ui.Display; import com.codename1.util.AsyncResource; import com.google.mlkit.vision.common.InputImage; @@ -54,28 +55,37 @@ && adapterClass(feature) != null @Override @SuppressWarnings({"rawtypes", "unchecked"}) - public AsyncResource analyze(VisionFeature feature, String backendId, - VisionImage image, VisionOptions options) { - AsyncResource out = new AsyncResource(); + public AsyncResource analyze(final VisionFeature feature, + String backendId, + final VisionImage image, + final VisionOptions options) { + final AsyncResource out = new AsyncResource(); if (!isSupported(feature, backendId)) { out.error(new VisionException(VisionException.UNSUPPORTED, feature + " is not included in this Android build")); return out; } - DecodedInput decoded = decode(image); - if (decoded == null) { - out.error(new VisionException(VisionException.INVALID_IMAGE, - "Vision input is not valid JPEG, PNG, NV21, or RGBA8888 data")); - return out; - } - try { - adapter(feature).analyze(decoded.input, decoded.width, decoded.height, - options, (AsyncResource) out); - } catch (Throwable error) { - out.error(new VisionException(VisionException.BACKEND_ERROR, - "Could not start the Android " + feature + " adapter", - error)); - } + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + try { + DecodedInput decoded = decode(image); + if (decoded == null) { + error(out, new VisionException( + VisionException.INVALID_IMAGE, + "Vision input is not valid JPEG, PNG, NV21, " + + "or RGBA8888 data")); + return; + } + adapter(feature).analyze(decoded.input, decoded.width, + decoded.height, options, (AsyncResource) out); + } catch (Throwable cause) { + error(out, new VisionException( + VisionException.BACKEND_ERROR, + "Could not start the Android " + feature + + " adapter", cause)); + } + } + }); return out; } @@ -102,7 +112,8 @@ private static DecodedInput decode(VisionImage image) { encoded, 0, encoded.length); return bitmap == null ? null : new DecodedInput( InputImage.fromBitmap(bitmap, image.getRotationDegrees()), - bitmap.getWidth(), bitmap.getHeight()); + bitmap.getWidth(), bitmap.getHeight(), + image.getRotationDegrees()); } byte[] pixels = image.getPixelsUnsafe(); int width = image.getWidth(); @@ -110,19 +121,25 @@ private static DecodedInput decode(VisionImage image) { if (pixels == null || width <= 0 || height <= 0) { return null; } + long pixelCount = (long) width * (long) height; + if (pixelCount > Integer.MAX_VALUE) { + return null; + } if (image.getFormat() == FrameFormat.NV21) { - if (pixels.length < width * height * 3 / 2) { + if ((width & 1) != 0 || (height & 1) != 0 + || pixels.length < pixelCount + pixelCount / 2) { return null; } return new DecodedInput(InputImage.fromByteArray( pixels, width, height, image.getRotationDegrees(), - InputImage.IMAGE_FORMAT_NV21), width, height); + InputImage.IMAGE_FORMAT_NV21), width, height, + image.getRotationDegrees()); } if (image.getFormat() == FrameFormat.RGBA8888) { - if (pixels.length < width * height * 4) { + if (pixelCount * 4 > pixels.length) { return null; } - int[] argb = new int[width * height]; + int[] argb = new int[(int) pixelCount]; for (int i = 0, p = 0; i < argb.length; i++, p += 4) { argb[i] = ((pixels[p + 3] & 255) << 24) | ((pixels[p] & 255) << 16) @@ -132,13 +149,15 @@ private static DecodedInput decode(VisionImage image) { Bitmap bitmap = Bitmap.createBitmap( argb, width, height, Bitmap.Config.ARGB_8888); return new DecodedInput(InputImage.fromBitmap( - bitmap, image.getRotationDegrees()), width, height); + bitmap, image.getRotationDegrees()), width, height, + image.getRotationDegrees()); } Bitmap bitmap = BitmapFactory.decodeByteArray( pixels, 0, pixels.length); return bitmap == null ? null : new DecodedInput( InputImage.fromBitmap(bitmap, image.getRotationDegrees()), - bitmap.getWidth(), bitmap.getHeight()); + bitmap.getWidth(), bitmap.getHeight(), + image.getRotationDegrees()); } private static final class DecodedInput { @@ -146,13 +165,27 @@ private static final class DecodedInput { final int width; final int height; - DecodedInput(InputImage input, int width, int height) { + DecodedInput(InputImage input, int width, int height, int rotation) { this.input = input; - this.width = width; - this.height = height; + if (rotation == 90 || rotation == 270) { + this.width = height; + this.height = width; + } else { + this.width = width; + this.height = height; + } } } + private static void error(final AsyncResource out, + final VisionException error) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.error(error); + } + }); + } + private static boolean isClassPresent(String className) { try { Class.forName(className); diff --git a/Ports/iOSPort/nativeSources/CN1Vision.m b/Ports/iOSPort/nativeSources/CN1Vision.m index cf4d9b51592..4c3b60e3714 100644 --- a/Ports/iOSPort/nativeSources/CN1Vision.m +++ b/Ports/iOSPort/nativeSources/CN1Vision.m @@ -251,6 +251,8 @@ static UIImageOrientation cn1UIImageOrientation(int rotation) { return cn1VisionJSON(@{@"error": @"Could not decode ML Kit image"}); } MLKVisionImage *vision = cn1MLKitImage(image, rotation); + CGSize resultSize = (rotation == 90 || rotation == 270) + ? CGSizeMake(image.size.height, image.size.width) : image.size; __block NSError *requestError = nil; dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); @@ -269,7 +271,7 @@ static UIImageOrientation cn1UIImageOrientation(int rotation) { NSMutableArray *items = [NSMutableArray array]; for (MLKTextBlock *block in result.blocks ?: @[]) { NSMutableDictionary *item = [NSMutableDictionary - dictionaryWithDictionary:cn1MLKitRect(block.frame, image.size)]; + dictionaryWithDictionary:cn1MLKitRect(block.frame, resultSize)]; item[@"text"] = block.text ?: @""; item[@"confidence"] = @1; [items addObject:item]; @@ -296,15 +298,15 @@ static UIImageOrientation cn1UIImageOrientation(int rotation) { NSMutableArray *items = [NSMutableArray array]; for (MLKBarcode *barcode in result ?: @[]) { NSMutableDictionary *item = [NSMutableDictionary - dictionaryWithDictionary:cn1MLKitRect(barcode.frame, image.size)]; + dictionaryWithDictionary:cn1MLKitRect(barcode.frame, resultSize)]; item[@"value"] = barcode.rawValue ?: [NSNull null]; item[@"format"] = cn1MLKitBarcodeFormat(barcode.format); NSMutableArray *corners = [NSMutableArray array]; for (NSValue *pointValue in barcode.cornerPoints ?: @[]) { CGPoint point = pointValue.CGPointValue; [corners addObject:@{ - @"x": @(point.x / image.size.width), - @"y": @(point.y / image.size.height) + @"x": @(point.x / resultSize.width), + @"y": @(point.y / resultSize.height) }]; } item[@"corners"] = corners; @@ -331,18 +333,18 @@ static UIImageOrientation cn1UIImageOrientation(int rotation) { NSMutableArray *items = [NSMutableArray array]; for (MLKFace *face in result ?: @[]) { NSMutableDictionary *item = [NSMutableDictionary - dictionaryWithDictionary:cn1MLKitRect(face.frame, image.size)]; + dictionaryWithDictionary:cn1MLKitRect(face.frame, resultSize)]; NSMutableDictionary *landmarks = [NSMutableDictionary dictionary]; cn1MLKitAddFaceLandmark(landmarks, @"leftEye", face, - MLKFaceLandmarkTypeLeftEye, image.size); + MLKFaceLandmarkTypeLeftEye, resultSize); cn1MLKitAddFaceLandmark(landmarks, @"rightEye", face, - MLKFaceLandmarkTypeRightEye, image.size); + MLKFaceLandmarkTypeRightEye, resultSize); cn1MLKitAddFaceLandmark(landmarks, @"noseBase", face, - MLKFaceLandmarkTypeNoseBase, image.size); + MLKFaceLandmarkTypeNoseBase, resultSize); cn1MLKitAddFaceLandmark(landmarks, @"mouthLeft", face, - MLKFaceLandmarkTypeMouthLeft, image.size); + MLKFaceLandmarkTypeMouthLeft, resultSize); cn1MLKitAddFaceLandmark(landmarks, @"mouthRight", face, - MLKFaceLandmarkTypeMouthRight, image.size); + MLKFaceLandmarkTypeMouthRight, resultSize); item[@"landmarks"] = landmarks; item[@"yaw"] = @(face.headEulerAngleY); item[@"pitch"] = @(face.headEulerAngleX); @@ -396,8 +398,8 @@ static UIImageOrientation cn1UIImageOrientation(int rotation) { for (MLKPoseLandmark *landmark in pose.landmarks ?: @[]) { [items addObject:@{ @"name": cn1MLKitPoseLandmarkName(landmark.type), - @"x": @(landmark.position.x / image.size.width), - @"y": @(landmark.position.y / image.size.height), + @"x": @(landmark.position.x / resultSize.width), + @"y": @(landmark.position.y / resultSize.height), @"confidence": @(landmark.inFrameLikelihood) }]; } @@ -808,6 +810,9 @@ static CGImageRef cn1VisionCreateRawImage(NSData *data, int width, int height, } memcpy(dest, source, pixelCount * 4); } else if (format == 1) { + if ((width & 1) != 0 || (height & 1) != 0) { + return NULL; + } if (data.length < pixelCount + pixelCount / 2) { return NULL; } From 37dec2998e388b76c5e2efc9efde4e46e225d279 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 00:55:02 +0300 Subject: [PATCH 27/80] Harden dynamic inference and vision state --- .../ai/vision/AbstractVisionAnalyzer.java | 3 +- .../com/codename1/ai/vision/VisionImage.java | 18 ++- .../codename1/ai/vision/VisionOptions.java | 6 + .../codename1/ai/vision/VisionPipeline.java | 9 +- .../impl/android/ai/AndroidInferenceImpl.java | 18 ++- .../codename1/ai/vision/VisionApiTest.java | 13 +- .../codename1/camera/VisionPipelineTest.java | 124 ++++++++++++++++++ 7 files changed, 172 insertions(+), 19 deletions(-) create mode 100644 maven/core-unittests/src/test/java/com/codename1/camera/VisionPipelineTest.java diff --git a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java index c1b5cff62e2..bb9dc91e342 100644 --- a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java +++ b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java @@ -34,7 +34,8 @@ abstract class AbstractVisionAnalyzer implements VisionAnalyzer { AbstractVisionAnalyzer(VisionFeature feature, VisionOptions options) { this.feature = feature; - this.options = options == null ? new VisionOptions() : options; + this.options = (options == null + ? new VisionOptions() : options).snapshot(); } /// @return whether this analyzer's feature/backend is available and open diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java index bfa7d774e56..3a50828efd2 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java @@ -64,8 +64,11 @@ public static VisionImage encoded(byte[] bytes) { /// @param width unrotated pixel width /// @param height unrotated pixel height /// @param format supported raw frame format - /// @param rotationDegrees clockwise display rotation + /// @param rotationDegrees clockwise display rotation; only 0, 90, 180, + /// or 270 degrees are supported /// @return immutable image input + /// @throws IllegalArgumentException if the data or dimensions are empty, + /// or if the rotation is not a quarter turn public static VisionImage pixels(byte[] bytes, int width, int height, FrameFormat format, int rotationDegrees) { if (bytes == null || bytes.length == 0 || width <= 0 || height <= 0) { @@ -77,6 +80,8 @@ public static VisionImage pixels(byte[] bytes, int width, int height, /// Copies a camera frame, including timestamp, format, and orientation. /// @param frame callback-owned frame /// @return detached immutable input safe for asynchronous analysis + /// @throws IllegalArgumentException if the frame reports a rotation other + /// than 0, 90, 180, or 270 degrees public static VisionImage fromCameraFrame(CameraFrame frame) { if (frame == null) { throw new NullPointerException("frame"); @@ -126,7 +131,7 @@ public int getHeight() { return height; } - /// @return normalized clockwise rotation in the range 0..359 + /// @return normalized clockwise rotation: 0, 90, 180, or 270 public int getRotationDegrees() { return rotationDegrees; } @@ -152,6 +157,13 @@ private static byte[] copy(byte[] value) { private static int normalizeRotation(int value) { int out = value % 360; - return out < 0 ? out + 360 : out; + if (out < 0) { + out += 360; + } + if (out % 90 != 0) { + throw new IllegalArgumentException( + "Rotation must be 0, 90, 180, or 270 degrees"); + } + return out; } } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java b/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java index 88b83f395c0..42710dc9cad 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java @@ -67,4 +67,10 @@ public float getMinimumConfidence() { public int getMaximumResults() { return maximumResults; } + + VisionOptions snapshot() { + return new VisionOptions().backend(backend) + .minimumConfidence(minimumConfidence) + .maximumResults(maximumResults); + } } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java index b4341887c01..fe6a452a55d 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java @@ -131,9 +131,12 @@ public void run() { pending = null; busy = next != null; } - notification.run(); - if (next != null) { - process(next); + try { + notification.run(); + } finally { + if (next != null) { + process(next); + } } } }); diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java index a3a2e392c5f..cf4c5aa55c9 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java @@ -166,21 +166,19 @@ public void run() { toBuffer(value, metadata.dataType()); } Map nativeOutputs = new HashMap(); - ByteBuffer[] outputBuffers = - new ByteBuffer[interpreter.getOutputTensorCount()]; - for (int i = 0; i < outputBuffers.length; i++) { - org.tensorflow.lite.Tensor metadata = interpreter.getOutputTensor(i); - ByteBuffer buffer = ByteBuffer.allocateDirect(metadata.numBytes()) - .order(ByteOrder.nativeOrder()); - outputBuffers[i] = buffer; - nativeOutputs.put(Integer.valueOf(i), buffer); + int outputCount = interpreter.getOutputTensorCount(); + for (int i = 0; i < outputCount; i++) { + // A null destination asks LiteRT to retain the result + // in its tensor. This is required for outputs whose + // shape is resolved or grows during invocation. + nativeOutputs.put(Integer.valueOf(i), null); } interpreter.runForMultipleInputsOutputs(nativeInputs, nativeOutputs); - final Tensor[] result = new Tensor[outputBuffers.length]; + final Tensor[] result = new Tensor[outputCount]; for (int i = 0; i < result.length; i++) { org.tensorflow.lite.Tensor metadata = interpreter.getOutputTensor(i); result[i] = fromBuffer(metadata.name(), metadata.shape(), - metadata.dataType(), outputBuffers[i]); + metadata.dataType(), metadata.asReadOnlyBuffer()); } Display.getInstance().callSerially(new Runnable() { public void run() { diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java index 2d3425f877d..dc4f13d7045 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java @@ -52,6 +52,9 @@ void pixelInputNormalizesRotation() { VisionImage image = VisionImage.pixels(new byte[] {1, 2, 3, 4}, 1, 1, FrameFormat.RGBA8888, -90); assertEquals(270, image.getRotationDegrees()); + assertThrows(IllegalArgumentException.class, + () -> VisionImage.pixels(new byte[] {1, 2, 3, 4}, + 1, 1, FrameFormat.RGBA8888, 45)); } @Test @@ -75,14 +78,20 @@ void analyzerForwardsFeatureBackendAndOptions() { implementation.setVisionImpl(backend); VisionOptions options = new VisionOptions() .backend(VisionBackends.mlKitBarcodeScanning()) - .minimumConfidence(.6f); + .minimumConfidence(.6f) + .maximumResults(3); TextRecognizer recognizer = new TextRecognizer(options); + options.backend(VisionBackends.auto()) + .minimumConfidence(.1f) + .maximumResults(1); TextRecognitionResult result = await(recognizer.process( VisionImage.encoded(new byte[] {1}))); assertEquals("hello", result.getText()); assertEquals(VisionFeature.TEXT_RECOGNITION, backend.feature); assertEquals("ml-kit", backend.backend); - assertSame(options, backend.options); + assertNotSame(options, backend.options); + assertEquals(.6f, backend.options.getMinimumConfidence()); + assertEquals(3, backend.options.getMaximumResults()); recognizer.close(); assertEquals(1, backend.closeCount); } diff --git a/maven/core-unittests/src/test/java/com/codename1/camera/VisionPipelineTest.java b/maven/core-unittests/src/test/java/com/codename1/camera/VisionPipelineTest.java new file mode 100644 index 00000000000..66dcc432000 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/camera/VisionPipelineTest.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.camera; + +import com.codename1.ai.vision.VisionAnalyzer; +import com.codename1.ai.vision.VisionImage; +import com.codename1.ai.vision.VisionPipeline; +import com.codename1.ai.vision.VisionPipelineListener; +import com.codename1.junit.UITestBase; +import com.codename1.util.AsyncResource; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +class VisionPipelineTest extends UITestBase { + private RecordingCameraImpl implementationBackend; + private CameraSession session; + private VisionPipeline pipeline; + + @BeforeEach + void openSession() { + implementationBackend = new RecordingCameraImpl(); + implementation.setCameraImpl(implementationBackend); + session = Camera.open(new CameraInfo("back", CameraFacing.BACK, + null, null, true, true), new CameraSessionOptions()); + } + + @AfterEach + void closeSession() { + if (pipeline != null) { + pipeline.close(); + } + if (session != null) { + session.close(); + } + } + + @Test + void listenerFailureDoesNotStrandPendingFrame() { + final RecordingAnalyzer analyzer = new RecordingAnalyzer(); + final int[] notifications = new int[1]; + pipeline = new VisionPipeline(session, analyzer, + new VisionPipelineListener() { + public void result(String value, VisionImage image) { + notifications[0]++; + if (notifications[0] == 1) { + throw new IllegalStateException( + "Application listener failed"); + } + } + + public void error(Throwable error) { + } + }); + + implementationBackend.lastFrameListener.onFrame(frame(1)); + implementationBackend.lastFrameListener.onFrame(frame(2)); + assertEquals(1, analyzer.operations.size()); + + analyzer.operations.get(0).complete("first"); + try { + flushSerialCalls(); + } catch (IllegalStateException expected) { + // The application exception may propagate from the EDT harness, + // but the pending operation must already have started. + } + assertEquals(2, analyzer.operations.size()); + + analyzer.operations.get(1).complete("second"); + flushSerialCalls(); + assertEquals(2, notifications[0]); + assertFalse(pipeline.isBusy()); + } + + private static CameraFrame frame(int value) { + return new CameraFrame(new byte[] {(byte) value}, null, + 1, 1, 0, value, FrameFormat.JPEG); + } + + private static final class RecordingAnalyzer + implements VisionAnalyzer { + final List> operations = + new ArrayList>(); + + public boolean isSupported() { + return true; + } + + public AsyncResource process(VisionImage image) { + AsyncResource operation = new AsyncResource(); + operations.add(operation); + return operation; + } + + public void close() { + } + } +} From 5cdca140cc45fd9b632ac599e23e15c01fd180b2 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 03:15:50 +0300 Subject: [PATCH 28/80] Fix Apple Vision barcode detection in simulators --- Ports/iOSPort/nativeSources/CN1Vision.m | 9 +++++++++ docs/ai-on-device-architecture.md | 11 +++++++++-- docs/developer-guide/Ai-And-Speech.asciidoc | 6 ++++++ .../main/resources/skill/references/ai-and-speech.md | 5 +++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/Ports/iOSPort/nativeSources/CN1Vision.m b/Ports/iOSPort/nativeSources/CN1Vision.m index 4c3b60e3714..602d28395bd 100644 --- a/Ports/iOSPort/nativeSources/CN1Vision.m +++ b/Ports/iOSPort/nativeSources/CN1Vision.m @@ -539,6 +539,15 @@ static CGImagePropertyOrientation cn1CGOrientation(int rotation) { } } else if (feature == 1) { VNDetectBarcodesRequest *request = [[VNDetectBarcodesRequest alloc] init]; +#if TARGET_OS_SIMULATOR + /* + * Recent iOS simulator runtimes can successfully perform the default + * barcode request while returning no observations for valid QR + * images. Revision 1 avoids that simulator-only Vision regression. + * Devices retain the latest revision and its additional symbologies. + */ + request.revision = VNDetectBarcodesRequestRevision1; +#endif if (![handler performRequests:@[request] error:&error]) { return cn1VisionError(error); } diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md index 75efc6d0261..9888f828a64 100644 --- a/docs/ai-on-device-architecture.md +++ b/docs/ai-on-device-architecture.md @@ -122,6 +122,11 @@ The iOS NEON implementation reports SIMD as unsupported in that x86_64 configuration and aliases its native entry points to the generic scalar implementation; device and arm64-simulator builds retain the NEON path. +Apple Vision barcode requests use revision 1 only when compiling for an iOS +simulator. Recent simulator runtimes can otherwise complete the default +request without returning observations for valid QR images. Physical devices +retain the current OS revision and its additional symbologies. + ## Image and camera contract `VisionImage` owns defensive copies of its input. It accepts encoded JPEG or @@ -158,8 +163,10 @@ the EDT. `scripts/hellocodenameone` registers three non-screenshot conformance tests: - `VisionOnDeviceApiTest` covers `VisionImage.fromCameraFrame()` ownership, - option normalization, capability queries for every analyzer, and close - semantics. + option normalization, capability queries for every analyzer, close + semantics, and—when a native barcode backend is available—a deterministic + QR decode through image marshalling, native detection, JSON parsing, format + normalization, and corner geometry. - `LanguageOnDeviceApiTest` covers language value/options contracts, capability queries for identification, translation, and smart reply, and immediate unsupported resources. diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index 569e7f8bcd5..1f01beab6bb 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -288,6 +288,12 @@ and Core Image. iOS can select ML Kit explicitly: include::../demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java[tag=ai-and-speech-on-device-vision,indent=0] ---- +On an iOS simulator, the Apple barcode backend uses Vision barcode +request revision 1. Newer simulator runtimes can otherwise complete a +barcode request without reporting valid QR codes. Physical devices use +the latest revision supplied by the OS. Test newer symbologies such as +Micro QR on a device, or select the matching ML Kit backend. + Create and reuse an analyzer when processing a sequence. Call `close()` when the analyzer is no longer needed. Before showing a feature, call its `isSupported()` method: availability can depend on the target, diff --git a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md index b90c267a70b..c0829365b95 100644 --- a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md +++ b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md @@ -330,6 +330,11 @@ Use `ModelCache.fetch(httpsUrl, cacheKey, sha256)` for models too large to bundle. iOS and Mac native default to Apple Vision; iOS also supports explicit ML Kit selection. Android uses ML Kit. +The Apple barcode backend uses Vision request revision 1 in the iOS +simulator because newer simulator runtimes can return no observations +for valid QR images. Devices use the latest OS revision. Validate newer +Apple symbologies on a device, or select the ML Kit barcode backend. + Dependency selection is per entry point. Referencing `TextRecognizer` retains only the Android text adapter/artifact; it does not pull barcode, face, labeling, pose, or segmentation. `LanguageIdentifier`, From a352e8191b0f7cc5d842670d4ff9aa7139067fbd Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 03:25:39 +0300 Subject: [PATCH 29/80] Reject insecure model download redirects --- .../codename1/ai/inference/ModelCache.java | 44 ++++++++++++++++--- docs/ai-on-device-architecture.md | 9 ++-- docs/developer-guide/Ai-And-Speech.asciidoc | 11 ++--- .../ai/inference/InferenceApiTest.java | 35 +++++++++++++++ .../skill/references/ai-and-speech.md | 5 ++- 5 files changed, 88 insertions(+), 16 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java index 1795033f22c..0dea1095077 100644 --- a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java +++ b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java @@ -35,8 +35,9 @@ import java.io.InputStream; /// Downloads a large model into app-private storage and exposes it as a -/// file-backed {@link ModelSource}. Downloads require HTTPS, use a temporary -/// file, and are promoted atomically only after optional digest verification. +/// file-backed {@link ModelSource}. The initial request and every redirect +/// require HTTPS. Downloads use a temporary file and are promoted atomically +/// only after optional digest verification. /// /// Supply a SHA-256 digest for third-party or remotely mutable models. Without /// a digest HTTPS authenticates the connection but does not pin the executable @@ -53,13 +54,13 @@ private ModelCache() { /// resumed because the portable network layer cannot prove that a server's /// partial response still represents the pinned model. /// - /// @param url HTTPS URL of the model + /// @param url HTTPS URL of the model; every redirect must also use HTTPS /// @param cacheKey stable cache name, independent of the URL /// @param sha256 optional lowercase or uppercase SHA-256 hex digest /// @return asynchronous cached model source public static AsyncResource fetch( final String url, final String cacheKey, final String sha256) { - if (url == null || !url.startsWith("https://")) { + if (!isHttpsUrl(url)) { throw new IllegalArgumentException("Model URL must use HTTPS"); } if (cacheKey == null || cacheKey.length() == 0) { @@ -119,7 +120,8 @@ private static void download(final Completion completion, final FileSystemStorage fs = FileSystemStorage.getInstance(); final String temporary = target + ".download"; prepareTemporary(fs, temporary); - final ConnectionRequest request = new ConnectionRequest(); + final ConnectionRequest request = new ModelDownloadRequest( + completion, fs, temporary); request.setPost(false); request.setFailSilently(true); request.setReadResponseForErrors(false); @@ -233,6 +235,38 @@ private static boolean isSha256(String value) { return true; } + static boolean isHttpsUrl(String url) { + return url != null && url.regionMatches(true, 0, + "https://", 0, "https://".length()); + } + + static final class ModelDownloadRequest extends ConnectionRequest { + private final Completion completion; + private final FileSystemStorage fs; + private final String temporary; + + ModelDownloadRequest(Completion completion, + FileSystemStorage fs, String temporary) { + this.completion = completion; + this.fs = fs; + this.temporary = temporary; + } + + @Override + public boolean onRedirect(String url) { + if (isHttpsUrl(url)) { + return false; + } + setKilled(true); + if (fs.exists(temporary)) { + fs.delete(temporary); + } + completion.fail(new IOException( + "Model download redirect must use HTTPS")); + return true; + } + } + static final class Completion { private final AsyncResource resource; private boolean done; diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md index 9888f828a64..3f20ccab7ee 100644 --- a/docs/ai-on-device-architecture.md +++ b/docs/ai-on-device-architecture.md @@ -149,10 +149,11 @@ actual backend id without exposing platform classes. `InferenceSession` supports named typed tensors, multiple inputs and outputs, input resizing, and reusable native sessions. Model sources are bytes, -resources, private files, or the HTTPS-only `ModelCache`. File sources are -opened by path without copying the model through the Java heap. The cache can -verify a SHA-256 digest before publishing a downloaded model; callers should -always supply the digest for mutable or third-party downloads. +resources, private files, or the HTTPS-only `ModelCache`. The cache rejects +redirects that downgrade a request to HTTP. File sources are opened by path +without copying the model through the Java heap. The cache can verify a +SHA-256 digest before publishing a downloaded model; callers should always +supply the digest for mutable or third-party downloads. All native sessions and analyzers must be closed. Expensive open, analysis, and inference work runs off the EDT; completion and error delivery return to diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index 1f01beab6bb..cecdb544294 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -377,11 +377,12 @@ the port boundary. Always close the session. Bundle small models as resources. For larger models use `ModelCache.fetch(url, cacheKey, sha256)`, which downloads over HTTPS, verifies the supplied SHA-256 digest, and reuses the app-private cached -file. The URL must use HTTPS. Supply the digest for every third-party or -remotely mutable model: an omitted digest provides transport security but -doesn't pin the executable model payload. Interrupted downloads restart -through a temporary `.download` file and are never promoted to the cache -until verification succeeds. +file. The initial URL and every redirect must use HTTPS; a redirect to +HTTP fails the resource and discards the temporary download. Supply the +digest for every third-party or remotely mutable model: an omitted digest +provides transport security but doesn't pin the executable model payload. +Interrupted downloads restart through a temporary `.download` file and are +never promoted to the cache until verification succeeds. === On-device language services diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index d6573f04063..503eb20cdf7 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -76,6 +76,41 @@ void modelCacheRejectsInsecureAndMalformedRequests() { "gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg")); } + @Test + void modelCacheRejectsInsecureRedirects() throws Exception { + FileSystemStorage fs = FileSystemStorage.getInstance(); + String temporary = fs.getAppHomePath() + + "model-cache-redirect-test.download"; + write(temporary, new byte[] {1, 2, 3}); + AsyncResource resource = new AsyncResource(); + AtomicReference error = new AtomicReference(); + resource.except(new SuccessCallback() { + @Override + public void onSucess(Throwable value) { + error.set(value); + } + }); + ModelCache.ModelDownloadRequest request = + new ModelCache.ModelDownloadRequest( + new ModelCache.Completion(resource), + fs, temporary); + + assertFalse(request.onRedirect( + "https://cdn.example.com/model.tflite")); + assertTrue(request.onRedirect( + "http://cdn.example.com/model.tflite")); + flushSerialCalls(); + assertFalse(fs.exists(temporary), + "a downgrade redirect must discard partial model data"); + assertNotNull(error.get(), + "a downgrade redirect must fail the model resource"); + assertTrue(error.get().getCause().getMessage().contains("HTTPS")); + assertTrue(ModelCache.isHttpsUrl( + "HTTPS://cdn.example.com/model.tflite")); + assertFalse(ModelCache.isHttpsUrl( + "http://cdn.example.com/model.tflite")); + } + @Test void modelCacheNamesDoNotAliasSanitizedKeys() { assertNotEquals(ModelCache.safeName("model/v1"), diff --git a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md index c0829365b95..390886c96d4 100644 --- a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md +++ b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md @@ -327,8 +327,9 @@ Translator.translate("Bonjour", "fr", "en", new LanguageOptions()) `VisionPipeline` to connect a reusable analyzer to camera frames; it copies callback-owned data and drops stale pending frames under load. Use `ModelCache.fetch(httpsUrl, cacheKey, sha256)` for models too large -to bundle. iOS and Mac native default to Apple Vision; iOS also supports -explicit ML Kit selection. Android uses ML Kit. +to bundle. The initial model URL and every redirect must remain HTTPS. +iOS and Mac native default to Apple Vision; iOS also supports explicit +ML Kit selection. Android uses ML Kit. The Apple barcode backend uses Vision request revision 1 in the iOS simulator because newer simulator runtimes can return no observations From d9bd7ea7360b22a822aee7c66d62cda97a3e7644 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 03:32:36 +0300 Subject: [PATCH 30/80] Resolve remaining AI value and cache races --- .../src/com/codename1/ai/Embedding.java | 20 +++- .../src/com/codename1/ai/ImagePart.java | 28 +++-- .../codename1/ai/inference/ModelCache.java | 102 +++++++++++++++++- docs/ai-on-device-architecture.md | 10 +- docs/developer-guide/Ai-And-Speech.asciidoc | 5 +- .../codename1/ai/ChatRequestBuilderTest.java | 21 ++++ .../ai/inference/InferenceApiTest.java | 37 +++++++ .../skill/references/ai-and-speech.md | 7 +- 8 files changed, 203 insertions(+), 27 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/Embedding.java b/CodenameOne/src/com/codename1/ai/Embedding.java index b3f5d245f56..f1349aa6f3b 100644 --- a/CodenameOne/src/com/codename1/ai/Embedding.java +++ b/CodenameOne/src/com/codename1/ai/Embedding.java @@ -22,25 +22,26 @@ */ package com.codename1.ai; -/// A single embedding vector. `index` matches the position of the -/// corresponding input string in the original request. /// One vector returned by an embedding provider. The vector is defensively /// copied on construction and access so callers cannot mutate stored results. +/// {@link #getIndex()} identifies the corresponding input position in a +/// batched {@link EmbeddingRequest}. public final class Embedding { private final float[] vector; private final int index; /// Creates an embedding result. - /// @param vector numeric embedding coordinates + /// @param vector numeric embedding coordinates, defensively copied; a + /// {@code null} value creates an empty vector /// @param index zero-based position of the corresponding request input public Embedding(float[] vector, int index) { - this.vector = vector == null ? new float[0] : vector; + this.vector = copy(vector); this.index = index; } /// @return a defensive copy of the embedding coordinates public float[] getVector() { - return vector; + return copy(vector); } /// @return zero-based position of this item in the request @@ -52,4 +53,13 @@ public int getIndex() { public int getDimensions() { return vector.length; } + + private static float[] copy(float[] value) { + if (value == null) { + return new float[0]; + } + float[] result = new float[value.length]; + System.arraycopy(value, 0, result, 0, value.length); + return result; + } } diff --git a/CodenameOne/src/com/codename1/ai/ImagePart.java b/CodenameOne/src/com/codename1/ai/ImagePart.java index 18f8e546237..dc50680ed0a 100644 --- a/CodenameOne/src/com/codename1/ai/ImagePart.java +++ b/CodenameOne/src/com/codename1/ai/ImagePart.java @@ -22,27 +22,26 @@ */ package com.codename1.ai; -/// An image attachment for a multi-modal [ChatMessage]. Construct from -/// raw bytes (the provider encodes them as base64 inline data) or from -/// a publicly-reachable URL -- both modes are accepted by OpenAI, -/// Anthropic, and Gemini. /// Image content within a multimodal {@link ChatMessage}. An image is either -/// inline encoded bytes with a MIME type or a provider-accessible URL. +/// inline encoded bytes with a MIME type or a provider-accessible URL. Inline +/// bytes are copied on construction and access so later caller mutations +/// cannot change a request that already contains this part. public final class ImagePart extends MessagePart { private final byte[] data; private final String mimeType; private final String url; - /// Inline image bytes. `mimeType` must be set (e.g. `"image/png"`, - /// `"image/jpeg"`); the providers reject inline images without it. - /// Creates an inline image part. + /// Creates an inline image part. The media type must describe the encoded + /// bytes, for example {@code image/png} or {@code image/jpeg}; providers + /// reject inline images without a type. + /// /// @param data encoded image bytes, defensively copied /// @param mimeType media type such as {@code image/png} public ImagePart(byte[] data, String mimeType) { if (data == null || mimeType == null) { throw new IllegalArgumentException("data and mimeType are required"); } - this.data = data; + this.data = copy(data); this.mimeType = mimeType; this.url = null; } @@ -61,7 +60,7 @@ public ImagePart(String url) { /// @return a defensive copy of inline bytes, or {@code null} for a URL image public byte[] getData() { - return data; + return copy(data); } /// @return MIME type of inline bytes, or {@code null} for a URL image @@ -78,4 +77,13 @@ public String getUrl() { public boolean isUrl() { return url != null; } + + private static byte[] copy(byte[] value) { + if (value == null) { + return null; + } + byte[] result = new byte[value.length]; + System.arraycopy(value, 0, result, 0, value.length); + return result; + } } diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java index 0dea1095077..c29c1df4cbe 100644 --- a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java +++ b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.io.InputStream; +import java.util.Hashtable; /// Downloads a large model into app-private storage and exposes it as a /// file-backed {@link ModelSource}. The initial request and every redirect @@ -45,6 +46,8 @@ /// {@link ModelSource#resource(String)}. public final class ModelCache { private static final int MAX_READABLE_CACHE_KEY_LENGTH = 160; + private static final Hashtable ACTIVE_FETCHES = + new Hashtable(); private ModelCache() { } @@ -53,6 +56,9 @@ private ModelCache() { /// A stale {@code .download} file is deleted and restarted rather than /// resumed because the portable network layer cannot prove that a server's /// partial response still represents the pinned model. + /// Concurrent requests for the same cache entry and content identity share + /// one operation. A different URL or digest using that cache key while the + /// first operation is active fails instead of racing on the temporary file. /// /// @param url HTTPS URL of the model; every redirect must also use HTTPS /// @param cacheKey stable cache name, independent of the URL @@ -70,16 +76,20 @@ public static AsyncResource fetch( throw new IllegalArgumentException("SHA-256 must contain 64 hex characters"); } - final AsyncResource out = new AsyncResource(); - final Completion completion = - new Completion(out); + final String fileName = safeName(cacheKey) + ".tflite"; + final FetchRegistration registration = + registerFetch(fileName, url, sha256); + if (!registration.owner) { + return registration.resource; + } + final AsyncResource out = registration.resource; + final Completion completion = registration.completion; Display.getInstance().scheduleBackgroundTask(new Runnable() { @Override public void run() { final FileSystemStorage fs = FileSystemStorage.getInstance(); final String directory = fs.getAppHomePath() + "ai-models/"; fs.mkdir(directory); - final String fileName = safeName(cacheKey) + ".tflite"; final String target = directory + fileName; try { if (fs.exists(target) && verify(target, sha256)) { @@ -106,6 +116,8 @@ public void run() { /// Fetches a model without content pinning. Prefer the three-argument /// overload for any model that is not versioned by the app itself. + /// Concurrent unpinned calls share an operation only when their URL and + /// cache key are identical. /// /// @param url HTTPS model URL /// @param cacheKey stable cache name @@ -240,6 +252,80 @@ static boolean isHttpsUrl(String url) { "https://", 0, "https://".length()); } + static FetchRegistration registerFetch( + String fileName, String url, String sha256) { + synchronized (ACTIVE_FETCHES) { + ActiveFetch active = ACTIVE_FETCHES.get(fileName); + if (active != null) { + if (active.matches(url, sha256)) { + return new FetchRegistration( + active.resource, null, false); + } + AsyncResource conflict = + new AsyncResource(); + new Completion(conflict).fail( + new IOException("A different model download is " + + "already using cache key " + fileName)); + return new FetchRegistration(conflict, null, false); + } + AsyncResource resource = + new AsyncResource(); + Completion completion = + new Completion(resource, fileName); + ACTIVE_FETCHES.put(fileName, + new ActiveFetch(url, sha256, resource)); + return new FetchRegistration(resource, completion, true); + } + } + + private static void releaseFetch( + String fileName, AsyncResource resource) { + if (fileName == null) { + return; + } + synchronized (ACTIVE_FETCHES) { + ActiveFetch active = ACTIVE_FETCHES.get(fileName); + if (active != null && active.resource == resource) { + ACTIVE_FETCHES.remove(fileName); + } + } + } + + static final class ActiveFetch { + private final String url; + private final String sha256; + private final AsyncResource resource; + + ActiveFetch(String url, String sha256, + AsyncResource resource) { + this.url = url; + this.sha256 = sha256; + this.resource = resource; + } + + boolean matches(String otherUrl, String otherSha256) { + if (sha256 != null || otherSha256 != null) { + return sha256 != null && otherSha256 != null + && sha256.equalsIgnoreCase(otherSha256); + } + return url.equals(otherUrl); + } + } + + static final class FetchRegistration { + final AsyncResource resource; + final Completion completion; + final boolean owner; + + FetchRegistration(AsyncResource resource, + Completion completion, + boolean owner) { + this.resource = resource; + this.completion = completion; + this.owner = owner; + } + } + static final class ModelDownloadRequest extends ConnectionRequest { private final Completion completion; private final FileSystemStorage fs; @@ -269,10 +355,16 @@ public boolean onRedirect(String url) { static final class Completion { private final AsyncResource resource; + private final String activeFileName; private boolean done; Completion(AsyncResource resource) { + this(resource, null); + } + + Completion(AsyncResource resource, String activeFileName) { this.resource = resource; + this.activeFileName = activeFileName; } synchronized void complete(final T value) { @@ -280,6 +372,7 @@ synchronized void complete(final T value) { return; } done = true; + releaseFetch(activeFileName, resource); Display.getInstance().callSerially(new Runnable() { @Override public void run() { @@ -293,6 +386,7 @@ synchronized void fail(final Throwable error) { return; } done = true; + releaseFetch(activeFileName, resource); Display.getInstance().callSerially(new Runnable() { @Override public void run() { diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md index 3f20ccab7ee..3fe8b136a82 100644 --- a/docs/ai-on-device-architecture.md +++ b/docs/ai-on-device-architecture.md @@ -150,10 +150,12 @@ actual backend id without exposing platform classes. `InferenceSession` supports named typed tensors, multiple inputs and outputs, input resizing, and reusable native sessions. Model sources are bytes, resources, private files, or the HTTPS-only `ModelCache`. The cache rejects -redirects that downgrade a request to HTTP. File sources are opened by path -without copying the model through the Java heap. The cache can verify a -SHA-256 digest before publishing a downloaded model; callers should always -supply the digest for mutable or third-party downloads. +redirects that downgrade a request to HTTP. Identical concurrent fetches for +one cache entry coalesce, while conflicting in-flight content identities fail +instead of sharing a temporary path. File sources are opened by path without +copying the model through the Java heap. The cache can verify a SHA-256 digest +before publishing a downloaded model; callers should always supply the digest +for mutable or third-party downloads. All native sessions and analyzers must be closed. Expensive open, analysis, and inference work runs off the EDT; completion and error delivery return to diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index cecdb544294..12178cfebb3 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -382,7 +382,10 @@ HTTP fails the resource and discards the temporary download. Supply the digest for every third-party or remotely mutable model: an omitted digest provides transport security but doesn't pin the executable model payload. Interrupted downloads restart through a temporary `.download` file and are -never promoted to the cache until verification succeeds. +never promoted to the cache until verification succeeds. Concurrent calls for +the same cache key and content identity share one download. A conflicting URL +or digest for a cache key that is already downloading fails rather than +writing to the same temporary file. === On-device language services diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/ChatRequestBuilderTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/ChatRequestBuilderTest.java index 2a17bfade45..cebf1f5a5c8 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/ChatRequestBuilderTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/ChatRequestBuilderTest.java @@ -109,4 +109,25 @@ void chatMessageGetTextConcatenatesTextParts() { // ChatView can render a stripped-down preview safely. assertEquals("hello\nworld", m.getText()); } + + @Test + void binaryAiValuesAreDefensivelyCopied() { + byte[] imageBytes = new byte[] {1, 2, 3}; + ImagePart image = new ImagePart(imageBytes, "image/png"); + imageBytes[0] = 9; + assertArrayEquals(new byte[] {1, 2, 3}, image.getData()); + byte[] returnedImage = image.getData(); + returnedImage[1] = 9; + assertArrayEquals(new byte[] {1, 2, 3}, image.getData()); + + float[] coordinates = new float[] {0.25f, 0.75f}; + Embedding embedding = new Embedding(coordinates, 0); + coordinates[0] = 9; + assertArrayEquals(new float[] {0.25f, 0.75f}, + embedding.getVector()); + float[] returnedVector = embedding.getVector(); + returnedVector[1] = 9; + assertArrayEquals(new float[] {0.25f, 0.75f}, + embedding.getVector()); + } } diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index 503eb20cdf7..d4542b1c820 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -111,6 +111,43 @@ public void onSucess(Throwable value) { "http://cdn.example.com/model.tflite")); } + @Test + void modelCacheCoalescesIdenticalConcurrentFetches() { + String fileName = "model-cache-coalescing-test.tflite"; + ModelCache.FetchRegistration first = ModelCache.registerFetch( + fileName, "https://one.example/model.tflite", null); + ModelCache.FetchRegistration duplicate = ModelCache.registerFetch( + fileName, "https://one.example/model.tflite", null); + assertTrue(first.owner); + assertFalse(duplicate.owner); + assertSame(first.resource, duplicate.resource); + + ModelCache.FetchRegistration conflict = ModelCache.registerFetch( + fileName, "https://two.example/model.tflite", null); + AtomicReference conflictError = + new AtomicReference(); + conflict.resource.except(new SuccessCallback() { + @Override + public void onSucess(Throwable value) { + conflictError.set(value); + } + }); + flushSerialCalls(); + assertNotNull(conflictError.get(), + "conflicting content for one cache key must fail"); + + first.completion.complete(ModelSource.file("test-model.tflite")); + flushSerialCalls(); + ModelCache.FetchRegistration afterCompletion = + ModelCache.registerFetch(fileName, + "https://two.example/model.tflite", null); + assertTrue(afterCompletion.owner, + "completion must release the cache-key download slot"); + afterCompletion.completion.complete( + ModelSource.file("test-model-2.tflite")); + flushSerialCalls(); + } + @Test void modelCacheNamesDoNotAliasSanitizedKeys() { assertNotEquals(ModelCache.safeName("model/v1"), diff --git a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md index 390886c96d4..a2fc55aef39 100644 --- a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md +++ b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md @@ -327,9 +327,10 @@ Translator.translate("Bonjour", "fr", "en", new LanguageOptions()) `VisionPipeline` to connect a reusable analyzer to camera frames; it copies callback-owned data and drops stale pending frames under load. Use `ModelCache.fetch(httpsUrl, cacheKey, sha256)` for models too large -to bundle. The initial model URL and every redirect must remain HTTPS. -iOS and Mac native default to Apple Vision; iOS also supports explicit -ML Kit selection. Android uses ML Kit. +to bundle. The initial model URL and every redirect must remain HTTPS; +identical concurrent fetches coalesce instead of sharing a temporary +file unsafely. iOS and Mac native default to Apple Vision; iOS also +supports explicit ML Kit selection. Android uses ML Kit. The Apple barcode backend uses Vision request revision 1 in the iOS simulator because newer simulator runtimes can return no observations From 90b25716cd56a818334cd747e4ae4402ad0c9017 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 03:37:22 +0300 Subject: [PATCH 31/80] Validate raw vision buffers before allocation --- Ports/iOSPort/nativeSources/CN1Vision.m | 55 +++++++++++++++++-------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/Ports/iOSPort/nativeSources/CN1Vision.m b/Ports/iOSPort/nativeSources/CN1Vision.m index 602d28395bd..f07816adfa6 100644 --- a/Ports/iOSPort/nativeSources/CN1Vision.m +++ b/Ports/iOSPort/nativeSources/CN1Vision.m @@ -809,32 +809,52 @@ static CGImageRef cn1VisionCreateRawImage(NSData *data, int width, int height, if (width <= 0 || height <= 0) { return NULL; } - NSUInteger pixelCount = (NSUInteger)width * (NSUInteger)height; - const uint8_t *source = data.bytes; - NSMutableData *rgba = [NSMutableData dataWithLength:pixelCount * 4]; - uint8_t *dest = rgba.mutableBytes; + if (format != 1 && format != 2) { + return NULL; + } + if (format == 1 && ((width & 1) != 0 || (height & 1) != 0)) { + return NULL; + } + NSUInteger imageWidth = (NSUInteger)width; + NSUInteger imageHeight = (NSUInteger)height; + if (imageWidth > NSUIntegerMax / imageHeight) { + return NULL; + } + NSUInteger pixelCount = imageWidth * imageHeight; + NSUInteger requiredLength; if (format == 2) { - if (data.length < pixelCount * 4) { - return NULL; - } - memcpy(dest, source, pixelCount * 4); - } else if (format == 1) { - if ((width & 1) != 0 || (height & 1) != 0) { + if (pixelCount > NSUIntegerMax / 4) { return NULL; } - if (data.length < pixelCount + pixelCount / 2) { + requiredLength = pixelCount * 4; + } else { + if (pixelCount > NSUIntegerMax - pixelCount / 2) { return NULL; } + requiredLength = pixelCount + pixelCount / 2; + } + if (data.length < requiredLength) { + return NULL; + } + const uint8_t *source = data.bytes; + NSMutableData *rgba = [NSMutableData dataWithLength:pixelCount * 4]; + uint8_t *dest = rgba.mutableBytes; + if (format == 2) { + memcpy(dest, source, pixelCount * 4); + } else { for (int y = 0; y < height; y++) { - int uvRow = width * height + (y >> 1) * width; + NSUInteger uvRow = pixelCount + + (NSUInteger)(y >> 1) * imageWidth; for (int x = 0; x < width; x++) { - int yy = source[y * width + x] & 255; - int uv = uvRow + (x & ~1); + int yy = source[(NSUInteger)y * imageWidth + + (NSUInteger)x] & 255; + NSUInteger uv = uvRow + (NSUInteger)(x & ~1); int v = (source[uv] & 255) - 128; int u = (source[uv + 1] & 255) - 128; int c = yy - 16; if (c < 0) c = 0; - NSUInteger p = ((NSUInteger)y * width + x) * 4; + NSUInteger p = ((NSUInteger)y * imageWidth + + (NSUInteger)x) * 4; dest[p] = cn1VisionClamp((298 * c + 409 * v + 128) >> 8); dest[p + 1] = cn1VisionClamp( (298 * c - 100 * u - 208 * v + 128) >> 8); @@ -843,13 +863,12 @@ static CGImageRef cn1VisionCreateRawImage(NSData *data, int width, int height, dest[p + 3] = 255; } } - } else { - return NULL; } CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); CGDataProviderRef provider = CGDataProviderCreateWithCFData( (CFDataRef)rgba); - CGImageRef cgImage = CGImageCreate(width, height, 8, 32, width * 4, + CGImageRef cgImage = CGImageCreate(imageWidth, imageHeight, 8, 32, + imageWidth * 4, colorSpace, kCGBitmapByteOrderDefault | kCGImageAlphaLast, provider, NULL, false, kCGRenderingIntentDefault); CGDataProviderRelease(provider); From fa9dce5b7e019b60016b3777c7c738b216786c62 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 03:46:51 +0300 Subject: [PATCH 32/80] Preserve inference inputs and label indices --- .../ai/inference/InferenceSession.java | 12 ++++++-- .../com/codename1/ai/vision/ImageLabel.java | 8 ++++- Ports/iOSPort/nativeSources/CN1Vision.m | 6 ++-- .../com/codename1/impl/ios/IOSVisionImpl.java | 3 +- .../ai/inference/InferenceApiTest.java | 29 +++++++++++++++++++ 5 files changed, 51 insertions(+), 7 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java index 39014670317..15bacd457f0 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -113,9 +113,13 @@ public TensorInfo[] getOutputs() { /// are matched by position. Calling {@link #close()} while this operation /// is pending prevents new work immediately but defers native release until /// the returned resource succeeds or fails. A session accepts one run at a - /// time because the underlying native interpreter is mutable. + /// time because the underlying native interpreter is mutable. The array + /// container is defensively copied before it is handed to the asynchronous + /// backend, so replacing an element after this method returns cannot change + /// the pending invocation. Each {@link Tensor} is itself immutable. /// - /// @param inputs one tensor for each model input + /// @param inputs one tensor for each model input; {@code null} is treated + /// as an empty input array /// @return asynchronous output tensors in model output order /// @throws IllegalStateException if the session is closed or already running public AsyncResource run(Tensor[] inputs) { @@ -128,8 +132,10 @@ public AsyncResource run(Tensor[] inputs) { } activeRuns++; try { + Tensor[] inputSnapshot = inputs == null + ? new Tensor[0] : inputs.clone(); result = implementation.run(handle, - inputs == null ? new Tensor[0] : inputs); + inputSnapshot); if (result == null) { throw new InferenceException( "Inference backend returned no asynchronous result"); diff --git a/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java b/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java index a3e63a7bc19..1a7c004067a 100644 --- a/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java +++ b/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java @@ -63,7 +63,13 @@ public float getConfidence() { return confidence; } - /// @return backend/model class index; not portable across models + /// Returns the backend/model class index when the selected classifier + /// exposes one. The index identifies a class in that specific model; it is + /// not portable across models or backends. Apple Vision does not expose a + /// numeric class index and therefore returns {@code -1}. Prefer + /// {@link #getText()} when writing backend-independent application logic. + /// + /// @return backend/model class index, or {@code -1} when unavailable public int getIndex() { return index; } diff --git a/Ports/iOSPort/nativeSources/CN1Vision.m b/Ports/iOSPort/nativeSources/CN1Vision.m index f07816adfa6..1785206b821 100644 --- a/Ports/iOSPort/nativeSources/CN1Vision.m +++ b/Ports/iOSPort/nativeSources/CN1Vision.m @@ -374,7 +374,8 @@ static UIImageOrientation cn1UIImageOrientation(int rotation) { for (MLKImageLabel *label in result ?: @[]) { [items addObject:@{ @"text": label.text ?: @"", - @"confidence": @(label.confidence) + @"confidence": @(label.confidence), + @"index": @(label.index) }]; } return cn1VisionJSON(@{@"items": items}); @@ -621,7 +622,8 @@ static CGImagePropertyOrientation cn1CGOrientation(int rotation) { for (VNClassificationObservation *observation in request.results) { [items addObject:@{ @"text": observation.identifier ?: @"", - @"confidence": @(observation.confidence) + @"confidence": @(observation.confidence), + @"index": @(-1) }]; } return cn1VisionJSON(@{@"items": items}); diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java index 5b59ecdaefd..4847070af41 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java @@ -201,7 +201,8 @@ private static T parse(VisionFeature feature, String json, for (int i = 0; i < values.length; i++) { Map value = (Map) items.get(i); values[i] = new ImageLabel(string(value, "text"), - number(value, "confidence"), i, metadata); + number(value, "confidence"), + integer(value, "index", -1), metadata); } return (T) values; } diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index d4542b1c820..195a0d97676 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -234,6 +234,33 @@ void sessionDefersCloseUntilPendingRunSettles() { assertThrows(IllegalStateException.class, session::getInputs); } + @Test + void sessionSnapshotsInputArrayBeforeAsyncInference() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + backend.pendingRun = new AsyncResource(); + implementation.setInferenceImpl(backend); + InferenceSession session = await(InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions())); + Tensor original = Tensor.floats("input", new int[] {1, 2}, + new float[] {1, 2}); + Tensor replacement = Tensor.floats("replacement", new int[] {1, 2}, + new float[] {9, 9}); + Tensor[] callerInputs = new Tensor[] {original}; + + AsyncResource run = session.run(callerInputs); + callerInputs[0] = replacement; + + assertNotSame(callerInputs, backend.receivedInputs); + assertSame(original, backend.receivedInputs[0], + "a pending backend must see the run() call-time inputs"); + backend.pendingRun.complete(new Tensor[] { + Tensor.floats("output", new int[] {1}, new float[] {3}) + }); + flushSerialCalls(); + assertTrue(run.isDone()); + session.close(); + } + private T await(AsyncResource resource) { final AtomicReference value = new AtomicReference(); resource.ready(new SuccessCallback() { @@ -252,6 +279,7 @@ private static final class RecordingInferenceImpl extends InferenceImpl { String resizedName; int closeCount; AsyncResource pendingRun; + Tensor[] receivedInputs; public boolean isSupported() { return true; @@ -277,6 +305,7 @@ public TensorInfo[] getOutputs(Object value) { } public AsyncResource run(Object value, Tensor[] inputs) { + receivedInputs = inputs; if (pendingRun != null) { return pendingRun; } From b51442cca99b7185866c3e2a11def6dc2731f112 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 03:58:47 +0300 Subject: [PATCH 33/80] Snapshot inference configuration and validate shapes --- .../ai/inference/InferenceOptions.java | 7 ++ .../ai/inference/InferenceSession.java | 68 ++++++++++++++++++- .../impl/android/ai/AndroidInferenceImpl.java | 20 ++++++ .../codename1/impl/ios/IOSInferenceImpl.java | 20 ++++++ .../ai/inference/InferenceApiTest.java | 43 ++++++++++++ 5 files changed, 157 insertions(+), 1 deletion(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java index 440a672b060..e183b81a733 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java @@ -81,4 +81,11 @@ public int getThreads() { public boolean isFallbackAllowed() { return allowFallback; } + + InferenceOptions snapshot() { + return new InferenceOptions() + .accelerator(accelerator) + .threads(threads) + .allowFallback(allowFallback); + } } diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java index 15bacd457f0..11abfee3262 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -55,6 +55,10 @@ public static boolean isSupported() { /// Opens and allocates a model session off the EDT. /// + /// The option values are copied before asynchronous backend work is + /// scheduled. Reusing or changing the supplied {@link InferenceOptions} + /// after this method returns therefore cannot alter the pending open. + /// /// @param source bytes, resource, or file containing a `.tflite` model /// @param options execution options; {@code null} uses defaults /// @return an asynchronous session, failed with {@link InferenceException} @@ -70,7 +74,9 @@ public static AsyncResource open(ModelSource source, out.error(new InferenceException("LiteRT inference is not supported")); return out; } - impl.open(source, options == null ? new InferenceOptions() : options) + InferenceOptions requested = options == null + ? new InferenceOptions() : options; + impl.open(source, requested.snapshot()) .ready(new SuccessCallback() { @Override public void onSucess(Object value) { @@ -122,6 +128,8 @@ public TensorInfo[] getOutputs() { /// as an empty input array /// @return asynchronous output tensors in model output order /// @throws IllegalStateException if the session is closed or already running + /// @throws IllegalArgumentException if an input name, count, or shape does + /// not match the model's current input metadata public AsyncResource run(Tensor[] inputs) { final AsyncResource result; synchronized (this) { @@ -134,6 +142,8 @@ public AsyncResource run(Tensor[] inputs) { try { Tensor[] inputSnapshot = inputs == null ? new Tensor[0] : inputs.clone(); + validateInputShapes(inputSnapshot, + implementation.getInputs(handle)); result = implementation.run(handle, inputSnapshot); if (result == null) { @@ -163,6 +173,62 @@ public void onSucess(Throwable error) { return result; } + private static void validateInputShapes(Tensor[] inputs, + TensorInfo[] metadata) { + if (metadata == null || inputs.length != metadata.length) { + throw new IllegalArgumentException("Expected " + + (metadata == null ? 0 : metadata.length) + + " input tensors but received " + inputs.length); + } + boolean[] resolved = new boolean[metadata.length]; + for (int i = 0; i < inputs.length; i++) { + Tensor tensor = inputs[i]; + if (tensor == null) { + throw new IllegalArgumentException( + "Input tensor " + i + " is null"); + } + int metadataPosition = i; + if (tensor.getName() != null) { + metadataPosition = -1; + for (int candidate = 0; candidate < metadata.length; + candidate++) { + if (tensor.getName().equals( + metadata[candidate].getName())) { + metadataPosition = candidate; + break; + } + } + if (metadataPosition < 0) { + throw new IllegalArgumentException( + "Unknown model input " + tensor.getName()); + } + } + TensorInfo expected = metadata[metadataPosition]; + if (resolved[metadataPosition]) { + throw new IllegalArgumentException("Model input " + + expected.getName() + " was supplied more than once"); + } + resolved[metadataPosition] = true; + if (!sameShape(tensor.getShape(), expected.getShape())) { + throw new IllegalArgumentException("Input " + + expected.getName() + " shape does not match the " + + "model metadata; call resizeInput() before run()"); + } + } + } + + private static boolean sameShape(int[] supplied, int[] expected) { + if (supplied.length != expected.length) { + return false; + } + for (int i = 0; i < supplied.length; i++) { + if (supplied[i] != expected[i]) { + return false; + } + } + return true; + } + /// Resizes an input and reallocates native tensors before the next run. /// /// This method throws while an asynchronous {@link #run(Tensor[])} is diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java index cf4c5aa55c9..9b6d562c7d8 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java @@ -162,6 +162,14 @@ public void run() { + " was supplied more than once"); } resolved[index] = true; + if (!sameShape(value.getShape(), + metadata.shape())) { + throw new IllegalArgumentException( + "Input " + metadata.name() + + " shape does not match the " + + "model metadata; call " + + "resizeInput() before run()"); + } nativeInputs[index] = toBuffer(value, metadata.dataType()); } @@ -213,6 +221,18 @@ private static Handle checked(Object value) { return (Handle) value; } + private static boolean sameShape(int[] supplied, int[] expected) { + if (supplied.length != expected.length) { + return false; + } + for (int i = 0; i < supplied.length; i++) { + if (supplied[i] != expected[i]) { + return false; + } + } + return true; + } + private static int inputIndex(Interpreter interpreter, String name) { for (int i = 0; i < interpreter.getInputTensorCount(); i++) { if (name == null || name.equals(interpreter.getInputTensor(i).name())) { diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java index e2055d3f3c7..2467d64164e 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java @@ -157,6 +157,14 @@ public void run() { + expected.getType() + " but received " + tensor.getType()); } + if (!sameShape(tensor.getShape(), + expected.getShape())) { + throw new IllegalArgumentException( + "Input " + expected.getName() + + " shape does not match the " + + "model metadata; call " + + "resizeInput() before run()"); + } } for (int i = 0; i < inputs.length; i++) { Tensor tensor = inputs[i]; @@ -377,6 +385,18 @@ private static int elementCount(int[] shape) { return out; } + private static boolean sameShape(int[] supplied, int[] expected) { + if (supplied.length != expected.length) { + return false; + } + for (int i = 0; i < supplied.length; i++) { + if (supplied[i] != expected[i]) { + return false; + } + } + return true; + } + private static int[] intArray(List values) { int[] out = new int[values.size()]; for (int i = 0; i < out.length; i++) { diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index 195a0d97676..77aaa65b940 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -207,6 +207,47 @@ void sessionLifecycleForwardsToBackend() { assertThrows(IllegalStateException.class, session::getInputs); } + @Test + void sessionSnapshotsOptionsBeforeAsyncOpen() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + implementation.setInferenceImpl(backend); + InferenceOptions options = new InferenceOptions() + .accelerator(InferenceOptions.Accelerator.GPU) + .threads(3) + .allowFallback(false); + + AsyncResource opening = InferenceSession.open( + ModelSource.bytes(new byte[] {1}), options); + options.accelerator(InferenceOptions.Accelerator.CPU) + .threads(8) + .allowFallback(true); + + assertNotSame(options, backend.openOptions); + assertEquals(InferenceOptions.Accelerator.GPU, + backend.openOptions.getAccelerator()); + assertEquals(3, backend.openOptions.getThreads()); + assertFalse(backend.openOptions.isFallbackAllowed()); + await(opening).close(); + } + + @Test + void sessionRejectsShapeMismatchBeforeBackendRun() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + implementation.setInferenceImpl(backend); + InferenceSession session = await(InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions())); + + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, () -> session.run( + new Tensor[] {Tensor.floats("input", + new int[] {2, 1}, + new float[] {1, 2})})); + assertTrue(error.getMessage().contains("resizeInput")); + assertNull(backend.receivedInputs, + "shape mismatch must not reach the asynchronous backend"); + session.close(); + } + @Test void sessionDefersCloseUntilPendingRunSettles() { RecordingInferenceImpl backend = new RecordingInferenceImpl(); @@ -280,12 +321,14 @@ private static final class RecordingInferenceImpl extends InferenceImpl { int closeCount; AsyncResource pendingRun; Tensor[] receivedInputs; + InferenceOptions openOptions; public boolean isSupported() { return true; } public AsyncResource open(ModelSource source, InferenceOptions options) { + openOptions = options; AsyncResource result = new AsyncResource(); result.complete(handle); return result; From b645b44af65951965b22e134aa7619bb8e7d5a42 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:08:23 +0300 Subject: [PATCH 34/80] Close cancelled sessions and snapshot language calls --- .../ai/inference/InferenceSession.java | 32 +++++++++++++++++-- .../ai/language/LanguageIdentifier.java | 7 ++-- .../ai/language/LanguageOptions.java | 6 ++++ .../com/codename1/ai/language/SmartReply.java | 8 +++-- .../com/codename1/ai/language/Translator.java | 5 ++- .../codename1/impl/ios/IOSLanguageImpl.java | 9 ++++-- .../ai/inference/InferenceApiTest.java | 23 +++++++++++++ .../ai/language/LanguageApiTest.java | 30 +++++++++++++++++ 8 files changed, 109 insertions(+), 11 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java index 11abfee3262..84fd1fefb45 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -58,6 +58,9 @@ public static boolean isSupported() { /// The option values are copied before asynchronous backend work is /// scheduled. Reusing or changing the supplied {@link InferenceOptions} /// after this method returns therefore cannot alter the pending open. + /// Canceling the returned resource prevents session publication; if the + /// native backend finishes opening afterward, its handle is closed + /// automatically. /// /// @param source bytes, resource, or file containing a `.tflite` model /// @param options execution options; {@code null} uses defaults @@ -68,7 +71,7 @@ public static AsyncResource open(ModelSource source, if (source == null) { throw new NullPointerException("source"); } - final AsyncResource out = new AsyncResource(); + final SessionOpenResource out = new SessionOpenResource(); final InferenceImpl impl = Display.getInstance().getInferenceBackend(); if (impl == null || !impl.isSupported()) { out.error(new InferenceException("LiteRT inference is not supported")); @@ -80,17 +83,40 @@ public static AsyncResource open(ModelSource source, .ready(new SuccessCallback() { @Override public void onSucess(Object value) { - out.complete(new InferenceSession(impl, value)); + out.publish(impl, value); } }).except(new SuccessCallback() { @Override public void onSucess(Throwable error) { - out.error(error); + out.fail(error); } }); return out; } + private static final class SessionOpenResource + extends AsyncResource { + @Override + public synchronized boolean cancel(boolean mayInterruptIfRunning) { + return super.cancel(mayInterruptIfRunning); + } + + synchronized void publish(InferenceImpl implementation, + Object nativeHandle) { + if (isCancelled()) { + implementation.close(nativeHandle); + return; + } + complete(new InferenceSession(implementation, nativeHandle)); + } + + synchronized void fail(Throwable error) { + if (!isCancelled()) { + error(error); + } + } + } + /// Returns the model's current input metadata. Shapes reflect the most /// recent successful {@link #resizeInput(String, int[])} call. /// diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java index 169d90a1d16..feb69ab7f18 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java @@ -47,13 +47,16 @@ public static boolean isSupported(LanguageOptions options) { actual.getBackend().getId()); } - /// Identifies possible languages off the EDT without uploading text. + /// Identifies possible languages off the EDT without uploading text. The + /// option values are copied before the backend starts, so later mutations + /// of the supplied object cannot alter the pending identification. /// @param text non-null text to classify /// @param options backend and confidence options, or {@code null} /// @return asynchronous ranked candidates; may be empty for undetermined text public static AsyncResource identify( String text, LanguageOptions options) { - LanguageOptions actual = options == null ? new LanguageOptions() : options; + LanguageOptions actual = (options == null + ? new LanguageOptions() : options).snapshot(); LanguageImpl impl = Display.getInstance().getLanguageBackend(); if (impl == null || !impl.isSupported("language-id", actual.getBackend().getId())) { diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java b/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java index acb277c2e69..65c29258baa 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java @@ -52,4 +52,10 @@ public LanguageBackend getBackend() { public float getMinimumConfidence() { return minimumConfidence; } + + LanguageOptions snapshot() { + return new LanguageOptions() + .backend(backend) + .minimumConfidence(minimumConfidence); + } } diff --git a/CodenameOne/src/com/codename1/ai/language/SmartReply.java b/CodenameOne/src/com/codename1/ai/language/SmartReply.java index e7a0ecb162c..50233fb91dd 100644 --- a/CodenameOne/src/com/codename1/ai/language/SmartReply.java +++ b/CodenameOne/src/com/codename1/ai/language/SmartReply.java @@ -47,12 +47,16 @@ public static boolean isSupported(LanguageOptions options) { "smart-reply", actual.getBackend().getId()); } + /// The conversation array and option values are copied before backend work + /// begins. {@link SmartReplyMessage} values are immutable. + /// /// @param conversation chronological messages, oldest first /// @param options backend options, or {@code null} /// @return asynchronous suggestions, possibly an empty array public static AsyncResource suggest(SmartReplyMessage[] conversation, LanguageOptions options) { - LanguageOptions actual = options == null ? new LanguageOptions() : options; + LanguageOptions actual = (options == null + ? new LanguageOptions() : options).snapshot(); LanguageImpl impl = Display.getInstance().getLanguageBackend(); if (impl == null || !impl.isSupported("smart-reply", actual.getBackend().getId())) { AsyncResource out = new AsyncResource(); @@ -60,7 +64,7 @@ public static AsyncResource suggest(SmartReplyMessage[] conversation, return out; } return impl.suggestReplies(conversation == null - ? new SmartReplyMessage[0] : conversation, + ? new SmartReplyMessage[0] : conversation.clone(), actual.getBackend().getId(), actual); } } diff --git a/CodenameOne/src/com/codename1/ai/language/Translator.java b/CodenameOne/src/com/codename1/ai/language/Translator.java index 243c15d57e6..9fc2b8ccfe4 100644 --- a/CodenameOne/src/com/codename1/ai/language/Translator.java +++ b/CodenameOne/src/com/codename1/ai/language/Translator.java @@ -46,6 +46,8 @@ public static boolean isSupported(LanguageOptions options) { actual.getBackend().getId()); } + /// Option values are copied before asynchronous backend work begins. + /// /// @param text source text /// @param sourceLanguage BCP-47/ML Kit source language tag /// @param targetLanguage BCP-47/ML Kit target language tag @@ -54,7 +56,8 @@ public static boolean isSupported(LanguageOptions options) { public static AsyncResource translate(String text, String sourceLanguage, String targetLanguage, LanguageOptions options) { - LanguageOptions actual = options == null ? new LanguageOptions() : options; + LanguageOptions actual = (options == null + ? new LanguageOptions() : options).snapshot(); LanguageImpl impl = Display.getInstance().getLanguageBackend(); if (impl == null || !impl.isSupported("translation", actual.getBackend().getId())) { AsyncResource out = new AsyncResource(); diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java index 6e53e4b3dd9..37f2759e200 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java @@ -62,18 +62,21 @@ public boolean isSupported(String feature, String backendId) { @Override public AsyncResource identify( final String text, String backendId, final LanguageOptions options) { - return identifyInBackground(text, "ml-kit".equals(backendId), options); + float minimumConfidence = options == null + ? 0 : options.getMinimumConfidence(); + return identifyInBackground(text, "ml-kit".equals(backendId), + minimumConfidence); } private static AsyncResource identifyInBackground( final String text, final boolean mlKit, - final LanguageOptions options) { + final float minimumConfidence) { final AsyncResource out = new AsyncResource(); run(out, new NativeCall() { public LanguageCandidate[] call() throws Exception { String json = IOSImplementation.nativeInstance.cn1LanguageIdentify( - text, options.getMinimumConfidence(), mlKit); + text, minimumConfidence, mlKit); Map root = parse(json); List values = list(root, "items"); LanguageCandidate[] result = new LanguageCandidate[values.size()]; diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index 77aaa65b940..efa69019be3 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -230,6 +230,25 @@ void sessionSnapshotsOptionsBeforeAsyncOpen() { await(opening).close(); } + @Test + void cancelledOpenClosesLateNativeHandle() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + backend.pendingOpen = new AsyncResource(); + implementation.setInferenceImpl(backend); + + AsyncResource opening = InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions()); + assertTrue(opening.cancel(false)); + backend.pendingOpen.complete(backend.handle); + flushSerialCalls(); + + assertTrue(opening.isCancelled()); + assertEquals(1, backend.closeCount, + "a late native handle must not be orphaned after cancellation"); + assertThrows(AsyncResource.AsyncExecutionException.class, + opening::get); + } + @Test void sessionRejectsShapeMismatchBeforeBackendRun() { RecordingInferenceImpl backend = new RecordingInferenceImpl(); @@ -322,6 +341,7 @@ private static final class RecordingInferenceImpl extends InferenceImpl { AsyncResource pendingRun; Tensor[] receivedInputs; InferenceOptions openOptions; + AsyncResource pendingOpen; public boolean isSupported() { return true; @@ -329,6 +349,9 @@ public boolean isSupported() { public AsyncResource open(ModelSource source, InferenceOptions options) { openOptions = options; + if (pendingOpen != null) { + return pendingOpen; + } AsyncResource result = new AsyncResource(); result.complete(handle); return result; diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java index d3302f70d7f..17682f93a3e 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java @@ -33,6 +33,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -77,10 +78,38 @@ void missingBackendReportsUnsupported() { assertThrows(AsyncResource.AsyncExecutionException.class, result::get); } + @Test + void languageOperationsSnapshotMutableArguments() { + RecordingLanguageImpl backend = new RecordingLanguageImpl(); + implementation.setLanguageImpl(backend); + LanguageOptions options = new LanguageOptions() + .backend(LanguageBackends.mlKitLanguageIdentification()) + .minimumConfidence(.35f); + + LanguageIdentifier.identify("bonjour", options); + options.backend(LanguageBackends.auto()).minimumConfidence(.9f); + + assertNotSame(options, backend.options); + assertEquals("ml-kit", backend.backend); + assertEquals(.35f, backend.options.getMinimumConfidence()); + + SmartReplyMessage original = new SmartReplyMessage( + "First", "remote", false, 1); + SmartReplyMessage replacement = new SmartReplyMessage( + "Replacement", "remote", false, 2); + SmartReplyMessage[] conversation = + new SmartReplyMessage[] {original}; + SmartReply.suggest(conversation, null); + conversation[0] = replacement; + assertNotSame(conversation, backend.conversation); + assertEquals(original, backend.conversation[0]); + } + private static final class RecordingLanguageImpl extends LanguageImpl { String feature; String backend; LanguageOptions options; + SmartReplyMessage[] conversation; public boolean isSupported(String value, String backendId) { feature = value; @@ -113,6 +142,7 @@ public AsyncResource suggestReplies( SmartReplyMessage[] conversation, String backendId, LanguageOptions value) { feature = "smart-reply"; + this.conversation = conversation; AsyncResource result = new AsyncResource(); result.complete(new String[] {"Yes"}); return result; From 63df084ea4554891ea0feda2d9e067f68f741bd3 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:11:21 +0300 Subject: [PATCH 35/80] Snapshot backend options before async work --- .../impl/android/ai/AndroidInferenceImpl.java | 18 ++++++++++-------- .../impl/android/ai/AndroidVisionImpl.java | 7 ++++++- .../codename1/impl/ios/IOSInferenceImpl.java | 17 +++++++++-------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java index 9b6d562c7d8..02b572f3572 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java @@ -65,21 +65,23 @@ public boolean isSupported() { public AsyncResource open(final ModelSource source, final InferenceOptions options) { final AsyncResource out = new AsyncResource(); + final InferenceOptions.Accelerator accelerator = + options.getAccelerator(); + final int threads = options.getThreads(); + final boolean allowFallback = options.isFallbackAllowed(); Display.getInstance().scheduleBackgroundTask(new Runnable() { public void run() { try { - InferenceOptions.Accelerator accelerator = - options.getAccelerator(); if ((accelerator == InferenceOptions.Accelerator.GPU || accelerator == InferenceOptions.Accelerator.CORE_ML) - && !options.isFallbackAllowed()) { + && !allowFallback) { throw new InferenceException(accelerator + " acceleration is unavailable on Android"); } ByteBuffer model = loadModel(source); Interpreter.Options nativeOptions = new Interpreter.Options(); - if (options.getThreads() > 0) { - nativeOptions.setNumThreads(options.getThreads()); + if (threads > 0) { + nativeOptions.setNumThreads(threads); } if (accelerator == InferenceOptions.Accelerator.NPU) { nativeOptions.setUseNNAPI(true); @@ -89,12 +91,12 @@ public void run() { interpreter = new Interpreter(model, nativeOptions); } catch (Throwable acceleratedFailure) { if (accelerator != InferenceOptions.Accelerator.NPU - || !options.isFallbackAllowed()) { + || !allowFallback) { throw acceleratedFailure; } Interpreter.Options cpuOptions = new Interpreter.Options(); - if (options.getThreads() > 0) { - cpuOptions.setNumThreads(options.getThreads()); + if (threads > 0) { + cpuOptions.setNumThreads(threads); } model.rewind(); interpreter = new Interpreter(model, cpuOptions); diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java index 9037f9c8efc..67a1c6281e0 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java @@ -60,6 +60,10 @@ public AsyncResource analyze(final VisionFeature feature, final VisionImage image, final VisionOptions options) { final AsyncResource out = new AsyncResource(); + final VisionOptions optionSnapshot = new VisionOptions() + .backend(options.getBackend()) + .minimumConfidence(options.getMinimumConfidence()) + .maximumResults(options.getMaximumResults()); if (!isSupported(feature, backendId)) { out.error(new VisionException(VisionException.UNSUPPORTED, feature + " is not included in this Android build")); @@ -77,7 +81,8 @@ public void run() { return; } adapter(feature).analyze(decoded.input, decoded.width, - decoded.height, options, (AsyncResource) out); + decoded.height, optionSnapshot, + (AsyncResource) out); } catch (Throwable cause) { error(out, new VisionException( VisionException.BACKEND_ERROR, diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java index 2467d64164e..ab3b4d7a182 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java @@ -64,13 +64,17 @@ public boolean isSupported() { public AsyncResource open(final ModelSource source, final InferenceOptions options) { final AsyncResource out = new AsyncResource(); - openInBackground(out, source, options); + openInBackground(out, source, options.getThreads(), + options.getAccelerator().ordinal(), + options.isFallbackAllowed()); return out; } private static void openInBackground(final AsyncResource out, final ModelSource source, - final InferenceOptions options) { + final int threads, + final int accelerator, + final boolean allowFallback) { Display.getInstance().scheduleBackgroundTask(new Runnable() { public void run() { try { @@ -78,13 +82,10 @@ public void run() { ? IOSImplementation.nativeInstance.cn1InferenceOpenFile( FileSystemStorage.getInstance().toNativePath( source.getPath()), - options.getThreads(), - options.getAccelerator().ordinal(), - options.isFallbackAllowed()) + threads, accelerator, allowFallback) : IOSImplementation.nativeInstance.cn1InferenceOpen( - loadModel(source), options.getThreads(), - options.getAccelerator().ordinal(), - options.isFallbackAllowed()); + loadModel(source), threads, accelerator, + allowFallback); Map root = parse(opened); int id = integer(root, "handle"); final Handle handle; From 3419ed2dea09ffddb444136490e9eb902b12c47f Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:22:11 +0300 Subject: [PATCH 36/80] Avoid array clone in ParparVM snapshots --- .../codename1/ai/inference/InferenceSession.java | 10 +++++++++- .../com/codename1/ai/language/SmartReply.java | 16 ++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java index 84fd1fefb45..1860456a2d3 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -167,7 +167,7 @@ public AsyncResource run(Tensor[] inputs) { activeRuns++; try { Tensor[] inputSnapshot = inputs == null - ? new Tensor[0] : inputs.clone(); + ? new Tensor[0] : copyInputs(inputs); validateInputShapes(inputSnapshot, implementation.getInputs(handle)); result = implementation.run(handle, @@ -243,6 +243,14 @@ private static void validateInputShapes(Tensor[] inputs, } } + private static Tensor[] copyInputs(Tensor[] inputs) { + Tensor[] copy = new Tensor[inputs.length]; + for (int i = 0; i < inputs.length; i++) { + copy[i] = inputs[i]; + } + return copy; + } + private static boolean sameShape(int[] supplied, int[] expected) { if (supplied.length != expected.length) { return false; diff --git a/CodenameOne/src/com/codename1/ai/language/SmartReply.java b/CodenameOne/src/com/codename1/ai/language/SmartReply.java index 50233fb91dd..23275c0c9d6 100644 --- a/CodenameOne/src/com/codename1/ai/language/SmartReply.java +++ b/CodenameOne/src/com/codename1/ai/language/SmartReply.java @@ -63,8 +63,20 @@ public static AsyncResource suggest(SmartReplyMessage[] conversation, out.error(new UnsupportedOperationException("smart reply is not supported")); return out; } - return impl.suggestReplies(conversation == null - ? new SmartReplyMessage[0] : conversation.clone(), + return impl.suggestReplies(copyConversation(conversation), actual.getBackend().getId(), actual); } + + private static SmartReplyMessage[] copyConversation( + SmartReplyMessage[] conversation) { + if (conversation == null) { + return new SmartReplyMessage[0]; + } + SmartReplyMessage[] copy = + new SmartReplyMessage[conversation.length]; + for (int i = 0; i < conversation.length; i++) { + copy[i] = conversation[i]; + } + return copy; + } } From 59b09ec16cd589ecbc19feb1be7309e42687e2ab Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:33:08 +0300 Subject: [PATCH 37/80] Preserve iOS barcode payload bytes --- Ports/iOSPort/nativeSources/CN1Vision.m | 3 +++ .../src/com/codename1/impl/ios/IOSVisionImpl.java | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Ports/iOSPort/nativeSources/CN1Vision.m b/Ports/iOSPort/nativeSources/CN1Vision.m index 1785206b821..d323d125199 100644 --- a/Ports/iOSPort/nativeSources/CN1Vision.m +++ b/Ports/iOSPort/nativeSources/CN1Vision.m @@ -300,6 +300,9 @@ static UIImageOrientation cn1UIImageOrientation(int rotation) { NSMutableDictionary *item = [NSMutableDictionary dictionaryWithDictionary:cn1MLKitRect(barcode.frame, resultSize)]; item[@"value"] = barcode.rawValue ?: [NSNull null]; + item[@"rawData"] = barcode.rawData == nil + ? [NSNull null] + : [barcode.rawData base64EncodedStringWithOptions:0]; item[@"format"] = cn1MLKitBarcodeFormat(barcode.format); NSMutableArray *corners = [NSMutableArray array]; for (NSValue *pointValue in barcode.cornerPoints ?: @[]) { diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java index 4847070af41..3ae8b389559 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java @@ -176,7 +176,8 @@ private static T parse(VisionFeature feature, String json, number(point, "y")); } values[i] = new Barcode(stringOrNull(value, "value"), - string(value, "format"), null, rect(value), + string(value, "format"), + decodeBase64OrNull(value, "rawData"), rect(value), corners, metadata); } return (T) values; @@ -334,6 +335,13 @@ private static byte[] nativeData(Map value) throws Exception { return Base64.decode(base64.getBytes("UTF-8")); } + private static byte[] decodeBase64OrNull(Map value, String key) + throws Exception { + String base64 = stringOrNull(value, key); + return base64 == null ? null + : Base64.decode(base64.getBytes("UTF-8")); + } + @Override public void close() { closed = true; From 382e8ba7f93ca11b880ab2d46e834c0f6a329441 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:42:42 +0300 Subject: [PATCH 38/80] Normalize Smart Reply participant IDs --- .../src/com/codename1/ai/language/SmartReplyMessage.java | 8 +++++--- .../impl/android/ai/AndroidSmartReplyAdapter.java | 4 +++- .../java/com/codename1/ai/language/LanguageApiTest.java | 3 ++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java b/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java index 46383b56e58..2acbabb7bda 100644 --- a/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java +++ b/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java @@ -31,13 +31,14 @@ public final class SmartReplyMessage { /// Creates one conversation turn used to generate reply suggestions. /// @param text message body; {@code null} becomes an empty string - /// @param participantId stable speaker id used to group conversation turns + /// @param participantId stable speaker id used to group conversation turns; + /// {@code null} becomes {@code "remote"} /// @param localUser whether this message was written by the current user /// @param timestampMillis message time in Unix epoch milliseconds public SmartReplyMessage(String text, String participantId, boolean localUser, long timestampMillis) { this.text = text == null ? "" : text; - this.participantId = participantId; + this.participantId = participantId == null ? "remote" : participantId; this.localUser = localUser; this.timestampMillis = timestampMillis; } @@ -47,7 +48,8 @@ public String getText() { return text; } - /// @return stable participant identifier used to group remote speakers + /// @return stable participant identifier used to group remote speakers, + /// never {@code null} public String getParticipantId() { return participantId; } diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java index ce57e48f41d..c9b5b392f2e 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java @@ -50,7 +50,9 @@ AsyncResource suggestReplies( message.getText(), message.getTimestampMillis()) : TextMessage.createForRemoteUser( message.getText(), message.getTimestampMillis(), - message.getParticipantId())); + message.getParticipantId() == null + ? "remote" + : message.getParticipantId())); } client.suggestReplies(messages).addOnSuccessListener( new OnSuccessListener() { diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java index 17682f93a3e..22bf96c1df3 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java @@ -94,7 +94,8 @@ void languageOperationsSnapshotMutableArguments() { assertEquals(.35f, backend.options.getMinimumConfidence()); SmartReplyMessage original = new SmartReplyMessage( - "First", "remote", false, 1); + "First", null, false, 1); + assertEquals("remote", original.getParticipantId()); SmartReplyMessage replacement = new SmartReplyMessage( "Replacement", "remote", false, 2); SmartReplyMessage[] conversation = From 86cc16558dade5037c41a9af8007798fd9c1caed Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:59:00 +0300 Subject: [PATCH 39/80] Raise iOS target for ML Kit pods --- docs/ai-on-device-architecture.md | 4 +++ docs/developer-guide/Ai-And-Speech.asciidoc | 6 ++++ .../com/codename1/builders/IPhoneBuilder.java | 5 +++ .../IPhoneBuilderDependencyConfigTest.java | 17 ++++++++++ .../build/shared/PlatformFeatureCatalog.java | 32 +++++++++++++++++++ .../shared/PlatformFeatureCatalogTest.java | 22 +++++++++++++ 6 files changed, 86 insertions(+) diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md index 3fe8b136a82..34bc13ff9d3 100644 --- a/docs/ai-on-device-architecture.md +++ b/docs/ai-on-device-architecture.md @@ -118,6 +118,10 @@ Pods projects. The repository's iOS UI and native-test runners also select `ARCHS=x86_64`, so Apple Silicon hosts use the supported simulator slice without requiring an application build hint. TensorFlow Lite's XCFramework does include an `arm64` simulator slice and does not trigger this fallback. +The same Google ML Kit catalog entries declare an iOS 15.5 deployment floor. +The iOS builder folds that value into its existing maximum-target calculation +before generating the app target and Podfile, preventing a lower application +hint from producing an unsatisfiable CocoaPods resolution. The iOS NEON implementation reports SIMD as unsupported in that x86_64 configuration and aliases its native entry points to the generic scalar implementation; device and arm64-simulator builds retain the NEON path. diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index 12178cfebb3..8d3bd6ef916 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -351,6 +351,9 @@ each class the application actually uses: The optional iOS ML Kit pod is added only when both the analyzer and its feature-specific selector are referenced. For example, `VisionBackends.mlKitBarcodeScanning()` adds only the barcode pod. +Current Google ML Kit iOS pods require iOS 15.5. Selecting one of these +optional backends automatically raises the effective application and +CocoaPods deployment target to 15.5 when the configured target is lower. Referencing one Android analyzer doesn't compile or package the other five adapters. @@ -408,6 +411,9 @@ keeps arm64 simulator builds native and adds no third-party dependency. identification into the ML Kit iOS pod. Translation and Smart Reply use their feature-scoped ML Kit components on both mobile platforms; those two iOS components currently require the x86_64 simulator slice. +The current Google ML Kit iOS pods also require iOS 15.5, so referencing +translation or Smart Reply, or opting identification into ML Kit, raises +the effective iOS deployment target to 15.5 automatically. Mac native and the other unsupported targets return `false` from the language service `isSupported()` checks. Completion and failure callbacks arrive on the EDT. diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java index b5759621a08..f1c669e8808 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java @@ -1060,6 +1060,11 @@ public void usesClassMethod(String cls, String method) { boolean includeApplePackageDependencies = !macNativeBuilder.isEnabled() || entry.iosDependenciesSupportMacCatalyst(); + if (includeApplePackageDependencies + && entry.iosMinimumDeploymentTarget() != null) { + addMinDeploymentTarget( + entry.iosMinimumDeploymentTarget()); + } if (includeApplePackageDependencies && !entry.iosDependenciesSupportArm64Simulator() && (!entry.iosPods().isEmpty() diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java index fb388802ab8..9a6fca21f73 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java @@ -196,6 +196,23 @@ void appendsEachRequiredAiFrameworkIndependently() throws Exception { assertEquals("ThirdPartyVision.framework;Vision.framework", value); } + @Test + void dependencyFloorRaisesExplicitDeploymentTarget() throws Exception { + IPhoneBuilder builder = new IPhoneBuilder(); + Method addTarget = IPhoneBuilder.class.getDeclaredMethod( + "addMinDeploymentTarget", String.class); + addTarget.setAccessible(true); + addTarget.invoke(builder, "15.5"); + + Method getTarget = IPhoneBuilder.class.getDeclaredMethod( + "getDeploymentTarget", BuildRequest.class); + getTarget.setAccessible(true); + String target = (String) getTarget.invoke(builder, + requestWithArgs("ios.deployment_target", "14.0")); + + assertEquals("15.5", target); + } + private BuildRequest requestWithArgs(String... kvPairs) { BuildRequest out = new BuildRequest(); for (int i = 0; i < kvPairs.length; i += 2) { diff --git a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java index 0a17052cadc..8e1e304cec5 100644 --- a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java +++ b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java @@ -91,6 +91,7 @@ public final class PlatformFeatureCatalog { .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKitTextRecognition") .iosPod("GoogleMLKit/TextRecognition") + .iosMinimumDeploymentTarget("15.5") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() .description("ML Kit iOS text-recognition backend")); @@ -103,6 +104,7 @@ public final class PlatformFeatureCatalog { .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKitBarcodeScanning") .iosPod("GoogleMLKit/BarcodeScanning") + .iosMinimumDeploymentTarget("15.5") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() .description("ML Kit iOS barcode backend")); @@ -115,6 +117,7 @@ public final class PlatformFeatureCatalog { .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKitFaceDetection") .iosPod("GoogleMLKit/FaceDetection") + .iosMinimumDeploymentTarget("15.5") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() .description("ML Kit iOS face-detection backend")); @@ -127,6 +130,7 @@ public final class PlatformFeatureCatalog { .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKitImageLabeling") .iosPod("GoogleMLKit/ImageLabeling") + .iosMinimumDeploymentTarget("15.5") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() .description("ML Kit iOS image-labeling backend")); @@ -139,6 +143,7 @@ public final class PlatformFeatureCatalog { .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKitPoseDetection") .iosPod("GoogleMLKit/PoseDetection") + .iosMinimumDeploymentTarget("15.5") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() .description("ML Kit iOS pose-detection backend")); @@ -152,6 +157,7 @@ public final class PlatformFeatureCatalog { .requiresMethod("com/codename1/ai/vision/VisionBackends", "mlKitSelfieSegmentation") .iosPod("GoogleMLKit/SegmentationSelfie") + .iosMinimumDeploymentTarget("15.5") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() .description("ML Kit iOS selfie-segmentation backend")); @@ -178,6 +184,7 @@ public final class PlatformFeatureCatalog { .requiresMethod("com/codename1/ai/language/LanguageBackends", "mlKitLanguageIdentification") .iosPod("GoogleMLKit/LanguageID") + .iosMinimumDeploymentTarget("15.5") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() .description("ML Kit iOS language-identification backend")); @@ -186,6 +193,7 @@ public final class PlatformFeatureCatalog { // link the small system NaturalLanguage framework. e.add(new Entry("com/codename1/ai/language/Translator") .iosPod("GoogleMLKit/Translate") + .iosMinimumDeploymentTarget("15.5") .iosFrameworks("NaturalLanguage") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() @@ -193,6 +201,7 @@ public final class PlatformFeatureCatalog { .description("On-device translation")); e.add(new Entry("com/codename1/ai/language/SmartReply") .iosPod("GoogleMLKit/SmartReply") + .iosMinimumDeploymentTarget("15.5") .iosFrameworks("NaturalLanguage") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() @@ -362,6 +371,7 @@ public static final class Entry { private final List androidMetaData = new ArrayList(); private boolean iosDependenciesSupportMacCatalyst = true; private boolean iosDependenciesSupportArm64Simulator = true; + private String iosMinimumDeploymentTarget; private boolean requiresBigUpload; private String description = ""; @@ -413,6 +423,11 @@ Entry iosPod(String pod) { return this; } + Entry iosMinimumDeploymentTarget(String target) { + iosMinimumDeploymentTarget = target; + return this; + } + Entry iosSpm(String identity, String url, String requirement, String... products) { iosSpm.add(new IosSpm(identity, url, requirement, Arrays.asList(products))); @@ -504,6 +519,23 @@ public boolean iosDependenciesSupportArm64Simulator() { return iosDependenciesSupportArm64Simulator; } + /** + * Returns the minimum iOS deployment target required by this entry's + * CocoaPod or Swift package payload. + * + *

The iOS builder combines this value with the application's + * requested deployment target and all other dependency floors, then + * uses the highest version for the generated app target and Podfile. + * A {@code null} value means that the entry does not impose an + * additional deployment floor.

+ * + * @return the required iOS version, such as {@code "15.5"}, or + * {@code null} when the dependency has no catalog-specific minimum + */ + public String iosMinimumDeploymentTarget() { + return iosMinimumDeploymentTarget; + } + public List iosFrameworks() { return Collections.unmodifiableList(iosFrameworks); } diff --git a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java index f98891e8965..c776e98860e 100644 --- a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java +++ b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java @@ -65,6 +65,28 @@ void explicitMlKitBackendAddsOnlyUsedFeaturePod() { assertTrue(foundTextPod); } + @Test + void currentIosMlKitPodsRequireIos155() { + PlatformFeatureCatalog.Accumulator acc = + new PlatformFeatureCatalog.Accumulator(); + acc.consume("com/codename1/ai/vision/TextRecognizer"); + acc.consumeMethod("com/codename1/ai/vision/VisionBackends", + "mlKitTextRecognition"); + acc.consume("com/codename1/ai/language/Translator"); + acc.consume("com/codename1/ai/language/SmartReply"); + + int podEntries = 0; + for (PlatformFeatureCatalog.Entry entry : acc.hits()) { + if (!entry.iosPods().isEmpty()) { + podEntries++; + assertEquals("15.5", + entry.iosMinimumDeploymentTarget(), + entry.description()); + } + } + assertEquals(3, podEntries); + } + @Test void speechRecognizerInjectsMicAndSpeechPlist() { List hits = PlatformFeatureCatalog.matchesFor( From 54a9200953f029a251e899c920e5e433a7a78ba1 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 05:11:34 +0300 Subject: [PATCH 40/80] Secure model cache downloads and promotion --- .../codename1/ai/inference/ModelCache.java | 71 ++++++++++++++++--- docs/ai-on-device-architecture.md | 14 ++-- docs/developer-guide/Ai-And-Speech.asciidoc | 9 +-- .../ai/inference/InferenceApiTest.java | 52 ++++++++++++++ .../skill/references/ai-and-speech.md | 6 +- 5 files changed, 130 insertions(+), 22 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java index c29c1df4cbe..331f07b6d51 100644 --- a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java +++ b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java @@ -36,8 +36,11 @@ import java.util.Hashtable; /// Downloads a large model into app-private storage and exposes it as a -/// file-backed {@link ModelSource}. The initial request and every redirect -/// require HTTPS. Downloads use a temporary file and are promoted atomically +/// file-backed {@link ModelSource}. The initial request requires HTTPS. +/// Ports that expose redirect responses reject any redirect to HTTP. Because +/// iOS follows redirects below the portable network layer, iOS downloads +/// require a SHA-256 digest so an unseen redirect cannot substitute the +/// executable model payload. Downloads use a temporary file and are promoted /// only after optional digest verification. /// /// Supply a SHA-256 digest for third-party or remotely mutable models. Without @@ -60,10 +63,14 @@ private ModelCache() { /// one operation. A different URL or digest using that cache key while the /// first operation is active fails instead of racing on the temporary file. /// - /// @param url HTTPS URL of the model; every redirect must also use HTTPS + /// @param url initial HTTPS URL of the model; observable redirects must + /// also use HTTPS /// @param cacheKey stable cache name, independent of the URL - /// @param sha256 optional lowercase or uppercase SHA-256 hex digest + /// @param sha256 lowercase or uppercase SHA-256 hex digest; optional on + /// ports that expose redirects and required on iOS /// @return asynchronous cached model source + /// @throws IllegalArgumentException if the URL, cache key, or digest is + /// invalid, or if the digest is omitted on iOS public static AsyncResource fetch( final String url, final String cacheKey, final String sha256) { if (!isHttpsUrl(url)) { @@ -75,6 +82,11 @@ public static AsyncResource fetch( if (sha256 != null && !isSha256(sha256)) { throw new IllegalArgumentException("SHA-256 must contain 64 hex characters"); } + if (sha256 == null && requiresPinnedModelDigest()) { + throw new IllegalArgumentException( + "iOS model downloads require a SHA-256 digest because " + + "redirects are followed below the portable network layer"); + } final String fileName = safeName(cacheKey) + ".tflite"; final FetchRegistration registration = @@ -114,14 +126,19 @@ public void run() { return out; } - /// Fetches a model without content pinning. Prefer the three-argument - /// overload for any model that is not versioned by the app itself. + /// Fetches a model without content pinning on ports that expose redirect + /// responses to the portable network layer. Prefer the three-argument + /// overload for any model that is not versioned by the app itself. iOS + /// rejects this overload because its native network stack follows + /// redirects before Codename One can validate their schemes. /// Concurrent unpinned calls share an operation only when their URL and /// cache key are identical. /// /// @param url HTTPS model URL /// @param cacheKey stable cache name /// @return asynchronous cached file source + /// @throws IllegalArgumentException if the URL or cache key is invalid, + /// or when called on iOS, where a digest is required public static AsyncResource fetch(String url, String cacheKey) { return fetch(url, cacheKey, null); } @@ -148,10 +165,8 @@ public void actionPerformed(NetworkEvent event) { public void run() { try { verifyDownloaded(fs, temporary, sha256); - if (fs.exists(target)) { - fs.delete(target); - } - fs.rename(temporary, fileName); + promoteDownloaded(fs, temporary, target, + fileName, sha256); completion.complete(ModelSource.file(target)); } catch (IOException error) { completion.fail(error); @@ -210,6 +225,38 @@ static void verifyDownloaded(FileSystemStorage fs, String temporary, } } + static void promoteDownloaded(FileSystemStorage fs, String temporary, + String target, String fileName, + String expected) throws IOException { + try { + if (fs.exists(target)) { + fs.delete(target); + if (fs.exists(target)) { + throw new IOException( + "Could not replace existing cached model"); + } + } + fs.rename(temporary, fileName); + if (!fs.exists(target)) { + throw new IOException( + "Downloaded model could not be promoted to the cache"); + } + if (!verify(target, expected)) { + fs.delete(target); + throw new IOException( + "Promoted model SHA-256 does not match"); + } + if (fs.exists(temporary)) { + fs.delete(temporary); + } + } catch (IOException error) { + if (fs.exists(temporary)) { + fs.delete(temporary); + } + throw error; + } + } + static String safeName(String value) { StringBuilder out = new StringBuilder(); for (int i = 0; i < value.length(); i++) { @@ -252,6 +299,10 @@ static boolean isHttpsUrl(String url) { "https://", 0, "https://".length()); } + static boolean requiresPinnedModelDigest() { + return "ios".equals(Display.getInstance().getPlatformName()); + } + static FetchRegistration registerFetch( String fileName, String url, String sha256) { synchronized (ACTIVE_FETCHES) { diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md index 34bc13ff9d3..97ce558a95a 100644 --- a/docs/ai-on-device-architecture.md +++ b/docs/ai-on-device-architecture.md @@ -154,12 +154,14 @@ actual backend id without exposing platform classes. `InferenceSession` supports named typed tensors, multiple inputs and outputs, input resizing, and reusable native sessions. Model sources are bytes, resources, private files, or the HTTPS-only `ModelCache`. The cache rejects -redirects that downgrade a request to HTTP. Identical concurrent fetches for -one cache entry coalesce, while conflicting in-flight content identities fail -instead of sharing a temporary path. File sources are opened by path without -copying the model through the Java heap. The cache can verify a SHA-256 digest -before publishing a downloaded model; callers should always supply the digest -for mutable or third-party downloads. +redirects that downgrade a request to HTTP on ports where redirects are +observable. iOS follows redirects below the portable network layer, so the +cache requires a SHA-256 digest for every iOS download and rejects its unpinned +overload. Identical concurrent fetches for one cache entry coalesce, while +conflicting in-flight content identities fail instead of sharing a temporary +path. File sources are opened by path without copying the model through the +Java heap. Cache promotion verifies that the final file exists and, when +supplied, still matches the requested digest before publishing its path. All native sessions and analyzers must be closed. Expensive open, analysis, and inference work runs off the EDT; completion and error delivery return to diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index 8d3bd6ef916..b724d49ee78 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -380,10 +380,11 @@ the port boundary. Always close the session. Bundle small models as resources. For larger models use `ModelCache.fetch(url, cacheKey, sha256)`, which downloads over HTTPS, verifies the supplied SHA-256 digest, and reuses the app-private cached -file. The initial URL and every redirect must use HTTPS; a redirect to -HTTP fails the resource and discards the temporary download. Supply the -digest for every third-party or remotely mutable model: an omitted digest -provides transport security but doesn't pin the executable model payload. +file. The initial URL must use HTTPS. Ports that expose redirect responses +reject a redirect to HTTP and discard the temporary download. iOS follows +redirects below the portable network layer, so iOS requires the SHA-256 +argument and rejects the unpinned two-argument overload. Supply the digest +on every platform for every third-party or remotely mutable model. Interrupted downloads restart through a temporary `.download` file and are never promoted to the cache until verification succeeds. Concurrent calls for the same cache key and content identity share one download. A conflicting URL diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index efa69019be3..9fd01c1e6a0 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -76,6 +76,17 @@ void modelCacheRejectsInsecureAndMalformedRequests() { "gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg")); } + @Test + void modelCacheRequiresDigestWhenIosHidesRedirects() { + implementation.setPlatformName("ios"); + assertTrue(ModelCache.requiresPinnedModelDigest()); + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> ModelCache.fetch( + "https://example.com/model.tflite", "ios-model")); + assertTrue(error.getMessage().contains("SHA-256")); + } + @Test void modelCacheRejectsInsecureRedirects() throws Exception { FileSystemStorage fs = FileSystemStorage.getInstance(); @@ -179,6 +190,47 @@ void modelCacheDiscardsPartialAndDigestMismatchFiles() throws Exception { assertFalse(fs.exists(temporary), "a digest mismatch must delete executable data"); } + @Test + void modelCacheVerifiesPromotionBeforePublishingPath() throws Exception { + FileSystemStorage fs = FileSystemStorage.getInstance(); + String directory = fs.getAppHomePath(); + String fileName = "model-cache-promoted-test.tflite"; + String target = directory + fileName; + String temporary = target + ".download"; + if (fs.exists(target)) { + fs.delete(target); + } + if (fs.exists(temporary)) { + fs.delete(temporary); + } + + assertThrows(java.io.IOException.class, () -> + ModelCache.promoteDownloaded(fs, temporary, target, + fileName, null)); + assertFalse(fs.exists(target), + "a failed rename must not publish a nonexistent cache path"); + + write(temporary, new byte[] {1, 2, 3}); + assertThrows(java.io.IOException.class, () -> + ModelCache.promoteDownloaded(fs, temporary, target, + fileName, + "00000000000000000000000000000000" + + "00000000000000000000000000000000")); + assertFalse(fs.exists(target), + "a final digest mismatch must remove the promoted path"); + assertFalse(fs.exists(temporary), + "a final digest mismatch must remove the temporary file"); + + write(temporary, new byte[] {1, 2, 3}); + ModelCache.promoteDownloaded(fs, temporary, target, fileName, + "039058c6f2c0cb492c533b0a4d14ef7" + + "7cc0f78abccced5287d84a1a2011cfb81"); + assertTrue(fs.exists(target)); + assertFalse(fs.exists(temporary)); + + fs.delete(target); + } + private static void write(String path, byte[] value) throws Exception { OutputStream output = FileSystemStorage.getInstance().openOutputStream(path); try { diff --git a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md index a2fc55aef39..b6f7e39000c 100644 --- a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md +++ b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md @@ -327,8 +327,10 @@ Translator.translate("Bonjour", "fr", "en", new LanguageOptions()) `VisionPipeline` to connect a reusable analyzer to camera frames; it copies callback-owned data and drops stale pending frames under load. Use `ModelCache.fetch(httpsUrl, cacheKey, sha256)` for models too large -to bundle. The initial model URL and every redirect must remain HTTPS; -identical concurrent fetches coalesce instead of sharing a temporary +to bundle. The initial model URL must remain HTTPS. Observable redirects +are rejected if they downgrade to HTTP; iOS requires the digest because +its network stack follows redirects below the portable callback. +Identical concurrent fetches coalesce instead of sharing a temporary file unsafely. iOS and Mac native default to Apple Vision; iOS also supports explicit ML Kit selection. Android uses ML Kit. From ebd1a3680db5a8119311bb755b8c9f2701845e46 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 05:23:01 +0300 Subject: [PATCH 41/80] Fix Android LiteRT output execution --- .../ai/inference/InferenceOptions.java | 8 ++++++- .../impl/android/ai/AndroidInferenceImpl.java | 22 ++++++++++++++----- docs/ai-on-device-architecture.md | 6 +++++ docs/developer-guide/Ai-And-Speech.asciidoc | 6 ++++- .../skill/references/ai-and-speech.md | 4 +++- 5 files changed, 38 insertions(+), 8 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java index e183b81a733..a413cc46e16 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java @@ -27,7 +27,9 @@ /// Accelerator requests are portable preferences rather than promises. /// With fallback enabled, a backend may execute on CPU when the requested /// delegate is unavailable. With fallback disabled, opening the session -/// fails instead of silently changing the execution target. +/// fails instead of silently changing the execution target. Android's NNAPI +/// runtime can mix NPU and CPU operations without reporting full delegation, +/// so Android rejects {@link Accelerator#NPU} when fallback is disabled. public final class InferenceOptions { /// Execution targets understood by the portable inference API. public enum Accelerator { @@ -60,6 +62,10 @@ public InferenceOptions threads(int value) { /// Controls whether opening may fall back from an unavailable accelerator /// to CPU execution. /// + /// On Android, setting this to {@code false} with + /// {@link Accelerator#NPU} rejects session creation because LiteRT cannot + /// prove that NNAPI delegated every operation. + /// /// @param value {@code true} to permit CPU fallback /// @return this options object public InferenceOptions allowFallback(boolean value) { diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java index 02b572f3572..b93f768e2b7 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java @@ -78,6 +78,13 @@ public void run() { throw new InferenceException(accelerator + " acceleration is unavailable on Android"); } + if (accelerator == InferenceOptions.Accelerator.NPU + && !allowFallback) { + throw new InferenceException( + "Strict NPU execution cannot be verified on " + + "Android; NNAPI may leave unsupported " + + "operations on the CPU"); + } ByteBuffer model = loadModel(source); Interpreter.Options nativeOptions = new Interpreter.Options(); if (threads > 0) { @@ -177,18 +184,23 @@ public void run() { } Map nativeOutputs = new HashMap(); int outputCount = interpreter.getOutputTensorCount(); + ByteBuffer[] outputBuffers = + new ByteBuffer[outputCount]; for (int i = 0; i < outputCount; i++) { - // A null destination asks LiteRT to retain the result - // in its tensor. This is required for outputs whose - // shape is resolved or grows during invocation. - nativeOutputs.put(Integer.valueOf(i), null); + org.tensorflow.lite.Tensor metadata = + interpreter.getOutputTensor(i); + ByteBuffer destination = ByteBuffer.allocateDirect( + metadata.numBytes()).order( + ByteOrder.nativeOrder()); + outputBuffers[i] = destination; + nativeOutputs.put(Integer.valueOf(i), destination); } interpreter.runForMultipleInputsOutputs(nativeInputs, nativeOutputs); final Tensor[] result = new Tensor[outputCount]; for (int i = 0; i < result.length; i++) { org.tensorflow.lite.Tensor metadata = interpreter.getOutputTensor(i); result[i] = fromBuffer(metadata.name(), metadata.shape(), - metadata.dataType(), metadata.asReadOnlyBuffer()); + metadata.dataType(), outputBuffers[i]); } Display.getInstance().callSerially(new Runnable() { public void run() { diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md index 97ce558a95a..8a5bc8b8ace 100644 --- a/docs/ai-on-device-architecture.md +++ b/docs/ai-on-device-architecture.md @@ -162,6 +162,12 @@ conflicting in-flight content identities fail instead of sharing a temporary path. File sources are opened by path without copying the model through the Java heap. Cache promotion verifies that the final file exists and, when supplied, still matches the requested digest before publishing its path. +Android supplies a direct, native-order destination buffer for every current +LiteRT output tensor before invoking `runForMultipleInputsOutputs`; null output +destinations are reserved for externally bound delegate buffers and are not +used by ordinary sessions. NNAPI is best effort when fallback is enabled. +Strict Android NPU requests are rejected because LiteRT cannot verify that +NNAPI delegated every operation instead of retaining CPU nodes. All native sessions and analyzers must be closed. Expensive open, analysis, and inference work runs off the EDT; completion and error delivery return to diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index b724d49ee78..acb97c2c6f3 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -363,7 +363,11 @@ five adapters. or a private file. Android uses LiteRT and can request NNAPI. iOS uses TensorFlow Lite Objective-C and may request the Core ML delegate. Both mobile implementations fall back to CPU unless -`allowFallback(false)` is set. The JavaSE simulator and targets without +`allowFallback(false)` is set. Android rejects strict +`Accelerator.NPU` sessions because NNAPI can mix delegated and CPU +operations without reporting that the entire graph reached the NPU. +With fallback enabled, NNAPI is a best-effort accelerator preference. +The JavaSE simulator and targets without a native LiteRT runtime, including Mac native, return `false` from `InferenceSession.isSupported()`. diff --git a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md index b6f7e39000c..9e7226d858d 100644 --- a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md +++ b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md @@ -347,7 +347,9 @@ On iOS, an ML Kit vision pod is selected only when both its analyzer and its matching feature-specific selector are referenced. For example, `VisionBackends.mlKitBarcodeScanning()` selects only the barcode pod. LiteRT is selected only by -`InferenceSession`. +`InferenceSession`. Android NNAPI acceleration is best effort when +fallback is enabled. Android rejects `Accelerator.NPU` together with +`allowFallback(false)` because LiteRT cannot prove full-graph delegation. Language identification defaults to Apple Natural Language on iOS and ML Kit on Android. Translation and Smart Reply use feature-scoped ML Kit From 7b0069033a4352bd9f97d722858428296ccc3f62 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 05:42:18 +0300 Subject: [PATCH 42/80] Honor dynamic outputs and strict NPU requests --- .../ai/inference/InferenceOptions.java | 11 ++-- .../impl/android/ai/AndroidInferenceImpl.java | 60 ++++++++++++++----- Ports/iOSPort/nativeSources/CN1Inference.m | 12 ++++ docs/ai-on-device-architecture.md | 15 +++-- docs/developer-guide/Ai-And-Speech.asciidoc | 10 ++-- .../builders/AndroidGradleBuilder.java | 11 ++++ .../AndroidAiSourceSelectionTest.java | 13 ++++ .../skill/references/ai-and-speech.md | 7 ++- 8 files changed, 108 insertions(+), 31 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java index a413cc46e16..077a0daf1f4 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java @@ -29,7 +29,9 @@ /// delegate is unavailable. With fallback disabled, opening the session /// fails instead of silently changing the execution target. Android's NNAPI /// runtime can mix NPU and CPU operations without reporting full delegation, -/// so Android rejects {@link Accelerator#NPU} when fallback is disabled. +/// and the iOS Core ML delegate can schedule work across the Neural Engine, +/// GPU, and CPU. Both mobile backends therefore reject +/// {@link Accelerator#NPU} when fallback is disabled. public final class InferenceOptions { /// Execution targets understood by the portable inference API. public enum Accelerator { @@ -62,9 +64,10 @@ public InferenceOptions threads(int value) { /// Controls whether opening may fall back from an unavailable accelerator /// to CPU execution. /// - /// On Android, setting this to {@code false} with - /// {@link Accelerator#NPU} rejects session creation because LiteRT cannot - /// prove that NNAPI delegated every operation. + /// On Android and iOS, setting this to {@code false} with + /// {@link Accelerator#NPU} rejects session creation. Neither LiteRT's + /// NNAPI delegate nor its Core ML delegate can prove that every operation + /// ran on the requested NPU instead of CPU or GPU. /// /// @param value {@code true} to permit CPU fallback /// @return this options object diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java index b93f768e2b7..08f32e59c66 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java @@ -40,6 +40,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; @@ -48,6 +49,8 @@ /** Android LiteRT backend. */ public final class AndroidInferenceImpl extends InferenceImpl { + private static volatile Method outputShapeRefreshMethod; + private static final class Handle { final Interpreter interpreter; @@ -182,25 +185,27 @@ public void run() { nativeInputs[index] = toBuffer(value, metadata.dataType()); } - Map nativeOutputs = new HashMap(); + Map nativeOutputs = + new HashMap(); int outputCount = interpreter.getOutputTensorCount(); - ByteBuffer[] outputBuffers = - new ByteBuffer[outputCount]; for (int i = 0; i < outputCount; i++) { - org.tensorflow.lite.Tensor metadata = - interpreter.getOutputTensor(i); - ByteBuffer destination = ByteBuffer.allocateDirect( - metadata.numBytes()).order( - ByteOrder.nativeOrder()); - outputBuffers[i] = destination; - nativeOutputs.put(Integer.valueOf(i), destination); + // LiteRT 1.0.1 treats a null output as invocation-only: + // the native tensor remains available through + // Tensor.asReadOnlyBuffer() after the run. This avoids + // allocating from a pre-run size that may still contain + // unresolved, value-dependent output dimensions. + nativeOutputs.put(Integer.valueOf(i), null); } - interpreter.runForMultipleInputsOutputs(nativeInputs, nativeOutputs); + interpreter.runForMultipleInputsOutputs(nativeInputs, + nativeOutputs); final Tensor[] result = new Tensor[outputCount]; for (int i = 0; i < result.length; i++) { - org.tensorflow.lite.Tensor metadata = interpreter.getOutputTensor(i); - result[i] = fromBuffer(metadata.name(), metadata.shape(), - metadata.dataType(), outputBuffers[i]); + org.tensorflow.lite.Tensor metadata = + interpreter.getOutputTensor(i); + refreshOutputShape(metadata); + result[i] = fromBuffer(metadata.name(), + metadata.shape(), metadata.dataType(), + metadata.asReadOnlyBuffer()); } Display.getInstance().callSerially(new Runnable() { public void run() { @@ -247,6 +252,33 @@ private static boolean sameShape(int[] supplied, int[] expected) { return true; } + private static void refreshOutputShape( + org.tensorflow.lite.Tensor metadata) { + try { + // TensorImpl caches shape(), and LiteRT refreshes that cache only + // when the invocation also reallocates tensors. Value-dependent + // output shapes can change without input reallocation, so refresh + // the package-private cache after every run. The builders retain + // this one method name for R8 release builds. + Method refresh = outputShapeRefreshMethod; + if (refresh == null) { + synchronized (AndroidInferenceImpl.class) { + refresh = outputShapeRefreshMethod; + if (refresh == null) { + refresh = metadata.getClass() + .getDeclaredMethod("refreshShape"); + refresh.setAccessible(true); + outputShapeRefreshMethod = refresh; + } + } + } + refresh.invoke(metadata); + } catch (Throwable error) { + throw new InferenceException( + "Could not refresh the LiteRT output shape", error); + } + } + private static int inputIndex(Interpreter interpreter, String name) { for (int i = 0; i < interpreter.getInputTensorCount(); i++) { if (name == null || name.equals(interpreter.getInputTensor(i).name())) { diff --git a/Ports/iOSPort/nativeSources/CN1Inference.m b/Ports/iOSPort/nativeSources/CN1Inference.m index 8897696a184..54aae618e8c 100644 --- a/Ports/iOSPort/nativeSources/CN1Inference.m +++ b/Ports/iOSPort/nativeSources/CN1Inference.m @@ -91,6 +91,18 @@ static void cn1InferenceEnsureHandles(void) { static NSString *cn1InferenceOpenPath(NSString *path, BOOL deleteModelOnClose, int threads, int accelerator, BOOL allowFallback) { + // TFLCoreMLDelegate does not expose a way to require Neural Engine-only + // execution. It may schedule unsupported operations on CPU or GPU, so a + // strict NPU request cannot honor the portable no-fallback contract. + if (accelerator == 3 && !allowFallback) { + if (deleteModelOnClose) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + } + return cn1InferenceJSON(@{ + @"error": @"Strict NPU execution cannot be verified on iOS; " + @"the Core ML delegate may use CPU or GPU" + }); + } TFLInterpreterOptions *options = [[TFLInterpreterOptions alloc] init]; if (threads > 0) options.numberOfThreads = (NSUInteger)threads; NSMutableArray *delegates = [NSMutableArray array]; diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md index 8a5bc8b8ace..62d56fcd9ff 100644 --- a/docs/ai-on-device-architecture.md +++ b/docs/ai-on-device-architecture.md @@ -162,12 +162,15 @@ conflicting in-flight content identities fail instead of sharing a temporary path. File sources are opened by path without copying the model through the Java heap. Cache promotion verifies that the final file exists and, when supplied, still matches the requested digest before publishing its path. -Android supplies a direct, native-order destination buffer for every current -LiteRT output tensor before invoking `runForMultipleInputsOutputs`; null output -destinations are reserved for externally bound delegate buffers and are not -used by ordinary sessions. NNAPI is best effort when fallback is enabled. -Strict Android NPU requests are rejected because LiteRT cannot verify that -NNAPI delegated every operation instead of retaining CPU nodes. +Android invokes LiteRT with null output destinations, then reads each result +from its native tensor buffer. This allows value-dependent output dimensions +to resolve before Codename One allocates or copies the result. LiteRT caches +output shapes when no input reallocation occurred, so the Android port +explicitly refreshes each output shape after invocation; both Android builders +retain that package-private LiteRT method name through R8. NPU selection is +best effort when fallback is enabled. Strict NPU requests are rejected on +Android because LiteRT cannot verify that NNAPI delegated every operation, +and on iOS because the Core ML delegate may schedule work on CPU or GPU. All native sessions and analyzers must be closed. Expensive open, analysis, and inference work runs off the EDT; completion and error delivery return to diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index acb97c2c6f3..8adfee887a3 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -363,10 +363,12 @@ five adapters. or a private file. Android uses LiteRT and can request NNAPI. iOS uses TensorFlow Lite Objective-C and may request the Core ML delegate. Both mobile implementations fall back to CPU unless -`allowFallback(false)` is set. Android rejects strict -`Accelerator.NPU` sessions because NNAPI can mix delegated and CPU -operations without reporting that the entire graph reached the NPU. -With fallback enabled, NNAPI is a best-effort accelerator preference. +`allowFallback(false)` is set. Android and iOS reject strict +`Accelerator.NPU` sessions: NNAPI can mix delegated and CPU operations, +and the Core ML delegate can schedule work across the Neural Engine, GPU, +and CPU. Neither reports that the entire graph ran on the requested NPU. +With fallback enabled, NPU selection is a best-effort accelerator +preference. The JavaSE simulator and targets without a native LiteRT runtime, including Mac native, return `false` from `InferenceSession.isSupported()`. diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java index a617300952b..7d1c2fa4d68 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java @@ -4635,6 +4635,8 @@ public void usesClassMethod(String cls, String method) { String keepAi = visionSupport || languageSupport || inferenceSupport ? "-keep class com.codename1.impl.android.ai.** { *; }\n\n" : ""; + String keepInferenceRuntime = + androidInferenceProguardRules(inferenceSupport); // workaround broken optimizer in proguard String proguardConfigOverride = "-dontusemixedcaseclassnames\n" + "-dontskipnonpubliclibraryclasses\n" @@ -4646,6 +4648,7 @@ public void usesClassMethod(String cls, String method) { + "-dontwarn com.google.android.gms.**\n" + keepFirebase + keepAi + + keepInferenceRuntime + "-keep class com.codename1.impl.android.AndroidBrowserComponentCallback {\n" + "*;\n" + "}\n\n" @@ -5359,6 +5362,14 @@ static String androidAiAdapterSource(String cls) { return null; } + static String androidInferenceProguardRules(boolean inferenceIncluded) { + return inferenceIncluded + ? "-keepclassmembers class org.tensorflow.lite.TensorImpl {\n" + + " void refreshShape();\n" + + "}\n\n" + : ""; + } + static String xmlize(String s) { s = s.replace("&", "&"); s = s.replace("<", "<"); diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java index 617a2f2ee33..e015378eb55 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java @@ -25,7 +25,9 @@ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; class AndroidAiSourceSelectionTest { @Test @@ -70,4 +72,15 @@ void ignoresSharedApiAndUnrelatedClasses() { assertNull(AndroidGradleBuilder.androidAiAdapterSource( "com/codename1/ui/Form")); } + + @Test + void retainsLiteRtOutputShapeRefreshOnlyForInference() { + String rules = + AndroidGradleBuilder.androidInferenceProguardRules(true); + assertTrue(rules.contains("org.tensorflow.lite.TensorImpl")); + assertTrue(rules.contains("void refreshShape();")); + assertFalse(AndroidGradleBuilder + .androidInferenceProguardRules(false) + .contains("TensorImpl")); + } } diff --git a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md index 9e7226d858d..660792173b9 100644 --- a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md +++ b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md @@ -347,9 +347,10 @@ On iOS, an ML Kit vision pod is selected only when both its analyzer and its matching feature-specific selector are referenced. For example, `VisionBackends.mlKitBarcodeScanning()` selects only the barcode pod. LiteRT is selected only by -`InferenceSession`. Android NNAPI acceleration is best effort when -fallback is enabled. Android rejects `Accelerator.NPU` together with -`allowFallback(false)` because LiteRT cannot prove full-graph delegation. +`InferenceSession`. NPU acceleration is best effort when fallback is +enabled. Android and iOS reject `Accelerator.NPU` together with +`allowFallback(false)`: NNAPI cannot prove full-graph delegation, while +the Core ML delegate may also schedule work on CPU or GPU. Language identification defaults to Apple Natural Language on iOS and ML Kit on Android. Translation and Smart Reply use feature-scoped ML Kit From 1ce782980b3726d5efc1298923de56cdd25929d7 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 05:45:44 +0300 Subject: [PATCH 43/80] Fix developer guide prose lint --- docs/developer-guide/Ai-And-Speech.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index 8adfee887a3..9fa6fd0c9f2 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -393,9 +393,9 @@ argument and rejects the unpinned two-argument overload. Supply the digest on every platform for every third-party or remotely mutable model. Interrupted downloads restart through a temporary `.download` file and are never promoted to the cache until verification succeeds. Concurrent calls for -the same cache key and content identity share one download. A conflicting URL -or digest for a cache key that is already downloading fails rather than -writing to the same temporary file. +the same cache key and content identity share one download. When a cache key +already has an active download, a request with a conflicting URL or digest +fails instead of sharing the temporary file. === On-device language services From 506f33bd34c01c0ebb615e542aba2e7fb1ea75e5 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 05:58:15 +0300 Subject: [PATCH 44/80] Handle inference cancellation and raw frames --- .../ai/inference/InferenceSession.java | 67 ++++++++++++++----- .../com/codename1/ai/vision/VisionImage.java | 17 ++++- .../src/com/codename1/camera/CameraFrame.java | 5 +- .../src/com/codename1/camera/FrameFormat.java | 11 ++- docs/ai-on-device-architecture.md | 8 ++- .../ai/inference/InferenceApiTest.java | 28 ++++++++ .../codename1/ai/vision/VisionApiTest.java | 19 ++++++ .../tests/VisionOnDeviceApiTest.java | 4 +- 8 files changed, 128 insertions(+), 31 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java index 1860456a2d3..e12d434c934 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -149,6 +149,9 @@ public TensorInfo[] getOutputs() { /// container is defensively copied before it is handed to the asynchronous /// backend, so replacing an element after this method returns cannot change /// the pending invocation. Each {@link Tensor} is itself immutable. + /// Canceling the returned resource forwards cancellation to the native + /// backend and releases the session's run slot, allowing a later + /// invocation or a pending {@link #close()} to proceed. /// /// @param inputs one tensor for each model input; {@code null} is treated /// as an empty input array @@ -157,7 +160,7 @@ public TensorInfo[] getOutputs() { /// @throws IllegalArgumentException if an input name, count, or shape does /// not match the model's current input metadata public AsyncResource run(Tensor[] inputs) { - final AsyncResource result; + final AsyncResource backendResult; synchronized (this) { ensureOpen(); if (activeRuns > 0) { @@ -170,9 +173,9 @@ public AsyncResource run(Tensor[] inputs) { ? new Tensor[0] : copyInputs(inputs); validateInputShapes(inputSnapshot, implementation.getInputs(handle)); - result = implementation.run(handle, + backendResult = implementation.run(handle, inputSnapshot); - if (result == null) { + if (backendResult == null) { throw new InferenceException( "Inference backend returned no asynchronous result"); } @@ -185,18 +188,7 @@ public AsyncResource run(Tensor[] inputs) { } } final PendingRun pending = new PendingRun(this); - result.ready(new SuccessCallback() { - @Override - public void onSucess(Tensor[] value) { - pending.finish(); - } - }).except(new SuccessCallback() { - @Override - public void onSucess(Throwable error) { - pending.finish(); - } - }); - return result; + return new SessionRunResource(backendResult, pending); } private static void validateInputShapes(Tensor[] inputs, @@ -343,4 +335,49 @@ synchronized void finish() { } } } + + private static final class SessionRunResource + extends AsyncResource { + private final AsyncResource backend; + private final PendingRun pending; + + SessionRunResource(AsyncResource backend, + PendingRun pending) { + this.backend = backend; + this.pending = pending; + backend.ready(new SuccessCallback() { + @Override + public void onSucess(Tensor[] value) { + try { + if (!isCancelled()) { + complete(value); + } + } finally { + SessionRunResource.this.pending.finish(); + } + } + }).except(new SuccessCallback() { + @Override + public void onSucess(Throwable error) { + try { + if (!isCancelled()) { + error(error); + } + } finally { + SessionRunResource.this.pending.finish(); + } + } + }); + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + boolean cancelled = super.cancel(mayInterruptIfRunning); + if (cancelled) { + backend.cancel(mayInterruptIfRunning); + pending.finish(); + } + return cancelled; + } + } } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java index 3a50828efd2..1b94a2e04da 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java @@ -77,7 +77,12 @@ public static VisionImage pixels(byte[] bytes, int width, int height, return new VisionImage(null, bytes, width, height, rotationDegrees, 0, format); } - /// Copies a camera frame, including timestamp, format, and orientation. + /// Copies the buffer selected by the camera frame's format, together with + /// its timestamp and orientation. JPEG frames copy only + /// {@link CameraFrame#getJpegBytes()}; NV21 and RGBA8888 frames copy only + /// {@link CameraFrame#getRawBytes()}. This preserves the requested raw + /// pipeline without retaining or preferring an encoded fallback. + /// /// @param frame callback-owned frame /// @return detached immutable input safe for asynchronous analysis /// @throws IllegalArgumentException if the frame reports a rotation other @@ -86,9 +91,15 @@ public static VisionImage fromCameraFrame(CameraFrame frame) { if (frame == null) { throw new NullPointerException("frame"); } - return new VisionImage(frame.getJpegBytes(), frame.getRawBytes(), + FrameFormat format = frame.getFormat() == null + ? FrameFormat.JPEG : frame.getFormat(); + byte[] encoded = format == FrameFormat.JPEG + ? frame.getJpegBytes() : null; + byte[] pixels = format == FrameFormat.JPEG + ? null : frame.getRawBytes(); + return new VisionImage(encoded, pixels, frame.getWidth(), frame.getHeight(), frame.getRotationDegrees(), - frame.getTimestampNanos(), frame.getFormat()); + frame.getTimestampNanos(), format); } /// @return a defensive copy of encoded bytes, or {@code null} for raw input diff --git a/CodenameOne/src/com/codename1/camera/CameraFrame.java b/CodenameOne/src/com/codename1/camera/CameraFrame.java index 58424b7f82a..b39c2e1aec2 100644 --- a/CodenameOne/src/com/codename1/camera/CameraFrame.java +++ b/CodenameOne/src/com/codename1/camera/CameraFrame.java @@ -52,8 +52,9 @@ public CameraFrame(byte[] jpegBytes, byte[] rawBytes, } /// JPEG-encoded bytes for this frame. Always non-null regardless of the - /// requested `FrameFormat`; the framework encodes raw frames into JPEG - /// on demand for AI module consumption. + /// requested `FrameFormat`; use this when encoded image data is needed. + /// `VisionImage#fromCameraFrame(CameraFrame)` instead selects the raw + /// buffer for NV21 and RGBA8888 frames. public byte[] getJpegBytes() { return jpegBytes; } diff --git a/CodenameOne/src/com/codename1/camera/FrameFormat.java b/CodenameOne/src/com/codename1/camera/FrameFormat.java index 99e44ce824e..a7cefe267e0 100644 --- a/CodenameOne/src/com/codename1/camera/FrameFormat.java +++ b/CodenameOne/src/com/codename1/camera/FrameFormat.java @@ -24,14 +24,13 @@ /// Pixel format requested for `CameraFrame` data delivered to a `FrameListener`. /// -/// `JPEG` is the default and the right choice for feeding frames into the -/// `com.codename1.ai.*` modules, all of which accept JPEG `byte[]` directly. -/// Raw formats (`NV21`, `RGBA8888`) are useful when an application performs -/// its own pixel processing and wants to avoid the JPEG encode/decode round-trip. +/// `JPEG` is the default and is useful when an application needs a portable +/// encoded image. Built-in vision analyzers also accept raw `NV21` and +/// `RGBA8888` frames directly; select a raw format for live analysis to avoid +/// a JPEG encode/decode round-trip. public enum FrameFormat { /// JPEG-encoded bytes available via `CameraFrame#getJpegBytes()`. - /// Always available regardless of the requested format; this is the - /// universal format for AI/vision module integration. + /// Always available regardless of the requested format. JPEG, /// YUV 4:2:0 NV21 layout available via `CameraFrame#getRawBytes()`. diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md index 62d56fcd9ff..18ae22ff104 100644 --- a/docs/ai-on-device-architecture.md +++ b/docs/ai-on-device-architecture.md @@ -141,9 +141,11 @@ encode/decode. `VisionImage.fromCameraFrame()` is safe beyond the `FrameListener.onFrame()` callback because it copies the callback-owned -arrays. `VisionPipeline` allows one request to run and retains only the newest -pending frame. This bounds memory and latency for live OCR, barcode, face, -pose, labeling, or segmentation. +buffer selected by the requested `FrameFormat`. Raw frames do not retain the +always-available JPEG fallback, so mobile backends consume NV21 or RGBA8888 +without accidentally selecting the encoded path. `VisionPipeline` allows one +request to run and retains only the newest pending frame. This bounds memory +and latency for live OCR, barcode, face, pose, labeling, or segmentation. Native results are converted to stable Codename One value types. Geometry is normalized to the top-left coordinate system. `VisionMetadata` carries the diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index 9fd01c1e6a0..2ac23c7e986 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -346,6 +346,34 @@ void sessionDefersCloseUntilPendingRunSettles() { assertThrows(IllegalStateException.class, session::getInputs); } + @Test + void cancelledRunReleasesSessionAndDeferredClose() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + backend.pendingRun = new AsyncResource(); + implementation.setInferenceImpl(backend); + InferenceSession session = await(InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions())); + AsyncResource run = session.run(new Tensor[] { + Tensor.floats("input", new int[] {1, 2}, + new float[] {1, 2}) + }); + + assertTrue(run.cancel(false)); + assertTrue(backend.pendingRun.isCancelled(), + "session cancellation must reach the backend operation"); + backend.pendingRun = new AsyncResource(); + AsyncResource next = session.run(new Tensor[] { + Tensor.floats("input", new int[] {1, 2}, + new float[] {3, 4}) + }); + session.close(); + assertEquals(0, backend.closeCount, + "the second pending run must still defer native close"); + assertTrue(next.cancel(false)); + assertEquals(1, backend.closeCount, + "cancelling the final run must complete deferred close"); + } + @Test void sessionSnapshotsInputArrayBeforeAsyncInference() { RecordingInferenceImpl backend = new RecordingInferenceImpl(); diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java index dc4f13d7045..74081d85f33 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java @@ -22,6 +22,7 @@ */ package com.codename1.ai.vision; +import com.codename1.camera.CameraFrame; import com.codename1.camera.FrameFormat; import com.codename1.impl.VisionImpl; import com.codename1.junit.UITestBase; @@ -57,6 +58,24 @@ void pixelInputNormalizesRotation() { 1, 1, FrameFormat.RGBA8888, 45)); } + @Test + void cameraFrameCopiesOnlyTheRequestedFormat() { + byte[] jpeg = new byte[] {1, 2}; + byte[] pixels = new byte[] {3, 4, 5, 6}; + VisionImage raw = VisionImage.fromCameraFrame(new CameraFrame( + jpeg, pixels, 1, 1, 0, 7L, FrameFormat.RGBA8888)); + jpeg[0] = 9; + pixels[0] = 9; + assertNull(raw.getEncodedBytes()); + assertArrayEquals(new byte[] {3, 4, 5, 6}, raw.getPixels()); + assertEquals(7L, raw.getTimestampNanos()); + + VisionImage encoded = VisionImage.fromCameraFrame(new CameraFrame( + jpeg, pixels, 1, 1, 0, 8L, FrameFormat.JPEG)); + assertNotNull(encoded.getEncodedBytes()); + assertNull(encoded.getPixels()); + } + @Test void backendMetadataIsOptionalAndImmutable() { Map values = new HashMap(); diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java index 1fdd5afd597..d0f1f64147d 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java @@ -131,8 +131,8 @@ private void checkImageOwnership() { VisionImage cameraImage = VisionImage.fromCameraFrame(frame); jpeg[0] = 99; pixels[0] = 99; - checkEqual(4, cameraImage.getEncodedBytes()[0], - "VisionImage must own camera JPEG bytes"); + check(cameraImage.getEncodedBytes() == null, + "raw VisionImage must not retain camera JPEG bytes"); checkEqual(10, cameraImage.getPixels()[0], "VisionImage must own camera pixel bytes"); checkEqual(1, cameraImage.getWidth(), "camera image width"); From f5efc486b4001a844486da37d8033241b643bb29 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:07:24 +0300 Subject: [PATCH 45/80] Keep native inference serialized after cancellation --- .../ai/inference/InferenceSession.java | 19 ++++----------- .../com/codename1/ai/vision/VisionImage.java | 18 +++++++++----- .../src/com/codename1/camera/CameraFrame.java | 13 ++++++---- docs/ai-on-device-architecture.md | 9 ++++--- .../ai/inference/InferenceApiTest.java | 24 ++++++++++--------- .../codename1/ai/vision/VisionApiTest.java | 6 +++++ .../tests/VisionOnDeviceApiTest.java | 12 ++++++++++ 7 files changed, 62 insertions(+), 39 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java index e12d434c934..f86cd779a24 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -149,9 +149,10 @@ public TensorInfo[] getOutputs() { /// container is defensively copied before it is handed to the asynchronous /// backend, so replacing an element after this method returns cannot change /// the pending invocation. Each {@link Tensor} is itself immutable. - /// Canceling the returned resource forwards cancellation to the native - /// backend and releases the session's run slot, allowing a later - /// invocation or a pending {@link #close()} to proceed. + /// Canceling the returned resource suppresses result publication but does + /// not interrupt an invocation that has already entered LiteRT. The + /// session remains busy, and a pending {@link #close()} remains deferred, + /// until the native operation actually succeeds or fails. /// /// @param inputs one tensor for each model input; {@code null} is treated /// as an empty input array @@ -338,12 +339,10 @@ synchronized void finish() { private static final class SessionRunResource extends AsyncResource { - private final AsyncResource backend; private final PendingRun pending; SessionRunResource(AsyncResource backend, PendingRun pending) { - this.backend = backend; this.pending = pending; backend.ready(new SuccessCallback() { @Override @@ -369,15 +368,5 @@ public void onSucess(Throwable error) { } }); } - - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - boolean cancelled = super.cancel(mayInterruptIfRunning); - if (cancelled) { - backend.cancel(mayInterruptIfRunning); - pending.finish(); - } - return cancelled; - } } } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java index 1b94a2e04da..58fa66cf8f5 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java @@ -80,8 +80,10 @@ public static VisionImage pixels(byte[] bytes, int width, int height, /// Copies the buffer selected by the camera frame's format, together with /// its timestamp and orientation. JPEG frames copy only /// {@link CameraFrame#getJpegBytes()}; NV21 and RGBA8888 frames copy only - /// {@link CameraFrame#getRawBytes()}. This preserves the requested raw - /// pipeline without retaining or preferring an encoded fallback. + /// {@link CameraFrame#getRawBytes()} when the port supplies that buffer. + /// If a port cannot supply the requested raw format, this method copies + /// the always-available JPEG fallback instead. This preserves the raw + /// pipeline without creating an empty image on JPEG-only camera ports. /// /// @param frame callback-owned frame /// @return detached immutable input safe for asynchronous analysis @@ -93,10 +95,14 @@ public static VisionImage fromCameraFrame(CameraFrame frame) { } FrameFormat format = frame.getFormat() == null ? FrameFormat.JPEG : frame.getFormat(); - byte[] encoded = format == FrameFormat.JPEG - ? frame.getJpegBytes() : null; - byte[] pixels = format == FrameFormat.JPEG - ? null : frame.getRawBytes(); + byte[] raw = frame.getRawBytes(); + boolean useRaw = format != FrameFormat.JPEG + && raw != null && raw.length > 0; + byte[] encoded = useRaw ? null : frame.getJpegBytes(); + byte[] pixels = useRaw ? raw : null; + if (!useRaw) { + format = FrameFormat.JPEG; + } return new VisionImage(encoded, pixels, frame.getWidth(), frame.getHeight(), frame.getRotationDegrees(), frame.getTimestampNanos(), format); diff --git a/CodenameOne/src/com/codename1/camera/CameraFrame.java b/CodenameOne/src/com/codename1/camera/CameraFrame.java index b39c2e1aec2..5a1c12f043b 100644 --- a/CodenameOne/src/com/codename1/camera/CameraFrame.java +++ b/CodenameOne/src/com/codename1/camera/CameraFrame.java @@ -60,8 +60,10 @@ public byte[] getJpegBytes() { } /// Raw pixel bytes in the format requested via - /// `CameraSessionOptions#frameFormat(FrameFormat)`. `null` when the - /// requested format was `FrameFormat#JPEG`. + /// `CameraSessionOptions#frameFormat(FrameFormat)`. This is `null` for + /// JPEG requests and may also be `null` when the current port cannot + /// expose the requested raw format. In that case + /// {@link #getJpegBytes()} remains available as a fallback. public byte[] getRawBytes() { return rawBytes; } @@ -88,8 +90,11 @@ public long getTimestampNanos() { return timestampNanos; } - /// Pixel format actually used for `#getRawBytes()`. JPEG bytes returned by - /// `#getJpegBytes()` are always JPEG-encoded regardless of this value. + /// Pixel format requested for this frame. A non-JPEG value describes + /// {@link #getRawBytes()} when that buffer is available; ports that cannot + /// expose raw pixels may return {@code null} there while still reporting + /// the requested format. JPEG bytes returned by {@link #getJpegBytes()} + /// are always JPEG-encoded regardless of this value. public FrameFormat getFormat() { return format; } diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md index 18ae22ff104..71bce95e58c 100644 --- a/docs/ai-on-device-architecture.md +++ b/docs/ai-on-device-architecture.md @@ -143,9 +143,12 @@ encode/decode. `FrameListener.onFrame()` callback because it copies the callback-owned buffer selected by the requested `FrameFormat`. Raw frames do not retain the always-available JPEG fallback, so mobile backends consume NV21 or RGBA8888 -without accidentally selecting the encoded path. `VisionPipeline` allows one -request to run and retains only the newest pending frame. This bounds memory -and latency for live OCR, barcode, face, pose, labeling, or segmentation. +without accidentally selecting the encoded path. When a camera port cannot +supply the requested raw buffer, `fromCameraFrame()` retains the JPEG fallback +and reports JPEG as the image format instead of creating an empty image. +`VisionPipeline` allows one request to run and retains only the newest pending +frame. This bounds memory and latency for live OCR, barcode, face, pose, +labeling, or segmentation. Native results are converted to stable Codename One value types. Geometry is normalized to the top-left coordinate system. `VisionMetadata` carries the diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index 2ac23c7e986..d8cb254ee07 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -347,7 +347,7 @@ void sessionDefersCloseUntilPendingRunSettles() { } @Test - void cancelledRunReleasesSessionAndDeferredClose() { + void cancelledRunStaysSerializedUntilNativeWorkSettles() { RecordingInferenceImpl backend = new RecordingInferenceImpl(); backend.pendingRun = new AsyncResource(); implementation.setInferenceImpl(backend); @@ -359,19 +359,21 @@ void cancelledRunReleasesSessionAndDeferredClose() { }); assertTrue(run.cancel(false)); - assertTrue(backend.pendingRun.isCancelled(), - "session cancellation must reach the backend operation"); - backend.pendingRun = new AsyncResource(); - AsyncResource next = session.run(new Tensor[] { - Tensor.floats("input", new int[] {1, 2}, - new float[] {3, 4}) - }); + assertFalse(backend.pendingRun.isCancelled(), + "outward cancellation must not interrupt a native invocation"); + assertThrows(IllegalStateException.class, () -> session.run( + new Tensor[] {Tensor.floats("input", new int[] {1, 2}, + new float[] {3, 4})})); session.close(); assertEquals(0, backend.closeCount, - "the second pending run must still defer native close"); - assertTrue(next.cancel(false)); + "cancellation must not close an interpreter still in use"); + backend.pendingRun.complete(new Tensor[] { + Tensor.floats("output", new int[] {1}, new float[] {3}) + }); + flushSerialCalls(); + assertTrue(run.isCancelled()); assertEquals(1, backend.closeCount, - "cancelling the final run must complete deferred close"); + "native settlement must complete deferred close"); } @Test diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java index 74081d85f33..f3d12761ff0 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java @@ -74,6 +74,12 @@ void cameraFrameCopiesOnlyTheRequestedFormat() { jpeg, pixels, 1, 1, 0, 8L, FrameFormat.JPEG)); assertNotNull(encoded.getEncodedBytes()); assertNull(encoded.getPixels()); + + VisionImage fallback = VisionImage.fromCameraFrame(new CameraFrame( + jpeg, null, 1, 1, 0, 9L, FrameFormat.NV21)); + assertNotNull(fallback.getEncodedBytes()); + assertNull(fallback.getPixels()); + assertEquals(FrameFormat.JPEG, fallback.getFormat()); } @Test diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java index d0f1f64147d..37a9700479f 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java @@ -143,6 +143,18 @@ private void checkImageOwnership() { "camera image timestamp"); check(cameraImage.getFormat() == FrameFormat.RGBA8888, "camera image format"); + + byte[] fallbackJpeg = new byte[] {50, 60}; + VisionImage fallbackImage = VisionImage.fromCameraFrame( + new CameraFrame(fallbackJpeg, null, 1, 1, 0, 1L, + FrameFormat.NV21)); + fallbackJpeg[0] = 99; + checkEqual(50, fallbackImage.getEncodedBytes()[0], + "JPEG-only port fallback must be copied"); + check(fallbackImage.getPixels() == null, + "JPEG-only port fallback must not expose raw pixels"); + check(fallbackImage.getFormat() == FrameFormat.JPEG, + "JPEG-only port fallback must report JPEG"); } private void checkOptions() { From 434e7822c7e1eb267855a16e9bdba31df09596e5 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:19:31 +0300 Subject: [PATCH 46/80] Guard pending vision work after close --- .../codename1/ai/vision/VisionPipeline.java | 23 +++++++++++---- .../codename1/camera/VisionPipelineTest.java | 28 +++++++++++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java index fe6a452a55d..1dae6f10671 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java @@ -80,9 +80,17 @@ private void accept(VisionImage image) { private void process(final VisionImage image) { final com.codename1.util.AsyncResource operation; try { - operation = analyzer.process(image); - if (operation == null) { - throw new IllegalStateException("Vision analyzer returned no operation"); + synchronized (this) { + if (closed) { + busy = false; + pending = null; + return; + } + operation = analyzer.process(image); + if (operation == null) { + throw new IllegalStateException( + "Vision analyzer returned no operation"); + } } } catch (final Throwable error) { onFinished(new Runnable() { @@ -144,7 +152,7 @@ public void run() { /// Returns whether a frame is currently being analyzed. /// - /// @return {@code true} while an analysis is in flight + /// @return {@code true} while the open pipeline has an analysis in flight public boolean isBusy() { synchronized (this) { return busy; @@ -152,8 +160,10 @@ public boolean isBusy() { } /// Stops accepting frames, detaches from the camera session, discards the - /// pending frame, and closes the analyzer. Calling this method more than - /// once has no effect. + /// pending frame, clears the busy state, and closes the analyzer. A pending + /// frame already selected for dispatch is rechecked and discarded before + /// it can reach the closed analyzer. Calling this method more than once + /// has no effect. @Override public void close() { synchronized (this) { @@ -161,6 +171,7 @@ public void close() { return; } closed = true; + busy = false; pending = null; } session.removeFrameListener(frameListener); diff --git a/maven/core-unittests/src/test/java/com/codename1/camera/VisionPipelineTest.java b/maven/core-unittests/src/test/java/com/codename1/camera/VisionPipelineTest.java index 66dcc432000..945416359f4 100644 --- a/maven/core-unittests/src/test/java/com/codename1/camera/VisionPipelineTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/camera/VisionPipelineTest.java @@ -98,6 +98,32 @@ public void error(Throwable error) { assertFalse(pipeline.isBusy()); } + @Test + void listenerCloseDiscardsAlreadySelectedPendingFrame() { + final RecordingAnalyzer analyzer = new RecordingAnalyzer(); + pipeline = new VisionPipeline(session, analyzer, + new VisionPipelineListener() { + public void result(String value, VisionImage image) { + pipeline.close(); + } + + public void error(Throwable error) { + } + }); + + implementationBackend.lastFrameListener.onFrame(frame(1)); + implementationBackend.lastFrameListener.onFrame(frame(2)); + assertEquals(1, analyzer.operations.size()); + + analyzer.operations.get(0).complete("first"); + flushSerialCalls(); + + assertEquals(1, analyzer.operations.size(), + "close from the listener must discard the pending frame"); + assertEquals(1, analyzer.closeCount); + assertFalse(pipeline.isBusy()); + } + private static CameraFrame frame(int value) { return new CameraFrame(new byte[] {(byte) value}, null, 1, 1, 0, value, FrameFormat.JPEG); @@ -107,6 +133,7 @@ private static final class RecordingAnalyzer implements VisionAnalyzer { final List> operations = new ArrayList>(); + int closeCount; public boolean isSupported() { return true; @@ -119,6 +146,7 @@ public AsyncResource process(VisionImage image) { } public void close() { + closeCount++; } } } From 747cecf9efc5b5754a236a75cdcc3d4b90b64af2 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 07:06:17 +0300 Subject: [PATCH 47/80] Define model download request identity --- .../codename1/ai/inference/ModelCache.java | 27 +++++++++++++++++++ .../ai/inference/InferenceApiTest.java | 27 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java index 331f07b6d51..0001f34f355 100644 --- a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java +++ b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java @@ -389,6 +389,33 @@ static final class ModelDownloadRequest extends ConnectionRequest { this.temporary = temporary; } + /// Compares the inherited URL and request arguments together with the + /// temporary destination path. Callback and storage-service instances + /// do not change the logical identity of a download. + /// + /// @param other request to compare + /// @return {@code true} when both requests download the same URL and + /// arguments to the same temporary path + @Override + public boolean equals(Object other) { + if (!super.equals(other)) { + return false; + } + ModelDownloadRequest request = (ModelDownloadRequest) other; + return temporary == null ? request.temporary == null + : temporary.equals(request.temporary); + } + + /// Produces a hash consistent with {@link #equals(Object)} from the + /// inherited request identity and temporary destination. + /// + /// @return hash of the logical download identity + @Override + public int hashCode() { + return 31 * super.hashCode() + + (temporary == null ? 0 : temporary.hashCode()); + } + @Override public boolean onRedirect(String url) { if (isHttpsUrl(url)) { diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index d8cb254ee07..13b4b2fca42 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -122,6 +122,33 @@ public void onSucess(Throwable value) { "http://cdn.example.com/model.tflite")); } + @Test + void modelDownloadRequestIdentityIncludesTemporaryPath() { + FileSystemStorage fs = FileSystemStorage.getInstance(); + ModelCache.ModelDownloadRequest first = + new ModelCache.ModelDownloadRequest( + new ModelCache.Completion( + new AsyncResource()), + fs, "model-a.download"); + ModelCache.ModelDownloadRequest same = + new ModelCache.ModelDownloadRequest( + new ModelCache.Completion( + new AsyncResource()), + fs, "model-a.download"); + ModelCache.ModelDownloadRequest differentTarget = + new ModelCache.ModelDownloadRequest( + new ModelCache.Completion( + new AsyncResource()), + fs, "model-b.download"); + first.setUrl("https://example.com/model.tflite"); + same.setUrl("https://example.com/model.tflite"); + differentTarget.setUrl("https://example.com/model.tflite"); + + assertEquals(first, same); + assertEquals(first.hashCode(), same.hashCode()); + assertNotEquals(first, differentTarget); + } + @Test void modelCacheCoalescesIdenticalConcurrentFetches() { String fileName = "model-cache-coalescing-test.tflite"; From cda000ae2cb342133b208747cc605bd12b33bb75 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 07:20:50 +0300 Subject: [PATCH 48/80] Correct on-device AI documentation --- .../generated/AiAndSpeechOnDeviceSnippet.java | 21 +- .../content/blog/platform-apis-in-the-core.md | 243 +++++------------- 2 files changed, 76 insertions(+), 188 deletions(-) diff --git a/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java b/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java index b69bd8d8674..7bb9fd46dba 100644 --- a/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java +++ b/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java @@ -41,7 +41,7 @@ class AiAndSpeechOnDeviceSnippet { void vision() { // tag::ai-and-speech-on-device-vision[] VisionOptions options = new VisionOptions() - .backend(VisionBackends.appleVision()) + .backend(VisionBackends.mlKitTextRecognition()) .minimumConfidence(0.5f); new TextRecognizer(options) @@ -61,9 +61,22 @@ void inference() { Tensor input = Tensor.floats("serving_default_input", new int[] {1, 224, 224, 3}, pixels); session.run(new Tensor[] {input}) - .ready(outputs -> consume(outputs[0])) - .except(error -> Log.e(error)); - }); + .ready(outputs -> { + try { + consume(outputs[0]); + } finally { + session.close(); + } + }) + .except(error -> { + try { + Log.e(error); + } finally { + session.close(); + } + }); + }) + .except(error -> Log.e(error)); // end::ai-and-speech-on-device-inference[] } diff --git a/docs/website/content/blog/platform-apis-in-the-core.md b/docs/website/content/blog/platform-apis-in-the-core.md index e7f83d76982..324c79ac73e 100644 --- a/docs/website/content/blog/platform-apis-in-the-core.md +++ b/docs/website/content/blog/platform-apis-in-the-core.md @@ -244,194 +244,64 @@ view.setInputListener(userText -> { `appendToLastMessage(...)` is the streaming entry point; it marshals through `callSerially` so deltas land on the EDT in order. `ConversationStore` persists the thread (the default backing is `Storage`; pluggable via a custom implementation if you would rather keep it in SQLite or push it to your server). -### The AI cn1libs +### On-device vision, language, and LiteRT -The core LLM stack is paired with a set of opt-in cn1libs that wrap specific on-device capabilities: Google ML Kit features, the TensorFlow Lite runtime, a local Whisper transcription engine, and an on-device Stable Diffusion model. Thirteen new cn1libs ship this release. +**Update (July 2026):** On-device vision, language services, and LiteRT +inference now ship as framework APIs. Do not add separate ML Kit or LiteRT +cn1lib dependencies. Import the APIs from `com.codename1.ai.vision`, +`com.codename1.ai.language`, and `com.codename1.ai.inference`. -These cn1libs are not yet listed in the Codename One Preferences cn1lib picker, so for the moment they are added by hand. Drop the matching dependency block into your project's `common/pom.xml` and rebuild. The build-time scanner does the rest: the iOS pod or Swift Package, the Android Gradle dependency, the plist usage strings (`NSCameraUsageDescription` for the vision libraries, `NSSpeechRecognitionUsageDescription` for Whisper, etc.), and the Android permissions (`android.permission.RECORD_AUDIO` for audio capture) are all injected automatically the first time the scanner sees the matching class on the classpath. - -For each cn1lib below, the dependency block is identical in shape; only the `` changes. The shared pattern is: - -```xml - - com.codenameone - - ${cn1.version} - -``` - -#### `cn1-ai-mlkit-text`: text recognition (OCR) - -**TL;DR.** Pull printed or handwritten text out of an image (a photo of a page, a sign, a receipt) entirely on-device. - -**Platforms.** iOS bridges to `GoogleMLKit/TextRecognition`. Android bridges to `com.google.mlkit:text-recognition`. The JavaSE simulator returns an unsupported error. - -**Use cases.** Receipt scanning, sign translation pipelines (combine with `cn1-ai-mlkit-translate`), accessibility tools that read printed text aloud, automated form ingestion. - -```java -byte[] jpeg = capturePhotoBytes(); -TextRecognizer.recognize(jpeg).onResult((text, err) -> { - if (err == null) Log.p("OCR: " + text); -}); -``` - -#### `cn1-ai-mlkit-barcode`: barcode and QR scanning - -**TL;DR.** Decodes QR, EAN, UPC, Data Matrix, PDF417, and the rest of the common 1D / 2D code families from a captured image. - -**Platforms.** iOS bridges to `MLKitBarcodeScanning`. Android bridges to `com.google.mlkit:barcode-scanning`. The JavaSE simulator returns an unsupported error. - -**Use cases.** Inventory scanning, ticket / boarding-pass readers, QR-driven onboarding flows, retail loyalty cards. - -```java -byte[] jpeg = capturePhotoBytes(); -BarcodeScanner.scan(jpeg).onResult((codes, err) -> { - if (err == null) { - for (String code : codes) Log.p("Found: " + code); - } -}); -``` - -#### `cn1-ai-mlkit-face`: face detection - -**TL;DR.** Returns bounding boxes for human faces detected in an image. Each face is reported as a packed `int[4]` (`x`, `y`, `width`, `height`). - -**Platforms.** iOS bridges to `MLKitFaceDetection`. Android bridges to `com.google.mlkit:face-detection`. - -**Use cases.** Auto-crop a contact photo, mosaic / blur bystanders in a group shot, drive a face-tracked overlay for AR-lite filters. +The builders inspect the feature classes and backend-selector methods that an +application actually uses. They retain only the matching native adapters and +add the required Android library, Apple framework, or optional iOS ML Kit pod. +For example, Android text recognition uses ML Kit automatically. Apple targets +default to Vision, while this explicit selector opts iOS into the text-only ML +Kit pod: ```java -FaceDetector.detect(jpeg).onResult((boxes, err) -> { - if (err != null) return; - for (int i = 0; i < boxes.length; i += 4) { - Log.p("face at " + boxes[i] + "," + boxes[i + 1] + " " - + boxes[i + 2] + "x" + boxes[i + 3]); - } -}); -``` +VisionOptions options = new VisionOptions() + .backend(VisionBackends.mlKitTextRecognition()); -#### `cn1-ai-mlkit-labeling`: image labeling - -**TL;DR.** "What is in this picture." Returns a list of descriptive labels for the image content. - -**Platforms.** iOS bridges to `MLKitImageLabeling`. Android bridges to `com.google.mlkit:image-labeling`. - -**Use cases.** Auto-tagging uploaded photos, content moderation pre-filters, content-based image search. - -```java -ImageLabeler.label(jpeg).onResult((labels, err) -> { - if (err == null) Log.p("labels: " + String.join(", ", labels)); -}); +new TextRecognizer(options) + .process(VisionImage.encoded(jpeg)) + .ready(result -> Log.p(result.getText())) + .except(error -> Log.e(error)); ``` -#### `cn1-ai-mlkit-translate`: on-device translation - -**TL;DR.** Translate short text between supported language pairs entirely on-device; no server round-trip, no API key, works offline. - -**Platforms.** iOS bridges to `MLKitTranslate`. Android bridges to `com.google.mlkit:translate`. Languages are identified by their ISO 639-1 codes (`en`, `fr`, `es`, ...). - -**Use cases.** Offline travel assistants, chat translation, accessibility readers for foreign signage (combine with `cn1-ai-mlkit-text`). - -```java -Translator.translate("Where is the train station?", "en", "fr") - .onResult((fr, err) -> { - if (err == null) Log.p(fr); // "Où est la gare ?" - }); -``` - -#### `cn1-ai-mlkit-smartreply`: short reply suggestions - -**TL;DR.** Generates short suggested replies for chat conversations, similar to Gmail's Smart Reply chips. - -**Platforms.** iOS bridges to `MLKitSmartReply`. Android bridges to `com.google.mlkit:smart-reply`. The input is a JSON array of `{role, message, timestamp, userId}` objects. - -**Use cases.** A "quick reply" row above the keyboard in your in-app chat, response suggestions in a CRM inbox. +`InferenceSession` owns native interpreter and delegate resources. Reuse a +session for repeated runs and close it when the owner is disposed. A one-shot +operation must close the session in both completion paths: ```java -String thread = "[{\"role\":\"remote\",\"message\":\"See you at 6?\"," - + "\"timestamp\":" + System.currentTimeMillis() + "," - + "\"userId\":\"u42\"}]"; - -SmartReply.suggest(thread).onResult((suggestions, err) -> { - if (err == null) { - for (String s : suggestions) Log.p("suggestion: " + s); - } -}); -``` - -#### `cn1-ai-mlkit-langid`: language identification - -**TL;DR.** Returns the most likely ISO 639-1 code for a given text, or `und` (undetermined) when the input is too short or ambiguous. - -**Platforms.** iOS bridges to `MLKitLanguageID`. Android bridges to `com.google.mlkit:language-id`. - -**Use cases.** Auto-route a customer-support message to the right team, pick the correct TTS voice for an arbitrary string, pre-screen input before running an expensive translate. - -```java -LanguageIdentifier.identify("Bonjour le monde").onResult((code, err) -> { - if (err == null) Log.p(code); // "fr" -}); -``` - -#### `cn1-ai-mlkit-pose`: pose detection - -**TL;DR.** Returns 33 skeletal landmarks per detected pose as a packed `float[3 * 33]` (`x`, `y`, `confidence` triples). - -**Platforms.** iOS bridges to `MLKitPoseDetection`. Android bridges to `com.google.mlkit:pose-detection`. - -**Use cases.** Fitness apps with form correction, dance / yoga timing analysis, gesture-driven controls. - -```java -PoseDetector.detect(jpeg).onResult((landmarks, err) -> { - if (err != null || landmarks.length < 99) return; - float noseX = landmarks[0], noseY = landmarks[1], noseConf = landmarks[2]; - Log.p("nose at (" + noseX + ", " + noseY + ") conf=" + noseConf); -}); -``` - -#### `cn1-ai-mlkit-segmentation`: selfie segmentation - -**TL;DR.** Returns a per-pixel mask separating the person in the foreground from the background as `byte[width * height]` (`0` = background, `255` = foreground). - -**Platforms.** iOS bridges to `MLKitSegmentationSelfie`. Android bridges to `com.google.mlkit:segmentation-selfie`. - -**Use cases.** Background replacement for video calls, sticker / portrait-mode effects, blur-the-background privacy filters. - -```java -SelfieSegmenter.segment(jpeg).onResult((mask, err) -> { - if (err == null) applyBackgroundReplacement(mask); -}); -``` - -#### `cn1-ai-mlkit-docscan`: document scanner - -**TL;DR.** Detects a rectangular document in a photo, perspective-corrects it, and writes the cropped JPEG to a temporary file. Returns the file path. - -**Platforms.** iOS uses Apple's VisionKit + Core Image rectangle detection (no extra pod). Android uses `com.google.android.gms:play-services-mlkit-document-scanner`. - -**Use cases.** "Scan to PDF" flows, expense apps that capture receipts, contract signing flows, ID-document capture. - -```java -DocumentScanner.scanToFile(jpeg).onResult((path, err) -> { - if (err == null) uploadDocument(path); -}); -``` - -#### `cn1-ai-tflite`: TensorFlow Lite interpreter - -**TL;DR.** A general-purpose on-device inference engine. Bring your own `.tflite` model and run it against a float32 input tensor. - -**Platforms.** iOS uses `TensorFlowLiteSwift` (Pods or Swift Package). Android uses `org.tensorflow:tensorflow-lite` + `tensorflow-lite-support`. - -**Use cases.** Any custom on-device ML model your team trains or pulls from TF Hub. Image classification, simple regression, recommendation pre-filters. - -```java -byte[] modelBytes = Util.readFully(Display.getInstance().getResourceAsStream(null, "/model.tflite")); -float[] input = featureVector(); -Interpreter.run(modelBytes, input).onResult((output, err) -> { - if (err == null) Log.p("model returned " + output.length + " values"); -}); -``` +InferenceSession.open(ModelSource.resource("/model.tflite"), + new InferenceOptions()) + .ready(session -> session.run(new Tensor[] {input}) + .ready(outputs -> { + try { + consume(outputs); + } finally { + session.close(); + } + }) + .except(error -> { + try { + Log.e(error); + } finally { + session.close(); + } + })) + .except(error -> Log.e(error)); +``` + +See [AI, Chat UI, and Speech](https://www.codenameone.com/developer-guide/#_ai_chat_ui_and_speech) +for the current API reference, feature matrix, camera pipeline, model cache, +and backend-selection examples. + +### Large optional AI cn1libs + +Vision, language, and LiteRT fit in core because their native dependencies can +be selected granularly by the builders. Runtimes with very large bundled data +or native libraries remain explicit dependencies. #### `cn1-ai-whisper`: speech-to-text via whisper.cpp @@ -466,15 +336,20 @@ StableDiffusion.generate("a teal hot-air balloon over Lisbon, watercolour", }); ``` -### Why these are cn1libs and not part of the core - -The core gets the AI plumbing every app that adopts AI at all wants: the LLM client, streaming, the chat UI, the secure storage primitive for credentials, the simulator Ollama redirect for offline iteration. +### Why these remain cn1libs -The cn1libs above are specialized verticals. Barcode scanning, document scanning, face detection, smart reply, pose detection, on-device translation, transcription, on-device image generation, each is genuinely useful, but only for some apps. They also each bring a non-trivial native dependency. The Google ML Kit Android frameworks are large; the iOS pods carry their own weight; the bundled `libwhisper.a` and the Stable Diffusion model are big. Pulling all of them into core would tax every app whether the feature is used or not. +The core provides the portable APIs and lets the builders retain native vision, +language, and LiteRT dependencies at feature or selector granularity. Whisper +and Stable Diffusion are different: they bundle unusually large native +libraries or model data that should remain an explicit application choice. The Stable Diffusion cn1lib in particular is large enough that the cloud build server cannot accept the upload at all (it trips the 2 GB pre-upload guard). That kind of opt-in does not belong in a dependency every app inherits. -The corresponding chapter, including the full `LlmClient` API table, the `ChatView` reference, the `SecureStorage` overloads, the simulator Ollama redirect, and the full cn1lib coverage, is at [AI, Chat UI, and Speech](https://www.codenameone.com/developer-guide/#_ai_chat_ui_and_speech) in the developer guide. +The corresponding chapter, including the full `LlmClient` API table, the +`ChatView` reference, the `SecureStorage` overloads, the simulator Ollama +redirect, the built-in on-device APIs, and the remaining cn1lib coverage, is +at [AI, Chat UI, and Speech](https://www.codenameone.com/developer-guide/#_ai_chat_ui_and_speech) +in the developer guide. ## OAuth and OIDC: the modern identity stack From ea0a9287e8fd83f1be9b799ad917548e19954cca Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 10:10:07 +0300 Subject: [PATCH 49/80] Apply catalog platform requirements --- .../builders/AndroidGradleBuilder.java | 3 + .../com/codename1/builders/IPhoneBuilder.java | 4 ++ .../build/shared/PlatformFeatureCatalog.java | 72 +++++++++++++++++++ .../shared/PlatformFeatureCatalogTest.java | 34 +++++++++ 4 files changed, 113 insertions(+) diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java index 7d1c2fa4d68..ed5b7c67871 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java @@ -1728,6 +1728,9 @@ public void usesClassMethod(String cls, String method) { aiExtraGradleDependencies.append(" implementation '").append(gav).append("'\n"); } } + if (aiAcc.minimumAndroidSdk() > 0) { + minSDK = maxInt(Integer.toString(aiAcc.minimumAndroidSdk()), minSDK); + } if (aiAcc.anyRequiresBigUpload()) { request.putArgument("cn1.ai.requiresBigUpload", "true"); } diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java index f1c669e8808..ae37e4604f3 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java @@ -2556,6 +2556,10 @@ public void usesClassMethod(String cls, String method) { } addLibs = appendFrameworks(addLibs, "Vision.framework", "CoreImage.framework", "CoreVideo.framework"); + for (String framework : aiAcc.iosFrameworks()) { + addLibs = appendFrameworks(addLibs, + framework + ".framework"); + } } if (usesCn1Language) { diff --git a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java index 8e1e304cec5..8774bd20229 100644 --- a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java +++ b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java @@ -86,6 +86,7 @@ public final class PlatformFeatureCatalog { e.add(new Entry("com/codename1/ai/vision/TextRecognizer") .iosFrameworks("Vision", "CoreImage") .androidGradle("com.google.mlkit:text-recognition:16.0.0") + .androidMinimumSdk(21) .description("Text recognition")); e.add(new Entry("com/codename1/ai/vision/TextRecognizer") .requiresMethod("com/codename1/ai/vision/VisionBackends", @@ -99,6 +100,7 @@ public final class PlatformFeatureCatalog { e.add(new Entry("com/codename1/ai/vision/BarcodeScanner") .iosFrameworks("Vision", "CoreImage") .androidGradle("com.google.mlkit:barcode-scanning:17.2.0") + .androidMinimumSdk(21) .description("Barcode scanning")); e.add(new Entry("com/codename1/ai/vision/BarcodeScanner") .requiresMethod("com/codename1/ai/vision/VisionBackends", @@ -112,6 +114,7 @@ public final class PlatformFeatureCatalog { e.add(new Entry("com/codename1/ai/vision/FaceDetector") .iosFrameworks("Vision", "CoreImage") .androidGradle("com.google.mlkit:face-detection:16.1.5") + .androidMinimumSdk(21) .description("Face detection")); e.add(new Entry("com/codename1/ai/vision/FaceDetector") .requiresMethod("com/codename1/ai/vision/VisionBackends", @@ -125,6 +128,7 @@ public final class PlatformFeatureCatalog { e.add(new Entry("com/codename1/ai/vision/ImageLabeler") .iosFrameworks("Vision", "CoreML") .androidGradle("com.google.mlkit:image-labeling:17.0.7") + .androidMinimumSdk(21) .description("Image labeling")); e.add(new Entry("com/codename1/ai/vision/ImageLabeler") .requiresMethod("com/codename1/ai/vision/VisionBackends", @@ -138,6 +142,7 @@ public final class PlatformFeatureCatalog { e.add(new Entry("com/codename1/ai/vision/PoseDetector") .iosFrameworks("Vision", "CoreML") .androidGradle("com.google.mlkit:pose-detection:18.0.0-beta3") + .androidMinimumSdk(21) .description("Pose detection")); e.add(new Entry("com/codename1/ai/vision/PoseDetector") .requiresMethod("com/codename1/ai/vision/VisionBackends", @@ -152,6 +157,7 @@ public final class PlatformFeatureCatalog { .iosFrameworks("Vision", "CoreML") .androidGradle( "com.google.mlkit:segmentation-selfie:16.0.0-beta5") + .androidMinimumSdk(21) .description("Selfie segmentation")); e.add(new Entry("com/codename1/ai/vision/SelfieSegmenter") .requiresMethod("com/codename1/ai/vision/VisionBackends", @@ -174,11 +180,13 @@ public final class PlatformFeatureCatalog { .iosDependenciesUnsupportedOnMacCatalyst() .iosFrameworks("CoreML", "Metal", "Accelerate") .androidGradle("com.google.ai.edge.litert:litert:1.0.1") + .androidMinimumSdk(21) .description("LiteRT inference")); e.add(new Entry("com/codename1/ai/language/LanguageIdentifier") .iosFrameworks("NaturalLanguage") .androidGradle("com.google.mlkit:language-id:17.0.6") + .androidMinimumSdk(21) .description("On-device language identification")); e.add(new Entry("com/codename1/ai/language/LanguageIdentifier") .requiresMethod("com/codename1/ai/language/LanguageBackends", @@ -198,6 +206,7 @@ public final class PlatformFeatureCatalog { .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() .androidGradle("com.google.mlkit:translate:17.0.3") + .androidMinimumSdk(21) .description("On-device translation")); e.add(new Entry("com/codename1/ai/language/SmartReply") .iosPod("GoogleMLKit/SmartReply") @@ -206,6 +215,7 @@ public final class PlatformFeatureCatalog { .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() .androidGradle("com.google.mlkit:smart-reply:17.0.4") + .androidMinimumSdk(21) .description("On-device smart reply")); e.add(new Entry("com/codename1/ai/whisper/") @@ -350,6 +360,46 @@ public boolean anyRequiresBigUpload() { } return false; } + + /** + * Returns the highest Android API level required by the matched + * catalog entries. + * + *

Builders should combine this value with the application's + * {@code android.min_sdk_version} and retain the larger value before + * writing either the manifest or Gradle configuration. This prevents + * Android's manifest merger from rejecting a dependency whose own + * minimum SDK is newer than Codename One's default.

+ * + * @return the required Android API level, or {@code 0} when none of + * the matched entries imposes an additional minimum + */ + public int minimumAndroidSdk() { + int minimum = 0; + for (Entry entry : hits()) { + minimum = Math.max(minimum, entry.androidMinimumSdk()); + } + return minimum; + } + + /** + * Returns the de-duplicated Apple system frameworks required by the + * matched entries. + * + *

Names are returned without the {@code .framework} suffix, exactly + * as they appear in the catalog. Builders can append the suffix while + * merging these values into an application's existing library list. + * The returned set is immutable.

+ * + * @return the framework names required by the observed API usage + */ + public Set iosFrameworks() { + Set frameworks = new LinkedHashSet(); + for (Entry entry : hits()) { + frameworks.addAll(entry.iosFrameworks()); + } + return Collections.unmodifiableSet(frameworks); + } } /** @@ -369,6 +419,7 @@ public static final class Entry { private final List androidPermissions = new ArrayList(); private final List androidFeatures = new ArrayList(); private final List androidMetaData = new ArrayList(); + private int androidMinimumSdk; private boolean iosDependenciesSupportMacCatalyst = true; private boolean iosDependenciesSupportArm64Simulator = true; private String iosMinimumDeploymentTarget; @@ -461,6 +512,11 @@ Entry androidGradle(String gav) { return this; } + Entry androidMinimumSdk(int apiLevel) { + androidMinimumSdk = apiLevel; + return this; + } + Entry androidPermissions(String... perms) { for (String p : perms) { androidPermissions.add(p); @@ -551,6 +607,22 @@ public List androidGradleDeps() { return Collections.unmodifiableList(androidGradle); } + /** + * Returns the minimum Android API level required by this entry's + * native dependencies. + * + *

The Android builder raises the application's requested minimum + * to at least this value before generating the manifest and Gradle + * configuration. A value of {@code 0} means that the entry does not + * impose an additional floor.

+ * + * @return the required Android API level, or {@code 0} when no + * catalog-specific minimum is required + */ + public int androidMinimumSdk() { + return androidMinimumSdk; + } + public List androidPermissions() { return Collections.unmodifiableList(androidPermissions); } diff --git a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java index c776e98860e..ad5d4bc8738 100644 --- a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java +++ b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java @@ -42,6 +42,7 @@ void builtInTextRecognizerMapsToAppleVisionAndAndroidMlKit() { assertTrue(e.iosFrameworks().contains("Vision")); assertTrue(e.iosPods().isEmpty()); assertTrue(e.androidGradleDeps().get(0).startsWith("com.google.mlkit:text-recognition")); + assertEquals(21, e.androidMinimumSdk()); assertTrue(e.iosPlistEntries().isEmpty(), "Still-image analysis must not imply camera permission"); } @@ -220,6 +221,39 @@ void inferenceUsesCoreMlEnabledObjectiveCPod() { "The official TensorFlow Lite Objective-C XCFramework has no Catalyst slice"); assertTrue(e.iosDependenciesSupportArm64Simulator(), "TensorFlow Lite's XCFramework includes an arm64 simulator slice"); + assertEquals(21, e.androidMinimumSdk(), + "LiteRT requires Android API 21"); + } + + @Test + void accumulatorCombinesAndroidFloorAndAppleFrameworks() { + PlatformFeatureCatalog.Accumulator acc = + new PlatformFeatureCatalog.Accumulator(); + acc.consume("com/codename1/ai/vision/DocumentScanner"); + acc.consume("com/codename1/ai/vision/ImageLabeler"); + + assertEquals(21, acc.minimumAndroidSdk()); + assertTrue(acc.iosFrameworks().contains("VisionKit")); + assertTrue(acc.iosFrameworks().contains("CoreML")); + assertTrue(acc.iosFrameworks().contains("Vision")); + } + + @Test + void everyMlKitAndLiteRtDependencyDeclaresAndroidFloor() { + int checked = 0; + for (PlatformFeatureCatalog.Entry entry + : PlatformFeatureCatalog.entries()) { + for (String dependency : entry.androidGradleDeps()) { + if (dependency.startsWith("com.google.mlkit:") + || dependency.startsWith( + "com.google.ai.edge.litert:")) { + checked++; + assertEquals(21, entry.androidMinimumSdk(), dependency); + } + } + } + assertEquals(10, checked, + "Expected all built-in AI Android dependencies"); } @Test From dee5804320b40a8a5ebee1007d26010c098493c8 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 10:22:27 +0300 Subject: [PATCH 50/80] Preserve catalog compatibility and plist defaults --- .../com/codename1/builders/IPhoneBuilder.java | 31 ++++-- .../IPhoneBuilderDependencyConfigTest.java | 32 ++++++ .../build/shared/PlatformFeatureCatalog.java | 105 ++++++++++++++++++ .../shared/PlatformFeatureCatalogTest.java | 30 ++++- 4 files changed, 186 insertions(+), 12 deletions(-) diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java index ae37e4604f3..0386ec261c7 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java @@ -356,7 +356,22 @@ private static String appendFrameworks(String libraries, } return out; } - + + private void applyCatalogPlistEntry(BuildRequest request, + String[] plistEntry) { + String privacyKey = plistEntry[0]; + String requestKey = "ios." + privacyKey; + String value = request.getArg(requestKey, null); + if (value == null) { + value = plistEntry[1]; + request.putArgument(requestKey, value); + } + if (value != null + && !privacyUsageDescriptions.containsKey(privacyKey)) { + privacyUsageDescriptions.put(privacyKey, value); + } + } + private int getDeploymentTargetInt(BuildRequest request) { String target = getDeploymentTarget(request); if (target.indexOf(".") > 0) { @@ -1101,10 +1116,7 @@ public void usesClassMethod(String cls, String method) { } } for (String[] plistEntry : entry.iosPlistEntries()) { - String key = "ios." + plistEntry[0]; - if (request.getArg(key, null) == null) { - request.putArgument(key, plistEntry[1]); - } + applyCatalogPlistEntry(request, plistEntry); } } if (spmPackages.length() > 0) { @@ -2544,6 +2556,11 @@ public void usesClassMethod(String cls, String method) { } } + for (String framework : aiAcc.iosFrameworks()) { + addLibs = appendFrameworks(addLibs, + framework + ".framework"); + } + if (usesCn1Vision) { try { replaceInFile(new File(buildinRes, @@ -2556,10 +2573,6 @@ public void usesClassMethod(String cls, String method) { } addLibs = appendFrameworks(addLibs, "Vision.framework", "CoreImage.framework", "CoreVideo.framework"); - for (String framework : aiAcc.iosFrameworks()) { - addLibs = appendFrameworks(addLibs, - framework + ".framework"); - } } if (usesCn1Language) { diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java index 9a6fca21f73..08f697a1944 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java @@ -24,8 +24,10 @@ import org.junit.jupiter.api.Test; +import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.List; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -213,6 +215,36 @@ void dependencyFloorRaisesExplicitDeploymentTarget() throws Exception { assertEquals("15.5", target); } + @Test + void catalogPrivacyDefaultsReachGeneratedPlistMap() throws Exception { + Method apply = IPhoneBuilder.class.getDeclaredMethod( + "applyCatalogPlistEntry", BuildRequest.class, + String[].class); + apply.setAccessible(true); + Field privacy = IPhoneBuilder.class.getDeclaredField( + "privacyUsageDescriptions"); + privacy.setAccessible(true); + + IPhoneBuilder defaultBuilder = new IPhoneBuilder(); + BuildRequest defaultRequest = new BuildRequest(); + apply.invoke(defaultBuilder, defaultRequest, + new String[] {"NSCameraUsageDescription", "Camera default"}); + assertEquals("Camera default", defaultRequest.getArg( + "ios.NSCameraUsageDescription", null)); + Map defaultMap = (Map) privacy.get(defaultBuilder); + assertEquals("Camera default", + defaultMap.get("NSCameraUsageDescription")); + + IPhoneBuilder explicitBuilder = new IPhoneBuilder(); + BuildRequest explicitRequest = requestWithArgs( + "ios.NSCameraUsageDescription", "Developer value"); + apply.invoke(explicitBuilder, explicitRequest, + new String[] {"NSCameraUsageDescription", "Camera default"}); + Map explicitMap = (Map) privacy.get(explicitBuilder); + assertEquals("Developer value", + explicitMap.get("NSCameraUsageDescription")); + } + private BuildRequest requestWithArgs(String... kvPairs) { BuildRequest out = new BuildRequest(); for (int i = 0; i < kvPairs.length; i += 2) { diff --git a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java index 8774bd20229..2fa3ef49878 100644 --- a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java +++ b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java @@ -79,6 +79,111 @@ public final class PlatformFeatureCatalog { .iosFrameworks("AVFAudio") .description("Text-to-speech")); + // Compatibility mappings for the retired AI cn1libs. The artifacts + // already published with these package names remain usable even + // though new applications should use the built-in vision, language, + // and inference APIs below. + e.add(new Entry("com/codename1/ai/mlkit/text/") + .iosPod("GoogleMLKit/TextRecognition") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle("com.google.mlkit:text-recognition:16.0.0") + .androidMinimumSdk(21) + .iosPlist("NSCameraUsageDescription", + "Used to recognise text from your camera.") + .description("Legacy ML Kit Text Recognition cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/barcode/") + .iosPod("GoogleMLKit/BarcodeScanning") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle("com.google.mlkit:barcode-scanning:17.2.0") + .androidMinimumSdk(21) + .iosPlist("NSCameraUsageDescription", + "Used to scan barcodes with your camera.") + .androidFeatures("android.hardware.camera") + .description("Legacy ML Kit Barcode Scanning cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/face/") + .iosPod("GoogleMLKit/FaceDetection") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle("com.google.mlkit:face-detection:16.1.5") + .androidMinimumSdk(21) + .iosPlist("NSCameraUsageDescription", + "Used to detect faces in images.") + .description("Legacy ML Kit Face Detection cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/labeling/") + .iosPod("GoogleMLKit/ImageLabeling") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle("com.google.mlkit:image-labeling:17.0.7") + .androidMinimumSdk(21) + .description("Legacy ML Kit Image Labeling cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/translate/") + .iosPod("GoogleMLKit/Translate") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle("com.google.mlkit:translate:17.0.1") + .androidMinimumSdk(21) + .description("Legacy ML Kit Translation cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/smartreply/") + .iosPod("GoogleMLKit/SmartReply") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle("com.google.mlkit:smart-reply:17.0.2") + .androidMinimumSdk(21) + .description("Legacy ML Kit Smart Reply cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/langid/") + .iosPod("GoogleMLKit/LanguageID") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle("com.google.mlkit:language-id:17.0.4") + .androidMinimumSdk(21) + .description("Legacy ML Kit Language ID cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/pose/") + .iosPod("GoogleMLKit/PoseDetection") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle("com.google.mlkit:pose-detection:18.0.0-beta3") + .androidMinimumSdk(21) + .description("Legacy ML Kit Pose Detection cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/segmentation/") + .iosPod("GoogleMLKit/SegmentationSelfie") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle( + "com.google.mlkit:segmentation-selfie:16.0.0-beta4") + .androidMinimumSdk(21) + .description("Legacy ML Kit Selfie Segmentation cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/docscan/") + .iosPod("GoogleMLKit/DocumentScanner") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .iosFrameworks("VisionKit") + .androidGradle( + "com.google.android.gms:play-services-mlkit-document-scanner:16.0.0-beta1") + .androidMinimumSdk(21) + .description("Legacy ML Kit Document Scanner cn1lib")); + e.add(new Entry("com/codename1/ai/tflite/") + .iosPod("TensorFlowLiteSwift") + .iosSpm("TensorFlowLiteSwift", + "https://github.com/tensorflow/tensorflow.git", + "from:2.13.0", "TensorFlowLite") + .androidGradle("org.tensorflow:tensorflow-lite:2.13.0") + .androidGradle( + "org.tensorflow:tensorflow-lite-support:0.4.4") + .androidMinimumSdk(21) + .description("Legacy TensorFlow Lite interpreter cn1lib")); + // Built-in vision APIs. Android uses ML Kit by default. iOS uses // Apple Vision/VisionKit unless VisionBackends.mlKit() is selected. // The compound entries require both the feature and selector method, diff --git a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java index ad5d4bc8738..b7149db0fa2 100644 --- a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java +++ b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java @@ -47,6 +47,26 @@ void builtInTextRecognizerMapsToAppleVisionAndAndroidMlKit() { "Still-image analysis must not imply camera permission"); } + @Test + void retiredCn1libsRetainPublishedDependencyMappings() { + List mlKit = + PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/mlkit/text/TextRecognizer"); + assertEquals(1, mlKit.size()); + assertTrue(mlKit.get(0).iosPods().contains( + "GoogleMLKit/TextRecognition")); + assertTrue(mlKit.get(0).androidGradleDeps().get(0).startsWith( + "com.google.mlkit:text-recognition:")); + + List tflite = + PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/tflite/Interpreter"); + assertEquals(1, tflite.size()); + assertFalse(tflite.get(0).iosPods().isEmpty()); + assertFalse(tflite.get(0).iosSpmSpecs().isEmpty()); + assertEquals(2, tflite.get(0).androidGradleDeps().size()); + } + @Test void explicitMlKitBackendAddsOnlyUsedFeaturePod() { PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); @@ -246,14 +266,18 @@ void everyMlKitAndLiteRtDependencyDeclaresAndroidFloor() { for (String dependency : entry.androidGradleDeps()) { if (dependency.startsWith("com.google.mlkit:") || dependency.startsWith( - "com.google.ai.edge.litert:")) { + "com.google.ai.edge.litert:") + || dependency.startsWith( + "com.google.android.gms:play-services-mlkit-") + || dependency.startsWith( + "org.tensorflow:tensorflow-lite")) { checked++; assertEquals(21, entry.androidMinimumSdk(), dependency); } } } - assertEquals(10, checked, - "Expected all built-in AI Android dependencies"); + assertEquals(22, checked, + "Expected built-in and compatibility AI dependencies"); } @Test From 770ada011bf5c3c1c74769d9eefef55b58770606 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 10:32:13 +0300 Subject: [PATCH 51/80] Allow trusted archive extraction routes --- .../java/com/codename1/builders/Executor.java | 7 +++- .../ExecutorArchiveExtractionTest.java | 36 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java index 5a2a8bbbb1e..cd1b91df85c 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java @@ -1399,8 +1399,13 @@ public void extractZip(InputStream source, File destDir, ExtractionFilter filter byte[] data = new byte[8192]; // write the files to the disk + resolveArchiveEntry(destDir, entryName); File destFile = filter.destFile(currentDir, entryName); - destFile = requireArchiveDestination(destDir, destFile, entryName); + if (destFile == null) { + throw new IOException("Extraction filter returned no destination for " + + entryName); + } + destFile = destFile.getCanonicalFile(); destFile.getParentFile().mkdirs(); FileOutputStream fos = new FileOutputStream(destFile); dest = new BufferedOutputStream(fos, data.length); diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java index fd8da34c4ba..8238f23b66c 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java @@ -22,9 +22,15 @@ */ package com.codename1.builders; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -71,4 +77,34 @@ void rejectsSiblingWithMatchingPrefix() { assertThrows(IOException.class, () -> Executor.resolveArchiveEntry( destination, "../archive-escape/payload.bin")); } + + @Test + void extractionFilterMayRouteSafeEntryToSibling() throws Exception { + File project = temporaryDirectory.resolve("project").toFile(); + File source = new File(project, "src"); + File settings = new File(project, + "codenameone_settings.properties"); + source.mkdirs(); + + new IPhoneBuilder().extractZip( + new ByteArrayInputStream(zipEntry( + "codenameone_settings.properties", + "codename1.mainName=MyApp")), + source, (path, fileName) -> settings); + + assertEquals("codename1.mainName=MyApp", + new String(Files.readAllBytes(settings.toPath()), + StandardCharsets.UTF_8)); + } + + private static byte[] zipEntry(String name, String contents) + throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + ZipOutputStream zip = new ZipOutputStream(bytes); + zip.putNextEntry(new ZipEntry(name)); + zip.write(contents.getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + zip.close(); + return bytes.toByteArray(); + } } From 88de186b40cb01fb2e08bec3385b941bebb2c1bc Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 10:42:54 +0300 Subject: [PATCH 52/80] Reject encoded formats in pixel images --- .../src/com/codename1/ai/vision/VisionImage.java | 9 ++++++++- .../java/com/codename1/ai/vision/VisionApiTest.java | 10 ++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java index 58fa66cf8f5..3a8d737811e 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java @@ -68,12 +68,19 @@ public static VisionImage encoded(byte[] bytes) { /// or 270 degrees are supported /// @return immutable image input /// @throws IllegalArgumentException if the data or dimensions are empty, - /// or if the rotation is not a quarter turn + /// if {@code format} is not {@link FrameFormat#NV21} or + /// {@link FrameFormat#RGBA8888}, or if the rotation is not a quarter turn public static VisionImage pixels(byte[] bytes, int width, int height, FrameFormat format, int rotationDegrees) { if (bytes == null || bytes.length == 0 || width <= 0 || height <= 0) { throw new IllegalArgumentException("Pixels and positive dimensions are required"); } + if (format != FrameFormat.NV21 + && format != FrameFormat.RGBA8888) { + throw new IllegalArgumentException( + "Raw pixels require NV21 or RGBA8888 format; " + + "use encoded() for JPEG or PNG data"); + } return new VisionImage(null, bytes, width, height, rotationDegrees, 0, format); } diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java index f3d12761ff0..16b53a0ac30 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java @@ -58,6 +58,16 @@ void pixelInputNormalizesRotation() { 1, 1, FrameFormat.RGBA8888, 45)); } + @Test + void pixelInputRejectsEncodedAndMissingFormats() { + byte[] raw = new byte[] {1, 2, 3, 4}; + assertThrows(IllegalArgumentException.class, + () -> VisionImage.pixels(raw, 1, 1, + FrameFormat.JPEG, 0)); + assertThrows(IllegalArgumentException.class, + () -> VisionImage.pixels(raw, 1, 1, null, 0)); + } + @Test void cameraFrameCopiesOnlyTheRequestedFormat() { byte[] jpeg = new byte[] {1, 2}; From 447dacc2a2f1d4b8d11002d1f73ad2cdd3e2e312 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 10:53:55 +0300 Subject: [PATCH 53/80] Reject overflowing segmentation masks --- .../src/com/codename1/ai/vision/SegmentationMask.java | 11 +++++++++-- .../java/com/codename1/ai/vision/VisionApiTest.java | 6 ++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java b/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java index cd6287e6b08..86fd14d337f 100644 --- a/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java +++ b/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java @@ -35,6 +35,9 @@ public final class SegmentationMask { /// @param width mask width in pixels /// @param height mask height in pixels /// @param confidence one foreground probability per pixel, defensively copied + /// @throws IllegalArgumentException if either dimension is negative, the + /// pixel count exceeds the maximum Java array length, or + /// {@code confidence} does not contain exactly one value per pixel public SegmentationMask(int width, int height, float[] confidence) { this(width, height, confidence, null); } @@ -44,10 +47,14 @@ public SegmentationMask(int width, int height, float[] confidence) { /// @param height mask height in pixels /// @param confidence one foreground probability per pixel, defensively copied /// @param metadata backend details, or {@code null} + /// @throws IllegalArgumentException if either dimension is negative, the + /// pixel count exceeds the maximum Java array length, or + /// {@code confidence} does not contain exactly one value per pixel public SegmentationMask(int width, int height, float[] confidence, VisionMetadata metadata) { - if (width < 0 || height < 0 - || confidence == null || confidence.length != width * height) { + long pixelCount = (long) width * (long) height; + if (width < 0 || height < 0 || pixelCount > Integer.MAX_VALUE + || confidence == null || confidence.length != pixelCount) { throw new IllegalArgumentException("Mask dimensions do not match its data"); } this.width = width; diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java index 16b53a0ac30..8bef48ae1a9 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java @@ -68,6 +68,12 @@ void pixelInputRejectsEncodedAndMissingFormats() { () -> VisionImage.pixels(raw, 1, 1, null, 0)); } + @Test + void segmentationMaskRejectsOverflowingPixelCount() { + assertThrows(IllegalArgumentException.class, + () -> new SegmentationMask(65536, 65536, new float[0])); + } + @Test void cameraFrameCopiesOnlyTheRequestedFormat() { byte[] jpeg = new byte[] {1, 2}; From 6b17ebc346966ea489a1f34f4deabf810929df49 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:10:39 +0300 Subject: [PATCH 54/80] Normalize Android translation language tags --- .../impl/android/ai/AndroidTranslationAdapter.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java index 5d76ae49c4b..57d5ca3cafd 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java @@ -30,6 +30,7 @@ import com.google.mlkit.nl.translate.Translation; import com.google.mlkit.nl.translate.Translator; import com.google.mlkit.nl.translate.TranslatorOptions; +import java.util.Locale; /** ML Kit translation; retained only for {@code Translator} users. */ final class AndroidTranslationAdapter extends AndroidLanguageAdapter { @@ -40,8 +41,9 @@ AsyncResource translate( final AsyncResource out = new AsyncResource(); TranslatorOptions translatorOptions = new TranslatorOptions.Builder() - .setSourceLanguage(sourceLanguage) - .setTargetLanguage(targetLanguage).build(); + .setSourceLanguage(normalizeLanguageTag(sourceLanguage)) + .setTargetLanguage(normalizeLanguageTag(targetLanguage)) + .build(); final Translator client = Translation.getClient(translatorOptions); client.downloadModelIfNeeded().onSuccessTask( @@ -57,4 +59,8 @@ public void onSuccess(String value) { }).addOnFailureListener(failure(out, client)); return out; } + + private static String normalizeLanguageTag(String languageTag) { + return languageTag.toLowerCase(Locale.ENGLISH); + } } From 66dcd872f76c47bb86226e76bf9c79a916f6bb93 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:41:29 +0300 Subject: [PATCH 55/80] Harden inference callbacks and language tags --- .../ai/inference/InferenceSession.java | 18 ++---- .../com/codename1/ai/language/Translator.java | 9 ++- .../android/ai/AndroidTranslationAdapter.java | 23 +++++-- Ports/iOSPort/nativeSources/CN1Language.m | 16 ++++- .../ai/inference/InferenceApiTest.java | 64 +++++++++++++++++++ 5 files changed, 110 insertions(+), 20 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java index f86cd779a24..5efdbc666a6 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -347,23 +347,17 @@ private static final class SessionRunResource backend.ready(new SuccessCallback() { @Override public void onSucess(Tensor[] value) { - try { - if (!isCancelled()) { - complete(value); - } - } finally { - SessionRunResource.this.pending.finish(); + SessionRunResource.this.pending.finish(); + if (!isCancelled()) { + complete(value); } } }).except(new SuccessCallback() { @Override public void onSucess(Throwable error) { - try { - if (!isCancelled()) { - error(error); - } - } finally { - SessionRunResource.this.pending.finish(); + SessionRunResource.this.pending.finish(); + if (!isCancelled()) { + error(error); } } }); diff --git a/CodenameOne/src/com/codename1/ai/language/Translator.java b/CodenameOne/src/com/codename1/ai/language/Translator.java index 9fc2b8ccfe4..24299c61e1d 100644 --- a/CodenameOne/src/com/codename1/ai/language/Translator.java +++ b/CodenameOne/src/com/codename1/ai/language/Translator.java @@ -49,8 +49,13 @@ public static boolean isSupported(LanguageOptions options) { /// Option values are copied before asynchronous backend work begins. /// /// @param text source text - /// @param sourceLanguage BCP-47/ML Kit source language tag - /// @param targetLanguage BCP-47/ML Kit target language tag + /// Current ML Kit backends accept a BCP-47 tag with an optional script or + /// region, such as {@code en-US}, and select the corresponding supported + /// base-language model ({@code en} in this example). The asynchronous + /// resource fails when either tag does not identify a supported model. + /// + /// @param sourceLanguage BCP-47 source language tag + /// @param targetLanguage BCP-47 target language tag /// @param options backend options, or {@code null} /// @return asynchronous translated text public static AsyncResource translate(String text, String sourceLanguage, diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java index 57d5ca3cafd..6c9273ac367 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java @@ -28,6 +28,7 @@ import com.google.android.gms.tasks.SuccessContinuation; import com.google.android.gms.tasks.Task; import com.google.mlkit.nl.translate.Translation; +import com.google.mlkit.nl.translate.TranslateLanguage; import com.google.mlkit.nl.translate.Translator; import com.google.mlkit.nl.translate.TranslatorOptions; import java.util.Locale; @@ -39,10 +40,17 @@ AsyncResource translate( final String text, String sourceLanguage, String targetLanguage, LanguageOptions options) { final AsyncResource out = new AsyncResource(); + String sourceCode = languageCode(sourceLanguage); + String targetCode = languageCode(targetLanguage); + if (sourceCode == null || targetCode == null) { + out.error(new IllegalArgumentException( + "Unsupported translation language")); + return out; + } TranslatorOptions translatorOptions = new TranslatorOptions.Builder() - .setSourceLanguage(normalizeLanguageTag(sourceLanguage)) - .setTargetLanguage(normalizeLanguageTag(targetLanguage)) + .setSourceLanguage(sourceCode) + .setTargetLanguage(targetCode) .build(); final Translator client = Translation.getClient(translatorOptions); @@ -60,7 +68,14 @@ public void onSuccess(String value) { return out; } - private static String normalizeLanguageTag(String languageTag) { - return languageTag.toLowerCase(Locale.ENGLISH); + private static String languageCode(String languageTag) { + if (languageTag == null) { + return null; + } + String language = Locale.forLanguageTag(languageTag).getLanguage(); + if (language.length() == 0) { + return null; + } + return TranslateLanguage.fromLanguageTag(language); } } diff --git a/Ports/iOSPort/nativeSources/CN1Language.m b/Ports/iOSPort/nativeSources/CN1Language.m index af27e9a3874..7f72835357e 100644 --- a/Ports/iOSPort/nativeSources/CN1Language.m +++ b/Ports/iOSPort/nativeSources/CN1Language.m @@ -121,11 +121,23 @@ #endif } +static NSString *cn1TranslateLanguageCode(NSString *languageTag) { + if (languageTag == nil) { + return nil; + } + NSString *normalized = [languageTag + stringByReplacingOccurrencesOfString:@"_" withString:@"-"]; + NSDictionary *components = + [NSLocale componentsFromLocaleIdentifier:normalized]; + NSString *language = components[NSLocaleLanguageCode]; + return language.length == 0 ? nil : language.lowercaseString; +} + static NSString *cn1TranslateLanguage(NSString *text, NSString *source, NSString *target) { #if defined(CN1_HAS_MLKIT_TRANSLATE) - MLKTranslateLanguage sourceLanguage = source.lowercaseString; - MLKTranslateLanguage targetLanguage = target.lowercaseString; + MLKTranslateLanguage sourceLanguage = cn1TranslateLanguageCode(source); + MLKTranslateLanguage targetLanguage = cn1TranslateLanguageCode(target); NSSet *supported = MLKTranslateAllLanguages(); if (sourceLanguage == nil || targetLanguage == nil || ![supported containsObject:sourceLanguage] || diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index 13b4b2fca42..27a2bc307cf 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -403,6 +403,70 @@ void cancelledRunStaysSerializedUntilNativeWorkSettles() { "native settlement must complete deferred close"); } + @Test + void settledRunCanStartAnotherRunFromSuccessCallback() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + backend.pendingRun = new AsyncResource(); + implementation.setInferenceImpl(backend); + final InferenceSession session = await(InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions())); + final AtomicReference> second = + new AtomicReference>(); + session.run(new Tensor[] { + Tensor.floats("input", new int[] {1, 2}, + new float[] {1, 2}) + }).ready(new SuccessCallback() { + public void onSucess(Tensor[] ignored) { + backend.pendingRun = null; + second.set(session.run(new Tensor[] { + Tensor.floats("input", new int[] {1, 2}, + new float[] {3, 4}) + })); + } + }); + + backend.pendingRun.complete(new Tensor[] { + Tensor.floats("output", new int[] {1}, new float[] {3}) + }); + flushSerialCalls(); + + assertNotNull(second.get()); + assertArrayEquals(new float[] {3}, + (float[]) await(second.get())[0].getData()); + session.close(); + } + + @Test + void settledRunCanStartAnotherRunFromFailureCallback() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + backend.pendingRun = new AsyncResource(); + implementation.setInferenceImpl(backend); + final InferenceSession session = await(InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions())); + final AtomicReference> second = + new AtomicReference>(); + session.run(new Tensor[] { + Tensor.floats("input", new int[] {1, 2}, + new float[] {1, 2}) + }).except(new SuccessCallback() { + public void onSucess(Throwable ignored) { + backend.pendingRun = null; + second.set(session.run(new Tensor[] { + Tensor.floats("input", new int[] {1, 2}, + new float[] {3, 4}) + })); + } + }); + + backend.pendingRun.error(new RuntimeException("expected")); + flushSerialCalls(); + + assertNotNull(second.get()); + assertArrayEquals(new float[] {3}, + (float[]) await(second.get())[0].getData()); + session.close(); + } + @Test void sessionSnapshotsInputArrayBeforeAsyncInference() { RecordingInferenceImpl backend = new RecordingInferenceImpl(); From 111d0bd66ab8c41beb07c26466bd5ce38585e99f Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:07:13 +0300 Subject: [PATCH 56/80] Preserve encoded vision image rotation --- .../com/codename1/ai/vision/VisionImage.java | 29 +++++++++++++++++-- docs/ai-on-device-architecture.md | 5 +++- docs/developer-guide/Ai-And-Speech.asciidoc | 10 +++++-- .../codename1/ai/vision/VisionApiTest.java | 9 ++++++ .../tests/VisionOnDeviceApiTest.java | 4 +++ 5 files changed, 51 insertions(+), 6 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java index 3a8d737811e..bacd3a96840 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java @@ -49,14 +49,39 @@ private VisionImage(byte[] encodedBytes, byte[] pixels, int width, int height, this.format = format == null ? FrameFormat.JPEG : format; } - /// Creates an encoded JPEG or PNG input. + /// Creates an encoded JPEG or PNG input whose stored pixels are already + /// upright. This method does not inspect EXIF orientation metadata. Use + /// {@link #encoded(byte[], int)} when the encoded image needs a clockwise + /// display rotation before analysis. + /// /// @param bytes complete encoded image, copied by this method /// @return immutable image input + /// @throws IllegalArgumentException if {@code bytes} is {@code null} or empty public static VisionImage encoded(byte[] bytes) { + return encoded(bytes, 0); + } + + /// Creates an encoded JPEG or PNG input with display orientation metadata. + /// The rotation describes how the stored pixels must be rotated clockwise + /// to appear upright; for example, pass {@code 90} for a portrait JPEG + /// whose pixels are stored in landscape orientation. Decoders on Android + /// and Apple platforms receive this value directly, so detected bounds and + /// points use the displayed orientation. This method does not parse EXIF; + /// callers loading an image outside {@link #fromCameraFrame(CameraFrame)} + /// must supply the EXIF-derived rotation when it is relevant. + /// + /// @param bytes complete encoded image, copied by this method + /// @param rotationDegrees clockwise display rotation; only 0, 90, 180, + /// or 270 degrees are supported + /// @return immutable image input + /// @throws IllegalArgumentException if {@code bytes} is {@code null} or + /// empty, or if the rotation is not a quarter turn + public static VisionImage encoded(byte[] bytes, int rotationDegrees) { if (bytes == null || bytes.length == 0) { throw new IllegalArgumentException("Image bytes must not be empty"); } - return new VisionImage(bytes, null, 0, 0, 0, 0, FrameFormat.JPEG); + return new VisionImage(bytes, null, 0, 0, rotationDegrees, 0, + FrameFormat.JPEG); } /// Creates raw NV21 or RGBA8888 input with display orientation metadata. diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md index 71bce95e58c..93dcfd7f0b7 100644 --- a/docs/ai-on-device-architecture.md +++ b/docs/ai-on-device-architecture.md @@ -137,7 +137,10 @@ retain the current OS revision and its additional symbologies. PNG, NV21, and RGBA8888. Android feeds raw NV21 directly to ML Kit and converts RGBA to a bitmap. Apple creates a `CGImage` directly from NV21 or RGBA memory and passes it to Vision or ML Kit without an intermediate JPEG -encode/decode. +encode/decode. `VisionImage.encoded(bytes, rotationDegrees)` carries the +clockwise display rotation for encoded gallery or file images; the +one-argument overload deliberately assumes upright stored pixels rather than +silently guessing or discarding EXIF orientation. `VisionImage.fromCameraFrame()` is safe beyond the `FrameListener.onFrame()` callback because it copies the callback-owned diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index 9fa6fd0c9f2..7c62880ef81 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -276,9 +276,13 @@ language. The vision API accepts JPEG, PNG, NV21, or RGBA8888 data through `VisionImage`. `VisionImage.fromCameraFrame(frame)` is normally the best way to connect a camera because it copies the frame's -callback-owned buffers before analysis becomes asynchronous. Each -analyzer returns stable Codename One result types. Bounds and points -use normalized coordinates in the range 0 through 1. +callback-owned buffers and preserves the frame rotation before analysis +becomes asynchronous. For an encoded gallery or file image whose pixels +need an EXIF display rotation, use +`VisionImage.encoded(bytes, rotationDegrees)`; the one-argument overload +assumes the stored pixels are already upright and does not inspect EXIF +metadata. Each analyzer returns stable Codename One result types. Bounds +and points use normalized coordinates in the range 0 through 1. Android uses ML Kit. iOS and Mac native builds default to Apple Vision and Core Image. iOS can select ML Kit explicitly: diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java index 8bef48ae1a9..69b0880bbdc 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java @@ -48,6 +48,15 @@ void visionImageDefensivelyCopiesInputAndOutput() { assertEquals(2, image.getEncodedBytes()[1]); } + @Test + void encodedInputPreservesExplicitDisplayRotation() { + VisionImage image = VisionImage.encoded( + new byte[] {1, 2, 3}, -90); + assertEquals(270, image.getRotationDegrees()); + assertThrows(IllegalArgumentException.class, + () -> VisionImage.encoded(new byte[] {1}, 45)); + } + @Test void pixelInputNormalizesRotation() { VisionImage image = VisionImage.pixels(new byte[] {1, 2, 3, 4}, diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java index 37a9700479f..c0936cf509a 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java @@ -123,6 +123,10 @@ private void checkImageOwnership() { encodedOutput[1] = 9; checkEqual(2, encoded.getEncodedBytes()[1], "VisionImage must copy encoded output"); + VisionImage rotatedEncoded = + VisionImage.encoded(new byte[] {7}, -90); + checkEqual(270, rotatedEncoded.getRotationDegrees(), + "encoded image rotation normalization"); byte[] jpeg = new byte[] {4, 5}; byte[] pixels = new byte[] {10, 20, 30, 40}; From e97371b29989914bd9bf6af631501aaf33c54721 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:17:25 +0300 Subject: [PATCH 57/80] Use native byte order for LiteRT outputs --- .../src/com/codename1/impl/android/ai/AndroidInferenceImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java index 08f32e59c66..a16e160bfa1 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java @@ -326,6 +326,7 @@ private static ByteBuffer toBuffer(Tensor value, DataType nativeType) { private static Tensor fromBuffer(String name, int[] shape, DataType nativeType, ByteBuffer value) { value.rewind(); + value.order(ByteOrder.nativeOrder()); TensorType type = type(nativeType); int count = elementCount(shape); if (type == TensorType.FLOAT32) { From f9b4cbd51d4b0cf9fc32ac0b89eff58d7902b0de Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:30:00 +0300 Subject: [PATCH 58/80] Isolate inference and model cache concurrency --- .../ai/inference/InferenceSession.java | 18 ++++++- .../codename1/ai/inference/ModelCache.java | 51 +++++++++++++++++-- docs/ai-on-device-architecture.md | 13 +++-- docs/developer-guide/Ai-And-Speech.asciidoc | 14 +++-- .../ai/inference/InferenceApiTest.java | 9 +++- 5 files changed, 90 insertions(+), 15 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java index 5efdbc666a6..011bd8a47d1 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -118,12 +118,16 @@ synchronized void fail(Throwable error) { } /// Returns the model's current input metadata. Shapes reflect the most - /// recent successful {@link #resizeInput(String, int[])} call. + /// recent successful {@link #resizeInput(String, int[])} call. Metadata + /// cannot be queried while {@link #run(Tensor[])} is pending because the + /// native interpreter is mutable and may be updating tensor state. /// /// @return a defensive copy of the input metadata array + /// @throws IllegalStateException if the session is closed or a run is pending public TensorInfo[] getInputs() { synchronized (this) { ensureOpen(); + ensureNotRunning("read input metadata"); return implementation.getInputs(handle); } } @@ -131,11 +135,16 @@ public TensorInfo[] getInputs() { /// Returns the model's current output metadata. Backends refresh this /// information after an invocation so models with dynamically resolved /// output dimensions report the shape used to decode the returned tensor. + /// Metadata cannot be queried while {@link #run(Tensor[])} is pending + /// because the native interpreter is mutable and may be updating tensor + /// state. /// /// @return a defensive copy of the output metadata array + /// @throws IllegalStateException if the session is closed or a run is pending public TensorInfo[] getOutputs() { synchronized (this) { ensureOpen(); + ensureNotRunning("read output metadata"); return implementation.getOutputs(handle); } } @@ -321,6 +330,13 @@ private void ensureOpen() { } } + private void ensureNotRunning(String operation) { + if (activeRuns > 0) { + throw new IllegalStateException( + "Cannot " + operation + " while inference is running"); + } + } + private static final class PendingRun { private final InferenceSession session; private boolean finished; diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java index 0001f34f355..b97d9b5cb5a 100644 --- a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java +++ b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java @@ -30,6 +30,7 @@ import com.codename1.ui.Display; import com.codename1.ui.events.ActionListener; import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; import java.io.IOException; import java.io.InputStream; @@ -60,8 +61,11 @@ private ModelCache() { /// resumed because the portable network layer cannot prove that a server's /// partial response still represents the pinned model. /// Concurrent requests for the same cache entry and content identity share - /// one operation. A different URL or digest using that cache key while the - /// first operation is active fails instead of racing on the temporary file. + /// one underlying operation, but each caller receives an independent + /// resource. Canceling one caller's resource suppresses only that caller's + /// notification and does not cancel the shared download or other callers. + /// A different URL or digest using that cache key while the first operation + /// is active fails instead of racing on the temporary file. /// /// @param url initial HTTPS URL of the model; observable redirects must /// also use HTTPS @@ -310,7 +314,7 @@ static FetchRegistration registerFetch( if (active != null) { if (active.matches(url, sha256)) { return new FetchRegistration( - active.resource, null, false); + subscribe(active.resource), null, false); } AsyncResource conflict = new AsyncResource(); @@ -325,10 +329,29 @@ static FetchRegistration registerFetch( new Completion(resource, fileName); ACTIVE_FETCHES.put(fileName, new ActiveFetch(url, sha256, resource)); - return new FetchRegistration(resource, completion, true); + return new FetchRegistration( + subscribe(resource), completion, true); } } + private static AsyncResource subscribe( + AsyncResource operation) { + final SubscriberResource subscriber = + new SubscriberResource(); + operation.ready(new SuccessCallback() { + @Override + public void onSucess(T value) { + subscriber.publish(value); + } + }).except(new SuccessCallback() { + @Override + public void onSucess(Throwable error) { + subscriber.fail(error); + } + }); + return subscriber; + } + private static void releaseFetch( String fileName, AsyncResource resource) { if (fileName == null) { @@ -363,6 +386,26 @@ boolean matches(String otherUrl, String otherSha256) { } } + private static final class SubscriberResource + extends AsyncResource { + @Override + public synchronized boolean cancel(boolean mayInterruptIfRunning) { + return super.cancel(mayInterruptIfRunning); + } + + synchronized void publish(T value) { + if (!isCancelled()) { + complete(value); + } + } + + synchronized void fail(Throwable error) { + if (!isCancelled()) { + error(error); + } + } + } + static final class FetchRegistration { final AsyncResource resource; final Completion completion; diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md index 93dcfd7f0b7..9647a0c6200 100644 --- a/docs/ai-on-device-architecture.md +++ b/docs/ai-on-device-architecture.md @@ -167,9 +167,12 @@ observable. iOS follows redirects below the portable network layer, so the cache requires a SHA-256 digest for every iOS download and rejects its unpinned overload. Identical concurrent fetches for one cache entry coalesce, while conflicting in-flight content identities fail instead of sharing a temporary -path. File sources are opened by path without copying the model through the -Java heap. Cache promotion verifies that the final file exists and, when -supplied, still matches the requested digest before publishing its path. +path. Each coalesced caller has an independent resource, so cancellation +suppresses only that caller's notification and never cancels the shared +download or another subscriber. File sources are opened by path without +copying the model through the Java heap. Cache promotion verifies that the +final file exists and, when supplied, still matches the requested digest +before publishing its path. Android invokes LiteRT with null output destinations, then reads each result from its native tensor buffer. This allows value-dependent output dimensions to resolve before Codename One allocates or copies the result. LiteRT caches @@ -182,7 +185,9 @@ and on iOS because the Core ML delegate may schedule work on CPU or GPU. All native sessions and analyzers must be closed. Expensive open, analysis, and inference work runs off the EDT; completion and error delivery return to -the EDT. +the EDT. A session rejects metadata queries, resizing, and another invocation +while a run is pending so no two API calls can touch the mutable native +interpreter concurrently. ## Permanent cross-platform coverage diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index 7c62880ef81..eb1795420b8 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -384,8 +384,10 @@ include::../demos/common/src/main/java/com/codenameone/developerguide/snippets/g Open is asynchronous because model initialization can allocate native resources. A session is reusable and supports multiple named inputs, -multiple outputs, and `resizeInput(...)`. Tensor data is copied across -the port boundary. Always close the session. +multiple outputs, and `resizeInput(...)`. A session serializes its mutable +native interpreter: while `run(...)` is pending, another run, input resize, +or input/output metadata query throws `IllegalStateException`. Tensor data is +copied across the port boundary. Always close the session. Bundle small models as resources. For larger models use `ModelCache.fetch(url, cacheKey, sha256)`, which downloads over HTTPS, @@ -397,9 +399,11 @@ argument and rejects the unpinned two-argument overload. Supply the digest on every platform for every third-party or remotely mutable model. Interrupted downloads restart through a temporary `.download` file and are never promoted to the cache until verification succeeds. Concurrent calls for -the same cache key and content identity share one download. When a cache key -already has an active download, a request with a conflicting URL or digest -fails instead of sharing the temporary file. +the same cache key and content identity share one download but receive +independent asynchronous resources. Canceling one caller suppresses only that +caller's notification; it does not cancel the shared download or the other +callers. When a cache key already has an active download, a request with a +conflicting URL or digest fails instead of sharing the temporary file. === On-device language services diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index 27a2bc307cf..d2c62764463 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -158,7 +158,9 @@ void modelCacheCoalescesIdenticalConcurrentFetches() { fileName, "https://one.example/model.tflite", null); assertTrue(first.owner); assertFalse(duplicate.owner); - assertSame(first.resource, duplicate.resource); + assertNotSame(first.resource, duplicate.resource, + "coalesced callers need independent cancellation handles"); + assertTrue(duplicate.resource.cancel(false)); ModelCache.FetchRegistration conflict = ModelCache.registerFetch( fileName, "https://two.example/model.tflite", null); @@ -176,6 +178,9 @@ public void onSucess(Throwable value) { first.completion.complete(ModelSource.file("test-model.tflite")); flushSerialCalls(); + assertEquals("test-model.tflite", first.resource.get().getPath()); + assertTrue(duplicate.resource.isCancelled(), + "one subscriber's cancellation must remain local"); ModelCache.FetchRegistration afterCompletion = ModelCache.registerFetch(fileName, "https://two.example/model.tflite", null); @@ -362,6 +367,8 @@ void sessionDefersCloseUntilPendingRunSettles() { new float[] {3, 4})})); assertThrows(IllegalStateException.class, () -> session.resizeInput("input", new int[] {1, 4})); + assertThrows(IllegalStateException.class, session::getInputs); + assertThrows(IllegalStateException.class, session::getOutputs); session.close(); assertEquals(0, backend.closeCount); backend.pendingRun.complete(new Tensor[] { From 2c5ecaa2e3bd7b6612889f67e96cf6c5c961d48f Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:27:43 +0300 Subject: [PATCH 59/80] Satisfy developer guide prose checks --- docs/developer-guide/Ai-And-Speech.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index eb1795420b8..2d2e54f3c70 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -280,7 +280,7 @@ callback-owned buffers and preserves the frame rotation before analysis becomes asynchronous. For an encoded gallery or file image whose pixels need an EXIF display rotation, use `VisionImage.encoded(bytes, rotationDegrees)`; the one-argument overload -assumes the stored pixels are already upright and does not inspect EXIF +assumes the stored pixels are already upright and doesn't inspect EXIF metadata. Each analyzer returns stable Codename One result types. Bounds and points use normalized coordinates in the range 0 through 1. @@ -401,7 +401,7 @@ Interrupted downloads restart through a temporary `.download` file and are never promoted to the cache until verification succeeds. Concurrent calls for the same cache key and content identity share one download but receive independent asynchronous resources. Canceling one caller suppresses only that -caller's notification; it does not cancel the shared download or the other +caller's notification; it doesn't cancel the shared download or the other callers. When a cache key already has an active download, a request with a conflicting URL or digest fails instead of sharing the temporary file. From 49d8c084b4623aff0978db16f1a07f1b896f6d45 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:35:49 +0300 Subject: [PATCH 60/80] Reject NaN AI confidence thresholds --- .../com/codename1/ai/language/LanguageOptions.java | 7 +++++++ .../src/com/codename1/ai/vision/VisionOptions.java | 8 +++++++- docs/developer-guide/Ai-And-Speech.asciidoc | 7 +++++-- .../com/codename1/ai/language/LanguageApiTest.java | 12 ++++++++++++ .../java/com/codename1/ai/vision/VisionApiTest.java | 12 ++++++++++++ 5 files changed, 43 insertions(+), 3 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java b/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java index 65c29258baa..89ea53fe0e1 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java @@ -36,9 +36,16 @@ public LanguageOptions backend(LanguageBackend value) { } /// Sets the minimum language-identification confidence, clamped to 0..1. + /// NaN has no meaningful ordering and is rejected instead of being passed + /// to a platform language identifier. /// @param value requested threshold /// @return this options object + /// @throws IllegalArgumentException if {@code value} is NaN public LanguageOptions minimumConfidence(float value) { + if (Float.isNaN(value)) { + throw new IllegalArgumentException( + "minimum confidence must be a number"); + } minimumConfidence = Math.max(0, Math.min(1, value)); return this; } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java b/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java index 42710dc9cad..f199198cc77 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java @@ -37,10 +37,16 @@ public VisionOptions backend(VisionBackend value) { return this; } - /// Sets the confidence threshold, clamped to 0..1. + /// Sets the confidence threshold, clamped to 0..1. NaN has no meaningful + /// ordering and is rejected instead of being passed to platform detectors. /// @param value requested threshold /// @return this options object + /// @throws IllegalArgumentException if {@code value} is NaN public VisionOptions minimumConfidence(float value) { + if (Float.isNaN(value)) { + throw new IllegalArgumentException( + "minimum confidence must be a number"); + } minimumConfidence = Math.max(0, Math.min(1, value)); return this; } diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index 2d2e54f3c70..ed1e7f7e046 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -283,6 +283,8 @@ need an EXIF display rotation, use assumes the stored pixels are already upright and doesn't inspect EXIF metadata. Each analyzer returns stable Codename One result types. Bounds and points use normalized coordinates in the range 0 through 1. +`VisionOptions.minimumConfidence(...)` clamps finite and infinite values to +that range and rejects NaN. Android uses ML Kit. iOS and Mac native builds default to Apple Vision and Core Image. iOS can select ML Kit explicitly: @@ -408,8 +410,9 @@ conflicting URL or digest fails instead of sharing the temporary file. === On-device language services `LanguageIdentifier`, `Translator`, and `SmartReply` share -`LanguageOptions`. Translation models are downloaded and cached by ML -Kit when a language pair is first used. Android and iOS builders +`LanguageOptions`. Its minimum-confidence threshold clamps to 0 through 1 and +rejects NaN. Translation models are downloaded and cached by ML Kit when a +language pair is first used. Android and iOS builders select `language-id`, `translate`, and `smart-reply` independently, so using language identification alone doesn't package the other two SDKs. diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java index 22bf96c1df3..a1315ada44f 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java @@ -38,6 +38,18 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class LanguageApiTest extends UITestBase { + @Test + void optionsRejectNaNConfidence() { + assertThrows(IllegalArgumentException.class, + () -> new LanguageOptions().minimumConfidence(Float.NaN)); + assertEquals(0f, new LanguageOptions() + .minimumConfidence(Float.NEGATIVE_INFINITY) + .getMinimumConfidence()); + assertEquals(1f, new LanguageOptions() + .minimumConfidence(Float.POSITIVE_INFINITY) + .getMinimumConfidence()); + } + @Test void languageOperationsForwardBackendAndOptions() { RecordingLanguageImpl backend = new RecordingLanguageImpl(); diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java index 69b0880bbdc..440879232c6 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java @@ -83,6 +83,18 @@ void segmentationMaskRejectsOverflowingPixelCount() { () -> new SegmentationMask(65536, 65536, new float[0])); } + @Test + void optionsRejectNaNConfidence() { + assertThrows(IllegalArgumentException.class, + () -> new VisionOptions().minimumConfidence(Float.NaN)); + assertEquals(0f, new VisionOptions() + .minimumConfidence(Float.NEGATIVE_INFINITY) + .getMinimumConfidence()); + assertEquals(1f, new VisionOptions() + .minimumConfidence(Float.POSITIVE_INFINITY) + .getMinimumConfidence()); + } + @Test void cameraFrameCopiesOnlyTheRequestedFormat() { byte[] jpeg = new byte[] {1, 2}; From a24869624fc5f805ffb03cf54a5f74a025e567f1 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:47:13 +0300 Subject: [PATCH 61/80] Guard microphone recording on tvOS --- Ports/iOSPort/nativeSources/IOSNative.m | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Ports/iOSPort/nativeSources/IOSNative.m b/Ports/iOSPort/nativeSources/IOSNative.m index fdbb4cc31d5..448b15a02be 100644 --- a/Ports/iOSPort/nativeSources/IOSNative.m +++ b/Ports/iOSPort/nativeSources/IOSNative.m @@ -8118,7 +8118,7 @@ JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_checkLocationUsage___R_boolean(CN1 } //native boolean checkMicrophoneUsage(); JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_checkMicrophoneUsage___R_boolean(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT me) { -#ifdef INCLUDE_MICROPHONE_USAGE +#if defined(INCLUDE_MICROPHONE_USAGE) && !TARGET_OS_TV return JAVA_TRUE; #else return JAVA_FALSE; @@ -8584,7 +8584,7 @@ void com_codename1_impl_ios_IOSNative_nsDataToByteArray___long_byte_1ARRAY(CN1_T JAVA_LONG com_codename1_impl_ios_IOSNative_createAudioUnit___java_lang_String_int_float_float_1ARRAY_R_long(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT path, JAVA_INT audioChannels, JAVA_FLOAT sampleRate, JAVA_OBJECT sampleBuffer) { -#ifdef INCLUDE_MICROPHONE_USAGE +#if defined(INCLUDE_MICROPHONE_USAGE) && !TARGET_OS_TV __block CN1AudioUnit* recorder = nil; __block NSString *exStr = nil; @@ -8674,7 +8674,7 @@ void com_codename1_impl_ios_IOSNative_destroyAudioUnit___long(CN1_THREAD_STATE_M JAVA_LONG com_codename1_impl_ios_IOSNative_createAudioRecorder___java_lang_String_java_lang_String_int_int_int_int(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT destinationFile, JAVA_OBJECT mimeType, JAVA_INT sampleRate, JAVA_INT bitRate, JAVA_INT channels, JAVA_INT maxDuration) { -#ifdef INCLUDE_MICROPHONE_USAGE +#if defined(INCLUDE_MICROPHONE_USAGE) && !TARGET_OS_TV __block AVAudioRecorder* recorder = nil; __block NSString *exStr = nil; From a28e626bfc691e8401f299ba702d8758433d54dc Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 28 Jul 2026 04:45:48 +0300 Subject: [PATCH 62/80] Guard low-level audio unit on watchOS --- Ports/iOSPort/nativeSources/IOSNative.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Ports/iOSPort/nativeSources/IOSNative.m b/Ports/iOSPort/nativeSources/IOSNative.m index 448b15a02be..6ae09c43d11 100644 --- a/Ports/iOSPort/nativeSources/IOSNative.m +++ b/Ports/iOSPort/nativeSources/IOSNative.m @@ -8584,7 +8584,7 @@ void com_codename1_impl_ios_IOSNative_nsDataToByteArray___long_byte_1ARRAY(CN1_T JAVA_LONG com_codename1_impl_ios_IOSNative_createAudioUnit___java_lang_String_int_float_float_1ARRAY_R_long(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT path, JAVA_INT audioChannels, JAVA_FLOAT sampleRate, JAVA_OBJECT sampleBuffer) { -#if defined(INCLUDE_MICROPHONE_USAGE) && !TARGET_OS_TV +#if defined(INCLUDE_MICROPHONE_USAGE) && !TARGET_OS_TV && !TARGET_OS_WATCH __block CN1AudioUnit* recorder = nil; __block NSString *exStr = nil; From 7be841b877cfbc99e052865c8445e55c93339dbc Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:32:12 +0300 Subject: [PATCH 63/80] Avoid allocation benchmark hangs on iOS simulator --- .../tests/CommonWorkloadBenchmarkTest.java | 44 ++++++++++++++----- .../conformance/port_status.py | 20 ++++++++- .../conformance/test_port_status.py | 37 ++++++++++++++++ 3 files changed, 90 insertions(+), 11 deletions(-) diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/CommonWorkloadBenchmarkTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/CommonWorkloadBenchmarkTest.java index 1fa93512e43..fe249ea193a 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/CommonWorkloadBenchmarkTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/CommonWorkloadBenchmarkTest.java @@ -24,8 +24,18 @@ package com.codenameone.examples.hellocodenameone.tests; import com.bench.CommonWorkloads; +import com.codename1.ui.CN; +import com.codename1.ui.Display; -/** Runs the canonical ParparVM common workloads inside every port app. */ +/** + * Runs the canonical ParparVM common workloads inside every port app. + * + *

The iOS simulator omits the allocation-heavy workloads because ParparVM's + * simulator mark-sweep collector can require several gigabytes for a single + * invocation. Device builds still exercise the complete workload set. Skipped + * workloads are emitted explicitly so that the port-status report distinguishes + * them from missing benchmark output.

+ */ public class CommonWorkloadBenchmarkTest extends BaseTest { private static final int WARMUP = 3; private static final int MEASUREMENTS = 5; @@ -54,15 +64,21 @@ public boolean runTest() { run("arrayRandom", new Workload() { public long run() { return CommonWorkloads.arrayRandom(); } }); - run("objectAllocation", new Workload() { - public long run() { return CommonWorkloads.objectAllocation(); } - }); - run("hashMapChurn", new Workload() { - public long run() { return CommonWorkloads.hashMapChurn(); } - }); - run("stringBuilding", new Workload() { - public long run() { return CommonWorkloads.stringBuilding(); } - }); + if (isIosSimulator()) { + skip("objectAllocation", "ios-simulator-gc-footprint"); + skip("hashMapChurn", "ios-simulator-gc-footprint"); + skip("stringBuilding", "ios-simulator-gc-footprint"); + } else { + run("objectAllocation", new Workload() { + public long run() { return CommonWorkloads.objectAllocation(); } + }); + run("hashMapChurn", new Workload() { + public long run() { return CommonWorkloads.hashMapChurn(); } + }); + run("stringBuilding", new Workload() { + public long run() { return CommonWorkloads.stringBuilding(); } + }); + } run("recursion", new Workload() { public long run() { return CommonWorkloads.recursion(); } }); @@ -96,6 +112,14 @@ private static void run(String id, Workload workload) { emit("benchmark id=" + id + " duration_ns=" + minimumNanos + " checksum=" + checksum); } + private static boolean isIosSimulator() { + return CN.isSimulator() && "ios".equals(Display.getInstance().getPlatformName()); + } + + private static void skip(String id, String reason) { + emit("skipped id=" + id + " reason=" + reason); + } + private static void emit(String value) { System.out.println("CN1SS:PERF:" + value); } diff --git a/scripts/hellocodenameone/conformance/port_status.py b/scripts/hellocodenameone/conformance/port_status.py index 8dc15f6abcf..e4e11980c26 100755 --- a/scripts/hellocodenameone/conformance/port_status.py +++ b/scripts/hellocodenameone/conformance/port_status.py @@ -33,6 +33,9 @@ PERF_BENCH_RE = re.compile( r"CN1SS:PERF:benchmark id=([A-Za-z0-9-]+) duration_ns=(\d+) checksum=(-?\d+)" ) +PERF_SKIP_RE = re.compile( + r"CN1SS:PERF:skipped id=([A-Za-z0-9-]+) reason=([A-Za-z0-9-]+)" +) PERF_COMPLETE_RE = re.compile( r"CN1SS:PERF:complete benchmark_version=(\d+) checksum=(-?\d+)" ) @@ -406,6 +409,7 @@ def parse_logs(paths: list[Path], states: dict[str, dict]) -> bool: def parse_performance(paths: list[Path], expected_benchmarks: list[str], binary_size: int | None) -> dict: benchmarks: dict[str, dict] = {} + skipped: dict[str, str] = {} benchmark_version: int | None = None suite_checksum: int | None = None for path in paths: @@ -421,11 +425,24 @@ def parse_performance(paths: list[Path], expected_benchmarks: list[str], binary_ "duration_ns": int(match.group(2)), "checksum": match.group(3), } + skipped.pop(benchmark_id, None) + match = PERF_SKIP_RE.search(line) + if match: + benchmark_id = match.group(1) + if benchmark_id not in expected_benchmarks: + raise ContractError( + f"Unknown skipped performance benchmark {benchmark_id} in {path}" + ) + if benchmark_id not in benchmarks: + skipped[benchmark_id] = match.group(2) match = PERF_COMPLETE_RE.search(line) if match: benchmark_version = int(match.group(1)) suite_checksum = int(match.group(2)) - missing = [item for item in expected_benchmarks if item not in benchmarks] + missing = [ + item for item in expected_benchmarks + if item not in benchmarks and item not in skipped + ] status = "complete" if ( not missing and benchmark_version is not None and suite_checksum is not None ) else "partial" @@ -434,6 +451,7 @@ def parse_performance(paths: list[Path], expected_benchmarks: list[str], binary_ "benchmark_version": benchmark_version, "method": "minimum of five measured runs after three in-process warm-ups", "benchmarks": {item: benchmarks[item] for item in expected_benchmarks if item in benchmarks}, + "skipped": {item: skipped[item] for item in expected_benchmarks if item in skipped}, "missing": missing, "suite_checksum": suite_checksum, } diff --git a/scripts/hellocodenameone/conformance/test_port_status.py b/scripts/hellocodenameone/conformance/test_port_status.py index 1c0d0cc1cb6..b5817f12521 100755 --- a/scripts/hellocodenameone/conformance/test_port_status.py +++ b/scripts/hellocodenameone/conformance/test_port_status.py @@ -114,6 +114,43 @@ def test_error_lines_allow_messages_or_no_message(self): self.assertEqual("fail", report["tests"]["StringApiTest"]["status"]) self.assertEqual(["suite-error"], report["tests"]["StringApiTest"]["reasons"]) + def test_performance_skips_are_complete_and_preserve_reasons(self): + expected = self.manifest["performance_benchmarks"] + skipped = {"objectAllocation", "hashMapChurn", "stringBuilding"} + log_text = "\n".join( + [ + *[ + ( + f"CN1SS:PERF:skipped id={benchmark} " + "reason=ios-simulator-gc-footprint" + if benchmark in skipped + else ( + f"CN1SS:PERF:benchmark id={benchmark} " + "duration_ns=12000000 checksum=42" + ) + ) + for benchmark in expected + ], + "CN1SS:PERF:complete benchmark_version=1 checksum=42", + ] + ) + with tempfile.TemporaryDirectory() as tmp: + log_path = Path(tmp) / "suite.log" + log_path.write_text(log_text, encoding="utf-8") + performance = port_status.parse_performance([log_path], expected, None) + + self.assertEqual("complete", performance["status"]) + self.assertEqual([], performance["missing"]) + self.assertEqual( + { + benchmark: "ios-simulator-gc-footprint" + for benchmark in expected + if benchmark in skipped + }, + performance["skipped"], + ) + self.assertNotIn("objectAllocation", performance["benchmarks"]) + def test_strict_report_errors_reject_failures_missing_tests_and_incomplete_suite(self): report = { "suite_finished": False, From 7f56cc96393c6f3756923117eb1a49ba562c1e02 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:48:57 +0300 Subject: [PATCH 64/80] Report iOS simulator runtime correctly --- Ports/iOSPort/nativeSources/IOSNative.m | 8 ++++++++ .../src/com/codename1/impl/ios/IOSImplementation.java | 5 +++++ Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java | 1 + 3 files changed, 14 insertions(+) diff --git a/Ports/iOSPort/nativeSources/IOSNative.m b/Ports/iOSPort/nativeSources/IOSNative.m index 6ae09c43d11..1185eec534c 100644 --- a/Ports/iOSPort/nativeSources/IOSNative.m +++ b/Ports/iOSPort/nativeSources/IOSNative.m @@ -6727,6 +6727,14 @@ JAVA_OBJECT com_codename1_impl_ios_IOSNative_getDeviceName__(CN1_THREAD_STATE_MU #endif // !TARGET_OS_WATCH } +JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_isSimulator___R_boolean(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject) { +#if TARGET_OS_SIMULATOR + return JAVA_TRUE; +#else + return JAVA_FALSE; +#endif +} + JAVA_OBJECT com_codename1_impl_ios_IOSNative_getDeviceHardwareModel__(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject) { // hw.machine is the hardware/marketing model identifier (e.g. "iPhone15,2"). // On the simulator it is the host arch; we map that to a stable label so the diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java index 35bda7d7edf..0773b0bda80 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java @@ -7870,6 +7870,11 @@ else if(largest > 2000) { } return dDensity; } + + @Override + public boolean isSimulator() { + return nativeInstance.isSimulator(); + } double ppi = 0; private static final double PPI_458 = 18.031496062992126; diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java index cd3f4b56dfb..67640f80ed8 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java @@ -478,6 +478,7 @@ native void fillGradient(int kind, int stopCount, float[] positions, float[] pre native String getUDID(); native String getOSVersion(); native String getDeviceName(); + native boolean isSimulator(); // The hardware/marketing model identifier (e.g. "iPhone15,2"). Unlike // getDeviceName() -- which returns the user-assigned device name and is // therefore personally identifying -- this is safe to use for analytics From 9af545665ecc79f56682dcb0d3d7ce68224d1750 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:34:07 +0300 Subject: [PATCH 65/80] Scope SIMD registry guard to JavaSE simulator --- .../hellocodenameone/tests/SimdApiTest.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/SimdApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/SimdApiTest.java index 36c07f55a6e..db06c7d5b21 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/SimdApiTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/SimdApiTest.java @@ -1,3 +1,26 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + package com.codenameone.examples.hellocodenameone.tests; import com.codename1.ui.CN; @@ -69,7 +92,7 @@ public boolean runTest() { return false; } - if (CN.isSimulator()) { + if (CN.isSimulator() && "javase".equals(CN.getPlatformName())) { try { simd.add(new int[4], new int[4], new int[4], 0, 4); fail("Expected simulator registry guard to reject non-alloc arrays"); From b431ad34082303afc2e887c6a98d58a58c24bf7d Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 29 Jul 2026 06:33:07 +0300 Subject: [PATCH 66/80] Address on-device AI review feedback --- .github/workflows/scripts-ios.yml | 21 +-- .../ai/inference/InferenceSession.java | 13 +- .../codename1/ai/inference/ModelSource.java | 13 ++ .../ai/language/LanguageBackends.java | 27 ++- .../ai/language/LanguageIdentifier.java | 85 +++++++-- .../ai/language/LanguageSession.java | 171 ++++++++++++++++++ .../com/codename1/ai/language/SmartReply.java | 91 ++++++++-- .../com/codename1/ai/language/Translator.java | 97 ++++++++-- .../codename1/ai/language/package-info.java | 9 +- .../codename1/ai/vision/BarcodeScanner.java | 5 +- .../codename1/ai/vision/DocumentScanner.java | 5 +- .../com/codename1/ai/vision/FaceDetector.java | 5 +- .../com/codename1/ai/vision/ImageLabeler.java | 5 +- .../com/codename1/ai/vision/PoseDetector.java | 5 +- .../codename1/ai/vision/SelfieSegmenter.java | 5 +- .../codename1/ai/vision/TextRecognizer.java | 5 +- .../codename1/ai/vision/VisionBackends.java | 7 + .../com/codename1/ai/vision/VisionImage.java | 21 ++- .../codename1/ai/vision/VisionOptions.java | 2 +- .../codename1/ai/vision/VisionPipeline.java | 72 ++++++-- .../impl/android/ai/AndroidInferenceImpl.java | 2 +- .../android/ai/AndroidLanguageAdapter.java | 12 +- .../android/ai/AndroidLanguageIdAdapter.java | 30 ++- .../impl/android/ai/AndroidLanguageImpl.java | 42 ++++- .../android/ai/AndroidSmartReplyAdapter.java | 23 ++- .../android/ai/AndroidTranslationAdapter.java | 38 +++- Ports/iOSPort/nativeSources/CN1Inference.m | 4 + Ports/iOSPort/nativeSources/CN1Language.m | 4 + Ports/iOSPort/nativeSources/CN1Vision.m | 4 + .../codename1/impl/ios/IOSImplementation.java | 4 +- .../codename1/impl/ios/IOSInferenceImpl.java | 2 +- .../com/codename1/impl/ios/IOSVisionImpl.java | 43 ++++- docs/ai-on-device-architecture.md | 47 +++-- .../generated/AiAndSpeechOnDeviceSnippet.java | 7 +- docs/developer-guide/Ai-And-Speech.asciidoc | 38 ++-- .../developer-guide/Working-With-iOS.asciidoc | 8 + .../builders/AndroidGradleBuilder.java | 6 + .../java/com/codename1/builders/Executor.java | 5 +- .../com/codename1/builders/IPhoneBuilder.java | 82 +++++++-- .../AndroidAiSourceSelectionTest.java | 3 + .../IPhoneBuilderDependencyConfigTest.java | 12 ++ .../ai/inference/InferenceApiTest.java | 15 +- .../ai/language/LanguageApiTest.java | 39 ++++ .../codename1/camera/VisionPipelineTest.java | 65 +++++++ .../build/shared/PlatformFeatureCatalog.java | 165 ++++++++++++++--- .../shared/PlatformFeatureCatalogTest.java | 58 +++++- .../tests/LanguageOnDeviceApiTest.java | 26 +++ .../skill/references/ai-and-speech.md | 12 +- 48 files changed, 1245 insertions(+), 215 deletions(-) create mode 100644 CodenameOne/src/com/codename1/ai/language/LanguageSession.java diff --git a/.github/workflows/scripts-ios.yml b/.github/workflows/scripts-ios.yml index f3555202f5c..fee02d9c421 100644 --- a/.github/workflows/scripts-ios.yml +++ b/.github/workflows/scripts-ios.yml @@ -240,13 +240,6 @@ jobs: CN1SS_FAIL_ON_MISMATCH: '1' CN1SS_ALLOWED_MISSING: '0' CN1SS_PORT_ID: ios-gl - # Google ML Kit excludes arm64 simulators, so this job uses ParparVM's - # x86 path. On a contended runner the allocation benchmark remained - # active after the generic 720s idle limit, with less than three - # minutes left under the 35-minute absolute cap. Keep strict missing- - # screenshot gating, but allow the active benchmark to complete. - CN1SS_SUITE_IDLE_TIMEOUT_SECONDS: '1200' - CN1SS_SUITE_TIMEOUT_SECONDS: '3000' run: | set -euo pipefail mkdir -p "${ARTIFACTS_DIR}" @@ -258,9 +251,9 @@ jobs: "${{ steps.build-ios-app.outputs.workspace }}" \ "" \ "${{ steps.build-ios-app.outputs.scheme }}" - # This job uses a 50-minute absolute suite limit and a 20-minute - # no-progress limit. The outer cap also includes its native rebuild - # and cleanup, so it leaves those phases headroom. + # The script enforces both a 35-minute absolute suite limit and a + # 12-minute no-progress limit. This outer cap also includes its + # native rebuild and cleanup, so it must leave those phases headroom. timeout-minutes: 65 - name: Upload iOS OpenGL port status @@ -471,14 +464,6 @@ jobs: CN1SS_REPORT_TITLE: 'iOS Metal screenshot updates' CN1SS_SUCCESS_MESSAGE: '✅ Native iOS Metal screenshot tests passed.' CN1SS_COMMENT_LOG_PREFIX: '[run-ios-device-tests-metal]' - # The x86 simulator runs CommonWorkloadBenchmarkTest through - # ParparVM's non-NEON path. Its object-allocation block took 555s on - # an otherwise identical successful run, but exceeded the generic - # 720s idle gate on a contended runner while the hang sample showed - # active GC/allocation work. Allow that benchmark variance here; - # the suite's independent 35-minute absolute limit still catches a - # genuinely stuck app. - CN1SS_SUITE_IDLE_TIMEOUT_SECONDS: '1200' # Strict screenshot gating: fail on any pixel mismatch and on any # missing screenshot. The Metal backend must produce the full # baseline set; a test that hangs (e.g. the DialogTheme texture diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java index 011bd8a47d1..33406176504 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -128,7 +128,7 @@ public TensorInfo[] getInputs() { synchronized (this) { ensureOpen(); ensureNotRunning("read input metadata"); - return implementation.getInputs(handle); + return copyMetadata(implementation.getInputs(handle)); } } @@ -145,10 +145,19 @@ public TensorInfo[] getOutputs() { synchronized (this) { ensureOpen(); ensureNotRunning("read output metadata"); - return implementation.getOutputs(handle); + return copyMetadata(implementation.getOutputs(handle)); } } + private static TensorInfo[] copyMetadata(TensorInfo[] metadata) { + if (metadata == null || metadata.length == 0) { + return new TensorInfo[0]; + } + TensorInfo[] copy = new TensorInfo[metadata.length]; + System.arraycopy(metadata, 0, copy, 0, metadata.length); + return copy; + } + /// Copies input tensors to native memory, invokes the model, and returns /// every output tensor. Named tensors are matched by name; unnamed tensors /// are matched by position. Calling {@link #close()} while this operation diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelSource.java b/CodenameOne/src/com/codename1/ai/inference/ModelSource.java index 610abfbf6aa..cb270b10bce 100644 --- a/CodenameOne/src/com/codename1/ai/inference/ModelSource.java +++ b/CodenameOne/src/com/codename1/ai/inference/ModelSource.java @@ -89,6 +89,19 @@ public byte[] getBytes() { return out; } + /// Returns the internal model byte array without copying it. + /// + /// This escape hatch is intended for native backend handoff and + /// memory-sensitive applications. The returned array must be treated as + /// read-only: modifying it changes the model source and violates this + /// class's immutability contract. Prefer {@link #getBytes()} unless the + /// additional full-model copy is known to be unacceptable. + /// + /// @return internal model bytes, or {@code null} for non-byte sources + public byte[] getBytesUnsafe() { + return bytes; + } + /// @return the file/resource path, or {@code null} for a byte source public String getPath() { return path; diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java b/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java index 3d3be29d9cc..00033d208c8 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java @@ -24,8 +24,13 @@ /// Selects native backends for language identification, translation, and /// Smart Reply. Automatic selection uses ML Kit on Android, Apple Natural -/// Language for iOS identification, and feature-scoped ML Kit components for -/// iOS translation and Smart Reply. +/// Language for iOS identification. iOS translation and Smart Reply require +/// their feature-specific ML Kit selector methods. +/// +/// The feature-specific methods are also build-time dependency markers. They +/// intentionally remain distinct even though they return the same runtime +/// backend id, because the builder scans each call site to include only the +/// requested ML Kit component. public final class LanguageBackends { private static final LanguageBackend AUTO = new Named("auto"); private static final LanguageBackend ML_KIT = new Named("ml-kit"); @@ -58,6 +63,24 @@ public static LanguageBackend mlKitLanguageIdentification() { return ML_KIT; } + /// Selects ML Kit translation. Calling this method is the build-time + /// marker that adds the translation pod on iOS; merely referencing + /// {@link Translator} does not add it or disable the arm64 simulator. + /// + /// @return the ML Kit translation selector + public static LanguageBackend mlKitTranslation() { + return ML_KIT; + } + + /// Selects ML Kit Smart Reply. Calling this method is the build-time + /// marker that adds the Smart Reply pod on iOS; merely referencing + /// {@link SmartReply} does not add it or disable the arm64 simulator. + /// + /// @return the ML Kit Smart Reply selector + public static LanguageBackend mlKitSmartReply() { + return ML_KIT; + } + private static final class Named implements LanguageBackend { private final String id; diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java index feb69ab7f18..95ee32a53c0 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java @@ -23,8 +23,8 @@ package com.codename1.ai.language; import com.codename1.impl.LanguageImpl; -import com.codename1.ui.Display; import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; /// Identifies possible languages entirely on device. Results are ranked by /// descending backend confidence and filtered by @@ -41,10 +41,18 @@ public static boolean isSupported() { /// @param options backend selection, or {@code null} for defaults /// @return whether the selected backend is available on this target public static boolean isSupported(LanguageOptions options) { - LanguageImpl impl = Display.getInstance().getLanguageBackend(); - LanguageOptions actual = options == null ? new LanguageOptions() : options; - return impl != null && impl.isSupported("language-id", - actual.getBackend().getId()); + return LanguageSession.isSupported("language-id", options); + } + + /// Opens a reusable language-identification session. Reusing a session + /// avoids repeatedly creating the native recognizer when classifying many + /// strings. Close it when no more identifications are pending. + /// + /// @param options backend and confidence options, or {@code null} + /// @return a reusable session that owns one native language backend + /// @throws UnsupportedOperationException if the selected backend is absent + public static Session open(LanguageOptions options) { + return new Session(LanguageSession.open("language-id", options)); } /// Identifies possible languages off the EDT without uploading text. The @@ -55,18 +63,67 @@ public static boolean isSupported(LanguageOptions options) { /// @return asynchronous ranked candidates; may be empty for undetermined text public static AsyncResource identify( String text, LanguageOptions options) { - LanguageOptions actual = (options == null - ? new LanguageOptions() : options).snapshot(); - LanguageImpl impl = Display.getInstance().getLanguageBackend(); - if (impl == null || !impl.isSupported("language-id", - actual.getBackend().getId())) { + final Session session; + try { + session = open(options); + } catch (RuntimeException error) { AsyncResource out = new AsyncResource(); - out.error(new UnsupportedOperationException( - "language-id is not supported")); + out.error(error); return out; } - return impl.identify(text == null ? "" : text, - actual.getBackend().getId(), actual); + AsyncResource result = session.identify(text); + closeWhenFinished(result, session); + return result; + } + + private static void closeWhenFinished(AsyncResource result, + final Session session) { + result.ready(new SuccessCallback() { + public void onSucess(T value) { + session.close(); + } + }).except(new SuccessCallback() { + public void onSucess(Throwable error) { + session.close(); + } + }); + } + + /// Reusable owner of a native language-identification backend. + /// + /// A session accepts multiple asynchronous requests. Calling + /// {@link #close()} prevents new requests immediately and defers native + /// release until requests already in progress have completed. + public static final class Session implements AutoCloseable { + private final LanguageSession session; + + private Session(LanguageSession session) { + this.session = session; + } + + /// Identifies languages without recreating the native backend. + /// + /// @param text text to classify; {@code null} is treated as empty + /// @return asynchronous ranked candidates, possibly empty + /// @throws IllegalStateException if this session is closed + public AsyncResource identify(final String text) { + return session.execute( + new LanguageSession.Operation() { + public AsyncResource run( + LanguageImpl implementation, + LanguageOptions options) { + return implementation.identify( + text == null ? "" : text, + options.getBackend().getId(), options); + } + }); + } + + /// Closes the session. This method is idempotent; native release is + /// deferred until pending identifications finish. + public void close() { + session.close(); + } } } diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageSession.java b/CodenameOne/src/com/codename1/ai/language/LanguageSession.java new file mode 100644 index 00000000000..097e9c0c85a --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/LanguageSession.java @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.language; + +import com.codename1.impl.LanguageImpl; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; + +/** + * Shared lifecycle implementation for the public feature-specific language + * sessions. This package-private type keeps native backends alive across + * repeated operations and defers release while an operation is pending. + */ +final class LanguageSession implements AutoCloseable { + interface Operation { + AsyncResource run(LanguageImpl implementation, + LanguageOptions options); + } + + private final LanguageImpl implementation; + private final LanguageOptions options; + private boolean closed; + private int activeOperations; + private boolean closePending; + + private LanguageSession(LanguageImpl implementation, + LanguageOptions options) { + this.implementation = implementation; + this.options = options; + } + + static boolean isSupported(String feature, LanguageOptions options) { + LanguageImpl implementation = + Display.getInstance().getLanguageBackend(); + if (implementation == null) { + return false; + } + try { + LanguageOptions actual = options == null + ? new LanguageOptions() : options; + return implementation.isSupported(feature, + actual.getBackend().getId()); + } finally { + implementation.close(); + } + } + + static LanguageSession open(String feature, LanguageOptions options) { + LanguageImpl implementation = + Display.getInstance().getLanguageBackend(); + LanguageOptions actual = (options == null + ? new LanguageOptions() : options).snapshot(); + if (implementation == null + || !implementation.isSupported(feature, + actual.getBackend().getId())) { + if (implementation != null) { + implementation.close(); + } + throw new UnsupportedOperationException( + feature + " is not supported"); + } + return new LanguageSession(implementation, actual); + } + + AsyncResource execute(Operation operation) { + final AsyncResource backendResult; + synchronized (this) { + if (closed) { + throw new IllegalStateException( + "Language session is closed"); + } + activeOperations++; + try { + backendResult = operation.run(implementation, options); + if (backendResult == null) { + throw new IllegalStateException( + "Language backend returned no asynchronous result"); + } + } catch (RuntimeException error) { + activeOperations--; + throw error; + } catch (Error error) { + activeOperations--; + throw error; + } + } + final AsyncResource result = new AsyncResource(); + final Completion completion = new Completion(); + backendResult.ready(new SuccessCallback() { + public void onSucess(T value) { + if (completion.finish()) { + result.complete(value); + operationFinished(); + } + } + }).except(new SuccessCallback() { + public void onSucess(Throwable error) { + if (completion.finish()) { + result.error(error); + operationFinished(); + } + } + }); + return result; + } + + private void operationFinished() { + LanguageImpl toClose = null; + synchronized (this) { + activeOperations--; + if (activeOperations == 0 && closePending) { + closePending = false; + toClose = implementation; + } + } + if (toClose != null) { + toClose.close(); + } + } + + public void close() { + LanguageImpl toClose = null; + synchronized (this) { + if (closed) { + return; + } + closed = true; + if (activeOperations == 0) { + toClose = implementation; + } else { + closePending = true; + } + } + if (toClose != null) { + toClose.close(); + } + } + + private static final class Completion { + private boolean finished; + + synchronized boolean finish() { + if (finished) { + return false; + } + finished = true; + return true; + } + } +} diff --git a/CodenameOne/src/com/codename1/ai/language/SmartReply.java b/CodenameOne/src/com/codename1/ai/language/SmartReply.java index 23275c0c9d6..17bb949ac60 100644 --- a/CodenameOne/src/com/codename1/ai/language/SmartReply.java +++ b/CodenameOne/src/com/codename1/ai/language/SmartReply.java @@ -23,8 +23,8 @@ package com.codename1.ai.language; import com.codename1.impl.LanguageImpl; -import com.codename1.ui.Display; import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; /// Produces short reply suggestions for a chronological conversation without /// uploading its messages. ML Kit may return no suggestions when the language @@ -38,13 +38,26 @@ public static boolean isSupported() { return isSupported(new LanguageOptions()); } + /// Tests whether Smart Reply is available for a backend selection. + /// + /// This capability check creates and closes a temporary native backend. + /// Retain a {@link Session} from {@link #open(LanguageOptions)} when the + /// application will immediately request repeated suggestions. + /// + /// @param options backend selection, or {@code null} for defaults /// @return whether the selected backend supports Smart Reply public static boolean isSupported(LanguageOptions options) { - LanguageOptions actual = options == null - ? new LanguageOptions() : options; - LanguageImpl impl = Display.getInstance().getLanguageBackend(); - return impl != null && impl.isSupported( - "smart-reply", actual.getBackend().getId()); + return LanguageSession.isSupported("smart-reply", options); + } + + /// Opens a reusable Smart Reply session. Reuse it for conversations that + /// need repeated suggestions to avoid recreating the native client. + /// + /// @param options backend options, or {@code null} + /// @return a reusable session that owns one native language backend + /// @throws UnsupportedOperationException if the selected backend is absent + public static Session open(LanguageOptions options) { + return new Session(LanguageSession.open("smart-reply", options)); } /// The conversation array and option values are copied before backend work @@ -55,16 +68,30 @@ public static boolean isSupported(LanguageOptions options) { /// @return asynchronous suggestions, possibly an empty array public static AsyncResource suggest(SmartReplyMessage[] conversation, LanguageOptions options) { - LanguageOptions actual = (options == null - ? new LanguageOptions() : options).snapshot(); - LanguageImpl impl = Display.getInstance().getLanguageBackend(); - if (impl == null || !impl.isSupported("smart-reply", actual.getBackend().getId())) { + final Session session; + try { + session = open(options); + } catch (RuntimeException error) { AsyncResource out = new AsyncResource(); - out.error(new UnsupportedOperationException("smart reply is not supported")); + out.error(error); return out; } - return impl.suggestReplies(copyConversation(conversation), - actual.getBackend().getId(), actual); + AsyncResource result = session.suggest(conversation); + closeWhenFinished(result, session); + return result; + } + + private static void closeWhenFinished(AsyncResource result, + final Session session) { + result.ready(new SuccessCallback() { + public void onSucess(T value) { + session.close(); + } + }).except(new SuccessCallback() { + public void onSucess(Throwable error) { + session.close(); + } + }); } private static SmartReplyMessage[] copyConversation( @@ -79,4 +106,42 @@ private static SmartReplyMessage[] copyConversation( } return copy; } + + /// Reusable owner of a native Smart Reply backend. + /// + /// Calling {@link #close()} prevents new requests immediately and defers + /// native release until suggestions already in progress have completed. + public static final class Session implements AutoCloseable { + private final LanguageSession session; + + private Session(LanguageSession session) { + this.session = session; + } + + /// Generates reply suggestions without recreating the native backend. + /// + /// @param conversation chronological messages, oldest first; + /// {@code null} is treated as an empty conversation + /// @return asynchronous suggestions, possibly empty + /// @throws IllegalStateException if this session is closed + public AsyncResource suggest( + final SmartReplyMessage[] conversation) { + final SmartReplyMessage[] snapshot = + copyConversation(conversation); + return session.execute(new LanguageSession.Operation() { + public AsyncResource run( + LanguageImpl implementation, + LanguageOptions options) { + return implementation.suggestReplies(snapshot, + options.getBackend().getId(), options); + } + }); + } + + /// Closes the session. This method is idempotent; native release is + /// deferred until pending suggestions finish. + public void close() { + session.close(); + } + } } diff --git a/CodenameOne/src/com/codename1/ai/language/Translator.java b/CodenameOne/src/com/codename1/ai/language/Translator.java index 24299c61e1d..f505f2ab73f 100644 --- a/CodenameOne/src/com/codename1/ai/language/Translator.java +++ b/CodenameOne/src/com/codename1/ai/language/Translator.java @@ -23,8 +23,8 @@ package com.codename1.ai.language; import com.codename1.impl.LanguageImpl; -import com.codename1.ui.Display; import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; /// Translates text on device with lazily installed language-pair models. /// The first request for a pair may take longer while ML Kit downloads the @@ -38,22 +38,35 @@ public static boolean isSupported() { return isSupported(new LanguageOptions()); } + /// Tests whether translation is available for a backend selection. + /// + /// This capability check creates and closes a temporary native backend. + /// Retain a {@link Session} from {@link #open(LanguageOptions)} when the + /// application will immediately perform repeated translations. + /// + /// @param options backend selection, or {@code null} for defaults /// @return whether the selected backend supports translation public static boolean isSupported(LanguageOptions options) { - LanguageOptions actual = options == null ? new LanguageOptions() : options; - LanguageImpl impl = Display.getInstance().getLanguageBackend(); - return impl != null && impl.isSupported("translation", - actual.getBackend().getId()); + return LanguageSession.isSupported("translation", options); } - /// Option values are copied before asynchronous backend work begins. + /// Opens a reusable translation session. Reuse it for repeated requests + /// so native translation clients and downloaded model state are retained. /// - /// @param text source text + /// @param options backend options, or {@code null} + /// @return a reusable session that owns one native language backend + /// @throws UnsupportedOperationException if the selected backend is absent + public static Session open(LanguageOptions options) { + return new Session(LanguageSession.open("translation", options)); + } + + /// Option values are copied before asynchronous backend work begins. /// Current ML Kit backends accept a BCP-47 tag with an optional script or /// region, such as {@code en-US}, and select the corresponding supported /// base-language model ({@code en} in this example). The asynchronous /// resource fails when either tag does not identify a supported model. /// + /// @param text source text; {@code null} is treated as empty /// @param sourceLanguage BCP-47 source language tag /// @param targetLanguage BCP-47 target language tag /// @param options backend options, or {@code null} @@ -61,15 +74,71 @@ public static boolean isSupported(LanguageOptions options) { public static AsyncResource translate(String text, String sourceLanguage, String targetLanguage, LanguageOptions options) { - LanguageOptions actual = (options == null - ? new LanguageOptions() : options).snapshot(); - LanguageImpl impl = Display.getInstance().getLanguageBackend(); - if (impl == null || !impl.isSupported("translation", actual.getBackend().getId())) { + final Session session; + try { + session = open(options); + } catch (RuntimeException error) { AsyncResource out = new AsyncResource(); - out.error(new UnsupportedOperationException("translation is not supported")); + out.error(error); return out; } - return impl.translate(text == null ? "" : text, sourceLanguage, targetLanguage, - actual.getBackend().getId(), actual); + AsyncResource result = session.translate(text, + sourceLanguage, targetLanguage); + closeWhenFinished(result, session); + return result; + } + + private static void closeWhenFinished(AsyncResource result, + final Session session) { + result.ready(new SuccessCallback() { + public void onSucess(T value) { + session.close(); + } + }).except(new SuccessCallback() { + public void onSucess(Throwable error) { + session.close(); + } + }); + } + + /// Reusable owner of a native translation backend. + /// + /// A session can serve multiple language pairs and retains native clients + /// between calls. Calling {@link #close()} prevents new requests and + /// defers release until pending translations finish. + public static final class Session implements AutoCloseable { + private final LanguageSession session; + + private Session(LanguageSession session) { + this.session = session; + } + + /// Translates text without recreating the native backend. + /// + /// @param text source text; {@code null} is treated as empty + /// @param sourceLanguage supported BCP-47 source language tag + /// @param targetLanguage supported BCP-47 target language tag + /// @return asynchronous translated text + /// @throws IllegalStateException if this session is closed + public AsyncResource translate(final String text, + final String sourceLanguage, + final String targetLanguage) { + return session.execute(new LanguageSession.Operation() { + public AsyncResource run( + LanguageImpl implementation, + LanguageOptions options) { + return implementation.translate( + text == null ? "" : text, sourceLanguage, + targetLanguage, options.getBackend().getId(), + options); + } + }); + } + + /// Closes the session. This method is idempotent; native release is + /// deferred until pending translations finish. + public void close() { + session.close(); + } } } diff --git a/CodenameOne/src/com/codename1/ai/language/package-info.java b/CodenameOne/src/com/codename1/ai/language/package-info.java index 09e12f7f923..e4510159efc 100644 --- a/CodenameOne/src/com/codename1/ai/language/package-info.java +++ b/CodenameOne/src/com/codename1/ai/language/package-info.java @@ -24,10 +24,11 @@ /// reply. /// ///

Android uses feature-scoped ML Kit components. On iOS, automatic -/// language identification uses Apple's Natural Language framework, while -/// translation and Smart Reply use their matching ML Kit components. An -/// application may opt into ML Kit language identification with -/// {@link com.codename1.ai.language.LanguageBackends#mlKitLanguageIdentification()}. +/// language identification uses Apple's Natural Language framework. An +/// application opts into each iOS ML Kit component with +/// {@link com.codename1.ai.language.LanguageBackends#mlKitLanguageIdentification()}, +/// {@link com.codename1.ai.language.LanguageBackends#mlKitTranslation()}, or +/// {@link com.codename1.ai.language.LanguageBackends#mlKitSmartReply()}. /// Each entry point and optional selector is scanned independently, so using /// one feature does not bundle the others. Translation model payloads are /// installed lazily by ML Kit. Other targets expose the same API with an diff --git a/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java b/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java index 577c4e98f7a..d9ca1e9162c 100644 --- a/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java +++ b/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java @@ -25,13 +25,14 @@ /// Creates reusable still-image or live-frame barcode analyzers. public final class BarcodeScanner extends AbstractVisionAnalyzer { /// Creates an analyzer using the platform default backend and options. - /// VisionOptions + /// @see VisionOptions public BarcodeScanner() { this(null); } /// Creates a reusable analyzer with explicit backend and result options. - /// options configuration captured by this analyzer; null uses defaults + /// @param options configuration captured by this analyzer; {@code null} + /// uses defaults public BarcodeScanner(VisionOptions options) { super(VisionFeature.BARCODE_SCANNING, options); } diff --git a/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java b/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java index 9d9a38a3822..94106796134 100644 --- a/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java +++ b/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java @@ -25,13 +25,14 @@ /// Creates reusable document-boundary and perspective-correction analyzers. public final class DocumentScanner extends AbstractVisionAnalyzer { /// Creates an analyzer using the platform default backend and options. - /// VisionOptions + /// @see VisionOptions public DocumentScanner() { this(null); } /// Creates a reusable analyzer with explicit backend and result options. - /// options configuration captured by this analyzer; null uses defaults + /// @param options configuration captured by this analyzer; {@code null} + /// uses defaults public DocumentScanner(VisionOptions options) { super(VisionFeature.DOCUMENT_SCANNING, options); } diff --git a/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java b/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java index 852b6ee18e0..63a8d6b82c0 100644 --- a/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java +++ b/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java @@ -25,13 +25,14 @@ /// Creates reusable on-device face analyzers. public final class FaceDetector extends AbstractVisionAnalyzer { /// Creates an analyzer using the platform default backend and options. - /// VisionOptions + /// @see VisionOptions public FaceDetector() { this(null); } /// Creates a reusable analyzer with explicit backend and result options. - /// options configuration captured by this analyzer; null uses defaults + /// @param options configuration captured by this analyzer; {@code null} + /// uses defaults public FaceDetector(VisionOptions options) { super(VisionFeature.FACE_DETECTION, options); } diff --git a/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java b/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java index 9bd368d250f..b607a7f03a8 100644 --- a/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java +++ b/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java @@ -25,13 +25,14 @@ /// Creates reusable on-device image classifiers. public final class ImageLabeler extends AbstractVisionAnalyzer { /// Creates an analyzer using the platform default backend and options. - /// VisionOptions + /// @see VisionOptions public ImageLabeler() { this(null); } /// Creates a reusable analyzer with explicit backend and result options. - /// options configuration captured by this analyzer; null uses defaults + /// @param options configuration captured by this analyzer; {@code null} + /// uses defaults public ImageLabeler(VisionOptions options) { super(VisionFeature.IMAGE_LABELING, options); } diff --git a/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java b/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java index 4748505b7b1..8c50777180d 100644 --- a/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java +++ b/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java @@ -25,13 +25,14 @@ /// Creates reusable body-pose analyzers. public final class PoseDetector extends AbstractVisionAnalyzer { /// Creates an analyzer using the platform default backend and options. - /// VisionOptions + /// @see VisionOptions public PoseDetector() { this(null); } /// Creates a reusable analyzer with explicit backend and result options. - /// options configuration captured by this analyzer; null uses defaults + /// @param options configuration captured by this analyzer; {@code null} + /// uses defaults public PoseDetector(VisionOptions options) { super(VisionFeature.POSE_DETECTION, options); } diff --git a/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java b/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java index 8b6d8eebccf..e31ab0a9b4c 100644 --- a/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java +++ b/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java @@ -25,13 +25,14 @@ /// Creates reusable foreground/person segmentation analyzers. public final class SelfieSegmenter extends AbstractVisionAnalyzer { /// Creates an analyzer using the platform default backend and options. - /// VisionOptions + /// @see VisionOptions public SelfieSegmenter() { this(null); } /// Creates a reusable analyzer with explicit backend and result options. - /// options configuration captured by this analyzer; null uses defaults + /// @param options configuration captured by this analyzer; {@code null} + /// uses defaults public SelfieSegmenter(VisionOptions options) { super(VisionFeature.SELFIE_SEGMENTATION, options); } diff --git a/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java b/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java index 7da6c711ef1..4bddef0788e 100644 --- a/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java +++ b/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java @@ -25,13 +25,14 @@ /// Creates reusable on-device OCR analyzers. public final class TextRecognizer extends AbstractVisionAnalyzer { /// Creates an analyzer using the platform default backend and options. - /// VisionOptions + /// @see VisionOptions public TextRecognizer() { this(null); } /// Creates a reusable analyzer with explicit backend and result options. - /// options configuration captured by this analyzer; null uses defaults + /// @param options configuration captured by this analyzer; {@code null} + /// uses defaults public TextRecognizer(VisionOptions options) { super(VisionFeature.TEXT_RECOGNITION, options); } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java b/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java index 7534cb4cbad..5828bd808b6 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java @@ -26,6 +26,13 @@ /// ML Kit on Android. Feature-specific ML Kit methods are separate so one /// selector never adds unrelated OCR, barcode, face, pose, labeling, or /// segmentation pods. +/// +/// Each ML Kit selector call is also a build-time dependency marker. The +/// methods intentionally return the same runtime backend id but must remain +/// distinct so the builder can include only the selected feature. Pass the +/// selector that corresponds to the analyzer being constructed; for example, +/// the face-detection selector cannot satisfy a text recognizer and that +/// mismatched analyzer reports unsupported. public final class VisionBackends { private static final VisionBackend AUTO = new NamedBackend("auto"); private static final VisionBackend APPLE = new NamedBackend("apple-vision"); diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java index bacd3a96840..7a17290a6a2 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java @@ -40,8 +40,15 @@ public final class VisionImage { private VisionImage(byte[] encodedBytes, byte[] pixels, int width, int height, int rotationDegrees, long timestampNanos, FrameFormat format) { - this.encodedBytes = copy(encodedBytes); - this.pixels = copy(pixels); + this(encodedBytes, pixels, width, height, rotationDegrees, + timestampNanos, format, true); + } + + private VisionImage(byte[] encodedBytes, byte[] pixels, int width, int height, + int rotationDegrees, long timestampNanos, + FrameFormat format, boolean copyData) { + this.encodedBytes = copyData ? copy(encodedBytes) : encodedBytes; + this.pixels = copyData ? copy(pixels) : pixels; this.width = width; this.height = height; this.rotationDegrees = normalizeRotation(rotationDegrees); @@ -49,6 +56,16 @@ private VisionImage(byte[] encodedBytes, byte[] pixels, int width, int height, this.format = format == null ? FrameFormat.JPEG : format; } + static VisionImage detachedCameraData(byte[] data, boolean raw, + int width, int height, + int rotationDegrees, + long timestampNanos, + FrameFormat format) { + return new VisionImage(raw ? null : data, raw ? data : null, + width, height, rotationDegrees, timestampNanos, + raw ? format : FrameFormat.JPEG, false); + } + /// Creates an encoded JPEG or PNG input whose stored pixels are already /// upright. This method does not inspect EXIF orientation metadata. Use /// {@link #encoded(byte[], int)} when the encoded image needs a clockwise diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java b/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java index f199198cc77..ac6e3d689e7 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java @@ -25,7 +25,7 @@ /// Common analyzer configuration. An analyzer captures the supplied options /// when constructed; reuse it for frames needing the same backend, confidence /// threshold, and result limit. -public class VisionOptions { +public final class VisionOptions { private VisionBackend backend = VisionBackends.auto(); private float minimumConfidence; private int maximumResults; diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java index 1dae6f10671..ce4649a79df 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java @@ -25,6 +25,7 @@ import com.codename1.camera.CameraFrame; import com.codename1.camera.CameraSession; import com.codename1.camera.FrameListener; +import com.codename1.camera.FrameFormat; import com.codename1.ui.Display; import com.codename1.util.SuccessCallback; @@ -38,7 +39,7 @@ public final class VisionPipeline implements AutoCloseable { private final VisionAnalyzer analyzer; private final VisionPipelineListener listener; private final FrameListener frameListener; - private VisionImage pending; + private PendingFrame pending; private boolean busy; private boolean closed; @@ -57,24 +58,27 @@ public VisionPipeline(CameraSession session, VisionAnalyzer analyzer, frameListener = new FrameListener() { @Override public void onFrame(CameraFrame frame) { - accept(VisionImage.fromCameraFrame(frame)); + accept(frame); } }; session.setFrameListener(frameListener); } - private void accept(VisionImage image) { + private void accept(CameraFrame frame) { synchronized (this) { if (closed) { return; } if (busy) { - pending = image; + if (pending == null) { + pending = new PendingFrame(); + } + pending.copyFrom(frame); return; } busy = true; } - process(image); + process(VisionImage.fromCameraFrame(frame)); } private void process(final VisionImage image) { @@ -86,11 +90,13 @@ private void process(final VisionImage image) { pending = null; return; } - operation = analyzer.process(image); - if (operation == null) { - throw new IllegalStateException( - "Vision analyzer returned no operation"); - } + } + // Native analyzers may complete synchronously or re-enter app + // code. Do not invoke them while holding the pipeline monitor. + operation = analyzer.process(image); + if (operation == null) { + throw new IllegalStateException( + "Vision analyzer returned no operation"); } } catch (final Throwable error) { onFinished(new Runnable() { @@ -128,22 +134,22 @@ private void onFinished(final Runnable notification) { Display.getInstance().callSerially(new Runnable() { @Override public void run() { - VisionImage next; + PendingFrame pendingFrame; synchronized (VisionPipeline.this) { if (closed) { busy = false; pending = null; return; } - next = pending; + pendingFrame = pending; pending = null; - busy = next != null; + busy = pendingFrame != null; } try { notification.run(); } finally { - if (next != null) { - process(next); + if (pendingFrame != null) { + process(pendingFrame.toImage()); } } } @@ -177,4 +183,40 @@ public void close() { session.removeFrameListener(frameListener); analyzer.close(); } + + private static final class PendingFrame { + private byte[] data; + private boolean raw; + private int width; + private int height; + private int rotationDegrees; + private long timestampNanos; + private FrameFormat format; + + void copyFrom(CameraFrame frame) { + FrameFormat requested = frame.getFormat() == null + ? FrameFormat.JPEG : frame.getFormat(); + byte[] rawBytes = frame.getRawBytes(); + raw = requested != FrameFormat.JPEG + && rawBytes != null && rawBytes.length > 0; + byte[] source = raw ? rawBytes : frame.getJpegBytes(); + if (source == null) { + source = new byte[0]; + } + if (data == null || data.length != source.length) { + data = new byte[source.length]; + } + System.arraycopy(source, 0, data, 0, source.length); + width = frame.getWidth(); + height = frame.getHeight(); + rotationDegrees = frame.getRotationDegrees(); + timestampNanos = frame.getTimestampNanos(); + format = raw ? requested : FrameFormat.JPEG; + } + + VisionImage toImage() { + return VisionImage.detachedCameraData(data, raw, width, height, + rotationDegrees, timestampNanos, format); + } + } } diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java index a16e160bfa1..2b0bb2e416b 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java @@ -371,7 +371,7 @@ private static ByteBuffer loadModel(ModelSource source) throws IOException { } byte[] bytes; if (source.getKind() == ModelSource.BYTES) { - bytes = source.getBytes(); + bytes = source.getBytesUnsafe(); } else { InputStream input = Display.getInstance().getResourceAsStream( AndroidInferenceImpl.class, source.getPath()); diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageAdapter.java index ca96cb8c558..f0f1e14edee 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageAdapter.java @@ -30,7 +30,7 @@ import com.google.android.gms.tasks.OnFailureListener; /** Shared contract and completion helpers for Android language adapters. */ -abstract class AndroidLanguageAdapter { +abstract class AndroidLanguageAdapter implements AutoCloseable { AsyncResource identify( String text, LanguageOptions options) { return unsupported("Language identification is not supported"); @@ -61,8 +61,7 @@ public void run() { }); } - static OnFailureListener failure(final AsyncResource out, - final AutoCloseable client) { + static OnFailureListener failure(final AsyncResource out) { return new OnFailureListener() { public void onFailure(final Exception error) { Display.getInstance().callSerially(new Runnable() { @@ -70,11 +69,10 @@ public void run() { out.error(error); } }); - try { - client.close(); - } catch (Exception ignored) { - } } }; } + + public void close() { + } } diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageIdAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageIdAdapter.java index bca3039ec33..a53114bd7ec 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageIdAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageIdAdapter.java @@ -35,16 +35,15 @@ /** ML Kit language ID; retained only for {@code LanguageIdentifier} users. */ final class AndroidLanguageIdAdapter extends AndroidLanguageAdapter { + private LanguageIdentifier client; + @Override AsyncResource identify( String text, LanguageOptions options) { final AsyncResource out = new AsyncResource(); - final LanguageIdentifier client = LanguageIdentification.getClient( - new LanguageIdentificationOptions.Builder() - .setConfidenceThreshold(options.getMinimumConfidence()) - .build()); - client.identifyPossibleLanguages(text).addOnSuccessListener( + final LanguageIdentifier current = client(options); + current.identifyPossibleLanguages(text).addOnSuccessListener( new OnSuccessListener>() { public void onSuccess(List values) { LanguageCandidate[] result = @@ -55,9 +54,26 @@ public void onSuccess(List values) { value.getLanguageTag(), value.getConfidence()); } complete(out, result); - client.close(); } - }).addOnFailureListener(failure(out, client)); + }).addOnFailureListener(failure(out)); return out; } + + private synchronized LanguageIdentifier client(LanguageOptions options) { + if (client == null) { + client = LanguageIdentification.getClient( + new LanguageIdentificationOptions.Builder() + .setConfidenceThreshold( + options.getMinimumConfidence()) + .build()); + } + return client; + } + + public synchronized void close() { + if (client != null) { + client.close(); + client = null; + } + } } diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageImpl.java index bab9094f966..380cfdd7f37 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageImpl.java @@ -34,6 +34,9 @@ */ public final class AndroidLanguageImpl extends LanguageImpl { private volatile boolean closed; + private AndroidLanguageAdapter languageIdAdapter; + private AndroidLanguageAdapter translationAdapter; + private AndroidLanguageAdapter smartReplyAdapter; @Override public boolean isSupported(String feature, String backendId) { @@ -75,30 +78,63 @@ public AsyncResource suggestReplies( : adapter.suggestReplies(conversation, options); } - private static AndroidLanguageAdapter adapter(String feature) { + private synchronized AndroidLanguageAdapter adapter(String feature) { + if (closed) { + return null; + } + AndroidLanguageAdapter cached; String className; if ("language-id".equals(feature)) { + cached = languageIdAdapter; className = "com.codename1.impl.android.ai.AndroidLanguageIdAdapter"; } else if ("translation".equals(feature)) { + cached = translationAdapter; className = "com.codename1.impl.android.ai.AndroidTranslationAdapter"; } else if ("smart-reply".equals(feature)) { + cached = smartReplyAdapter; className = "com.codename1.impl.android.ai.AndroidSmartReplyAdapter"; } else { return null; } + if (cached != null) { + return cached; + } try { - return (AndroidLanguageAdapter) + cached = (AndroidLanguageAdapter) Class.forName(className).newInstance(); + if ("language-id".equals(feature)) { + languageIdAdapter = cached; + } else if ("translation".equals(feature)) { + translationAdapter = cached; + } else { + smartReplyAdapter = cached; + } + return cached; } catch (Throwable ignored) { return null; } } @Override - public void close() { + public synchronized void close() { + if (closed) { + return; + } closed = true; + closeAdapter(languageIdAdapter); + closeAdapter(translationAdapter); + closeAdapter(smartReplyAdapter); + languageIdAdapter = null; + translationAdapter = null; + smartReplyAdapter = null; + } + + private static void closeAdapter(AndroidLanguageAdapter adapter) { + if (adapter != null) { + adapter.close(); + } } } diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java index c9b5b392f2e..2aa6754f07d 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java @@ -37,11 +37,13 @@ /** ML Kit Smart Reply; retained only for {@code SmartReply} users. */ final class AndroidSmartReplyAdapter extends AndroidLanguageAdapter { + private SmartReplyGenerator client; + @Override AsyncResource suggestReplies( SmartReplyMessage[] conversation, LanguageOptions options) { final AsyncResource out = new AsyncResource(); - final SmartReplyGenerator client = SmartReply.getClient(); + final SmartReplyGenerator current = client(); List messages = new ArrayList(); for (int i = 0; i < conversation.length; i++) { SmartReplyMessage message = conversation[i]; @@ -54,7 +56,7 @@ AsyncResource suggestReplies( ? "remote" : message.getParticipantId())); } - client.suggestReplies(messages).addOnSuccessListener( + current.suggestReplies(messages).addOnSuccessListener( new OnSuccessListener() { public void onSuccess(SmartReplySuggestionResult value) { List suggestions = @@ -64,9 +66,22 @@ public void onSuccess(SmartReplySuggestionResult value) { result[i] = suggestions.get(i).getText(); } complete(out, result); - client.close(); } - }).addOnFailureListener(failure(out, client)); + }).addOnFailureListener(failure(out)); return out; } + + private synchronized SmartReplyGenerator client() { + if (client == null) { + client = SmartReply.getClient(); + } + return client; + } + + public synchronized void close() { + if (client != null) { + client.close(); + client = null; + } + } } diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java index 6c9273ac367..493491ee91a 100644 --- a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java @@ -31,10 +31,15 @@ import com.google.mlkit.nl.translate.TranslateLanguage; import com.google.mlkit.nl.translate.Translator; import com.google.mlkit.nl.translate.TranslatorOptions; +import java.util.HashMap; import java.util.Locale; +import java.util.Map; /** ML Kit translation; retained only for {@code Translator} users. */ final class AndroidTranslationAdapter extends AndroidLanguageAdapter { + private final Map clients = + new HashMap(); + @Override AsyncResource translate( final String text, String sourceLanguage, String targetLanguage, @@ -47,13 +52,7 @@ AsyncResource translate( "Unsupported translation language")); return out; } - TranslatorOptions translatorOptions = - new TranslatorOptions.Builder() - .setSourceLanguage(sourceCode) - .setTargetLanguage(targetCode) - .build(); - final Translator client = - Translation.getClient(translatorOptions); + final Translator client = client(sourceCode, targetCode); client.downloadModelIfNeeded().onSuccessTask( new SuccessContinuation() { public Task then(Void ignored) { @@ -62,12 +61,33 @@ public Task then(Void ignored) { }).addOnSuccessListener(new OnSuccessListener() { public void onSuccess(String value) { complete(out, value); - client.close(); } - }).addOnFailureListener(failure(out, client)); + }).addOnFailureListener(failure(out)); return out; } + private synchronized Translator client(String sourceCode, + String targetCode) { + String key = sourceCode + "\n" + targetCode; + Translator client = clients.get(key); + if (client == null) { + TranslatorOptions options = new TranslatorOptions.Builder() + .setSourceLanguage(sourceCode) + .setTargetLanguage(targetCode) + .build(); + client = Translation.getClient(options); + clients.put(key, client); + } + return client; + } + + public synchronized void close() { + for (Translator client : clients.values()) { + client.close(); + } + clients.clear(); + } + private static String languageCode(String languageTag) { if (languageTag == null) { return null; diff --git a/Ports/iOSPort/nativeSources/CN1Inference.m b/Ports/iOSPort/nativeSources/CN1Inference.m index 54aae618e8c..464d75198e9 100644 --- a/Ports/iOSPort/nativeSources/CN1Inference.m +++ b/Ports/iOSPort/nativeSources/CN1Inference.m @@ -23,6 +23,10 @@ #import "CodenameOne_GLViewController.h" #import "xmlvm.h" +#if !__has_feature(objc_arc) +#error CN1Inference.m requires ARC (-fobjc-arc) +#endif + #if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV #import #import "java_lang_String.h" diff --git a/Ports/iOSPort/nativeSources/CN1Language.m b/Ports/iOSPort/nativeSources/CN1Language.m index 7f72835357e..981c9464d08 100644 --- a/Ports/iOSPort/nativeSources/CN1Language.m +++ b/Ports/iOSPort/nativeSources/CN1Language.m @@ -23,6 +23,10 @@ #import "CodenameOne_GLViewController.h" #import "xmlvm.h" +#if !__has_feature(objc_arc) +#error CN1Language.m requires ARC (-fobjc-arc) +#endif + #if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV #import #import diff --git a/Ports/iOSPort/nativeSources/CN1Vision.m b/Ports/iOSPort/nativeSources/CN1Vision.m index d323d125199..9411aaa5791 100644 --- a/Ports/iOSPort/nativeSources/CN1Vision.m +++ b/Ports/iOSPort/nativeSources/CN1Vision.m @@ -23,6 +23,10 @@ #import "CodenameOne_GLViewController.h" #import "xmlvm.h" +#if !__has_feature(objc_arc) +#error CN1Vision.m requires ARC (-fobjc-arc) +#endif + #if defined(INCLUDE_CN1_VISION) && !TARGET_OS_WATCH && !TARGET_OS_TV #import #import diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java index 0773b0bda80..e40f91a957d 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java @@ -4766,7 +4766,9 @@ public com.codename1.impl.ARImpl createARImpl() { @Override public com.codename1.impl.VisionImpl createVisionImpl() { - for (int feature = 0; feature <= 6; feature++) { + for (int feature = 0; + feature < com.codename1.ai.vision.VisionFeature.values().length; + feature++) { if (nativeInstance.cn1VisionIsSupported(feature, false) || nativeInstance.cn1VisionIsSupported(feature, true)) { return new IOSVisionImpl(); diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java index ab3b4d7a182..8a8dc299615 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java @@ -435,7 +435,7 @@ private static int integer(Map value, String key) { private static byte[] loadModel(ModelSource source) throws IOException { if (source.getKind() == ModelSource.BYTES) { - return source.getBytes(); + return source.getBytesUnsafe(); } InputStream input; input = Display.getInstance().getResourceAsStream( diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java index 3ae8b389559..17a3eecddc5 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java @@ -36,6 +36,7 @@ import com.codename1.ai.vision.VisionOptions; import com.codename1.ai.vision.VisionPoint; import com.codename1.ai.vision.VisionRect; +import com.codename1.camera.FrameFormat; import com.codename1.impl.VisionImpl; import com.codename1.io.JSONParser; import com.codename1.ui.Display; @@ -60,7 +61,7 @@ public boolean isSupported(VisionFeature feature, String backendId) { return false; } return IOSImplementation.nativeInstance.cn1VisionIsSupported( - feature.ordinal(), "ml-kit".equals(backendId)); + nativeFeatureId(feature), "ml-kit".equals(backendId)); } @Override @@ -81,7 +82,7 @@ public AsyncResource analyze(final VisionFeature feature, final int width = encoded == null ? image.getWidth() : 0; final int height = encoded == null ? image.getHeight() : 0; final int frameFormat = encoded == null - ? image.getFormat().ordinal() : 0; + ? nativeFrameFormatId(image.getFormat()) : 0; if (input == null || input.length == 0) { out.error(new VisionException(VisionException.INVALID_IMAGE, "Apple Vision requires encoded, NV21, or RGBA8888 input")); @@ -107,7 +108,7 @@ private static void analyzeInBackground(final AsyncResource out, public void run() { try { String json = IOSImplementation.nativeInstance.cn1VisionAnalyze( - input, feature.ordinal(), mlKit, + input, nativeFeatureId(feature), mlKit, rotation, width, height, frameFormat); if (json == null || json.length() == 0) { throw new VisionException(VisionException.BACKEND_ERROR, @@ -134,6 +135,42 @@ public void run() { }); } + private static int nativeFeatureId(VisionFeature feature) { + switch (feature) { + case TEXT_RECOGNITION: + return 0; + case BARCODE_SCANNING: + return 1; + case FACE_DETECTION: + return 2; + case IMAGE_LABELING: + return 3; + case POSE_DETECTION: + return 4; + case SELFIE_SEGMENTATION: + return 5; + case DOCUMENT_SCANNING: + return 6; + default: + throw new IllegalArgumentException( + "Unsupported vision feature " + feature); + } + } + + private static int nativeFrameFormatId(FrameFormat format) { + switch (format) { + case JPEG: + return 0; + case NV21: + return 1; + case RGBA8888: + return 2; + default: + throw new IllegalArgumentException( + "Unsupported camera frame format " + format); + } + } + @SuppressWarnings({"rawtypes", "unchecked"}) private static T parse(VisionFeature feature, String json, String backendId, float minimumConfidence, diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md index 9647a0c6200..ea1280d7f96 100644 --- a/docs/ai-on-device-architecture.md +++ b/docs/ai-on-device-architecture.md @@ -35,7 +35,10 @@ scanner is an interactive Activity camera flow, while the core `maven/platform-feature-catalog` is the authoritative dependency registry. `PlatformFeatureCatalog.Accumulator` consumes exact class and method -references from the existing bytecode scanners. +references from the existing bytecode scanners. It rejects unrelated symbols +at consume time and memoizes its immutable hit set behind a dirty flag, so +memory and repeated builder queries scale with catalog matches rather than +total application bytecode. The local Maven plugin and BuildDaemon both consume the same source contract: @@ -53,6 +56,17 @@ copying the new result. This is required for granular removal: a plain directory merge would retain an old Podfile, workspace, Pods directory, or optional native source after the application stops referencing a feature. +Catalog-selected CocoaPods are deduplicated by pod name. An existing +user-declared spec appears first and wins, preserving constraints such as +`GoogleMLKit/TextRecognition ~> 7.0`. Explicit `ios.dependencyManager=spm` +and `none` modes reject incompatible declared or catalog-selected +dependencies instead of silently omitting them. + +The companion archive extraction hardening is intentionally broader than AI: +every Android/build archive entry is canonicalized and checked for traversal +before extraction. Trusted extraction filters may still route a validated +entry to a deliberate sibling such as the project settings file. + ### Android source granularity The optional Android sources are excluded from `maven/android` compilation @@ -97,8 +111,10 @@ when it observes both: On iOS, language identification defaults to the system Natural Language framework. Calling `LanguageBackends.mlKitLanguageIdentification()` opts -into `GoogleMLKit/LanguageID`; translation and Smart Reply map independently -to `GoogleMLKit/Translate` and `GoogleMLKit/SmartReply`. `InferenceSession` selects +into `GoogleMLKit/LanguageID`. Translation and Smart Reply pods are selected +only by calls to `LanguageBackends.mlKitTranslation()` and +`LanguageBackends.mlKitSmartReply()`. Merely referencing their API classes +does not change simulator architectures. `InferenceSession` selects `TensorFlowLiteObjC/CoreML`. The native implementation is disabled for watchOS and tvOS. Mac native uses @@ -114,10 +130,9 @@ Google ML Kit's iOS binary frameworks contain device `arm64` and simulator constraint independently from Catalyst support. When one of those entries is selected, both builders set `EXCLUDED_ARCHS[sdk=iphonesimulator*]=arm64` on the generated application and -Pods projects. The repository's iOS UI and native-test runners also select -`ARCHS=x86_64`, so Apple Silicon hosts use the supported simulator slice -without requiring an application build hint. TensorFlow Lite's XCFramework -does include an `arm64` simulator slice and does not trigger this fallback. +Pods projects and emit a diagnostic identifying the selected dependency. +TensorFlow Lite's XCFramework does include an `arm64` simulator slice and +does not trigger this fallback. The same Google ML Kit catalog entries declare an iOS 15.5 deployment floor. The iOS builder folds that value into its existing maximum-target calculation before generating the app target and Podfile, preventing a lower application @@ -150,8 +165,10 @@ without accidentally selecting the encoded path. When a camera port cannot supply the requested raw buffer, `fromCameraFrame()` retains the JPEG fallback and reports JPEG as the image format instead of creating an empty image. `VisionPipeline` allows one request to run and retains only the newest pending -frame. This bounds memory and latency for live OCR, barcode, face, pose, -labeling, or segmentation. +frame. Superseding same-sized pending frames reuse one detached buffer, and +analyzer invocation occurs outside the pipeline monitor. This bounds memory, +latency, and lock re-entry for live OCR, barcode, face, pose, labeling, or +segmentation. Native results are converted to stable Codename One value types. Geometry is normalized to the top-left coordinate system. `VisionMetadata` carries the @@ -183,11 +200,13 @@ best effort when fallback is enabled. Strict NPU requests are rejected on Android because LiteRT cannot verify that NNAPI delegated every operation, and on iOS because the Core ML delegate may schedule work on CPU or GPU. -All native sessions and analyzers must be closed. Expensive open, analysis, -and inference work runs off the EDT; completion and error delivery return to -the EDT. A session rejects metadata queries, resizing, and another invocation -while a run is pending so no two API calls can touch the mutable native -interpreter concurrently. +All native sessions and analyzers must be closed. Language APIs expose +reusable feature sessions in addition to static one-shot methods; Android +sessions cache their ML Kit client or translation clients by language pair. +Expensive open, analysis, and inference work runs off the EDT; completion and +error delivery return to the EDT. An inference session rejects metadata +queries, resizing, and another invocation while a run is pending so no two API +calls can touch the mutable native interpreter concurrently. ## Permanent cross-platform coverage diff --git a/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java b/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java index 7bb9fd46dba..67926897934 100644 --- a/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java +++ b/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java @@ -26,6 +26,7 @@ import com.codename1.ai.inference.InferenceSession; import com.codename1.ai.inference.ModelSource; import com.codename1.ai.inference.Tensor; +import com.codename1.ai.language.LanguageBackends; import com.codename1.ai.language.LanguageOptions; import com.codename1.ai.language.Translator; import com.codename1.ai.vision.TextRecognizer; @@ -82,9 +83,11 @@ void inference() { void language() { // tag::ai-and-speech-on-device-language[] - if (Translator.isSupported()) { + LanguageOptions options = new LanguageOptions() + .backend(LanguageBackends.mlKitTranslation()); + if (Translator.isSupported(options)) { Translator.translate("Where is the station?", "en", "fr", - new LanguageOptions()) + options) .ready(value -> Log.p(value)) .except(error -> Log.e(error)); } diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index ed1e7f7e046..49840a2985c 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -308,7 +308,13 @@ linked backend, and OS version. `VisionPipeline` attaches an analyzer to a `CameraSession` frame stream and keeps only the newest pending frame. Live OCR and pose detection therefore can't build an unbounded queue when inference is slower than -the camera. Results and errors are delivered on the Codename One EDT. +the camera. While an analysis is busy, the pipeline reuses the pending +buffer when newer same-sized frames replace it. Results and errors are +delivered on the Codename One EDT. + +`CameraFrame.getRawBytes()` can be `null` even after requesting NV21 or +RGBA8888 when a port cannot expose that format. JPEG bytes remain available; +`VisionImage.fromCameraFrame(...)` automatically selects that fallback. `DocumentScanner` currently performs still-image rectangle correction on Apple platforms. Google's Android document scanner is an interactive @@ -425,13 +431,21 @@ include::../demos/common/src/main/java/com/codenameone/developerguide/snippets/g `LanguageBackends.auto()` selects ML Kit on Android and Apple's Natural Language framework for language identification on iOS. The Apple default keeps arm64 simulator builds native and adds no third-party dependency. -`LanguageBackends.mlKitLanguageIdentification()` opts language -identification into the ML Kit iOS pod. Translation and Smart Reply use -their feature-scoped ML Kit components on both mobile platforms; those -two iOS components currently require the x86_64 simulator slice. -The current Google ML Kit iOS pods also require iOS 15.5, so referencing -translation or Smart Reply, or opting identification into ML Kit, raises -the effective iOS deployment target to 15.5 automatically. +`LanguageBackends.mlKitLanguageIdentification()` opts identification into +the ML Kit iOS pod. iOS translation and Smart Reply are explicit opt-ins: +use `LanguageBackends.mlKitTranslation()` or +`LanguageBackends.mlKitSmartReply()` respectively. Merely referencing +`Translator` or `SmartReply` doesn't add a pod or exclude the arm64 +simulator. Selecting one of those ML Kit backends requires the x86_64 +simulator slice and raises an iOS deployment target below 15.5. + +The static language methods are convenient for one operation and close their +temporary backend after either success or failure. For repeated work, call +`LanguageIdentifier.open(...)`, `Translator.open(...)`, or +`SmartReply.open(...)`, reuse the returned feature session, and close it when +the last pending operation completes. Reusable sessions retain backend state +and, on Android, cache ML Kit clients and translation clients per language +pair. Mac native and the other unsupported targets return `false` from the language service `isSupported()` checks. Completion and failure callbacks arrive on the EDT. @@ -466,10 +480,10 @@ platform-specific integer arrays. `cn1-ai-whisper` and and models are intentionally large. Remove the retired cn1lib dependency before adding the built-in class. -Keeping both packages can leave duplicate ML Kit versions in an -Android or CocoaPods dependency graph. The builders don't ship -compatibility aliases because those aliases would keep the -old native-interface and packaging contracts alive. +During migration, the builders deduplicate identical CocoaPods names and +preserve an explicitly declared user version constraint. Removing the old +library is still required because compatibility aliases would keep its old +native-interface and packaging contracts alive. ==== Putting it all together diff --git a/docs/developer-guide/Working-With-iOS.asciidoc b/docs/developer-guide/Working-With-iOS.asciidoc index 71889dac989..37da30909f3 100644 --- a/docs/developer-guide/Working-With-iOS.asciidoc +++ b/docs/developer-guide/Working-With-iOS.asciidoc @@ -276,6 +276,14 @@ Supported need formats are `from:`, `exact:`, `branch:`, `revision:`, and `range `ios.dependencyManager=auto` preserves backward compatibility. Existing projects with `ios.pods` continue to use CocoaPods. Projects with `ios.spm.*` use SPM. If both hint families are present, both are applied. +Explicit dependency-manager modes are validated rather than silently dropping +dependencies. `ios.dependencyManager=spm` fails when the project or a built-in +API requires a CocoaPod; use `auto` or `both`. Likewise, +`ios.dependencyManager=none` fails while pod or SPM hints are present; remove +the dependency hints only if suppression is intentional, or switch to +`auto`. This is a migration-visible change for older projects that used +`spm` or `none` to ignore declared native dependencies. + === Including dynamic frameworks If you need to use a dynamic framework (for example: SomeThirdPartySDK.framework), and it isn't available through CocoaPods, then you can add it to your project by zipping up the framework and copying it to your native/ios directory. diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java index ed5b7c67871..51b09c27519 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java @@ -1329,6 +1329,9 @@ public boolean build(File sourceZip, final BuildRequest request) throws BuildExc // features to xPermissions right now. final PlatformFeatureCatalog.Accumulator aiAcc = new PlatformFeatureCatalog.Accumulator(); + // scanClassesForPermissions invokes callbacks serially on this build + // thread. The mutable feature flags and source set are intentionally + // unsynchronized; parallelizing the scanner requires revisiting them. try { scanClassesForPermissions(dummyClassesDir, new Executor.ClassScanner() { @@ -5362,6 +5365,9 @@ static String androidAiAdapterSource(String cls) { if ("com/codename1/ai/language/SmartReply".equals(cls)) { return "AndroidSmartReplyAdapter.java"; } + // DocumentScanner is Apple-only. Returning null intentionally prunes + // the Android vision backend for an app that references only it; the + // public API then reports UNSUPPORTED without a native dependency. return null; } diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java index cd1b91df85c..693a391385b 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java @@ -1398,7 +1398,10 @@ public void extractZip(InputStream source, File destDir, ExtractionFilter filter int count; byte[] data = new byte[8192]; - // write the files to the disk + // Validate the untrusted archive name before the trusted + // filter sees it. The filter deliberately owns final routing + // and may select a sibling (for example, project settings), + // so its canonical result is not constrained to destDir. resolveArchiveEntry(destDir, entryName); File destFile = filter.destFile(currentDir, entryName); if (destFile == null) { diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java index 0386ec261c7..10a7c15606c 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java @@ -357,6 +357,37 @@ private static String appendFrameworks(String libraries, return out; } + static String appendPodSpecIfAbsent(String pods, String candidate) { + String out = pods == null ? "" : pods; + String candidateName = podName(candidate); + for (String existing : out.split("[,;]")) { + if (candidateName.equalsIgnoreCase(podName(existing))) { + return out; + } + } + return out.length() == 0 ? candidate : out + "," + candidate; + } + + static String deduplicatePodSpecs(String pods) { + String out = ""; + if (pods == null) { + return out; + } + for (String podSpec : pods.split("[,;]")) { + podSpec = podSpec.trim(); + if (podSpec.length() > 0) { + out = appendPodSpecIfAbsent(out, podSpec); + } + } + return out; + } + + private static String podName(String podSpec) { + String value = podSpec == null ? "" : podSpec.trim(); + int separator = value.indexOf(' '); + return separator < 0 ? value : value.substring(0, separator).trim(); + } + private void applyCatalogPlistEntry(BuildRequest request, String[] plistEntry) { String privacyKey = plistEntry[0]; @@ -384,6 +415,11 @@ private int getDeploymentTargetInt(BuildRequest request) { @Override public boolean build(File sourceZip, BuildRequest request) throws BuildException { + // Builder instances are normally single-use, but keep scan-derived + // native feature state deterministic if an instance is reused. + usesCn1Vision = false; + usesCn1Language = false; + usesCn1Inference = false; Stopwatch stopwatch = new Stopwatch(); addMinDeploymentTarget(DEFAULT_MIN_DEPLOYMENT_VERSION); if (request.getArg("ios.deployment_target", null) == null) { @@ -1059,7 +1095,8 @@ public void usesClassMethod(String cls, String method) { // iosPods string and SPM entries through the request build // hints, so the IOSDependencyManager.resolve() call below can // pick them up consistently with manually-declared deps. - if (!aiAcc.hits().isEmpty()) { + Set platformFeatureHits = aiAcc.hits(); + if (!platformFeatureHits.isEmpty()) { // Prefer SPM when the project already uses SPM and the // entry exposes an SPM spec; otherwise pods. A handful // of ML Kit libs are pods-only -- those force pods on @@ -1067,7 +1104,7 @@ public void usesClassMethod(String cls, String method) { // upgrade the effective mode to BOTH below). boolean projectPrefersSpm = dependencyConfig.usesSwiftPackages() && !dependencyConfig.usesCocoaPods(); StringBuilder spmPackages = new StringBuilder(request.getArg("ios.spm.packages", "")); - for (PlatformFeatureCatalog.Entry entry : aiAcc.hits()) { + for (PlatformFeatureCatalog.Entry entry : platformFeatureHits) { // Third-party AI packages may omit Catalyst or arm64 // simulator slices. Keep framework-only Apple Vision enabled // for macNative and record the simulator constraint for the @@ -1085,6 +1122,10 @@ public void usesClassMethod(String cls, String method) { && (!entry.iosPods().isEmpty() || !entry.iosSpmSpecs().isEmpty())) { excludeArm64Simulator = true; + log("Catalog-selected iOS dependency \"" + + entry.description() + + "\" has no arm64 simulator slice. The generated " + + "project will use the x86_64 simulator architecture."); } boolean handledViaSpm = false; if (includeApplePackageDependencies @@ -1111,14 +1152,16 @@ public void usesClassMethod(String cls, String method) { } if (includeApplePackageDependencies && !handledViaSpm) { for (String pod : entry.iosPods()) { - if (iosPods.length() > 0) iosPods += ","; - iosPods += pod; + // User-declared specs are already first in iosPods, so + // they win when the catalog requests the same pod. + iosPods = appendPodSpecIfAbsent(iosPods, pod); } } for (String[] plistEntry : entry.iosPlistEntries()) { applyCatalogPlistEntry(request, plistEntry); } } + iosPods = deduplicatePodSpecs(iosPods); if (spmPackages.length() > 0) { request.putArgument("ios.spm.packages", spmPackages.toString()); } @@ -2534,7 +2577,8 @@ public void usesClassMethod(String cls, String method) { // Augmented reality: uncomment INCLUDE_CN1_AR so the CN1AR // natives (ARKit + ARSCNView) compile in, and link ARKit / // SceneKit explicitly -- neither is default-linked and the - // PlatformFeatureCatalog iosFrameworks field is documentation-only. + // Catalog frameworks are linked below. This explicit AR block also + // controls the native compile define and remains for compatibility. // Apps that never reference com.codename1.ar leave the define // commented out so no ARKit symbol is referenced, which keeps // Apple's API-usage scan quiet and tvOS/watchOS slices clean. @@ -3325,6 +3369,20 @@ public void usesClassMethod(String cls, String method) { } + String arcPhaseFixScript = + usesCn1Vision || usesCn1Language || usesCn1Inference + ? " arc_sources = ['CN1Vision.m', 'CN1Language.m', 'CN1Inference.m']\n" + + " main_target.source_build_phase.files.each do |bf|\n" + + " ref = bf.file_ref\n" + + " name = ref && File.basename(ref.path || ref.name || '')\n" + + " next unless arc_sources.include?(name)\n" + + " settings = bf.settings || {}\n" + + " flags = settings['COMPILER_FLAGS'].to_s.split\n" + + " flags << '-fobjc-arc' unless flags.include?('-fobjc-arc')\n" + + " settings['COMPILER_FLAGS'] = flags.join(' ')\n" + + " bf.settings = settings\n" + + " end\n" + : ""; String createSchemesScript = "#!/usr/bin/env ruby\n" + "require 'xcodeproj'\n" + "require 'pathname'\n" + @@ -3435,17 +3493,7 @@ public void usesClassMethod(String cls, String method) { + " end\n" + " raise \"Swift files/resources still present in Copy Bundle Resources: #{names.join(', ')}\"\n" + " end\n" - + " arc_sources = ['CN1Vision.m', 'CN1Language.m', 'CN1Inference.m']\n" - + " main_target.source_build_phase.files.each do |bf|\n" - + " ref = bf.file_ref\n" - + " name = ref && File.basename(ref.path || ref.name || '')\n" - + " next unless arc_sources.include?(name)\n" - + " settings = bf.settings || {}\n" - + " flags = settings['COMPILER_FLAGS'].to_s.split\n" - + " flags << '-fobjc-arc' unless flags.include?('-fobjc-arc')\n" - + " settings['COMPILER_FLAGS'] = flags.join(' ')\n" - + " bf.settings = settings\n" - + " end\n" + + arcPhaseFixScript + " end\n" + "rescue => e\n" + " puts \"Error while correcting Swift build phases: #{$!}\"\n" @@ -3481,7 +3529,7 @@ public void usesClassMethod(String cls, String method) { return false; } String podFileContents = "target '" + request.getMainClass() + "' do\n"; - String[] pods = iosPods.split("[,;]"); + String[] pods = deduplicatePodSpecs(iosPods).split("[,;]"); for (String podLib : pods) { podLib = podLib.trim(); if (podLib.isEmpty()) { diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java index e015378eb55..92f47cdd870 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java @@ -67,6 +67,9 @@ void selectsOneSourcePerLanguageFeature() { @Test void ignoresSharedApiAndUnrelatedClasses() { + assertNull(AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/DocumentScanner"), + "DocumentScanner is Apple-only and must not retain an Android backend"); assertNull(AndroidGradleBuilder.androidAiAdapterSource( "com/codename1/ai/vision/VisionPipeline")); assertNull(AndroidGradleBuilder.androidAiAdapterSource( diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java index 08f697a1944..41f5550d158 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java @@ -198,6 +198,18 @@ void appendsEachRequiredAiFrameworkIndependently() throws Exception { assertEquals("ThirdPartyVision.framework;Vision.framework", value); } + @Test + void catalogPodsDoNotOverrideUserVersionSpecs() { + String pods = IPhoneBuilder.appendPodSpecIfAbsent( + "GoogleMLKit/TextRecognition ~> 7.0,OtherPod", + "GoogleMLKit/TextRecognition"); + assertEquals("GoogleMLKit/TextRecognition ~> 7.0,OtherPod", pods); + + assertEquals("GoogleMLKit/TextRecognition ~> 7.0,OtherPod", + IPhoneBuilder.deduplicatePodSpecs( + pods + ",GoogleMLKit/TextRecognition;OtherPod")); + } + @Test void dependencyFloorRaisesExplicitDeploymentTarget() throws Exception { IPhoneBuilder builder = new IPhoneBuilder(); diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index d2c62764463..9f512ab83c8 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -51,6 +51,15 @@ void tensorsAreImmutableAndValidateShape() { new int[] {Integer.MAX_VALUE, 2}, new byte[] {1})); } + @Test + void modelSourceOffersExplicitReadOnlyZeroCopyAccess() { + byte[] source = new byte[] {1, 2, 3}; + ModelSource model = ModelSource.bytes(source); + assertNotSame(source, model.getBytesUnsafe()); + assertSame(model.getBytesUnsafe(), model.getBytesUnsafe()); + assertNotSame(model.getBytesUnsafe(), model.getBytes()); + } + @Test void modelCacheCompletionIsSingleShot() { AsyncResource resource = new AsyncResource(); @@ -278,7 +287,11 @@ void sessionLifecycleForwardsToBackend() { implementation.setInferenceImpl(backend); InferenceSession session = await(InferenceSession.open( ModelSource.bytes(new byte[] {1}), new InferenceOptions().threads(2))); - assertEquals(2, session.getInputs()[0].getShape()[1]); + TensorInfo[] inputs = session.getInputs(); + assertEquals(2, inputs[0].getShape()[1]); + assertNotSame(inputs, session.getInputs()); + inputs[0] = null; + assertNotNull(session.getInputs()[0]); Tensor[] output = await(session.run(new Tensor[] { Tensor.floats("input", new int[] {1, 2}, new float[] {1, 2}) })); diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java index a1315ada44f..c04e1e364a5 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java @@ -118,11 +118,47 @@ void languageOperationsSnapshotMutableArguments() { assertEquals(original, backend.conversation[0]); } + @Test + void reusableSessionRetainsBackendUntilClosed() { + RecordingLanguageImpl backend = new RecordingLanguageImpl(); + implementation.setLanguageImpl(backend); + + Translator.Session session = Translator.open( + new LanguageOptions().backend( + LanguageBackends.mlKitTranslation())); + assertEquals("hello", + session.translate("bonjour", "fr", "en").get()); + assertEquals("hello", + session.translate("salut", "fr", "en").get()); + assertEquals(2, backend.translationCalls); + assertEquals(0, backend.closeCount); + + session.close(); + session.close(); + assertEquals(1, backend.closeCount); + assertThrows(IllegalStateException.class, + () -> session.translate("bonjour", "fr", "en")); + } + + @Test + void staticConvenienceOperationClosesItsEphemeralBackend() { + RecordingLanguageImpl backend = new RecordingLanguageImpl(); + implementation.setLanguageImpl(backend); + + assertEquals("hello", + Translator.translate("bonjour", "fr", "en", + new LanguageOptions().backend( + LanguageBackends.mlKitTranslation())).get()); + assertEquals(1, backend.closeCount); + } + private static final class RecordingLanguageImpl extends LanguageImpl { String feature; String backend; LanguageOptions options; SmartReplyMessage[] conversation; + int translationCalls; + int closeCount; public boolean isSupported(String value, String backendId) { feature = value; @@ -132,6 +168,7 @@ public boolean isSupported(String value, String backendId) { public AsyncResource identify( String text, String backendId, LanguageOptions value) { + feature = "language-id"; backend = backendId; options = value; AsyncResource result = @@ -146,6 +183,7 @@ public AsyncResource translate( String text, String sourceLanguage, String targetLanguage, String backendId, LanguageOptions value) { feature = "translation"; + translationCalls++; AsyncResource result = new AsyncResource(); result.complete("hello"); return result; @@ -162,6 +200,7 @@ public AsyncResource suggestReplies( } public void close() { + closeCount++; } } } diff --git a/maven/core-unittests/src/test/java/com/codename1/camera/VisionPipelineTest.java b/maven/core-unittests/src/test/java/com/codename1/camera/VisionPipelineTest.java index 945416359f4..acccc3fa26a 100644 --- a/maven/core-unittests/src/test/java/com/codename1/camera/VisionPipelineTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/camera/VisionPipelineTest.java @@ -124,6 +124,69 @@ public void error(Throwable error) { assertFalse(pipeline.isBusy()); } + @Test + void busyPipelineKeepsOnlyNewestPendingFrame() { + final RecordingAnalyzer analyzer = new RecordingAnalyzer(); + pipeline = new VisionPipeline(session, analyzer, + new VisionPipelineListener() { + public void result(String value, VisionImage image) { + } + + public void error(Throwable error) { + } + }); + + implementationBackend.lastFrameListener.onFrame(frame(1)); + implementationBackend.lastFrameListener.onFrame(frame(2)); + implementationBackend.lastFrameListener.onFrame(frame(3)); + analyzer.operations.get(0).complete("first"); + flushSerialCalls(); + + assertEquals(2, analyzer.operations.size()); + assertEquals(Integer.valueOf(3), analyzer.inputValues.get(1)); + } + + @Test + void synchronousAnalyzerFailureDoesNotStallPipeline() { + final int[] errors = new int[1]; + VisionAnalyzer analyzer = new VisionAnalyzer() { + int calls; + + public boolean isSupported() { + return true; + } + + public AsyncResource process(VisionImage image) { + calls++; + if (calls == 1) { + throw new IllegalStateException("synchronous failure"); + } + AsyncResource result = + new AsyncResource(); + result.complete("recovered"); + return result; + } + + public void close() { + } + }; + pipeline = new VisionPipeline(session, analyzer, + new VisionPipelineListener() { + public void result(String value, VisionImage image) { + } + + public void error(Throwable error) { + errors[0]++; + } + }); + + implementationBackend.lastFrameListener.onFrame(frame(1)); + implementationBackend.lastFrameListener.onFrame(frame(2)); + flushSerialCalls(); + assertEquals(1, errors[0]); + assertFalse(pipeline.isBusy()); + } + private static CameraFrame frame(int value) { return new CameraFrame(new byte[] {(byte) value}, null, 1, 1, 0, value, FrameFormat.JPEG); @@ -133,6 +196,7 @@ private static final class RecordingAnalyzer implements VisionAnalyzer { final List> operations = new ArrayList>(); + final List inputValues = new ArrayList(); int closeCount; public boolean isSupported() { @@ -140,6 +204,7 @@ public boolean isSupported() { } public AsyncResource process(VisionImage image) { + inputValues.add((int) image.getEncodedBytesUnsafe()[0]); AsyncResource operation = new AsyncResource(); operations.add(operation); return operation; diff --git a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java index 2fa3ef49878..5d60009331d 100644 --- a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java +++ b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java @@ -48,7 +48,17 @@ */ public final class PlatformFeatureCatalog { + private static final String MLKIT_LANGUAGE_ID = + "com.google.mlkit:language-id:17.0.6"; + private static final String MLKIT_TRANSLATE = + "com.google.mlkit:translate:17.0.3"; + private static final String MLKIT_SMART_REPLY = + "com.google.mlkit:smart-reply:17.0.4"; + private static final String MLKIT_SELFIE_SEGMENTATION = + "com.google.mlkit:segmentation-selfie:16.0.0-beta5"; private static final List ENTRIES; + private static final List CLASS_PREFIXES; + private static final Set METHOD_KEYS; static { List e = new ArrayList(); @@ -76,7 +86,7 @@ public final class PlatformFeatureCatalog { .description("On-device speech-to-text")); e.add(new Entry("com/codename1/media/TextToSpeech") - .iosFrameworks("AVFAudio") + .iosFrameworks("AVFoundation") .description("Text-to-speech")); // Compatibility mappings for the retired AI cn1libs. The artifacts @@ -127,7 +137,7 @@ public final class PlatformFeatureCatalog { .iosMinimumDeploymentTarget("15.5") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() - .androidGradle("com.google.mlkit:translate:17.0.1") + .androidGradle(MLKIT_TRANSLATE) .androidMinimumSdk(21) .description("Legacy ML Kit Translation cn1lib")); e.add(new Entry("com/codename1/ai/mlkit/smartreply/") @@ -135,7 +145,7 @@ public final class PlatformFeatureCatalog { .iosMinimumDeploymentTarget("15.5") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() - .androidGradle("com.google.mlkit:smart-reply:17.0.2") + .androidGradle(MLKIT_SMART_REPLY) .androidMinimumSdk(21) .description("Legacy ML Kit Smart Reply cn1lib")); e.add(new Entry("com/codename1/ai/mlkit/langid/") @@ -143,7 +153,7 @@ public final class PlatformFeatureCatalog { .iosMinimumDeploymentTarget("15.5") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() - .androidGradle("com.google.mlkit:language-id:17.0.4") + .androidGradle(MLKIT_LANGUAGE_ID) .androidMinimumSdk(21) .description("Legacy ML Kit Language ID cn1lib")); e.add(new Entry("com/codename1/ai/mlkit/pose/") @@ -159,8 +169,7 @@ public final class PlatformFeatureCatalog { .iosMinimumDeploymentTarget("15.5") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() - .androidGradle( - "com.google.mlkit:segmentation-selfie:16.0.0-beta4") + .androidGradle(MLKIT_SELFIE_SEGMENTATION) .androidMinimumSdk(21) .description("Legacy ML Kit Selfie Segmentation cn1lib")); e.add(new Entry("com/codename1/ai/mlkit/docscan/") @@ -260,8 +269,7 @@ public final class PlatformFeatureCatalog { e.add(new Entry("com/codename1/ai/vision/SelfieSegmenter") .iosFrameworks("Vision", "CoreML") - .androidGradle( - "com.google.mlkit:segmentation-selfie:16.0.0-beta5") + .androidGradle(MLKIT_SELFIE_SEGMENTATION) .androidMinimumSdk(21) .description("Selfie segmentation")); e.add(new Entry("com/codename1/ai/vision/SelfieSegmenter") @@ -290,7 +298,7 @@ public final class PlatformFeatureCatalog { e.add(new Entry("com/codename1/ai/language/LanguageIdentifier") .iosFrameworks("NaturalLanguage") - .androidGradle("com.google.mlkit:language-id:17.0.6") + .androidGradle(MLKIT_LANGUAGE_ID) .androidMinimumSdk(21) .description("On-device language identification")); e.add(new Entry("com/codename1/ai/language/LanguageIdentifier") @@ -305,23 +313,31 @@ public final class PlatformFeatureCatalog { // the ML Kit adapters, so every target that compiles this source must // link the small system NaturalLanguage framework. e.add(new Entry("com/codename1/ai/language/Translator") + .iosFrameworks("NaturalLanguage") + .androidGradle(MLKIT_TRANSLATE) + .androidMinimumSdk(21) + .description("On-device translation")); + e.add(new Entry("com/codename1/ai/language/Translator") + .requiresMethod("com/codename1/ai/language/LanguageBackends", + "mlKitTranslation") .iosPod("GoogleMLKit/Translate") .iosMinimumDeploymentTarget("15.5") - .iosFrameworks("NaturalLanguage") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() - .androidGradle("com.google.mlkit:translate:17.0.3") + .description("ML Kit iOS translation backend")); + e.add(new Entry("com/codename1/ai/language/SmartReply") + .iosFrameworks("NaturalLanguage") + .androidGradle(MLKIT_SMART_REPLY) .androidMinimumSdk(21) - .description("On-device translation")); + .description("On-device smart reply")); e.add(new Entry("com/codename1/ai/language/SmartReply") + .requiresMethod("com/codename1/ai/language/LanguageBackends", + "mlKitSmartReply") .iosPod("GoogleMLKit/SmartReply") .iosMinimumDeploymentTarget("15.5") - .iosFrameworks("NaturalLanguage") .iosDependenciesUnsupportedOnMacCatalyst() .iosDependenciesUnsupportedOnArm64Simulator() - .androidGradle("com.google.mlkit:smart-reply:17.0.4") - .androidMinimumSdk(21) - .description("On-device smart reply")); + .description("ML Kit iOS Smart Reply backend")); e.add(new Entry("com/codename1/ai/whisper/") .iosFrameworks("Accelerate") @@ -350,6 +366,7 @@ public final class PlatformFeatureCatalog { .androidGradle("androidx.camera:camera-lifecycle:1.3.4") .androidGradle("androidx.camera:camera-view:1.3.4") .androidGradle("androidx.camera:camera-video:1.3.4") + .androidMinimumSdk(21) .description("Cross-platform camera (preview + frames + photo + video)")); // First-class Bluetooth (com.codename1.bluetooth.*): CoreBluetooth @@ -398,12 +415,27 @@ public final class PlatformFeatureCatalog { .description("Cross-platform augmented reality (world/image/face tracking)")); ENTRIES = Collections.unmodifiableList(e); + Set classPrefixes = new LinkedHashSet(); + Set methodKeys = new LinkedHashSet(); + for (Entry entry : e) { + classPrefixes.add(entry.classPrefix); + if (entry.hasMethodRequirement()) { + methodKeys.add(entry.methodOwner + "#" + entry.methodName); + } + } + CLASS_PREFIXES = Collections.unmodifiableList( + new ArrayList(classPrefixes)); + METHOD_KEYS = Collections.unmodifiableSet(methodKeys); } private PlatformFeatureCatalog() { } - /** All registered entries. Mostly useful for tests and tooling. */ + /** + * Returns every registered platform feature in declaration order. + * + * @return immutable catalog entry list used by builders and tooling + */ public static List entries() { return ENTRIES; } @@ -413,6 +445,9 @@ public static List entries() { * given internal-form class name (slashes, not dots). When the * prefix ends with a slash, package-prefix matching is used; * otherwise an exact class match is required. + * + * @param internalClassName JVM internal-form class name + * @return immutable matching entries that have no method requirement */ public static List matchesFor(String internalClassName) { if (internalClassName == null) { @@ -434,29 +469,68 @@ public static List matchesFor(String internalClassName) { public static final class Accumulator { private final Set classes = new LinkedHashSet(); private final Set methods = new LinkedHashSet(); + private Set cachedHits = Collections.emptySet(); + private boolean dirty = true; + /** + * Records a class only when it can match at least one catalog entry. + * The scanner calls this for the full application and framework + * graph, so filtering here keeps memory proportional to catalog usage + * rather than application size. + * + * @param internalClassName JVM internal-form class name + */ public void consume(String internalClassName) { - if (internalClassName != null) { - classes.add(internalClassName); + if (isCatalogClass(internalClassName) + && classes.add(internalClassName)) { + dirty = true; } } + /** + * Records a method reference only when an entry requires that exact + * owner and method name. + * + * @param internalClassName JVM internal-form owner name + * @param methodName referenced method name + */ public void consumeMethod(String internalClassName, String methodName) { if (internalClassName != null && methodName != null) { - methods.add(internalClassName + "#" + methodName); + String key = internalClassName + "#" + methodName; + if (METHOD_KEYS.contains(key) && methods.add(key)) { + dirty = true; + } } } + /** + * Returns the immutable matched-entry set. The result is recomputed + * only after a relevant class or method is newly observed. + * + * @return matched catalog entries in declaration order + */ public Set hits() { + if (!dirty) { + return cachedHits; + } Set hits = new LinkedHashSet(); for (Entry entry : ENTRIES) { if (entry.requirementsMet(classes, methods)) { hits.add(entry); } } - return hits; + cachedHits = Collections.unmodifiableSet(hits); + dirty = false; + return cachedHits; } + /** + * Tests whether any observed feature requires the builder's larger + * upload allowance. + * + * @return {@code true} when at least one matched entry is marked as a + * large upload + */ public boolean anyRequiresBigUpload() { for (Entry e : hits()) { if (e.requiresBigUpload) { @@ -507,6 +581,22 @@ public Set iosFrameworks() { } } + private static boolean isCatalogClass(String internalClassName) { + if (internalClassName == null) { + return false; + } + for (String prefix : CLASS_PREFIXES) { + if (prefix.endsWith("/")) { + if (internalClassName.startsWith(prefix)) { + return true; + } + } else if (internalClassName.equals(prefix)) { + return true; + } + } + return false; + } + /** * A single registry record. Mutable while the table is being * built (the fluent setters); semantically immutable once exposed @@ -651,14 +741,17 @@ Entry description(String d) { return this; } + /** @return exact class or package prefix that activates this entry */ public String classPrefix() { return classPrefix; } + /** @return immutable CocoaPods dependency specifications */ public List iosPods() { return Collections.unmodifiableList(iosPods); } + /** @return immutable Swift Package Manager dependency descriptors */ public List iosSpmSpecs() { return Collections.unmodifiableList(iosSpm); } @@ -666,6 +759,8 @@ public List iosSpmSpecs() { /** * Whether this entry's CocoaPod/SPM payload has a Mac Catalyst slice. * System frameworks are not affected by this flag. + * + * @return {@code true} when catalog dependencies support Catalyst */ public boolean iosDependenciesSupportMacCatalyst() { return iosDependenciesSupportMacCatalyst; @@ -675,6 +770,9 @@ public boolean iosDependenciesSupportMacCatalyst() { * Whether this entry's CocoaPod/SPM payload has an arm64 iOS * Simulator slice. Dependencies that return {@code false} still * support the x86_64 simulator. + * + * @return {@code true} when catalog dependencies support arm64 + * simulator builds */ public boolean iosDependenciesSupportArm64Simulator() { return iosDependenciesSupportArm64Simulator; @@ -697,17 +795,22 @@ public String iosMinimumDeploymentTarget() { return iosMinimumDeploymentTarget; } + /** @return immutable Apple system-framework names without suffixes */ public List iosFrameworks() { return Collections.unmodifiableList(iosFrameworks); } /** Each entry is {key, defaultValue}. The builder injects the * value only if the app hasn't already declared one for the - * same key in its build hints. */ + * same key in its build hints. + * + * @return immutable list of two-element property-list entries + */ public List iosPlistEntries() { return Collections.unmodifiableList(iosPlist); } + /** @return immutable Android Gradle dependency coordinates */ public List androidGradleDeps() { return Collections.unmodifiableList(androidGradle); } @@ -728,35 +831,49 @@ public int androidMinimumSdk() { return androidMinimumSdk; } + /** @return immutable Android manifest permission names */ public List androidPermissions() { return Collections.unmodifiableList(androidPermissions); } + /** @return immutable Android manifest feature names */ public List androidFeatures() { return Collections.unmodifiableList(androidFeatures); } /** Each entry is {name, value}: an application-level manifest * <meta-data> element the Android builder injects unless the - * app already declares the same name. */ + * app already declares the same name. + * + * @return immutable list of two-element manifest metadata entries + */ public List androidMetaDataEntries() { return Collections.unmodifiableList(androidMetaData); } + /** @return whether this entry requires the larger upload allowance */ public boolean requiresBigUpload() { return requiresBigUpload; } + /** @return user-readable feature or dependency description */ public String description() { return description; } } - /** Swift Package Manager dependency descriptor. */ + /** + * Immutable Swift Package Manager dependency descriptor consumed by the + * iOS builder. + */ public static final class IosSpm { + /** Stable package identity used for de-duplication. */ public final String identity; + /** Package repository URL. */ public final String url; + /** Builder-formatted version or branch requirement. */ public final String requirement; + /** Immutable package product names linked into the app. */ public final List products; IosSpm(String identity, String url, String requirement, List products) { diff --git a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java index b7149db0fa2..370c9584b44 100644 --- a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java +++ b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java @@ -24,11 +24,15 @@ import org.junit.jupiter.api.Test; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; class PlatformFeatureCatalogTest { @@ -105,6 +109,21 @@ void currentIosMlKitPodsRequireIos155() { entry.description()); } } + assertEquals(1, podEntries, + "Language class references alone must preserve the arm64 simulator"); + + acc.consumeMethod("com/codename1/ai/language/LanguageBackends", + "mlKitTranslation"); + acc.consumeMethod("com/codename1/ai/language/LanguageBackends", + "mlKitSmartReply"); + podEntries = 0; + for (PlatformFeatureCatalog.Entry entry : acc.hits()) { + if (!entry.iosPods().isEmpty()) { + podEntries++; + assertEquals("15.5", entry.iosMinimumDeploymentTarget(), + entry.description()); + } + } assertEquals(3, podEntries); } @@ -126,7 +145,7 @@ void textToSpeechInjectsNoPermissions() { "com/codename1/media/TextToSpeech"); assertEquals(1, hits.size()); PlatformFeatureCatalog.Entry e = hits.get(0); - assertTrue(e.iosFrameworks().contains("AVFAudio")); + assertTrue(e.iosFrameworks().contains("AVFoundation")); assertTrue(e.androidPermissions().isEmpty(), "TTS is built-in on every supported OS -- no permission needed"); assertTrue(e.iosPlistEntries().isEmpty(), @@ -277,7 +296,8 @@ void everyMlKitAndLiteRtDependencyDeclaresAndroidFloor() { } } assertEquals(22, checked, - "Expected built-in and compatibility AI dependencies"); + "If an AI dependency is intentionally added or removed, " + + "update this lock count after verifying its Android floor"); } @Test @@ -350,16 +370,42 @@ void documentCorrectionDoesNotInjectUnusedAndroidScanner() { } @Test - void accumulatorDeduplicates() { - // Same class twice in the same scan shouldn't add the entry - // twice -- otherwise we'd inject duplicate Gradle / pod - // lines on the wire. + void accumulatorFiltersUnrelatedSymbolsAndMemoizesHits() { PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); + for (int i = 0; i < 10000; i++) { + acc.consume("example/unrelated/Class" + i); + acc.consumeMethod("example/unrelated/Class" + i, "method" + i); + } + Set empty = acc.hits(); + assertTrue(empty.isEmpty()); + assertSame(empty, acc.hits(), + "Unchanged scans should reuse the memoized result"); + acc.consume("com/codename1/ai/vision/TextRecognizer"); acc.consume("com/codename1/ai/vision/TextRecognizer"); assertEquals(1, acc.hits().size()); } + @Test + void catalogUsesOneVersionPerAndroidArtifact() { + Map versions = new HashMap(); + for (PlatformFeatureCatalog.Entry entry + : PlatformFeatureCatalog.entries()) { + for (String dependency : entry.androidGradleDeps()) { + String[] parts = dependency.split(":"); + if (parts.length < 3) { + continue; + } + String artifact = parts[0] + ":" + parts[1]; + String previous = versions.put(artifact, parts[2]); + if (previous != null) { + assertEquals(previous, parts[2], + "Version drift for " + artifact); + } + } + } + } + @Test void arApiInjectsArKitCameraAndArCore() { List hits = PlatformFeatureCatalog.matchesFor( diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java index b42582294e5..b9892e80304 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java @@ -55,6 +55,7 @@ public boolean runTest() { try { checkValuesAndOptions(); checkCapabilities(); + checkReusableSessionLifecycle(); done(); return true; } catch (Throwable t) { @@ -117,6 +118,31 @@ private void checkCapabilities() { } } + private void checkReusableSessionLifecycle() { + LanguageIdentifier.Session identifier = + LanguageIdentifier.open(new LanguageOptions()); + Translator.Session translator = + Translator.open(new LanguageOptions()); + SmartReply.Session smartReply = + SmartReply.open(new LanguageOptions()); + identifier.close(); + translator.close(); + smartReply.close(); + // Closing is intentionally idempotent so owners can use one cleanup path. + identifier.close(); + translator.close(); + smartReply.close(); + + assertImmediateFailure(identifier.identify("hello"), + "closed language-identification session"); + assertImmediateFailure(translator.translate( + "bonjour", "fr", "en"), "closed translation session"); + assertImmediateFailure(smartReply.suggest( + new SmartReplyMessage[] { + new SmartReplyMessage("Hello", "remote", false, 1) + }), "closed smart-reply session"); + } + private void assertImmediateFailure(AsyncResource resource, String label) { check(resource.isDone(), label diff --git a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md index 660792173b9..66c58eab62b 100644 --- a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md +++ b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md @@ -319,7 +319,9 @@ InferenceSession.open( new InferenceOptions()) .ready(session -> session.run(new Tensor[] {input})); -Translator.translate("Bonjour", "fr", "en", new LanguageOptions()) +LanguageOptions translationOptions = new LanguageOptions() + .backend(LanguageBackends.mlKitTranslation()); +Translator.translate("Bonjour", "fr", "en", translationOptions) .ready(result -> Log.p(result)); ``` @@ -353,8 +355,12 @@ enabled. Android and iOS reject `Accelerator.NPU` together with the Core ML delegate may also schedule work on CPU or GPU. Language identification defaults to Apple Natural Language on iOS and ML -Kit on Android. Translation and Smart Reply use feature-scoped ML Kit -components. Call `isSupported()` before exposing a feature. Vision has native backends +Kit on Android. iOS translation and Smart Reply require their +`mlKitTranslation()` and `mlKitSmartReply()` selectors; referencing the API +class alone does not disable the arm64 simulator. The static methods own a +one-shot backend. For repeated operations, reuse and close the feature session +returned by `LanguageIdentifier.open()`, `Translator.open()`, or +`SmartReply.open()`. Call `isSupported()` before exposing a feature. Vision has native backends on Android, iOS, and Mac native. Language services and LiteRT inference have native backends on Android and iOS. JavaSE, JavaScript, native Windows/Linux, watchOS, and tvOS return unsupported; Mac native returns From c827d9560534fcd6d52d5b9f1d7aec7824697874 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 29 Jul 2026 06:35:35 +0300 Subject: [PATCH 67/80] Use markdown docs for language session --- .../src/com/codename1/ai/language/LanguageSession.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageSession.java b/CodenameOne/src/com/codename1/ai/language/LanguageSession.java index 097e9c0c85a..c1126820f53 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageSession.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageSession.java @@ -27,11 +27,9 @@ import com.codename1.util.AsyncResource; import com.codename1.util.SuccessCallback; -/** - * Shared lifecycle implementation for the public feature-specific language - * sessions. This package-private type keeps native backends alive across - * repeated operations and defers release while an operation is pending. - */ +/// Shared lifecycle implementation for the public feature-specific language +/// sessions. This package-private type keeps native backends alive across +/// repeated operations and defers release while an operation is pending. final class LanguageSession implements AutoCloseable { interface Operation { AsyncResource run(LanguageImpl implementation, From b9bc8aab8d3861e904d04ef0bfc237a8bde461cc Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 29 Jul 2026 06:42:51 +0300 Subject: [PATCH 68/80] Validate specially routed archive entries --- .../java/com/codename1/builders/Executor.java | 16 ++++++--- .../ExecutorArchiveExtractionTest.java | 35 +++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java index 693a391385b..0eb485669f1 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java @@ -1245,15 +1245,21 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD while ((entry = zis.getNextEntry()) != null) { String entryName = entry.getName(); + // Validate the original ZIP name before any archive-specific + // branch can route it into a generated tar and continue. + resolveArchiveEntry(resDir, entryName); + if(!"html.tar".equals(entryName)&& (entryName.startsWith("html") || entryName.startsWith("/html"))) { if(entry.isDirectory()) { continue; } - + entryName = entryName.substring(5); + // The stripped name becomes a member of another archive, + // so validate it independently against that archive's root. + resolveArchiveEntry(resDir, entryName); if(tos == null) { tos = new TarOutputStream(new FileOutputStream(new File(resDir, "html.tar"))); } - entryName = entryName.substring(5); TarEntry tEntry = new TarEntry(new File(entryName), entryName); tEntry.setSize(entry.getSize()); debug("Packaging entry " + entryName + " size: " + entry.getSize()); @@ -1270,10 +1276,11 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD continue; } int podSpecsPrefix = entryName.startsWith("podspecs/") ? "podspecs/".length() : "/podspecs/".length(); + entryName = entryName.substring(podSpecsPrefix); + resolveArchiveEntry(resDir, entryName); if(podspecTos == null) { podspecTos = new TarOutputStream(new FileOutputStream(new File(resDir, "podspecs.tar"))); } - entryName = entryName.substring(podSpecsPrefix); TarEntry tEntry = new TarEntry(new File(entryName), entryName); tEntry.setSize(entry.getSize()); debug("Packaging entry " + entryName + " size: " + entry.getSize()); @@ -1292,10 +1299,11 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD continue; } int libPrefix = entryName.startsWith("javase.lib/") ? "javase.lib/".length() : "/javase.lib/".length(); + entryName = entryName.substring(libPrefix); + resolveArchiveEntry(resDir, entryName); if(libTos == null) { libTos = new TarOutputStream(new FileOutputStream(new File(resDir, "javase.lib.tar"))); } - entryName = entryName.substring(libPrefix); TarEntry tEntry = new TarEntry(new File(entryName), entryName); tEntry.setSize(entry.getSize()); debug("Packaging entry " + entryName + " size: " + entry.getSize()); diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java index 8238f23b66c..a920e0ed27a 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java @@ -35,6 +35,7 @@ import org.junit.jupiter.api.io.TempDir; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; class ExecutorArchiveExtractionTest { @@ -97,6 +98,21 @@ void extractionFilterMayRouteSafeEntryToSibling() throws Exception { StandardCharsets.UTF_8)); } + @Test + void rejectsTraversalBeforeArchiveSpecificRouting() throws Exception { + assertSpecialEntryRejected("html/../../payload", "html.tar"); + assertSpecialEntryRejected("javase.lib/../../payload", + "javase.lib.tar"); + } + + @Test + void rejectsTraversalInStrippedTarMemberName() throws Exception { + assertSpecialEntryRejected("html/../payload", "html.tar"); + assertSpecialEntryRejected("podspecs/../payload", "podspecs.tar"); + assertSpecialEntryRejected("javase.lib/../payload", + "javase.lib.tar"); + } + private static byte[] zipEntry(String name, String contents) throws IOException { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); @@ -107,4 +123,23 @@ private static byte[] zipEntry(String name, String contents) zip.close(); return bytes.toByteArray(); } + + private void assertSpecialEntryRejected(String entryName, + String generatedArchive) throws Exception { + File root = temporaryDirectory.resolve("special-" + + Math.abs(entryName.hashCode())).toFile(); + File classes = new File(root, "classes"); + File resources = new File(root, "resources"); + File sources = new File(root, "sources"); + classes.mkdirs(); + resources.mkdirs(); + sources.mkdirs(); + + new IPhoneBuilder().unzip( + new ByteArrayInputStream(zipEntry(entryName, "payload")), + classes, resources, sources); + + assertFalse(new File(resources, generatedArchive).exists(), + "Unsafe entry must not create " + generatedArchive); + } } From 2ec1b8df68a4d336509cda5ab0c82847aad06a77 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 29 Jul 2026 06:52:51 +0300 Subject: [PATCH 69/80] Fix AI operation lifecycle races --- .../ai/language/LanguageIdentifier.java | 29 ++++----- .../ai/language/LanguageSession.java | 50 ++++++++++++++- .../com/codename1/ai/language/SmartReply.java | 30 ++++----- .../com/codename1/ai/language/Translator.java | 32 ++++------ .../ai/vision/AbstractVisionAnalyzer.java | 6 +- .../codename1/ai/vision/VisionAnalyzer.java | 3 + .../ai/language/LanguageApiTest.java | 53 ++++++++++++++- .../codename1/ai/vision/VisionApiTest.java | 64 +++++++++++++++++++ .../TestCodenameOneImplementation.java | 16 +++++ 9 files changed, 221 insertions(+), 62 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java index 95ee32a53c0..1b035b98051 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java @@ -24,7 +24,6 @@ import com.codename1.impl.LanguageImpl; import com.codename1.util.AsyncResource; -import com.codename1.util.SuccessCallback; /// Identifies possible languages entirely on device. Results are ranked by /// descending backend confidence and filtered by @@ -58,6 +57,8 @@ public static Session open(LanguageOptions options) { /// Identifies possible languages off the EDT without uploading text. The /// option values are copied before the backend starts, so later mutations /// of the supplied object cannot alter the pending identification. + /// Cancelling the returned resource suppresses late result callbacks; the + /// temporary backend is released after its pending native work settles. /// @param text non-null text to classify /// @param options backend and confidence options, or {@code null} /// @return asynchronous ranked candidates; may be empty for undetermined text @@ -72,22 +73,7 @@ public static AsyncResource identify( out.error(error); return out; } - AsyncResource result = session.identify(text); - closeWhenFinished(result, session); - return result; - } - - private static void closeWhenFinished(AsyncResource result, - final Session session) { - result.ready(new SuccessCallback() { - public void onSucess(T value) { - session.close(); - } - }).except(new SuccessCallback() { - public void onSucess(Throwable error) { - session.close(); - } - }); + return session.identify(text, true); } /// Reusable owner of a native language-identification backend. @@ -103,11 +89,18 @@ private Session(LanguageSession session) { } /// Identifies languages without recreating the native backend. + /// Cancelling the returned resource suppresses late callbacks without + /// closing this reusable session. /// /// @param text text to classify; {@code null} is treated as empty /// @return asynchronous ranked candidates, possibly empty /// @throws IllegalStateException if this session is closed public AsyncResource identify(final String text) { + return identify(text, false); + } + + private AsyncResource identify( + final String text, boolean closeWhenFinished) { return session.execute( new LanguageSession.Operation() { public AsyncResource run( @@ -117,7 +110,7 @@ public AsyncResource run( text == null ? "" : text, options.getBackend().getId(), options); } - }); + }, closeWhenFinished); } /// Closes the session. This method is idempotent; native release is diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageSession.java b/CodenameOne/src/com/codename1/ai/language/LanguageSession.java index c1126820f53..5613b0f7a32 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageSession.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageSession.java @@ -82,6 +82,11 @@ static LanguageSession open(String feature, LanguageOptions options) { } AsyncResource execute(Operation operation) { + return execute(operation, false); + } + + AsyncResource execute(Operation operation, + final boolean closeWhenFinished) { final AsyncResource backendResult; synchronized (this) { if (closed) { @@ -103,20 +108,27 @@ AsyncResource execute(Operation operation) { throw error; } } - final AsyncResource result = new AsyncResource(); + final OperationResource result = + new OperationResource(this, closeWhenFinished); final Completion completion = new Completion(); backendResult.ready(new SuccessCallback() { public void onSucess(T value) { if (completion.finish()) { - result.complete(value); + if (closeWhenFinished) { + close(); + } operationFinished(); + result.publish(value); } } }).except(new SuccessCallback() { public void onSucess(Throwable error) { if (completion.finish()) { - result.error(error); + if (closeWhenFinished) { + close(); + } operationFinished(); + result.fail(error); } } }); @@ -155,6 +167,38 @@ public void close() { } } + private static final class OperationResource + extends AsyncResource { + private final LanguageSession owner; + private final boolean closeWhenFinished; + + OperationResource(LanguageSession owner, + boolean closeWhenFinished) { + this.owner = owner; + this.closeWhenFinished = closeWhenFinished; + } + + public synchronized boolean cancel(boolean mayInterruptIfRunning) { + boolean cancelled = super.cancel(mayInterruptIfRunning); + if (cancelled && closeWhenFinished) { + owner.close(); + } + return cancelled; + } + + synchronized void publish(T value) { + if (!isCancelled()) { + complete(value); + } + } + + synchronized void fail(Throwable error) { + if (!isCancelled()) { + error(error); + } + } + } + private static final class Completion { private boolean finished; diff --git a/CodenameOne/src/com/codename1/ai/language/SmartReply.java b/CodenameOne/src/com/codename1/ai/language/SmartReply.java index 17bb949ac60..2a14ea8539f 100644 --- a/CodenameOne/src/com/codename1/ai/language/SmartReply.java +++ b/CodenameOne/src/com/codename1/ai/language/SmartReply.java @@ -24,7 +24,6 @@ import com.codename1.impl.LanguageImpl; import com.codename1.util.AsyncResource; -import com.codename1.util.SuccessCallback; /// Produces short reply suggestions for a chronological conversation without /// uploading its messages. ML Kit may return no suggestions when the language @@ -62,6 +61,8 @@ public static Session open(LanguageOptions options) { /// The conversation array and option values are copied before backend work /// begins. {@link SmartReplyMessage} values are immutable. + /// Cancelling the returned resource suppresses late result callbacks; the + /// temporary backend is released after its pending native work settles. /// /// @param conversation chronological messages, oldest first /// @param options backend options, or {@code null} @@ -76,22 +77,7 @@ public static AsyncResource suggest(SmartReplyMessage[] conversation, out.error(error); return out; } - AsyncResource result = session.suggest(conversation); - closeWhenFinished(result, session); - return result; - } - - private static void closeWhenFinished(AsyncResource result, - final Session session) { - result.ready(new SuccessCallback() { - public void onSucess(T value) { - session.close(); - } - }).except(new SuccessCallback() { - public void onSucess(Throwable error) { - session.close(); - } - }); + return session.suggest(conversation, true); } private static SmartReplyMessage[] copyConversation( @@ -119,6 +105,8 @@ private Session(LanguageSession session) { } /// Generates reply suggestions without recreating the native backend. + /// Cancelling the returned resource suppresses late callbacks without + /// closing this reusable session. /// /// @param conversation chronological messages, oldest first; /// {@code null} is treated as an empty conversation @@ -126,6 +114,12 @@ private Session(LanguageSession session) { /// @throws IllegalStateException if this session is closed public AsyncResource suggest( final SmartReplyMessage[] conversation) { + return suggest(conversation, false); + } + + private AsyncResource suggest( + final SmartReplyMessage[] conversation, + boolean closeWhenFinished) { final SmartReplyMessage[] snapshot = copyConversation(conversation); return session.execute(new LanguageSession.Operation() { @@ -135,7 +129,7 @@ public AsyncResource run( return implementation.suggestReplies(snapshot, options.getBackend().getId(), options); } - }); + }, closeWhenFinished); } /// Closes the session. This method is idempotent; native release is diff --git a/CodenameOne/src/com/codename1/ai/language/Translator.java b/CodenameOne/src/com/codename1/ai/language/Translator.java index f505f2ab73f..3f97041f3c0 100644 --- a/CodenameOne/src/com/codename1/ai/language/Translator.java +++ b/CodenameOne/src/com/codename1/ai/language/Translator.java @@ -24,7 +24,6 @@ import com.codename1.impl.LanguageImpl; import com.codename1.util.AsyncResource; -import com.codename1.util.SuccessCallback; /// Translates text on device with lazily installed language-pair models. /// The first request for a pair may take longer while ML Kit downloads the @@ -65,6 +64,8 @@ public static Session open(LanguageOptions options) { /// region, such as {@code en-US}, and select the corresponding supported /// base-language model ({@code en} in this example). The asynchronous /// resource fails when either tag does not identify a supported model. + /// Cancelling the returned resource suppresses late result callbacks; the + /// temporary backend is released after its pending native work settles. /// /// @param text source text; {@code null} is treated as empty /// @param sourceLanguage BCP-47 source language tag @@ -82,23 +83,7 @@ public static AsyncResource translate(String text, String sourceLanguage out.error(error); return out; } - AsyncResource result = session.translate(text, - sourceLanguage, targetLanguage); - closeWhenFinished(result, session); - return result; - } - - private static void closeWhenFinished(AsyncResource result, - final Session session) { - result.ready(new SuccessCallback() { - public void onSucess(T value) { - session.close(); - } - }).except(new SuccessCallback() { - public void onSucess(Throwable error) { - session.close(); - } - }); + return session.translate(text, sourceLanguage, targetLanguage, true); } /// Reusable owner of a native translation backend. @@ -114,6 +99,8 @@ private Session(LanguageSession session) { } /// Translates text without recreating the native backend. + /// Cancelling the returned resource suppresses late callbacks without + /// closing this reusable session. /// /// @param text source text; {@code null} is treated as empty /// @param sourceLanguage supported BCP-47 source language tag @@ -123,6 +110,13 @@ private Session(LanguageSession session) { public AsyncResource translate(final String text, final String sourceLanguage, final String targetLanguage) { + return translate(text, sourceLanguage, targetLanguage, false); + } + + private AsyncResource translate(final String text, + final String sourceLanguage, + final String targetLanguage, + boolean closeWhenFinished) { return session.execute(new LanguageSession.Operation() { public AsyncResource run( LanguageImpl implementation, @@ -132,7 +126,7 @@ public AsyncResource run( targetLanguage, options.getBackend().getId(), options); } - }); + }, closeWhenFinished); } /// Closes the session. This method is idempotent; native release is diff --git a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java index bb9dc91e342..04e46da8931 100644 --- a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java +++ b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java @@ -40,7 +40,7 @@ abstract class AbstractVisionAnalyzer implements VisionAnalyzer { /// @return whether this analyzer's feature/backend is available and open @Override - public final boolean isSupported() { + public final synchronized boolean isSupported() { VisionImpl impl = implementation(); return impl != null && impl.isSupported(feature, options.getBackend().getId()); } @@ -49,7 +49,7 @@ public final boolean isSupported() { /// @param image immutable encoded or raw input /// @return asynchronous typed result @Override - public final AsyncResource process(VisionImage image) { + public final synchronized AsyncResource process(VisionImage image) { if (image == null) { throw new NullPointerException("image"); } @@ -68,7 +68,7 @@ public final AsyncResource process(VisionImage image) { /// Idempotently releases the retained native backend. @Override - public final void close() { + public final synchronized void close() { closed = true; if (implementation != null) { implementation.close(); diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java b/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java index a5e126e7a4c..84e5de95020 100644 --- a/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java @@ -27,6 +27,9 @@ /// Reusable, closable on-device analyzer for still images or camera frames. /// Implementations may retain native detectors and models between calls, so /// create one analyzer per stream/workflow and close it when finished. +/// Backend creation, request scheduling, capability checks, and close are +/// serialized so a concurrent close cannot leave a newly created backend +/// attached after the analyzer has closed. public interface VisionAnalyzer extends AutoCloseable { /// Tests the exact feature/backend pair configured for this analyzer. /// @return {@code true} when the current target supports it diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java index c04e1e364a5..113091c3b98 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java @@ -29,8 +29,11 @@ import com.codename1.impl.LanguageImpl; import com.codename1.junit.UITestBase; import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; import org.junit.jupiter.api.Test; +import java.util.concurrent.atomic.AtomicInteger; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotSame; @@ -152,6 +155,48 @@ void staticConvenienceOperationClosesItsEphemeralBackend() { assertEquals(1, backend.closeCount); } + @Test + void cancelledOperationSuppressesLateBackendCallbacksAndCloses() + throws Exception { + assertCancelledSettlementDoesNotPublish(false); + assertCancelledSettlementDoesNotPublish(true); + } + + private void assertCancelledSettlementDoesNotPublish(boolean fail) { + RecordingLanguageImpl backend = new RecordingLanguageImpl(); + backend.deferTranslation = true; + implementation.setLanguageImpl(backend); + final AtomicInteger successes = new AtomicInteger(); + final AtomicInteger failures = new AtomicInteger(); + + AsyncResource result = Translator.translate( + "bonjour", "fr", "en", + new LanguageOptions().backend( + LanguageBackends.mlKitTranslation())); + result.ready(new SuccessCallback() { + public void onSucess(String value) { + successes.incrementAndGet(); + } + }).except(new SuccessCallback() { + public void onSucess(Throwable error) { + failures.incrementAndGet(); + } + }); + + assertTrue(result.cancel(false)); + if (fail) { + backend.pendingTranslation.error( + new IllegalStateException("late failure")); + } else { + backend.pendingTranslation.complete("late success"); + } + + assertTrue(result.isCancelled()); + assertEquals(0, successes.get()); + assertEquals(0, failures.get()); + assertEquals(1, backend.closeCount); + } + private static final class RecordingLanguageImpl extends LanguageImpl { String feature; String backend; @@ -159,6 +204,8 @@ private static final class RecordingLanguageImpl extends LanguageImpl { SmartReplyMessage[] conversation; int translationCalls; int closeCount; + boolean deferTranslation; + AsyncResource pendingTranslation; public boolean isSupported(String value, String backendId) { feature = value; @@ -185,7 +232,11 @@ public AsyncResource translate( feature = "translation"; translationCalls++; AsyncResource result = new AsyncResource(); - result.complete("hello"); + if (deferTranslation) { + pendingTranslation = result; + } else { + result.complete("hello"); + } return result; } diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java index 440879232c6..e05f6f15a60 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java @@ -32,6 +32,8 @@ import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.*; @@ -167,6 +169,68 @@ void unsupportedAnalyzerReturnsFailedResource() { assertThrows(AsyncResource.AsyncExecutionException.class, result::get); } + @Test + void analyzerCreationAndCloseAreSerialized() throws Exception { + final RecordingVisionImpl backend = new RecordingVisionImpl(); + implementation.setVisionImpl(backend); + final CountDownLatch creationEntered = new CountDownLatch(1); + final CountDownLatch allowCreation = new CountDownLatch(1); + implementation.setVisionImplCreationHook(new Runnable() { + public void run() { + creationEntered.countDown(); + try { + allowCreation.await(); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new RuntimeException(error); + } + } + }); + final TextRecognizer recognizer = new TextRecognizer(); + final AtomicReference failure = + new AtomicReference(); + Thread processor = new Thread(new Runnable() { + public void run() { + try { + recognizer.process( + VisionImage.encoded(new byte[] {1})); + } catch (Throwable error) { + failure.set(error); + } + } + }, "vision-process"); + Thread closer = new Thread(new Runnable() { + public void run() { + recognizer.close(); + } + }, "vision-close"); + + processor.start(); + assertTrue(creationEntered.await(2, TimeUnit.SECONDS)); + closer.start(); + try { + long deadline = System.currentTimeMillis() + 2000; + while (closer.getState() != Thread.State.BLOCKED + && closer.isAlive() + && System.currentTimeMillis() < deadline) { + Thread.yield(); + } + assertEquals(Thread.State.BLOCKED, closer.getState()); + } finally { + allowCreation.countDown(); + } + processor.join(2000); + closer.join(2000); + + assertFalse(processor.isAlive()); + assertFalse(closer.isAlive()); + assertNull(failure.get()); + assertEquals(1, backend.closeCount); + assertThrows(IllegalStateException.class, + () -> recognizer.process( + VisionImage.encoded(new byte[] {1}))); + } + private T await(AsyncResource resource) { final AtomicReference value = new AtomicReference(); resource.ready(new SuccessCallback() { diff --git a/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java b/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java index 0ee88888071..287dcb61a92 100644 --- a/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java +++ b/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java @@ -937,6 +937,7 @@ public com.codename1.impl.ARImpl createARImpl() { } private com.codename1.impl.VisionImpl visionImpl; + private Runnable visionImplCreationHook; private com.codename1.impl.InferenceImpl inferenceImpl; private com.codename1.impl.LanguageImpl languageImpl; @@ -944,8 +945,22 @@ public void setVisionImpl(com.codename1.impl.VisionImpl visionImpl) { this.visionImpl = visionImpl; } + /** + * Installs a test hook invoked immediately before the vision backend is + * returned from {@link #createVisionImpl()}. Tests can use this to hold + * backend creation at a deterministic concurrency boundary. + * + * @param hook hook to invoke, or {@code null} to clear it + */ + public void setVisionImplCreationHook(Runnable hook) { + visionImplCreationHook = hook; + } + @Override public com.codename1.impl.VisionImpl createVisionImpl() { + if (visionImplCreationHook != null) { + visionImplCreationHook.run(); + } return visionImpl; } @@ -1338,6 +1353,7 @@ public void reset() { largerTextScale = 1f; cameraImpl = null; arImpl = null; + visionImplCreationHook = null; motionSensorManager = null; platformName = "test"; } From 84838ce04194bb1d7977a9263aeb6480918d785e Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 29 Jul 2026 06:57:14 +0300 Subject: [PATCH 70/80] Cover final on-device AI review feedback --- .../ai/language/LanguageApiTest.java | 41 +++++++++++++++++++ .../build/shared/PlatformFeatureCatalog.java | 1 + .../shared/PlatformFeatureCatalogTest.java | 2 + 3 files changed, 44 insertions(+) diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java index 113091c3b98..60ea51a3376 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java @@ -162,6 +162,47 @@ void cancelledOperationSuppressesLateBackendCallbacksAndCloses() assertCancelledSettlementDoesNotPublish(true); } + @Test + void throwingCallbackCannotDeferSessionClose() { + assertThrowingCallbackStillCloses(false); + assertThrowingCallbackStillCloses(true); + } + + private void assertThrowingCallbackStillCloses(boolean fail) { + RecordingLanguageImpl backend = new RecordingLanguageImpl(); + backend.deferTranslation = true; + implementation.setLanguageImpl(backend); + Translator.Session session = Translator.open( + new LanguageOptions().backend( + LanguageBackends.mlKitTranslation())); + AsyncResource result = session.translate( + "bonjour", "fr", "en"); + if (fail) { + result.except(new SuccessCallback() { + public void onSucess(Throwable error) { + throw new IllegalStateException("application callback"); + } + }); + } else { + result.ready(new SuccessCallback() { + public void onSucess(String value) { + throw new IllegalStateException("application callback"); + } + }); + } + session.close(); + + assertThrows(IllegalStateException.class, () -> { + if (fail) { + backend.pendingTranslation.error( + new IllegalArgumentException("backend failure")); + } else { + backend.pendingTranslation.complete("hello"); + } + }); + assertEquals(1, backend.closeCount); + } + private void assertCancelledSettlementDoesNotPublish(boolean fail) { RecordingLanguageImpl backend = new RecordingLanguageImpl(); backend.deferTranslation = true; diff --git a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java index 5d60009331d..4da4061e87a 100644 --- a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java +++ b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java @@ -393,6 +393,7 @@ public final class PlatformFeatureCatalog { e.add(new Entry("com/codename1/ai/imagegen/StableDiffusion") .iosFrameworks("CoreML", "Vision") .androidGradle("com.microsoft.onnxruntime:onnxruntime-android:1.16.3") + .androidMinimumSdk(21) .markBigUpload() .description("On-device Stable Diffusion (local-build only)")); diff --git a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java index 370c9584b44..4110478d811 100644 --- a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java +++ b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java @@ -173,6 +173,8 @@ void stableDiffusionFlagsBigUpload() { acc.consume("com/codename1/ai/imagegen/StableDiffusion"); assertTrue(acc.anyRequiresBigUpload(), "On-device SD ships a 1-2 GB Core ML model -- cloud builds must abort with a friendly message"); + assertEquals(21, acc.minimumAndroidSdk(), + "ONNX Runtime Android requires API 21"); } @Test From 91c05ad379d1c6f29d30e0bb9a647666a1026cf2 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 29 Jul 2026 07:07:42 +0300 Subject: [PATCH 71/80] Fix developer guide style findings --- docs/developer-guide/Ai-And-Speech.asciidoc | 2 +- docs/developer-guide/Working-With-iOS.asciidoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index 49840a2985c..abdda3fe28c 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -313,7 +313,7 @@ buffer when newer same-sized frames replace it. Results and errors are delivered on the Codename One EDT. `CameraFrame.getRawBytes()` can be `null` even after requesting NV21 or -RGBA8888 when a port cannot expose that format. JPEG bytes remain available; +RGBA8888 when a port can't expose that format. JPEG bytes remain available; `VisionImage.fromCameraFrame(...)` automatically selects that fallback. `DocumentScanner` currently performs still-image rectangle correction on diff --git a/docs/developer-guide/Working-With-iOS.asciidoc b/docs/developer-guide/Working-With-iOS.asciidoc index 37da30909f3..ff327851b20 100644 --- a/docs/developer-guide/Working-With-iOS.asciidoc +++ b/docs/developer-guide/Working-With-iOS.asciidoc @@ -276,7 +276,7 @@ Supported need formats are `from:`, `exact:`, `branch:`, `revision:`, and `range `ios.dependencyManager=auto` preserves backward compatibility. Existing projects with `ios.pods` continue to use CocoaPods. Projects with `ios.spm.*` use SPM. If both hint families are present, both are applied. -Explicit dependency-manager modes are validated rather than silently dropping +Explicit dependency-manager modes are validated rather than dropping dependencies. `ios.dependencyManager=spm` fails when the project or a built-in API requires a CocoaPod; use `auto` or `both`. Likewise, `ios.dependencyManager=none` fails while pod or SPM hints are present; remove From 69528197c891d2efdc9a9ae5145b71a059d2552f Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 29 Jul 2026 07:26:39 +0300 Subject: [PATCH 72/80] Fix language port-status lifecycle assertion --- .../tests/LanguageOnDeviceApiTest.java | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java index b9892e80304..71512751c02 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java @@ -133,14 +133,35 @@ private void checkReusableSessionLifecycle() { translator.close(); smartReply.close(); - assertImmediateFailure(identifier.identify("hello"), - "closed language-identification session"); - assertImmediateFailure(translator.translate( - "bonjour", "fr", "en"), "closed translation session"); - assertImmediateFailure(smartReply.suggest( - new SmartReplyMessage[] { + assertClosedSessionThrows(new ClosedSessionOperation() { + public void run() { + identifier.identify("hello"); + } + }, "closed language-identification session"); + assertClosedSessionThrows(new ClosedSessionOperation() { + public void run() { + translator.translate("bonjour", "fr", "en"); + } + }, "closed translation session"); + assertClosedSessionThrows(new ClosedSessionOperation() { + public void run() { + smartReply.suggest(new SmartReplyMessage[] { new SmartReplyMessage("Hello", "remote", false, 1) - }), "closed smart-reply session"); + }); + } + }, "closed smart-reply session"); + } + + private void assertClosedSessionThrows(ClosedSessionOperation operation, + String label) { + try { + operation.run(); + } catch (IllegalStateException expected) { + // The documented closed-session contract. + return; + } + throw new IllegalStateException(label + + " unexpectedly accepted a new request"); } private void assertImmediateFailure(AsyncResource resource, @@ -175,4 +196,8 @@ private void checkEqual(float expected, float actual, String label) { + expected + " got " + actual); } } + + private interface ClosedSessionOperation { + void run(); + } } From 7a0d622531ceb24964f8e7f6122c793ba362fcbb Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 29 Jul 2026 07:51:02 +0300 Subject: [PATCH 73/80] Satisfy language lifecycle quality checks --- .../src/com/codename1/ai/language/LanguageIdentifier.java | 4 +++- .../src/com/codename1/ai/language/LanguageSession.java | 4 ++++ CodenameOne/src/com/codename1/ai/language/SmartReply.java | 4 +++- CodenameOne/src/com/codename1/ai/language/Translator.java | 4 +++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java index 1b035b98051..e61e6ed7283 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java @@ -64,7 +64,7 @@ public static Session open(LanguageOptions options) { /// @return asynchronous ranked candidates; may be empty for undetermined text public static AsyncResource identify( String text, LanguageOptions options) { - final Session session; + final Session session; //NOPMD CloseResource - async result owns closure try { session = open(options); } catch (RuntimeException error) { @@ -103,6 +103,7 @@ private AsyncResource identify( final String text, boolean closeWhenFinished) { return session.execute( new LanguageSession.Operation() { + @Override public AsyncResource run( LanguageImpl implementation, LanguageOptions options) { @@ -115,6 +116,7 @@ public AsyncResource run( /// Closes the session. This method is idempotent; native release is /// deferred until pending identifications finish. + @Override public void close() { session.close(); } diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageSession.java b/CodenameOne/src/com/codename1/ai/language/LanguageSession.java index 5613b0f7a32..0cb7d789a3a 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageSession.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageSession.java @@ -112,6 +112,7 @@ AsyncResource execute(Operation operation, new OperationResource(this, closeWhenFinished); final Completion completion = new Completion(); backendResult.ready(new SuccessCallback() { + @Override public void onSucess(T value) { if (completion.finish()) { if (closeWhenFinished) { @@ -122,6 +123,7 @@ public void onSucess(T value) { } } }).except(new SuccessCallback() { + @Override public void onSucess(Throwable error) { if (completion.finish()) { if (closeWhenFinished) { @@ -149,6 +151,7 @@ private void operationFinished() { } } + @Override public void close() { LanguageImpl toClose = null; synchronized (this) { @@ -178,6 +181,7 @@ private static final class OperationResource this.closeWhenFinished = closeWhenFinished; } + @Override public synchronized boolean cancel(boolean mayInterruptIfRunning) { boolean cancelled = super.cancel(mayInterruptIfRunning); if (cancelled && closeWhenFinished) { diff --git a/CodenameOne/src/com/codename1/ai/language/SmartReply.java b/CodenameOne/src/com/codename1/ai/language/SmartReply.java index 2a14ea8539f..3b363102ad8 100644 --- a/CodenameOne/src/com/codename1/ai/language/SmartReply.java +++ b/CodenameOne/src/com/codename1/ai/language/SmartReply.java @@ -69,7 +69,7 @@ public static Session open(LanguageOptions options) { /// @return asynchronous suggestions, possibly an empty array public static AsyncResource suggest(SmartReplyMessage[] conversation, LanguageOptions options) { - final Session session; + final Session session; //NOPMD CloseResource - async result owns closure try { session = open(options); } catch (RuntimeException error) { @@ -123,6 +123,7 @@ private AsyncResource suggest( final SmartReplyMessage[] snapshot = copyConversation(conversation); return session.execute(new LanguageSession.Operation() { + @Override public AsyncResource run( LanguageImpl implementation, LanguageOptions options) { @@ -134,6 +135,7 @@ public AsyncResource run( /// Closes the session. This method is idempotent; native release is /// deferred until pending suggestions finish. + @Override public void close() { session.close(); } diff --git a/CodenameOne/src/com/codename1/ai/language/Translator.java b/CodenameOne/src/com/codename1/ai/language/Translator.java index 3f97041f3c0..ee543a7fb8a 100644 --- a/CodenameOne/src/com/codename1/ai/language/Translator.java +++ b/CodenameOne/src/com/codename1/ai/language/Translator.java @@ -75,7 +75,7 @@ public static Session open(LanguageOptions options) { public static AsyncResource translate(String text, String sourceLanguage, String targetLanguage, LanguageOptions options) { - final Session session; + final Session session; //NOPMD CloseResource - async result owns closure try { session = open(options); } catch (RuntimeException error) { @@ -118,6 +118,7 @@ private AsyncResource translate(final String text, final String targetLanguage, boolean closeWhenFinished) { return session.execute(new LanguageSession.Operation() { + @Override public AsyncResource run( LanguageImpl implementation, LanguageOptions options) { @@ -131,6 +132,7 @@ public AsyncResource run( /// Closes the session. This method is idempotent; native release is /// deferred until pending translations finish. + @Override public void close() { session.close(); } From b291314ae9690543bbd732d2c85399a79e17bebb Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 29 Jul 2026 07:54:46 +0300 Subject: [PATCH 74/80] Reject strict Core ML fallback on iOS --- .../codename1/ai/inference/InferenceOptions.java | 11 +++++++---- Ports/iOSPort/nativeSources/CN1Inference.m | 16 ++++++++++------ docs/developer-guide/Ai-And-Speech.asciidoc | 3 +++ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java index 077a0daf1f4..00d0485d2fa 100644 --- a/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java @@ -31,7 +31,9 @@ /// runtime can mix NPU and CPU operations without reporting full delegation, /// and the iOS Core ML delegate can schedule work across the Neural Engine, /// GPU, and CPU. Both mobile backends therefore reject -/// {@link Accelerator#NPU} when fallback is disabled. +/// {@link Accelerator#NPU} when fallback is disabled. iOS also rejects strict +/// {@link Accelerator#CORE_ML} sessions because the delegate does not report +/// whether unsupported model operations remained on LiteRT's CPU path. public final class InferenceOptions { /// Execution targets understood by the portable inference API. public enum Accelerator { @@ -65,9 +67,10 @@ public InferenceOptions threads(int value) { /// to CPU execution. /// /// On Android and iOS, setting this to {@code false} with - /// {@link Accelerator#NPU} rejects session creation. Neither LiteRT's - /// NNAPI delegate nor its Core ML delegate can prove that every operation - /// ran on the requested NPU instead of CPU or GPU. + /// {@link Accelerator#NPU} rejects session creation. On iOS it also + /// rejects {@link Accelerator#CORE_ML}. Neither LiteRT's NNAPI delegate + /// nor its Core ML delegate can prove that every operation ran on the + /// requested accelerator instead of CPU or another processor. /// /// @param value {@code true} to permit CPU fallback /// @return this options object diff --git a/Ports/iOSPort/nativeSources/CN1Inference.m b/Ports/iOSPort/nativeSources/CN1Inference.m index 464d75198e9..fae51e072e8 100644 --- a/Ports/iOSPort/nativeSources/CN1Inference.m +++ b/Ports/iOSPort/nativeSources/CN1Inference.m @@ -95,16 +95,20 @@ static void cn1InferenceEnsureHandles(void) { static NSString *cn1InferenceOpenPath(NSString *path, BOOL deleteModelOnClose, int threads, int accelerator, BOOL allowFallback) { - // TFLCoreMLDelegate does not expose a way to require Neural Engine-only - // execution. It may schedule unsupported operations on CPU or GPU, so a - // strict NPU request cannot honor the portable no-fallback contract. - if (accelerator == 3 && !allowFallback) { + // TFLCoreMLDelegate does not report whether it delegated the whole graph. + // Unsupported operations may remain on LiteRT's CPU path, and delegated + // operations may use the Neural Engine, GPU, or CPU. Strict NPU and + // Core ML requests therefore cannot honor the no-fallback contract. + if ((accelerator == 3 || accelerator == 4) && !allowFallback) { if (deleteModelOnClose) { [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; } + NSString *target = accelerator == 3 ? @"NPU" : @"Core ML"; return cn1InferenceJSON(@{ - @"error": @"Strict NPU execution cannot be verified on iOS; " - @"the Core ML delegate may use CPU or GPU" + @"error": [NSString stringWithFormat: + @"Strict %@ execution cannot be verified on iOS; " + @"the Core ML delegate may leave unsupported operations " + @"on CPU", target] }); } TFLInterpreterOptions *options = [[TFLInterpreterOptions alloc] init]; diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index abdda3fe28c..13ea3d402c8 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -381,6 +381,9 @@ and the Core ML delegate can schedule work across the Neural Engine, GPU, and CPU. Neither reports that the entire graph ran on the requested NPU. With fallback enabled, NPU selection is a best-effort accelerator preference. +On iOS, strict `Accelerator.CORE_ML` sessions are also rejected because +the delegate doesn't report whether unsupported model operations remained +on LiteRT's CPU path. The JavaSE simulator and targets without a native LiteRT runtime, including Mac native, return `false` from `InferenceSession.isSupported()`. From 1aa8a90ad68a6cc17a0f42083d1415214896c0cb Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 29 Jul 2026 08:45:51 +0300 Subject: [PATCH 75/80] Make unsupported language test portable --- .../hellocodenameone/tests/LanguageOnDeviceApiTest.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java index 71512751c02..b951c5bef84 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java @@ -168,13 +168,8 @@ private void assertImmediateFailure(AsyncResource resource, String label) { check(resource.isDone(), label + " unsupported fallback must complete immediately"); - try { - resource.get(); - throw new IllegalStateException(label - + " unsupported fallback unexpectedly succeeded"); - } catch (AsyncResource.AsyncExecutionException expected) { - // The documented unsupported-resource contract. - } + check(!resource.isReady(), label + + " unsupported fallback unexpectedly succeeded"); } private void check(boolean value, String label) { From 67239bf6eb0bbf4cdc4e5c0bbb766eab248ae58e Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:23:34 +0300 Subject: [PATCH 76/80] Gate language lifecycle tests by capability --- .../tests/LanguageOnDeviceApiTest.java | 44 +++++++++++++------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java index b951c5bef84..9adf439cf6c 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java @@ -54,8 +54,8 @@ public boolean shouldTakeScreenshot() { public boolean runTest() { try { checkValuesAndOptions(); - checkCapabilities(); - checkReusableSessionLifecycle(); + boolean[] capabilities = checkCapabilities(); + checkReusableSessionLifecycle(capabilities); done(); return true; } catch (Throwable t) { @@ -94,7 +94,7 @@ private void checkValuesAndOptions() { "smart-reply timestamp"); } - private void checkCapabilities() { + private boolean[] checkCapabilities() { boolean languageId = LanguageIdentifier.isSupported(); boolean translation = Translator.isSupported(); boolean smartReply = SmartReply.isSupported(); @@ -116,33 +116,51 @@ private void checkCapabilities() { new SmartReplyMessage("Hello", "remote", false, 1) }, null), "smart reply"); } + return new boolean[] {languageId, translation, smartReply}; } - private void checkReusableSessionLifecycle() { + private void checkReusableSessionLifecycle(boolean[] capabilities) { + if (capabilities[0]) { + checkLanguageIdentifierSessionLifecycle(); + } + if (capabilities[1]) { + checkTranslatorSessionLifecycle(); + } + if (capabilities[2]) { + checkSmartReplySessionLifecycle(); + } + } + + private void checkLanguageIdentifierSessionLifecycle() { LanguageIdentifier.Session identifier = LanguageIdentifier.open(new LanguageOptions()); - Translator.Session translator = - Translator.open(new LanguageOptions()); - SmartReply.Session smartReply = - SmartReply.open(new LanguageOptions()); identifier.close(); - translator.close(); - smartReply.close(); // Closing is intentionally idempotent so owners can use one cleanup path. identifier.close(); - translator.close(); - smartReply.close(); - assertClosedSessionThrows(new ClosedSessionOperation() { public void run() { identifier.identify("hello"); } }, "closed language-identification session"); + } + + private void checkTranslatorSessionLifecycle() { + Translator.Session translator = + Translator.open(new LanguageOptions()); + translator.close(); + translator.close(); assertClosedSessionThrows(new ClosedSessionOperation() { public void run() { translator.translate("bonjour", "fr", "en"); } }, "closed translation session"); + } + + private void checkSmartReplySessionLifecycle() { + SmartReply.Session smartReply = + SmartReply.open(new LanguageOptions()); + smartReply.close(); + smartReply.close(); assertClosedSessionThrows(new ClosedSessionOperation() { public void run() { smartReply.suggest(new SmartReplyMessage[] { From d64364f1c9cb88f7ee6932dac45183efd0bd9d4e Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:12:56 +0300 Subject: [PATCH 77/80] Harden AI asynchronous failure isolation --- .../codename1/ai/inference/ModelCache.java | 13 ++++- .../ai/language/LanguageSession.java | 26 +++++++-- .../ai/inference/InferenceApiTest.java | 57 +++++++++++++++++++ .../ai/language/LanguageApiTest.java | 24 ++++++++ 4 files changed, 113 insertions(+), 7 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java index b97d9b5cb5a..e09da307260 100644 --- a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java +++ b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java @@ -24,6 +24,7 @@ import com.codename1.io.ConnectionRequest; import com.codename1.io.FileSystemStorage; +import com.codename1.io.Log; import com.codename1.io.NetworkEvent; import com.codename1.io.NetworkManager; import com.codename1.security.Hash; @@ -341,12 +342,20 @@ private static AsyncResource subscribe( operation.ready(new SuccessCallback() { @Override public void onSucess(T value) { - subscriber.publish(value); + try { + subscriber.publish(value); + } catch (Throwable callbackFailure) { + Log.e(callbackFailure); + } } }).except(new SuccessCallback() { @Override public void onSucess(Throwable error) { - subscriber.fail(error); + try { + subscriber.fail(error); + } catch (Throwable callbackFailure) { + Log.e(callbackFailure); + } } }); return subscriber; diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageSession.java b/CodenameOne/src/com/codename1/ai/language/LanguageSession.java index 0cb7d789a3a..9ead9a5b712 100644 --- a/CodenameOne/src/com/codename1/ai/language/LanguageSession.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageSession.java @@ -87,7 +87,9 @@ AsyncResource execute(Operation operation) { AsyncResource execute(Operation operation, final boolean closeWhenFinished) { - final AsyncResource backendResult; + AsyncResource startedResult = null; + RuntimeException synchronousFailure = null; + Error synchronousError = null; synchronized (this) { if (closed) { throw new IllegalStateException( @@ -95,21 +97,35 @@ AsyncResource execute(Operation operation, } activeOperations++; try { - backendResult = operation.run(implementation, options); - if (backendResult == null) { + startedResult = operation.run(implementation, options); + if (startedResult == null) { throw new IllegalStateException( "Language backend returned no asynchronous result"); } } catch (RuntimeException error) { activeOperations--; - throw error; + synchronousFailure = error; } catch (Error error) { activeOperations--; - throw error; + synchronousError = error; } } final OperationResource result = new OperationResource(this, closeWhenFinished); + if (synchronousFailure != null) { + if (closeWhenFinished) { + close(); + } + result.fail(synchronousFailure); + return result; + } + if (synchronousError != null) { + if (closeWhenFinished) { + close(); + } + throw synchronousError; + } + final AsyncResource backendResult = startedResult; final Completion completion = new Completion(); backendResult.ready(new SuccessCallback() { @Override diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java index 9f512ab83c8..6e44e5bd4ca 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -200,6 +200,63 @@ public void onSucess(Throwable value) { flushSerialCalls(); } + @Test + void modelCacheIsolatesCoalescedSubscriberCallbacks() { + assertModelCacheSubscriberIsolation(false); + assertModelCacheSubscriberIsolation(true); + } + + private void assertModelCacheSubscriberIsolation(boolean fail) { + String suffix = fail ? "failure" : "success"; + String fileName = "model-cache-subscriber-" + suffix + ".tflite"; + ModelCache.FetchRegistration first = ModelCache.registerFetch( + fileName, "https://one.example/model.tflite", null); + ModelCache.FetchRegistration second = ModelCache.registerFetch( + fileName, "https://one.example/model.tflite", null); + AtomicReference secondNotification = + new AtomicReference(); + if (fail) { + first.resource.except(new SuccessCallback() { + @Override + public void onSucess(Throwable value) { + throw new IllegalStateException( + "first subscriber callback failed"); + } + }); + second.resource.except(new SuccessCallback() { + @Override + public void onSucess(Throwable value) { + secondNotification.set(value); + } + }); + first.completion.fail(new IllegalStateException( + "download failed")); + } else { + first.resource.ready(new SuccessCallback() { + @Override + public void onSucess(ModelSource value) { + throw new IllegalStateException( + "first subscriber callback failed"); + } + }); + second.resource.ready(new SuccessCallback() { + @Override + public void onSucess(ModelSource value) { + secondNotification.set(value); + } + }); + first.completion.complete( + ModelSource.file("test-model.tflite")); + } + flushSerialCalls(); + + assertTrue(first.resource.isDone()); + assertTrue(second.resource.isDone(), + "a throwing subscriber must not block later subscribers"); + assertNotNull(secondNotification.get(), + "the later subscriber must still receive its notification"); + } + @Test void modelCacheNamesDoNotAliasSanitizedKeys() { assertNotEquals(ModelCache.safeName("model/v1"), diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java index 60ea51a3376..e6daad301dc 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java @@ -155,6 +155,25 @@ void staticConvenienceOperationClosesItsEphemeralBackend() { assertEquals(1, backend.closeCount); } + @Test + void synchronousBackendFailureClosesEphemeralBackend() { + RecordingLanguageImpl backend = new RecordingLanguageImpl(); + backend.throwTranslation = true; + implementation.setLanguageImpl(backend); + + AsyncResource result = Translator.translate( + "bonjour", "fr", "en", + new LanguageOptions().backend( + LanguageBackends.mlKitTranslation())); + + assertTrue(result.isDone()); + AsyncResource.AsyncExecutionException error = assertThrows( + AsyncResource.AsyncExecutionException.class, result::get); + assertEquals("native initialization failed", + error.getCause().getMessage()); + assertEquals(1, backend.closeCount); + } + @Test void cancelledOperationSuppressesLateBackendCallbacksAndCloses() throws Exception { @@ -246,6 +265,7 @@ private static final class RecordingLanguageImpl extends LanguageImpl { int translationCalls; int closeCount; boolean deferTranslation; + boolean throwTranslation; AsyncResource pendingTranslation; public boolean isSupported(String value, String backendId) { @@ -272,6 +292,10 @@ public AsyncResource translate( String backendId, LanguageOptions value) { feature = "translation"; translationCalls++; + if (throwTranslation) { + throw new IllegalStateException( + "native initialization failed"); + } AsyncResource result = new AsyncResource(); if (deferTranslation) { pendingTranslation = result; From 9aca72009fb58b01a9b1c7105a3126383ba0ef8b Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:28:04 +0300 Subject: [PATCH 78/80] Harden vision cancellation and watch builds --- .../ai/vision/AbstractVisionAnalyzer.java | 39 ++++++++++++++- Ports/iOSPort/nativeSources/CN1Inference.m | 2 +- Ports/iOSPort/nativeSources/CN1Language.m | 2 +- Ports/iOSPort/nativeSources/CN1Vision.m | 2 +- .../codename1/ai/vision/VisionApiTest.java | 48 +++++++++++++++++++ 5 files changed, 89 insertions(+), 4 deletions(-) diff --git a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java index 04e46da8931..8293dbb38b9 100644 --- a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java +++ b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java @@ -25,6 +25,7 @@ import com.codename1.impl.VisionImpl; import com.codename1.ui.Display; import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; abstract class AbstractVisionAnalyzer implements VisionAnalyzer { private final VisionFeature feature; @@ -63,7 +64,9 @@ public final synchronized AsyncResource process(VisionImage image) { feature + " is not supported by " + options.getBackend().getId())); return out; } - return impl.analyze(feature, options.getBackend().getId(), image, options); + AsyncResource backend = impl.analyze(feature, + options.getBackend().getId(), image, options); + return new AnalysisResource(backend); } /// Idempotently releases the retained native backend. @@ -82,4 +85,38 @@ private VisionImpl implementation() { } return implementation; } + + private static final class AnalysisResource + extends AsyncResource { + AnalysisResource(AsyncResource backend) { + backend.ready(new SuccessCallback() { + @Override + public void onSucess(T value) { + publish(value); + } + }).except(new SuccessCallback() { + @Override + public void onSucess(Throwable error) { + fail(error); + } + }); + } + + @Override + public synchronized boolean cancel(boolean mayInterruptIfRunning) { + return super.cancel(mayInterruptIfRunning); + } + + private synchronized void publish(T value) { + if (!isCancelled()) { + complete(value); + } + } + + private synchronized void fail(Throwable error) { + if (!isCancelled()) { + error(error); + } + } + } } diff --git a/Ports/iOSPort/nativeSources/CN1Inference.m b/Ports/iOSPort/nativeSources/CN1Inference.m index fae51e072e8..8094171dda8 100644 --- a/Ports/iOSPort/nativeSources/CN1Inference.m +++ b/Ports/iOSPort/nativeSources/CN1Inference.m @@ -23,11 +23,11 @@ #import "CodenameOne_GLViewController.h" #import "xmlvm.h" +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV #if !__has_feature(objc_arc) #error CN1Inference.m requires ARC (-fobjc-arc) #endif -#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV #import #import "java_lang_String.h" diff --git a/Ports/iOSPort/nativeSources/CN1Language.m b/Ports/iOSPort/nativeSources/CN1Language.m index 981c9464d08..e039ddddc16 100644 --- a/Ports/iOSPort/nativeSources/CN1Language.m +++ b/Ports/iOSPort/nativeSources/CN1Language.m @@ -23,11 +23,11 @@ #import "CodenameOne_GLViewController.h" #import "xmlvm.h" +#if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV #if !__has_feature(objc_arc) #error CN1Language.m requires ARC (-fobjc-arc) #endif -#if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV #import #import #import "java_lang_String.h" diff --git a/Ports/iOSPort/nativeSources/CN1Vision.m b/Ports/iOSPort/nativeSources/CN1Vision.m index 9411aaa5791..3a6820fdc12 100644 --- a/Ports/iOSPort/nativeSources/CN1Vision.m +++ b/Ports/iOSPort/nativeSources/CN1Vision.m @@ -23,11 +23,11 @@ #import "CodenameOne_GLViewController.h" #import "xmlvm.h" +#if defined(INCLUDE_CN1_VISION) && !TARGET_OS_WATCH && !TARGET_OS_TV #if !__has_feature(objc_arc) #error CN1Vision.m requires ARC (-fobjc-arc) #endif -#if defined(INCLUDE_CN1_VISION) && !TARGET_OS_WATCH && !TARGET_OS_TV #import #import #import diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java index e05f6f15a60..1e559f5a732 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java @@ -169,6 +169,34 @@ void unsupportedAnalyzerReturnsFailedResource() { assertThrows(AsyncResource.AsyncExecutionException.class, result::get); } + @Test + void cancelledAnalysisSuppressesLateBackendCallbacks() { + DeferredVisionImpl backend = new DeferredVisionImpl(); + implementation.setVisionImpl(backend); + TextRecognizer recognizer = new TextRecognizer(); + AsyncResource result = recognizer.process( + VisionImage.encoded(new byte[] {1})); + final int[] callbackCount = new int[1]; + result.ready(new SuccessCallback() { + public void onSucess(TextRecognitionResult value) { + callbackCount[0]++; + } + }).except(new SuccessCallback() { + public void onSucess(Throwable error) { + callbackCount[0]++; + } + }); + + assertTrue(result.cancel(false)); + backend.pending.complete(new TextRecognitionResult("late", null)); + backend.pending.error(new RuntimeException("later error")); + flushSerialCalls(); + + assertTrue(result.isCancelled()); + assertEquals(0, callbackCount[0]); + recognizer.close(); + } + @Test void analyzerCreationAndCloseAreSerialized() throws Exception { final RecordingVisionImpl backend = new RecordingVisionImpl(); @@ -269,4 +297,24 @@ public void close() { closeCount++; } } + + private static final class DeferredVisionImpl extends VisionImpl { + final AsyncResource pending = + new AsyncResource(); + + public boolean isSupported(VisionFeature feature, String backendId) { + return true; + } + + @SuppressWarnings("unchecked") + public AsyncResource analyze(VisionFeature feature, + String backendId, + VisionImage image, + VisionOptions options) { + return (AsyncResource) pending; + } + + public void close() { + } + } } From bfedb7e8c78ca7a981e83cf6807649c3614e60b4 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:12:45 +0300 Subject: [PATCH 79/80] Preserve legacy cn1lib virtual paths --- .../java/com/codename1/builders/Executor.java | 16 ++++-- .../ExecutorArchiveExtractionTest.java | 55 +++++++++++++++++-- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java index 0eb485669f1..661ffedd5fd 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java @@ -1245,15 +1245,14 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD while ((entry = zis.getNextEntry()) != null) { String entryName = entry.getName(); - // Validate the original ZIP name before any archive-specific - // branch can route it into a generated tar and continue. - resolveArchiveEntry(resDir, entryName); - - if(!"html.tar".equals(entryName)&& (entryName.startsWith("html") || entryName.startsWith("/html"))) { + if (entryName.startsWith("html/") + || entryName.startsWith("/html/")) { if(entry.isDirectory()) { continue; } - entryName = entryName.substring(5); + int htmlPrefix = entryName.startsWith("html/") + ? "html/".length() : "/html/".length(); + entryName = entryName.substring(htmlPrefix); // The stripped name becomes a member of another archive, // so validate it independently against that archive's root. resolveArchiveEntry(resDir, entryName); @@ -1316,6 +1315,11 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD continue; } + // Entries outside the supported virtual cn1lib namespaces + // retain their original ZIP name, which must be relative and + // remain inside the extraction root. + resolveArchiveEntry(resDir, entryName); + if (entry.isDirectory()) { File dir = resolveArchiveEntry(classesDir, entryName); dir.mkdirs(); diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java index a920e0ed27a..79b80ed8a77 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java @@ -29,6 +29,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.zip.CRC32; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import org.junit.jupiter.api.Test; @@ -37,6 +38,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; class ExecutorArchiveExtractionTest { @TempDir @@ -111,14 +113,36 @@ void rejectsTraversalInStrippedTarMemberName() throws Exception { assertSpecialEntryRejected("podspecs/../payload", "podspecs.tar"); assertSpecialEntryRejected("javase.lib/../payload", "javase.lib.tar"); + assertSpecialEntryRejected("/html/../payload", "html.tar"); + assertSpecialEntryRejected("/podspecs/../payload", "podspecs.tar"); + assertSpecialEntryRejected("/javase.lib/../payload", + "javase.lib.tar"); + } + + @Test + void preservesSupportedLeadingSlashVirtualEntries() throws Exception { + assertSpecialEntryAccepted("/html/index.html", "html.tar"); + assertSpecialEntryAccepted("/podspecs/library.podspec", + "podspecs.tar"); + assertSpecialEntryAccepted("/javase.lib/library.jar", + "javase.lib.tar"); } private static byte[] zipEntry(String name, String contents) throws IOException { + byte[] payload = contents.getBytes(StandardCharsets.UTF_8); + CRC32 checksum = new CRC32(); + checksum.update(payload); + ZipEntry entry = new ZipEntry(name); + entry.setMethod(ZipEntry.STORED); + entry.setSize(payload.length); + entry.setCompressedSize(payload.length); + entry.setCrc(checksum.getValue()); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); ZipOutputStream zip = new ZipOutputStream(bytes); - zip.putNextEntry(new ZipEntry(name)); - zip.write(contents.getBytes(StandardCharsets.UTF_8)); + zip.putNextEntry(entry); + zip.write(payload); zip.closeEntry(); zip.close(); return bytes.toByteArray(); @@ -126,8 +150,29 @@ private static byte[] zipEntry(String name, String contents) private void assertSpecialEntryRejected(String entryName, String generatedArchive) throws Exception { - File root = temporaryDirectory.resolve("special-" + File root = specialRoot(entryName); + File resources = unzipSpecialEntry(root, entryName); + + assertFalse(new File(resources, generatedArchive).exists(), + "Unsafe entry must not create " + generatedArchive); + } + + private void assertSpecialEntryAccepted(String entryName, + String generatedArchive) throws Exception { + File root = specialRoot(entryName); + File resources = unzipSpecialEntry(root, entryName); + + assertTrue(new File(resources, generatedArchive).isFile(), + "Supported entry must create " + generatedArchive); + } + + private File specialRoot(String entryName) { + return temporaryDirectory.resolve("special-" + Math.abs(entryName.hashCode())).toFile(); + } + + private static File unzipSpecialEntry(File root, String entryName) + throws Exception { File classes = new File(root, "classes"); File resources = new File(root, "resources"); File sources = new File(root, "sources"); @@ -138,8 +183,6 @@ private void assertSpecialEntryRejected(String entryName, new IPhoneBuilder().unzip( new ByteArrayInputStream(zipEntry(entryName, "payload")), classes, resources, sources); - - assertFalse(new File(resources, generatedArchive).exists(), - "Unsafe entry must not create " + generatedArchive); + return resources; } } From 8f122ca3e2898341bb00d8821bdc84e1b1231e0e Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:35:33 +0300 Subject: [PATCH 80/80] Preserve historical AI post context --- .../workflows/release-on-maven-central.yml | 22 +- docs/ai-on-device-architecture.md | 261 ------------------ .../content/blog/platform-apis-in-the-core.md | 250 +++++++++++++---- 3 files changed, 205 insertions(+), 328 deletions(-) delete mode 100644 docs/ai-on-device-architecture.md diff --git a/.github/workflows/release-on-maven-central.yml b/.github/workflows/release-on-maven-central.yml index 2259751af64..2f61e0b69df 100644 --- a/.github/workflows/release-on-maven-central.yml +++ b/.github/workflows/release-on-maven-central.yml @@ -61,24 +61,30 @@ jobs: # "Deployment failed while publishing" even when the bundle was # actually accepted and published. As a safety net for that # false-positive case, poll Maven Central for the key artifacts: - # the codenameone-maven-plugin (proxy for the core release) plus - # both archetypes. Skipped when the deploy already reported - # success (the artifact may still be propagating from Sonatype - # Central to repo1; that propagation can take 30+ minutes and - # isn't worth blocking on). + # the codenameone-maven-plugin (proxy for the core release), its + # platform-feature-catalog dependency, and both archetypes. The + # catalog is a separate reactor artifact used by the local builders + # for built-in platform dependency selection, so confirming only the + # plugin could leave a released plugin with an unavailable runtime + # dependency. Skipped when the deploy already reported success (the + # artifacts may still be propagating from Sonatype Central to repo1; + # that propagation can take 30+ minutes and isn't worth blocking on). set +e for i in $(seq 1 90); do plugin_code=$(curl -s -o /dev/null -w "%{http_code}" \ "https://repo1.maven.org/maven2/com/codenameone/codenameone-maven-plugin/${GITHUB_REF_NAME}/codenameone-maven-plugin-${GITHUB_REF_NAME}.pom") + catalog_code=$(curl -s -o /dev/null -w "%{http_code}" \ + "https://repo1.maven.org/maven2/com/codenameone/platform-feature-catalog/${GITHUB_REF_NAME}/platform-feature-catalog-${GITHUB_REF_NAME}.pom") app_code=$(curl -s -o /dev/null -w "%{http_code}" \ "https://repo1.maven.org/maven2/com/codenameone/cn1app-archetype/${GITHUB_REF_NAME}/cn1app-archetype-${GITHUB_REF_NAME}.pom") lib_code=$(curl -s -o /dev/null -w "%{http_code}" \ "https://repo1.maven.org/maven2/com/codenameone/cn1lib-archetype/${GITHUB_REF_NAME}/cn1lib-archetype-${GITHUB_REF_NAME}.pom") - if [ "$plugin_code" = "200" ] && [ "$app_code" = "200" ] && [ "$lib_code" = "200" ]; then - echo "Confirmed codenameone-maven-plugin + cn1{app,lib}-archetype ${GITHUB_REF_NAME} on Maven Central" + if [ "$plugin_code" = "200" ] && [ "$catalog_code" = "200" ] && \ + [ "$app_code" = "200" ] && [ "$lib_code" = "200" ]; then + echo "Confirmed plugin + platform-feature-catalog + cn1{app,lib}-archetype ${GITHUB_REF_NAME} on Maven Central" exit 0 fi - echo "[$i/90] Waiting on Maven Central (plugin=$plugin_code, cn1app=$app_code, cn1lib=$lib_code)" + echo "[$i/90] Waiting on Maven Central (plugin=$plugin_code, catalog=$catalog_code, cn1app=$app_code, cn1lib=$lib_code)" sleep 20 done echo "Artifacts ${GITHUB_REF_NAME} did not appear on Maven Central within 30 minutes" diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md deleted file mode 100644 index ea1280d7f96..00000000000 --- a/docs/ai-on-device-architecture.md +++ /dev/null @@ -1,261 +0,0 @@ -# Built-in on-device AI architecture - -This document is the maintainer reference for the built-in vision, -language, and LiteRT APIs. User-facing examples live in -`docs/developer-guide/Ai-And-Speech.asciidoc`. - -## Scope - -The public surface is in: - -- `CodenameOne/src/com/codename1/ai/vision` -- `CodenameOne/src/com/codename1/ai/language` -- `CodenameOne/src/com/codename1/ai/inference` - -The API is part of `codenameone-core` on every target. The default -`CodenameOneImplementation` factories return `null`, so an unsupported -target has deterministic `isSupported() == false` behavior and never falls -back to a cloud service. - -Native backends are: - -| Target | Vision | Language | `.tflite` inference | -| --- | --- | --- | --- | -| Android | ML Kit | ML Kit | LiteRT | -| iOS | Apple Vision/Core Image; optional ML Kit | Apple Natural Language for identification; ML Kit for translation, Smart Reply, and optional identification | TensorFlow Lite Objective-C; optional Core ML delegate | -| Mac native | Apple Vision/Core Image through Mac Catalyst | Unsupported fallback | Unsupported fallback | -| JavaSE, JavaScript, native Windows/Linux | Unsupported fallback | Unsupported fallback | Unsupported fallback | -| watchOS, tvOS | Unsupported fallback | Unsupported fallback | Unsupported fallback | - -Document correction is intentionally Apple-only. The Google ML Kit document -scanner is an interactive Activity camera flow, while the core -`DocumentScanner` contract analyzes an existing `VisionImage`. - -## Build-time selection - -`maven/platform-feature-catalog` is the authoritative dependency registry. -`PlatformFeatureCatalog.Accumulator` consumes exact class and method -references from the existing bytecode scanners. It rejects unrelated symbols -at consume time and memoizes its immutable hit set behind a dirty flag, so -memory and repeated builder queries scale with catalog matches rather than -total application bytecode. - -The local Maven plugin and BuildDaemon both consume the same source contract: - -- `maven/codenameone-maven-plugin/.../AndroidGradleBuilder.java` -- `maven/codenameone-maven-plugin/.../IPhoneBuilder.java` -- `BuildDaemon/src/com/codename1/build/daemon/AndroidGradleBuilder.java` -- `BuildDaemon/src/com/codename1/build/daemon/IPhoneBuilder.java` - -BuildDaemon carries a source copy of `PlatformFeatureCatalog.java` because it -is built and deployed separately. Keep that copy byte-for-byte equal to the -Maven module source and run `cmp` as part of validation. - -Local source builds replace the generated iOS/Mac project directory before -copying the new result. This is required for granular removal: a plain -directory merge would retain an old Podfile, workspace, Pods directory, or -optional native source after the application stops referencing a feature. - -Catalog-selected CocoaPods are deduplicated by pod name. An existing -user-declared spec appears first and wins, preserving constraints such as -`GoogleMLKit/TextRecognition ~> 7.0`. Explicit `ios.dependencyManager=spm` -and `none` modes reject incompatible declared or catalog-selected -dependencies instead of silently omitting them. - -The companion archive extraction hardening is intentionally broader than AI: -every Android/build archive entry is canonicalized and checked for traversal -before extraction. Trusted extraction filters may still route a validated -entry to a deliberate sibling such as the project settings file. - -### Android source granularity - -The optional Android sources are excluded from `maven/android` compilation -and compiled in the generated application after dependencies are selected. -The port bundle contains: - -- one reflection-loaded group dispatcher (`AndroidVisionImpl` or - `AndroidLanguageImpl`); -- one dependency-neutral group adapter base; -- one source file per concrete ML Kit feature; -- one `AndroidInferenceImpl`, because LiteRT is already a single dependency. - -`AndroidGradleBuilder.androidAiAdapterSource()` maps each exact public entry -point to one source. `pruneOptionalAiSources()` deletes every unselected -source before Gradle compiles the generated application. R8 keeps only the -remaining `com.codename1.impl.android.ai` classes so reflection cannot rename -the selected dispatcher or adapter. - -| Core entry point | Retained Android source | Gradle dependency | -| --- | --- | --- | -| `TextRecognizer` | `AndroidTextRecognitionAdapter.java` | `com.google.mlkit:text-recognition:16.0.0` | -| `BarcodeScanner` | `AndroidBarcodeScanningAdapter.java` | `com.google.mlkit:barcode-scanning:17.2.0` | -| `FaceDetector` | `AndroidFaceDetectionAdapter.java` | `com.google.mlkit:face-detection:16.1.5` | -| `ImageLabeler` | `AndroidImageLabelingAdapter.java` | `com.google.mlkit:image-labeling:17.0.7` | -| `PoseDetector` | `AndroidPoseDetectionAdapter.java` | `com.google.mlkit:pose-detection:18.0.0-beta3` | -| `SelfieSegmenter` | `AndroidSelfieSegmentationAdapter.java` | `com.google.mlkit:segmentation-selfie:16.0.0-beta5` | -| `LanguageIdentifier` | `AndroidLanguageIdAdapter.java` | `com.google.mlkit:language-id:17.0.6` | -| `Translator` | `AndroidTranslationAdapter.java` | `com.google.mlkit:translate:17.0.3` | -| `SmartReply` | `AndroidSmartReplyAdapter.java` | `com.google.mlkit:smart-reply:17.0.4` | -| `InferenceSession` | `AndroidInferenceImpl.java` | `com.google.ai.edge.litert:litert:1.0.1` | - -### Apple method granularity - -`CN1Vision.m` and `CN1Language.m` use `__has_include` around each ML Kit -component. The class scanner always links the small Apple system frameworks -needed by a referenced analyzer. It adds a Google ML Kit vision pod only -when it observes both: - -1. the concrete analyzer class; and -2. a call to the matching feature-specific selector, such as - `VisionBackends.mlKitTextRecognition()`. - -On iOS, language identification defaults to the system Natural Language -framework. Calling `LanguageBackends.mlKitLanguageIdentification()` opts -into `GoogleMLKit/LanguageID`. Translation and Smart Reply pods are selected -only by calls to `LanguageBackends.mlKitTranslation()` and -`LanguageBackends.mlKitSmartReply()`. Merely referencing their API classes -does not change simulator architectures. `InferenceSession` selects -`TensorFlowLiteObjC/CoreML`. - -The native implementation is disabled for watchOS and tvOS. Mac native uses -the Catalyst slice of the framework-only Apple Vision implementation. The -official Google ML Kit and TensorFlow Lite Objective-C packages do not ship -Mac Catalyst slices, so their catalog entries are marked incompatible with -Catalyst. The builders omit those package dependencies from a Mac-native -build; language and inference consequently use their explicit unsupported -stubs there. - -Google ML Kit's iOS binary frameworks contain device `arm64` and simulator -`x86_64` slices, but no `arm64` simulator slice. Catalog entries declare that -constraint independently from Catalyst support. When one of those entries is -selected, both builders set -`EXCLUDED_ARCHS[sdk=iphonesimulator*]=arm64` on the generated application and -Pods projects and emit a diagnostic identifying the selected dependency. -TensorFlow Lite's XCFramework does include an `arm64` simulator slice and -does not trigger this fallback. -The same Google ML Kit catalog entries declare an iOS 15.5 deployment floor. -The iOS builder folds that value into its existing maximum-target calculation -before generating the app target and Podfile, preventing a lower application -hint from producing an unsatisfiable CocoaPods resolution. -The iOS NEON implementation reports SIMD as unsupported in that x86_64 -configuration and aliases its native entry points to the generic scalar -implementation; device and arm64-simulator builds retain the NEON path. - -Apple Vision barcode requests use revision 1 only when compiling for an iOS -simulator. Recent simulator runtimes can otherwise complete the default -request without returning observations for valid QR images. Physical devices -retain the current OS revision and its additional symbologies. - -## Image and camera contract - -`VisionImage` owns defensive copies of its input. It accepts encoded JPEG or -PNG, NV21, and RGBA8888. Android feeds raw NV21 directly to ML Kit and -converts RGBA to a bitmap. Apple creates a `CGImage` directly from NV21 or -RGBA memory and passes it to Vision or ML Kit without an intermediate JPEG -encode/decode. `VisionImage.encoded(bytes, rotationDegrees)` carries the -clockwise display rotation for encoded gallery or file images; the -one-argument overload deliberately assumes upright stored pixels rather than -silently guessing or discarding EXIF orientation. - -`VisionImage.fromCameraFrame()` is safe beyond the -`FrameListener.onFrame()` callback because it copies the callback-owned -buffer selected by the requested `FrameFormat`. Raw frames do not retain the -always-available JPEG fallback, so mobile backends consume NV21 or RGBA8888 -without accidentally selecting the encoded path. When a camera port cannot -supply the requested raw buffer, `fromCameraFrame()` retains the JPEG fallback -and reports JPEG as the image format instead of creating an empty image. -`VisionPipeline` allows one request to run and retains only the newest pending -frame. Superseding same-sized pending frames reuse one detached buffer, and -analyzer invocation occurs outside the pipeline monitor. This bounds memory, -latency, and lock re-entry for live OCR, barcode, face, pose, labeling, or -segmentation. - -Native results are converted to stable Codename One value types. Geometry is -normalized to the top-left coordinate system. `VisionMetadata` carries the -actual backend id without exposing platform classes. - -## Inference contract - -`InferenceSession` supports named typed tensors, multiple inputs and outputs, -input resizing, and reusable native sessions. Model sources are bytes, -resources, private files, or the HTTPS-only `ModelCache`. The cache rejects -redirects that downgrade a request to HTTP on ports where redirects are -observable. iOS follows redirects below the portable network layer, so the -cache requires a SHA-256 digest for every iOS download and rejects its unpinned -overload. Identical concurrent fetches for one cache entry coalesce, while -conflicting in-flight content identities fail instead of sharing a temporary -path. Each coalesced caller has an independent resource, so cancellation -suppresses only that caller's notification and never cancels the shared -download or another subscriber. File sources are opened by path without -copying the model through the Java heap. Cache promotion verifies that the -final file exists and, when supplied, still matches the requested digest -before publishing its path. -Android invokes LiteRT with null output destinations, then reads each result -from its native tensor buffer. This allows value-dependent output dimensions -to resolve before Codename One allocates or copies the result. LiteRT caches -output shapes when no input reallocation occurred, so the Android port -explicitly refreshes each output shape after invocation; both Android builders -retain that package-private LiteRT method name through R8. NPU selection is -best effort when fallback is enabled. Strict NPU requests are rejected on -Android because LiteRT cannot verify that NNAPI delegated every operation, -and on iOS because the Core ML delegate may schedule work on CPU or GPU. - -All native sessions and analyzers must be closed. Language APIs expose -reusable feature sessions in addition to static one-shot methods; Android -sessions cache their ML Kit client or translation clients by language pair. -Expensive open, analysis, and inference work runs off the EDT; completion and -error delivery return to the EDT. An inference session rejects metadata -queries, resizing, and another invocation while a run is pending so no two API -calls can touch the mutable native interpreter concurrently. - -## Permanent cross-platform coverage - -`scripts/hellocodenameone` registers three non-screenshot conformance tests: - -- `VisionOnDeviceApiTest` covers `VisionImage.fromCameraFrame()` ownership, - option normalization, capability queries for every analyzer, close - semantics, and—when a native barcode backend is available—a deterministic - QR decode through image marshalling, native detection, JSON parsing, format - normalization, and corner geometry. -- `LanguageOnDeviceApiTest` covers language value/options contracts, - capability queries for identification, translation, and smart reply, and - immediate unsupported resources. -- `InferenceOnDeviceApiTest` covers immutable tensors and model sources, - validation and options, runtime capability reporting, and the unsupported - session-open contract. - -The tests avoid permission prompts, mutable model downloads, and large bundled -models. Their concrete API references are nevertheless part of the application -bytecode, so the Android and Apple source-build jobs exercise the same granular -builder selection used by applications. The tests map to independent -`on-device-vision`, `on-device-language`, and `on-device-inference` features in -`docs/website/data/port_status.json`; fresh per-port reports replace the -bootstrap `not-run` results after the updated suite runs on `master`. - -## Adding a feature - -1. Add the vendor-neutral API and result types to core. -2. Extend the appropriate implementation SPI under - `CodenameOne/src/com/codename1/impl`. -3. Add one Android adapter source and one exact - `androidAiAdapterSource()` mapping. -4. Add the smallest Android dependency to `PlatformFeatureCatalog`. -5. Add the Apple implementation behind a feature-specific compile guard. -6. Add the system framework and optional pod mapping to the catalog. -7. Copy the catalog into BuildDaemon and update both builders if source - selection changes. -8. Add core API tests, catalog tests, builder-selection tests, Java 8 Android - compilation, Objective-C syntax compilation, CocoaPods integration, and - Xcode device/Catalyst builds as applicable. Some Google ML Kit releases - cannot link an arm64 simulator even though the sources compile there. - -Use an isolated Maven repository for validation: - -```sh -mvn -Dmaven.repo.local=/private/tmp/cn1-ai-validation-m2 ... -``` - -Large model families and native runtimes such as Whisper and Stable Diffusion -remain opt-in cn1libs. The core/builder approach is intended for APIs whose -code can be selected cheaply and whose model payloads are supplied by the OS, -downloaded lazily by the native SDK, or supplied by the application. diff --git a/docs/website/content/blog/platform-apis-in-the-core.md b/docs/website/content/blog/platform-apis-in-the-core.md index 324c79ac73e..586ac6f4c77 100644 --- a/docs/website/content/blog/platform-apis-in-the-core.md +++ b/docs/website/content/blog/platform-apis-in-the-core.md @@ -244,64 +244,201 @@ view.setInputListener(userText -> { `appendToLastMessage(...)` is the streaming entry point; it marshals through `callSerially` so deltas land on the EDT in order. `ConversationStore` persists the thread (the default backing is `Storage`; pluggable via a custom implementation if you would rather keep it in SQLite or push it to your server). -### On-device vision, language, and LiteRT +### The AI cn1libs -**Update (July 2026):** On-device vision, language services, and LiteRT -inference now ship as framework APIs. Do not add separate ML Kit or LiteRT -cn1lib dependencies. Import the APIs from `com.codename1.ai.vision`, -`com.codename1.ai.language`, and `com.codename1.ai.inference`. +> **Editor's note (July 2026):** This section describes the APIs as they +> shipped when this post was published. Vision, language services, and LiteRT +> inference have since moved into core with builder-selected native +> dependencies; Whisper and Stable Diffusion remain cn1libs. See +> [AI, Chat UI, and Speech](https://www.codenameone.com/developer-guide/#_ai_chat_ui_and_speech) +> for the current APIs and migration guidance. -The builders inspect the feature classes and backend-selector methods that an -application actually uses. They retain only the matching native adapters and -add the required Android library, Apple framework, or optional iOS ML Kit pod. -For example, Android text recognition uses ML Kit automatically. Apple targets -default to Vision, while this explicit selector opts iOS into the text-only ML -Kit pod: +The core LLM stack is paired with a set of opt-in cn1libs that wrap specific on-device capabilities: Google ML Kit features, the TensorFlow Lite runtime, a local Whisper transcription engine, and an on-device Stable Diffusion model. Thirteen new cn1libs ship this release. + +These cn1libs are not yet listed in the Codename One Preferences cn1lib picker, so for the moment they are added by hand. Drop the matching dependency block into your project's `common/pom.xml` and rebuild. The build-time scanner does the rest: the iOS pod or Swift Package, the Android Gradle dependency, the plist usage strings (`NSCameraUsageDescription` for the vision libraries, `NSSpeechRecognitionUsageDescription` for Whisper, etc.), and the Android permissions (`android.permission.RECORD_AUDIO` for audio capture) are all injected automatically the first time the scanner sees the matching class on the classpath. + +For each cn1lib below, the dependency block is identical in shape; only the `` changes. The shared pattern is: + +```xml + + com.codenameone + + ${cn1.version} + +``` + +#### `cn1-ai-mlkit-text`: text recognition (OCR) + +**TL;DR.** Pull printed or handwritten text out of an image (a photo of a page, a sign, a receipt) entirely on-device. + +**Platforms.** iOS bridges to `GoogleMLKit/TextRecognition`. Android bridges to `com.google.mlkit:text-recognition`. The JavaSE simulator returns an unsupported error. + +**Use cases.** Receipt scanning, sign translation pipelines (combine with `cn1-ai-mlkit-translate`), accessibility tools that read printed text aloud, automated form ingestion. + +```java +byte[] jpeg = capturePhotoBytes(); +TextRecognizer.recognize(jpeg).onResult((text, err) -> { + if (err == null) Log.p("OCR: " + text); +}); +``` + +#### `cn1-ai-mlkit-barcode`: barcode and QR scanning + +**TL;DR.** Decodes QR, EAN, UPC, Data Matrix, PDF417, and the rest of the common 1D / 2D code families from a captured image. + +**Platforms.** iOS bridges to `MLKitBarcodeScanning`. Android bridges to `com.google.mlkit:barcode-scanning`. The JavaSE simulator returns an unsupported error. + +**Use cases.** Inventory scanning, ticket / boarding-pass readers, QR-driven onboarding flows, retail loyalty cards. + +```java +byte[] jpeg = capturePhotoBytes(); +BarcodeScanner.scan(jpeg).onResult((codes, err) -> { + if (err == null) { + for (String code : codes) Log.p("Found: " + code); + } +}); +``` + +#### `cn1-ai-mlkit-face`: face detection + +**TL;DR.** Returns bounding boxes for human faces detected in an image. Each face is reported as a packed `int[4]` (`x`, `y`, `width`, `height`). + +**Platforms.** iOS bridges to `MLKitFaceDetection`. Android bridges to `com.google.mlkit:face-detection`. + +**Use cases.** Auto-crop a contact photo, mosaic / blur bystanders in a group shot, drive a face-tracked overlay for AR-lite filters. + +```java +FaceDetector.detect(jpeg).onResult((boxes, err) -> { + if (err != null) return; + for (int i = 0; i < boxes.length; i += 4) { + Log.p("face at " + boxes[i] + "," + boxes[i + 1] + " " + + boxes[i + 2] + "x" + boxes[i + 3]); + } +}); +``` + +#### `cn1-ai-mlkit-labeling`: image labeling + +**TL;DR.** "What is in this picture." Returns a list of descriptive labels for the image content. + +**Platforms.** iOS bridges to `MLKitImageLabeling`. Android bridges to `com.google.mlkit:image-labeling`. + +**Use cases.** Auto-tagging uploaded photos, content moderation pre-filters, content-based image search. + +```java +ImageLabeler.label(jpeg).onResult((labels, err) -> { + if (err == null) Log.p("labels: " + String.join(", ", labels)); +}); +``` + +#### `cn1-ai-mlkit-translate`: on-device translation + +**TL;DR.** Translate short text between supported language pairs entirely on-device; no server round-trip, no API key, works offline. + +**Platforms.** iOS bridges to `MLKitTranslate`. Android bridges to `com.google.mlkit:translate`. Languages are identified by their ISO 639-1 codes (`en`, `fr`, `es`, ...). + +**Use cases.** Offline travel assistants, chat translation, accessibility readers for foreign signage (combine with `cn1-ai-mlkit-text`). + +```java +Translator.translate("Where is the train station?", "en", "fr") + .onResult((fr, err) -> { + if (err == null) Log.p(fr); // "Où est la gare ?" + }); +``` + +#### `cn1-ai-mlkit-smartreply`: short reply suggestions + +**TL;DR.** Generates short suggested replies for chat conversations, similar to Gmail's Smart Reply chips. + +**Platforms.** iOS bridges to `MLKitSmartReply`. Android bridges to `com.google.mlkit:smart-reply`. The input is a JSON array of `{role, message, timestamp, userId}` objects. + +**Use cases.** A "quick reply" row above the keyboard in your in-app chat, response suggestions in a CRM inbox. + +```java +String thread = "[{\"role\":\"remote\",\"message\":\"See you at 6?\"," + + "\"timestamp\":" + System.currentTimeMillis() + "," + + "\"userId\":\"u42\"}]"; + +SmartReply.suggest(thread).onResult((suggestions, err) -> { + if (err == null) { + for (String s : suggestions) Log.p("suggestion: " + s); + } +}); +``` + +#### `cn1-ai-mlkit-langid`: language identification + +**TL;DR.** Returns the most likely ISO 639-1 code for a given text, or `und` (undetermined) when the input is too short or ambiguous. + +**Platforms.** iOS bridges to `MLKitLanguageID`. Android bridges to `com.google.mlkit:language-id`. + +**Use cases.** Auto-route a customer-support message to the right team, pick the correct TTS voice for an arbitrary string, pre-screen input before running an expensive translate. + +```java +LanguageIdentifier.identify("Bonjour le monde").onResult((code, err) -> { + if (err == null) Log.p(code); // "fr" +}); +``` + +#### `cn1-ai-mlkit-pose`: pose detection + +**TL;DR.** Returns 33 skeletal landmarks per detected pose as a packed `float[3 * 33]` (`x`, `y`, `confidence` triples). + +**Platforms.** iOS bridges to `MLKitPoseDetection`. Android bridges to `com.google.mlkit:pose-detection`. + +**Use cases.** Fitness apps with form correction, dance / yoga timing analysis, gesture-driven controls. ```java -VisionOptions options = new VisionOptions() - .backend(VisionBackends.mlKitTextRecognition()); +PoseDetector.detect(jpeg).onResult((landmarks, err) -> { + if (err != null || landmarks.length < 99) return; + float noseX = landmarks[0], noseY = landmarks[1], noseConf = landmarks[2]; + Log.p("nose at (" + noseX + ", " + noseY + ") conf=" + noseConf); +}); +``` + +#### `cn1-ai-mlkit-segmentation`: selfie segmentation + +**TL;DR.** Returns a per-pixel mask separating the person in the foreground from the background as `byte[width * height]` (`0` = background, `255` = foreground). + +**Platforms.** iOS bridges to `MLKitSegmentationSelfie`. Android bridges to `com.google.mlkit:segmentation-selfie`. -new TextRecognizer(options) - .process(VisionImage.encoded(jpeg)) - .ready(result -> Log.p(result.getText())) - .except(error -> Log.e(error)); +**Use cases.** Background replacement for video calls, sticker / portrait-mode effects, blur-the-background privacy filters. + +```java +SelfieSegmenter.segment(jpeg).onResult((mask, err) -> { + if (err == null) applyBackgroundReplacement(mask); +}); ``` -`InferenceSession` owns native interpreter and delegate resources. Reuse a -session for repeated runs and close it when the owner is disposed. A one-shot -operation must close the session in both completion paths: +#### `cn1-ai-mlkit-docscan`: document scanner + +**TL;DR.** Detects a rectangular document in a photo, perspective-corrects it, and writes the cropped JPEG to a temporary file. Returns the file path. + +**Platforms.** iOS uses Apple's VisionKit + Core Image rectangle detection (no extra pod). Android uses `com.google.android.gms:play-services-mlkit-document-scanner`. + +**Use cases.** "Scan to PDF" flows, expense apps that capture receipts, contract signing flows, ID-document capture. ```java -InferenceSession.open(ModelSource.resource("/model.tflite"), - new InferenceOptions()) - .ready(session -> session.run(new Tensor[] {input}) - .ready(outputs -> { - try { - consume(outputs); - } finally { - session.close(); - } - }) - .except(error -> { - try { - Log.e(error); - } finally { - session.close(); - } - })) - .except(error -> Log.e(error)); -``` - -See [AI, Chat UI, and Speech](https://www.codenameone.com/developer-guide/#_ai_chat_ui_and_speech) -for the current API reference, feature matrix, camera pipeline, model cache, -and backend-selection examples. - -### Large optional AI cn1libs - -Vision, language, and LiteRT fit in core because their native dependencies can -be selected granularly by the builders. Runtimes with very large bundled data -or native libraries remain explicit dependencies. +DocumentScanner.scanToFile(jpeg).onResult((path, err) -> { + if (err == null) uploadDocument(path); +}); +``` + +#### `cn1-ai-tflite`: TensorFlow Lite interpreter + +**TL;DR.** A general-purpose on-device inference engine. Bring your own `.tflite` model and run it against a float32 input tensor. + +**Platforms.** iOS uses `TensorFlowLiteSwift` (Pods or Swift Package). Android uses `org.tensorflow:tensorflow-lite` + `tensorflow-lite-support`. + +**Use cases.** Any custom on-device ML model your team trains or pulls from TF Hub. Image classification, simple regression, recommendation pre-filters. + +```java +byte[] modelBytes = Util.readFully(Display.getInstance().getResourceAsStream(null, "/model.tflite")); +float[] input = featureVector(); +Interpreter.run(modelBytes, input).onResult((output, err) -> { + if (err == null) Log.p("model returned " + output.length + " values"); +}); +``` #### `cn1-ai-whisper`: speech-to-text via whisper.cpp @@ -336,20 +473,15 @@ StableDiffusion.generate("a teal hot-air balloon over Lisbon, watercolour", }); ``` -### Why these remain cn1libs +### Why these are cn1libs and not part of the core + +The core gets the AI plumbing every app that adopts AI at all wants: the LLM client, streaming, the chat UI, the secure storage primitive for credentials, the simulator Ollama redirect for offline iteration. -The core provides the portable APIs and lets the builders retain native vision, -language, and LiteRT dependencies at feature or selector granularity. Whisper -and Stable Diffusion are different: they bundle unusually large native -libraries or model data that should remain an explicit application choice. +The cn1libs above are specialized verticals. Barcode scanning, document scanning, face detection, smart reply, pose detection, on-device translation, transcription, on-device image generation, each is genuinely useful, but only for some apps. They also each bring a non-trivial native dependency. The Google ML Kit Android frameworks are large; the iOS pods carry their own weight; the bundled `libwhisper.a` and the Stable Diffusion model are big. Pulling all of them into core would tax every app whether the feature is used or not. The Stable Diffusion cn1lib in particular is large enough that the cloud build server cannot accept the upload at all (it trips the 2 GB pre-upload guard). That kind of opt-in does not belong in a dependency every app inherits. -The corresponding chapter, including the full `LlmClient` API table, the -`ChatView` reference, the `SecureStorage` overloads, the simulator Ollama -redirect, the built-in on-device APIs, and the remaining cn1lib coverage, is -at [AI, Chat UI, and Speech](https://www.codenameone.com/developer-guide/#_ai_chat_ui_and_speech) -in the developer guide. +The corresponding chapter, including the full `LlmClient` API table, the `ChatView` reference, the `SecureStorage` overloads, the simulator Ollama redirect, and the full cn1lib coverage, is at [AI, Chat UI, and Speech](https://www.codenameone.com/developer-guide/#_ai_chat_ui_and_speech) in the developer guide. ## OAuth and OIDC: the modern identity stack