diff --git a/.github/workflows/ai-cn1lib-native-check.yml b/.github/workflows/ai-cn1lib-native-check.yml index 929947c6328..2d41910c144 100644 --- a/.github/workflows/ai-cn1lib-native-check.yml +++ b/.github/workflows/ai-cn1lib-native-check.yml @@ -1,23 +1,23 @@ -name: AI cn1lib iOS xcodebuild check +name: Large AI cn1lib iOS xcodebuild check -# Per cn1lib: synthesise a tiny Xcode project that links the bundled -# `nativeios/*.m` against the real `GoogleMLKit/*` (or TFLite / whisper) -# pod, run `pod install`, then `xcodebuild build`. Catches API drift, -# missing imports, and missing pod hints early -- on the same macOS -# runners we already use for build-ios. +# The lightweight vision, language, and LiteRT bridges now live in core and +# are exercised by the normal port/build tests. Keep this isolated native +# check for the two deliberately large cn1libs. on: workflow_dispatch: pull_request: branches: [master] paths: - - 'maven/cn1-ai-**' + - 'maven/cn1-ai-whisper/**' + - 'maven/cn1-ai-stablediffusion/**' - 'scripts/gen-ai-cn1libs.py' - '.github/workflows/ai-cn1lib-native-check.yml' push: branches: [master] paths: - - 'maven/cn1-ai-**' + - 'maven/cn1-ai-whisper/**' + - 'maven/cn1-ai-stablediffusion/**' - 'scripts/gen-ai-cn1libs.py' jobs: @@ -32,17 +32,6 @@ jobs: fail-fast: false matrix: include: - - { lib: cn1-ai-mlkit-text, pod: 'GoogleMLKit/TextRecognition' } - - { lib: cn1-ai-mlkit-barcode, pod: 'GoogleMLKit/BarcodeScanning' } - - { lib: cn1-ai-mlkit-face, pod: 'GoogleMLKit/FaceDetection' } - - { lib: cn1-ai-mlkit-labeling, pod: 'GoogleMLKit/ImageLabeling' } - - { lib: cn1-ai-mlkit-translate, pod: 'GoogleMLKit/Translate' } - - { lib: cn1-ai-mlkit-smartreply, pod: 'GoogleMLKit/SmartReply' } - - { lib: cn1-ai-mlkit-langid, pod: 'GoogleMLKit/LanguageID' } - - { lib: cn1-ai-mlkit-pose, pod: 'GoogleMLKit/PoseDetection' } - - { lib: cn1-ai-mlkit-segmentation, pod: 'GoogleMLKit/SegmentationSelfie' } - - { lib: cn1-ai-mlkit-docscan, pod: '' } # VisionKit/CoreImage only - - { lib: cn1-ai-tflite, pod: 'TensorFlowLiteObjC' } - { lib: cn1-ai-whisper, pod: '' } # links static libwhisper.a - { lib: cn1-ai-stablediffusion, pod: '' } # links Swift runner steps: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index bf3190c7d73..a2819144e71 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -184,17 +184,6 @@ jobs: git config --global --add safe.directory "$GITHUB_WORKSPACE" all_libs=( - cn1-ai-mlkit-text - cn1-ai-mlkit-barcode - cn1-ai-mlkit-face - cn1-ai-mlkit-labeling - cn1-ai-mlkit-translate - cn1-ai-mlkit-smartreply - cn1-ai-mlkit-langid - cn1-ai-mlkit-pose - cn1-ai-mlkit-segmentation - cn1-ai-mlkit-docscan - cn1-ai-tflite cn1-ai-whisper cn1-ai-stablediffusion cn1-admob @@ -203,17 +192,6 @@ jobs: cn1-ads-mock ) ai_libs=( - cn1-ai-mlkit-text - cn1-ai-mlkit-barcode - cn1-ai-mlkit-face - cn1-ai-mlkit-labeling - cn1-ai-mlkit-translate - cn1-ai-mlkit-smartreply - cn1-ai-mlkit-langid - cn1-ai-mlkit-pose - cn1-ai-mlkit-segmentation - cn1-ai-mlkit-docscan - cn1-ai-tflite cn1-ai-whisper cn1-ai-stablediffusion ) diff --git a/CodenameOne/src/com/codename1/ai/ChatMessage.java b/CodenameOne/src/com/codename1/ai/ChatMessage.java index 5ddc222b77a..98464eee911 100644 --- a/CodenameOne/src/com/codename1/ai/ChatMessage.java +++ b/CodenameOne/src/com/codename1/ai/ChatMessage.java @@ -39,10 +39,19 @@ public final class ChatMessage { private final String name; private final String toolCallId; + /// Creates a message with no assistant tool calls or provider name. + /// @param role speaker role; required + /// @param parts ordered multimodal content; {@code null} becomes empty public ChatMessage(Role role, List parts) { this(role, parts, null, null, null); } + /// Creates a fully specified normalized message. + /// @param role speaker role; required + /// @param parts ordered content parts; {@code null} becomes empty + /// @param toolCalls assistant tool calls; {@code null} becomes empty + /// @param name optional provider-visible participant name + /// @param toolCallId tool call answered by a {@link Role#TOOL} message public ChatMessage(Role role, List parts, List toolCalls, String name, String toolCallId) { if (role == null) { @@ -57,20 +66,29 @@ public ChatMessage(Role role, List parts, List toolCalls, this.toolCallId = toolCallId; } + /// @param text system instruction text + /// @return a single-part system message public static ChatMessage system(String text) { return single(Role.SYSTEM, new TextPart(text)); } + /// @param text user input text + /// @return a single-part user message public static ChatMessage user(String text) { return single(Role.USER, new TextPart(text)); } + /// @param text assistant output text + /// @return a single-part assistant message public static ChatMessage assistant(String text) { return single(Role.ASSISTANT, new TextPart(text)); } /// Builds a USER message containing both a text and image part -- /// the common multi-modal pattern. + /// @param text optional text accompanying the image + /// @param image image content sent to the model + /// @return multimodal user message public static ChatMessage userWithImage(String text, ImagePart image) { List parts = new ArrayList(2); if (text != null && text.length() > 0) { @@ -81,6 +99,9 @@ public static ChatMessage userWithImage(String text, ImagePart image) { } /// Builds a TOOL message wrapping the result of a previous tool call. + /// @param toolCallId provider id of the answered tool call + /// @param resultJson application result encoded as JSON + /// @return tool-role result message public static ChatMessage toolResult(String toolCallId, String resultJson) { return new ChatMessage(Role.TOOL, Arrays.asList(new ToolResultPart(toolCallId, resultJson)), @@ -93,22 +114,27 @@ private static ChatMessage single(Role r, MessagePart p) { return new ChatMessage(r, parts); } + /// @return speaker role public Role getRole() { return role; } + /// @return immutable content parts in message order public List getParts() { return parts; } + /// @return immutable assistant tool calls public List getToolCalls() { return toolCalls; } + /// @return optional participant name, or {@code null} public String getName() { return name; } + /// @return answered tool-call id for tool messages, or {@code null} public String getToolCallId() { return toolCallId; } @@ -116,6 +142,7 @@ public String getToolCallId() { /// Convenience: concatenates the text of every [TextPart]. Image /// and tool-result parts are skipped. Useful for `ChatView` /// rendering when you don't care about multi-modal content. + /// @return text parts joined with newlines public String getText() { StringBuilder sb = new StringBuilder(); for (MessagePart p : parts) { diff --git a/CodenameOne/src/com/codename1/ai/ChatRequest.java b/CodenameOne/src/com/codename1/ai/ChatRequest.java index d6458e5a323..b0cc8e357fa 100644 --- a/CodenameOne/src/com/codename1/ai/ChatRequest.java +++ b/CodenameOne/src/com/codename1/ai/ChatRequest.java @@ -67,54 +67,68 @@ private ChatRequest(Builder b) { this.safetyFilter = b.safetyFilter; } + /// Starts an empty request builder. At least one message is required. + /// @return a new builder public static Builder builder() { return new Builder(); } + /// @return requested provider model, or {@code null} for the client default public String getModel() { return model; } + /// @return immutable conversation messages in provider order public List getMessages() { return messages; } + /// @return sampling temperature, or {@code null} for the provider default public Float getTemperature() { return temperature; } + /// @return maximum generated tokens, or {@code null} for the provider default public Integer getMaxTokens() { return maxTokens; } + /// @return nucleus-sampling probability, or {@code null} when unspecified public Float getTopP() { return topP; } + /// @return immutable stop sequences; empty when none were requested public List getStopSequences() { return stopSequences; } + /// @return deterministic sampling seed, or {@code null} when unspecified public Long getSeed() { return seed; } + /// @return requested text/JSON response format, or {@code null} public ResponseFormat getResponseFormat() { return responseFormat; } + /// @return immutable tools offered to the model public List getTools() { return tools; } + /// @return tool-selection policy, or {@code null} for provider behavior public ToolChoice getToolChoice() { return toolChoice; } + /// @return immutable provider metadata attached to this request public Map getMetadata() { return metadata; } + /// @return client-side preflight filter, or {@code null} when disabled public SafetyFilter getSafetyFilter() { return safetyFilter; } @@ -138,6 +152,8 @@ public Builder toBuilder() { return b; } + /// Mutable fluent builder for {@link ChatRequest}. Collection arguments + /// are copied when the immutable request is built. public static final class Builder { private String model; private List messages = new ArrayList(); @@ -155,72 +171,114 @@ public static final class Builder { Builder() { } + /// Selects a provider model. + /// @param model provider model id; {@code null} uses the client default + /// @return this builder public Builder model(String model) { this.model = model; return this; } + /// Replaces the conversation history. + /// @param messages messages in chronological order; {@code null} clears them + /// @return this builder public Builder messages(List messages) { this.messages = messages == null ? new ArrayList() : new ArrayList(messages); return this; } + /// Appends one conversation message. + /// @param m message to append + /// @return this builder public Builder addMessage(ChatMessage m) { this.messages.add(m); return this; } + /// Sets sampling temperature. + /// @param t provider-supported temperature, or {@code null} to omit + /// @return this builder public Builder temperature(Float t) { this.temperature = t; return this; } + /// Limits response length. + /// @param n maximum generated token count, or {@code null} to omit + /// @return this builder public Builder maxTokens(Integer n) { this.maxTokens = n; return this; } + /// Sets nucleus sampling probability. + /// @param p provider-supported probability, or {@code null} to omit + /// @return this builder public Builder topP(Float p) { this.topP = p; return this; } + /// Sets sequences that terminate generation. + /// @param stops stop strings, or {@code null} for none + /// @return this builder public Builder stopSequences(List stops) { this.stopSequences = stops; return this; } + /// Requests deterministic sampling where supported. + /// @param seed provider sampling seed, or {@code null} to omit + /// @return this builder public Builder seed(Long seed) { this.seed = seed; return this; } + /// Requests text or structured JSON output. + /// @param f desired format, or {@code null} for provider default + /// @return this builder public Builder responseFormat(ResponseFormat f) { this.responseFormat = f; return this; } + /// Replaces the callable tools advertised to the model. + /// @param tools tool definitions, or {@code null} for none + /// @return this builder public Builder tools(List tools) { this.tools = tools; return this; } + /// Controls whether and which tool the model may call. + /// @param choice selection policy, or {@code null} for provider default + /// @return this builder public Builder toolChoice(ToolChoice choice) { this.toolChoice = choice; return this; } + /// Attaches provider-specific request metadata. + /// @param meta metadata copied into the request, or {@code null} + /// @return this builder public Builder metadata(Map meta) { this.metadata = meta; return this; } + /// Installs a client-side filter evaluated before network submission. + /// @param f safety policy, or {@code null} to disable it + /// @return this builder public Builder safetyFilter(SafetyFilter f) { this.safetyFilter = f; return this; } + /// Validates and creates an immutable request. + /// @return completed request + /// @throws IllegalStateException when no messages were supplied public ChatRequest build() { if (messages.isEmpty()) { throw new IllegalStateException("at least one message is required"); diff --git a/CodenameOne/src/com/codename1/ai/ChatResponse.java b/CodenameOne/src/com/codename1/ai/ChatResponse.java index 53f08263b0a..a60dd24817a 100644 --- a/CodenameOne/src/com/codename1/ai/ChatResponse.java +++ b/CodenameOne/src/com/codename1/ai/ChatResponse.java @@ -40,6 +40,12 @@ public final class ChatResponse { private final Usage usage; private final String modelUsed; + /// Creates a normalized terminal response. + /// @param assistantMessage aggregated assistant output, or {@code null} + /// @param toolCalls requested tool calls; {@code null} becomes empty + /// @param finishReason normalized provider finish reason + /// @param usage token accounting, or {@code null} + /// @param modelUsed provider-reported model id, or {@code null} public ChatResponse(ChatMessage assistantMessage, List toolCalls, String finishReason, Usage usage, String modelUsed) { this.assistantMessage = assistantMessage; @@ -50,28 +56,34 @@ public ChatResponse(ChatMessage assistantMessage, List toolCalls, this.modelUsed = modelUsed; } + /// @return normalized assistant message, including any multimodal parts public ChatMessage getAssistantMessage() { return assistantMessage; } + /// @return immutable tool calls requested by the model public List getToolCalls() { return toolCalls; } /// Convenience: the assembled assistant text. Equivalent to /// `getAssistantMessage().getText()` when there is one. + /// @return assistant text, or an empty string without a message public String getText() { return assistantMessage == null ? "" : assistantMessage.getText(); } + /// @return provider finish reason such as {@code stop} or {@code length} public String getFinishReason() { return finishReason; } + /// @return token accounting, or {@code null} when the provider omitted it public Usage getUsage() { return usage; } + /// @return provider-reported model id, or {@code null} when omitted public String getModelUsed() { return modelUsed; } diff --git a/CodenameOne/src/com/codename1/ai/ConversationStore.java b/CodenameOne/src/com/codename1/ai/ConversationStore.java index a3b72d7191b..3cf8d38baa0 100644 --- a/CodenameOne/src/com/codename1/ai/ConversationStore.java +++ b/CodenameOne/src/com/codename1/ai/ConversationStore.java @@ -45,6 +45,8 @@ public final class ConversationStore { private final String storageKey; + /// Creates a store using one app-private {@link Storage} key. + /// @param storageKey non-empty key dedicated to this conversation public ConversationStore(String storageKey) { if (storageKey == null || storageKey.length() == 0) { throw new IllegalArgumentException("storageKey is required"); @@ -52,6 +54,9 @@ public ConversationStore(String storageKey) { this.storageKey = storageKey; } + /// Replaces the persisted history. + /// @param messages chronological history; {@code null} stores an empty list + /// @throws IOException when JSON encoding or storage fails public void save(List messages) throws IOException { List serialized = new ArrayList(messages == null ? 0 : messages.size()); if (messages != null) { @@ -90,6 +95,10 @@ public void save(List messages) throws IOException { Storage.getInstance().writeObject(storageKey, payload); } + /// Loads persisted history. Missing or incompatible data is treated as an + /// empty conversation so upgrades do not prevent application startup. + /// @return mutable chronological message list + /// @throws IOException when stored JSON cannot be decoded public List load() throws IOException { Object raw = Storage.getInstance().readObject(storageKey); if (raw == null) { @@ -130,10 +139,12 @@ public List load() throws IOException { return out; } + /// Deletes the persisted conversation. public void clear() { Storage.getInstance().deleteStorageFile(storageKey); } + /// @return app-private storage key used by this store public String getStorageKey() { return storageKey; } diff --git a/CodenameOne/src/com/codename1/ai/Embedding.java b/CodenameOne/src/com/codename1/ai/Embedding.java index bb49091a2b9..f1349aa6f3b 100644 --- a/CodenameOne/src/com/codename1/ai/Embedding.java +++ b/CodenameOne/src/com/codename1/ai/Embedding.java @@ -22,26 +22,44 @@ */ package com.codename1.ai; -/// A single embedding vector. `index` matches the position of the -/// corresponding input string in the original request. +/// One vector returned by an embedding provider. The vector is defensively +/// copied on construction and access so callers cannot mutate stored results. +/// {@link #getIndex()} identifies the corresponding input position in a +/// batched {@link EmbeddingRequest}. public final class Embedding { private final float[] vector; private final int index; + /// Creates an embedding result. + /// @param vector numeric embedding coordinates, defensively copied; a + /// {@code null} value creates an empty vector + /// @param index zero-based position of the corresponding request input public Embedding(float[] vector, int index) { - this.vector = vector == null ? new float[0] : vector; + this.vector = copy(vector); this.index = index; } + /// @return a defensive copy of the embedding coordinates public float[] getVector() { - return vector; + return copy(vector); } + /// @return zero-based position of this item in the request public int getIndex() { return index; } + /// @return number of coordinates in the embedding public int getDimensions() { return vector.length; } + + private static float[] copy(float[] value) { + if (value == null) { + return new float[0]; + } + float[] result = new float[value.length]; + System.arraycopy(value, 0, result, 0, value.length); + return result; + } } diff --git a/CodenameOne/src/com/codename1/ai/EmbeddingRequest.java b/CodenameOne/src/com/codename1/ai/EmbeddingRequest.java index d8cf2b7e0bf..97fcbdfe1cc 100644 --- a/CodenameOne/src/com/codename1/ai/EmbeddingRequest.java +++ b/CodenameOne/src/com/codename1/ai/EmbeddingRequest.java @@ -42,27 +42,35 @@ private EmbeddingRequest(Builder b) { this.dimensions = b.dimensions; } + /// @return a new empty embedding request builder public static Builder builder() { return new Builder(); } - /// Convenience for the single-input case. + /// Creates a single-input request. + /// @param model provider model id, or {@code null} for the client default + /// @param text text to embed + /// @return immutable embedding request public static EmbeddingRequest of(String model, String text) { return builder().model(model).inputs(Arrays.asList(text)).build(); } + /// @return requested model id, or {@code null} for the client default public String getModel() { return model; } + /// @return immutable input strings in request order public List getInputs() { return inputs; } + /// @return requested output dimensions, or {@code null} for model default public Integer getDimensions() { return dimensions; } + /// Mutable builder for an immutable {@link EmbeddingRequest}. public static final class Builder { private String model; private List inputs = new ArrayList(); @@ -71,27 +79,38 @@ public static final class Builder { Builder() { } + /// @param m provider model id, or {@code null} for client default + /// @return this builder public Builder model(String m) { this.model = m; return this; } + /// Replaces all input strings. + /// @param in strings to embed; {@code null} clears the list + /// @return this builder public Builder inputs(List in) { this.inputs = in == null ? new ArrayList() : new ArrayList(in); return this; } + /// @param s input string to append + /// @return this builder public Builder addInput(String s) { this.inputs.add(s); return this; } + /// @param d requested vector dimensions, or {@code null} for model default + /// @return this builder public Builder dimensions(Integer d) { this.dimensions = d; return this; } + /// @return validated immutable request + /// @throws IllegalStateException when no input strings were supplied public EmbeddingRequest build() { if (inputs.isEmpty()) { throw new IllegalStateException("at least one input is required"); diff --git a/CodenameOne/src/com/codename1/ai/EmbeddingResponse.java b/CodenameOne/src/com/codename1/ai/EmbeddingResponse.java index d2553a48e96..d47f0fa6108 100644 --- a/CodenameOne/src/com/codename1/ai/EmbeddingResponse.java +++ b/CodenameOne/src/com/codename1/ai/EmbeddingResponse.java @@ -26,11 +26,17 @@ import java.util.Collections; import java.util.List; +/// Immutable provider response containing embeddings, token accounting, and +/// the model name reported by the service. public final class EmbeddingResponse { private final List data; private final Usage usage; private final String modelUsed; + /// Creates a normalized embedding response. + /// @param data embedding items in provider response order; {@code null} becomes empty + /// @param usage provider token accounting, or {@code null} when omitted + /// @param modelUsed model identifier returned by the provider public EmbeddingResponse(List data, Usage usage, String modelUsed) { this.data = data == null ? Collections.emptyList() : Collections.unmodifiableList(new ArrayList(data)); @@ -38,14 +44,17 @@ public EmbeddingResponse(List data, Usage usage, String modelUsed) { this.modelUsed = modelUsed; } + /// @return immutable embedding items in request order public List getData() { return data; } + /// @return provider token accounting, or {@code null} when unavailable public Usage getUsage() { return usage; } + /// @return provider-reported model identifier, or {@code null} when omitted public String getModelUsed() { return modelUsed; } diff --git a/CodenameOne/src/com/codename1/ai/GenerateImageRequest.java b/CodenameOne/src/com/codename1/ai/GenerateImageRequest.java index 1f308e137e8..c22635c97ba 100644 --- a/CodenameOne/src/com/codename1/ai/GenerateImageRequest.java +++ b/CodenameOne/src/com/codename1/ai/GenerateImageRequest.java @@ -23,6 +23,9 @@ package com.codename1.ai; /// Request payload for [ImageGenerator#generate(GenerateImageRequest)]. +/// Mutable image-generation request shared by the supported providers. +/// Provider-specific values that are unsupported may be ignored or rejected +/// by the selected {@link ImageGenerator}. public final class GenerateImageRequest { private final String prompt; private String model; @@ -32,6 +35,8 @@ public final class GenerateImageRequest { private int count = 1; private Long seed; + /// Creates a request with required prompt text. + /// @param prompt description of the image to generate public GenerateImageRequest(String prompt) { if (prompt == null || prompt.length() == 0) { throw new IllegalArgumentException("prompt is required"); @@ -39,19 +44,24 @@ public GenerateImageRequest(String prompt) { this.prompt = prompt; } + /// @return image description sent to the provider public String getPrompt() { return prompt; } + /// @return requested model id, or {@code null} for generator default public String getModel() { return model; } + /// @param model provider model id, or {@code null} for generator default + /// @return this request public GenerateImageRequest setModel(String model) { this.model = model; return this; } + /// @return requested dimensions such as {@code 1024x1024} public String getSize() { return size; } @@ -63,24 +73,31 @@ public GenerateImageRequest setSize(String size) { return this; } + /// @return provider style value, or {@code null} when unspecified public String getStyle() { return style; } + /// @param style provider-specific rendering style, or {@code null} + /// @return this request public GenerateImageRequest setStyle(String style) { this.style = style; return this; } + /// @return provider quality tier, or {@code null} when unspecified public String getQuality() { return quality; } + /// @param quality provider-specific quality tier, or {@code null} + /// @return this request public GenerateImageRequest setQuality(String quality) { this.quality = quality; return this; } + /// @return requested image count public int getCount() { return count; } @@ -92,10 +109,13 @@ public GenerateImageRequest setCount(int count) { return this; } + /// @return deterministic seed, or {@code null} when unspecified public Long getSeed() { return seed; } + /// @param seed deterministic seed where supported, or {@code null} + /// @return this request public GenerateImageRequest setSeed(Long seed) { this.seed = seed; return this; diff --git a/CodenameOne/src/com/codename1/ai/ImageGenerator.java b/CodenameOne/src/com/codename1/ai/ImageGenerator.java index dc0e6fc8418..e78eceed084 100644 --- a/CodenameOne/src/com/codename1/ai/ImageGenerator.java +++ b/CodenameOne/src/com/codename1/ai/ImageGenerator.java @@ -50,6 +50,9 @@ /// ``` public abstract class ImageGenerator { + /// Creates an OpenAI image generator. + /// @param apiKey OpenAI API key + /// @return configured generator public static ImageGenerator openai(String apiKey) { return new OpenAiImageGenerator(apiKey); } @@ -57,6 +60,8 @@ public static ImageGenerator openai(String apiKey) { /// Replicate runs a wide catalog of third-party image models /// (SDXL, Flux, etc.) behind a uniform REST API. Pass the API /// token from `https://replicate.com/account`. + /// @param apiKey Replicate API token + /// @return configured generator public static ImageGenerator replicate(String apiKey) { return new ReplicateImageGenerator(apiKey); } @@ -65,6 +70,7 @@ public static ImageGenerator replicate(String apiKey) { /// `cn1-ai-stablediffusion`; without it this returns an /// `AsyncResource` that completes with /// `UnsupportedOperationException`. + /// @return optional on-device generator or an unsupported stub public static ImageGenerator onDevice() { // Lazy lookup so app code can compile even without the // cn1lib. The cn1lib registers an implementation via @@ -82,6 +88,9 @@ public AsyncResource generate(GenerateImageRequest req) { }; } + /// Generates an image without blocking the EDT. + /// @param req provider-neutral generation request + /// @return asynchronous generated image public abstract AsyncResource generate(GenerateImageRequest req); // --------------------- OpenAI --------------------- diff --git a/CodenameOne/src/com/codename1/ai/ImagePart.java b/CodenameOne/src/com/codename1/ai/ImagePart.java index df8a81e0426..dc50680ed0a 100644 --- a/CodenameOne/src/com/codename1/ai/ImagePart.java +++ b/CodenameOne/src/com/codename1/ai/ImagePart.java @@ -22,27 +22,33 @@ */ package com.codename1.ai; -/// An image attachment for a multi-modal [ChatMessage]. Construct from -/// raw bytes (the provider encodes them as base64 inline data) or from -/// a publicly-reachable URL -- both modes are accepted by OpenAI, -/// Anthropic, and Gemini. +/// Image content within a multimodal {@link ChatMessage}. An image is either +/// inline encoded bytes with a MIME type or a provider-accessible URL. Inline +/// bytes are copied on construction and access so later caller mutations +/// cannot change a request that already contains this part. public final class ImagePart extends MessagePart { private final byte[] data; private final String mimeType; private final String url; - /// Inline image bytes. `mimeType` must be set (e.g. `"image/png"`, - /// `"image/jpeg"`); the providers reject inline images without it. + /// Creates an inline image part. The media type must describe the encoded + /// bytes, for example {@code image/png} or {@code image/jpeg}; providers + /// reject inline images without a type. + /// + /// @param data encoded image bytes, defensively copied + /// @param mimeType media type such as {@code image/png} public ImagePart(byte[] data, String mimeType) { if (data == null || mimeType == null) { throw new IllegalArgumentException("data and mimeType are required"); } - this.data = data; + this.data = copy(data); this.mimeType = mimeType; this.url = null; } /// Remote image by URL. Only HTTPS is portable across providers. + /// Creates a remotely addressed image part. + /// @param url provider-accessible HTTPS or data URL public ImagePart(String url) { if (url == null) { throw new IllegalArgumentException("url is required"); @@ -52,19 +58,32 @@ public ImagePart(String url) { this.url = url; } + /// @return a defensive copy of inline bytes, or {@code null} for a URL image public byte[] getData() { - return data; + return copy(data); } + /// @return MIME type of inline bytes, or {@code null} for a URL image public String getMimeType() { return mimeType; } + /// @return provider-accessible URL, or {@code null} for an inline image public String getUrl() { return url; } + /// @return {@code true} when this part references a URL instead of bytes public boolean isUrl() { return url != null; } + + private static byte[] copy(byte[] value) { + if (value == null) { + return null; + } + byte[] result = new byte[value.length]; + System.arraycopy(value, 0, result, 0, value.length); + return result; + } } diff --git a/CodenameOne/src/com/codename1/ai/LlmChatBinding.java b/CodenameOne/src/com/codename1/ai/LlmChatBinding.java index b4618cc219f..932219ba686 100644 --- a/CodenameOne/src/com/codename1/ai/LlmChatBinding.java +++ b/CodenameOne/src/com/codename1/ai/LlmChatBinding.java @@ -69,6 +69,12 @@ public final class LlmChatBinding { private LlmChatBinding() { } + /// Connects a chat view to a streaming client. Each submitted user message + /// replays the current view history while retaining the base request's + /// model, sampling, tool, metadata, and safety settings. + /// @param view chat UI to drive + /// @param client provider client used for every turn + /// @param baseRequest request template and optional initial messages public static void bind(final ChatView view, final LlmClient client, final ChatRequest baseRequest) { diff --git a/CodenameOne/src/com/codename1/ai/LlmClient.java b/CodenameOne/src/com/codename1/ai/LlmClient.java index fea905929a4..4bf5f1dec79 100644 --- a/CodenameOne/src/com/codename1/ai/LlmClient.java +++ b/CodenameOne/src/com/codename1/ai/LlmClient.java @@ -80,28 +80,44 @@ protected LlmClient(String baseUrl) { /// OpenAI / OpenAI-compatible (Together, Groq, Fireworks, vLLM, /// Ollama, etc.). Uses the public endpoint by default; override /// with [#setBaseUrl(String)]. + /// @param apiKey provider API key + /// @return configured client public static LlmClient openai(String apiKey) { return SimulatorRedirect.maybeWrap(new OpenAiClient(apiKey, DEFAULT_OPENAI_URL)); } + /// Creates a client for Anthropic's Messages API. + /// @param apiKey Anthropic API key + /// @return configured client public static LlmClient anthropic(String apiKey) { return SimulatorRedirect.maybeWrap(new AnthropicClient(apiKey, DEFAULT_ANTHROPIC_URL)); } + /// Creates a client for Gemini's OpenAI-compatible endpoint. + /// @param apiKey Google AI API key + /// @return configured client public static LlmClient gemini(String apiKey) { return SimulatorRedirect.maybeWrap(new GeminiClient(apiKey, DEFAULT_GEMINI_URL)); } /// Default Ollama install: `http://localhost:11434/v1`, model /// `llama3.2`. + /// @return local Ollama client public static LlmClient ollama() { return ollama("llama3.2"); } + /// Creates a client for the default local Ollama endpoint. + /// @param defaultModel model used when requests omit one + /// @return local Ollama client public static LlmClient ollama(String defaultModel) { return ollama(DEFAULT_OLLAMA_URL, defaultModel); } + /// Creates an Ollama client for a custom endpoint. + /// @param baseUrl OpenAI-compatible endpoint root + /// @param defaultModel model used when requests omit one + /// @return configured Ollama client public static LlmClient ollama(String baseUrl, String defaultModel) { OpenAiClient c = new OpenAiClient("ollama", baseUrl); c.setDefaultModel(defaultModel); @@ -111,6 +127,10 @@ public static LlmClient ollama(String baseUrl, String defaultModel) { /// Generic OpenAI-compatible endpoint (llama.cpp server, vLLM, /// LM Studio, a custom proxy). `apiKey` may be empty for local /// services that don't authenticate. + /// @param baseUrl OpenAI-compatible endpoint root + /// @param apiKey API key, or empty for an unauthenticated service + /// @param defaultModel model used when requests omit one + /// @return configured compatible client public static LlmClient localOpenAiCompatible(String baseUrl, String apiKey, String defaultModel) { OpenAiClient c = new OpenAiClient(apiKey == null ? "" : apiKey, baseUrl); c.setDefaultModel(defaultModel); @@ -120,33 +140,48 @@ public static LlmClient localOpenAiCompatible(String baseUrl, String apiKey, Str /// Non-streaming chat. Equivalent to `chatStream` with a no-op /// listener but optimized -- the provider skips the SSE response /// and returns a single JSON object. + /// @param req immutable chat request + /// @return asynchronous normalized provider response public abstract AsyncResource chat(ChatRequest req); /// Streaming chat. `listener` fires for every content delta / /// tool-call fragment on the EDT. The returned `AsyncResource` /// completes with the aggregated final response once the stream /// ends; cancel it to close the underlying socket. + /// @param req immutable chat request + /// @param listener incremental callbacks delivered on the EDT + /// @return asynchronous aggregated response public abstract AsyncResource chatStream(ChatRequest req, StreamingListener listener); + /// Creates embeddings for one or more strings. + /// @param req immutable embedding request + /// @return asynchronous normalized embedding response public abstract AsyncResource embed(EmbeddingRequest req); /// One of `"openai"`, `"anthropic"`, `"gemini"`, `"ollama"`, /// `"local"`. Used by `ChatView` and tests to vary behaviour by /// provider. + /// @return stable provider family id public abstract String getProvider(); + /// @return current provider endpoint root public String getBaseUrl() { return baseUrl; } + /// Overrides the provider endpoint for proxies or compatible services. + /// @param baseUrl endpoint root used by subsequent calls public void setBaseUrl(String baseUrl) { this.baseUrl = baseUrl; } + /// @return network timeout in milliseconds public int getHttpTimeoutMs() { return httpTimeoutMs; } + /// Sets the network timeout for subsequent calls. + /// @param httpTimeoutMs timeout in milliseconds public void setHttpTimeoutMs(int httpTimeoutMs) { this.httpTimeoutMs = httpTimeoutMs; } diff --git a/CodenameOne/src/com/codename1/ai/LlmException.java b/CodenameOne/src/com/codename1/ai/LlmException.java index 6caf452f400..c4f5994981c 100644 --- a/CodenameOne/src/com/codename1/ai/LlmException.java +++ b/CodenameOne/src/com/codename1/ai/LlmException.java @@ -100,19 +100,39 @@ public enum ErrorType { private final ErrorType type; private final int retryAfterSeconds; + /// Creates an unclassified provider failure. + /// @param message user-readable failure description public LlmException(String message) { this(message, -1, null, null, null, ErrorType.UNKNOWN, -1); } + /// Creates an unclassified failure with its underlying cause. + /// @param message user-readable failure description + /// @param cause network, parsing, or provider cause public LlmException(String message, Throwable cause) { this(message, -1, null, null, cause, ErrorType.UNKNOWN, -1); } + /// Creates a classified provider failure without retry timing. + /// @param message user-readable failure description + /// @param httpStatus HTTP status, or {@code -1} without a response + /// @param providerErrorCode provider-specific machine code + /// @param rawBody raw response body retained for diagnostics + /// @param cause originating error, or {@code null} + /// @param type portable failure classification public LlmException(String message, int httpStatus, String providerErrorCode, String rawBody, Throwable cause, ErrorType type) { this(message, httpStatus, providerErrorCode, rawBody, cause, type, -1); } + /// Creates a fully classified provider failure. + /// @param message user-readable failure description + /// @param httpStatus HTTP status, or {@code -1} without a response + /// @param providerErrorCode provider-specific machine code + /// @param rawBody raw response body retained for diagnostics + /// @param cause originating error, or {@code null} + /// @param type portable failure classification + /// @param retryAfterSeconds server-requested delay, or {@code -1} public LlmException(String message, int httpStatus, String providerErrorCode, String rawBody, Throwable cause, ErrorType type, int retryAfterSeconds) { @@ -134,14 +154,17 @@ public ErrorType getType() { return type; } + /// @return HTTP status, or {@code -1} when no response was received public int getHttpStatus() { return httpStatus; } + /// @return provider-specific error code, or {@code null} public String getProviderErrorCode() { return providerErrorCode; } + /// @return raw response body for diagnostics, or {@code null} public String getRawBody() { return rawBody; } diff --git a/CodenameOne/src/com/codename1/ai/PromptTemplate.java b/CodenameOne/src/com/codename1/ai/PromptTemplate.java index 7bfb090dfa6..6769d42f5fc 100644 --- a/CodenameOne/src/com/codename1/ai/PromptTemplate.java +++ b/CodenameOne/src/com/codename1/ai/PromptTemplate.java @@ -45,15 +45,25 @@ private PromptTemplate(String template) { this.template = template; } + /// Starts a template instance. + /// @param template text containing optional {@code {name}} placeholders + /// @return mutable template values bound to the supplied text public static PromptTemplate of(String template) { return new PromptTemplate(template); } + /// Binds one placeholder. A {@code null} value renders as an empty string. + /// @param key placeholder name without braces + /// @param value replacement value + /// @return this template public PromptTemplate put(String key, String value) { values.put(key, value == null ? "" : value); return this; } + /// Adds all supplied placeholder values. + /// @param map replacements keyed by placeholder name; {@code null} is ignored + /// @return this template public PromptTemplate putAll(Map map) { if (map != null) { values.putAll(map); @@ -64,6 +74,7 @@ public PromptTemplate putAll(Map map) { /// Renders the final string. Unknown placeholders are left /// intact (`{like_this}`) so they're easy to spot in test /// output -- silently dropping them tends to hide bugs. + /// @return rendered prompt text public String build() { StringBuilder out = new StringBuilder(template.length() + 32); int i = 0; @@ -88,11 +99,13 @@ public String build() { } /// Convenience: render and wrap as a [ChatMessage] with USER role. + /// @return rendered user message public ChatMessage asUser() { return ChatMessage.user(build()); } /// Convenience: render and wrap as a [ChatMessage] with SYSTEM role. + /// @return rendered system message public ChatMessage asSystem() { return ChatMessage.system(build()); } diff --git a/CodenameOne/src/com/codename1/ai/RetryPolicy.java b/CodenameOne/src/com/codename1/ai/RetryPolicy.java index 00beb3fd0ca..cbfef885cba 100644 --- a/CodenameOne/src/com/codename1/ai/RetryPolicy.java +++ b/CodenameOne/src/com/codename1/ai/RetryPolicy.java @@ -60,23 +60,34 @@ private RetryPolicy(int maxAttempts, long initialDelayMs, long maxDelayMs, /// 4 attempts, starting at 500 ms, doubling, capped at 30 s, with /// jitter. Good default for chat workloads. + /// @return recommended transient-failure policy public static RetryPolicy exponentialBackoff() { return new RetryPolicy(4, 500L, 30000L, 2.0, true); } /// No retries -- failures are returned to the caller as-is. + /// @return single-attempt policy public static RetryPolicy none() { return new RetryPolicy(1, 0L, 0L, 1.0, false); } + /// Creates a bounded exponential retry policy. + /// @param maxAttempts total attempts including the initial call + /// @param initialDelayMs delay before the first retry + /// @param maxDelayMs upper bound for computed delays + /// @param multiplier growth factor applied after each attempt + /// @param jitter whether to randomize each delay from zero to its bound + /// @return normalized retry policy public static RetryPolicy custom(int maxAttempts, long initialDelayMs, long maxDelayMs, double multiplier, boolean jitter) { return new RetryPolicy(maxAttempts, initialDelayMs, maxDelayMs, multiplier, jitter); } - /// Inspect a thrown exception and decide whether to retry. Apps - /// can override to add provider-specific rules (e.g. retry on a - /// custom 5xx code). + /// Inspects a thrown exception and decides whether this policy permits + /// another attempt. + /// @param t operation failure + /// @param attemptsSoFar attempts already made, including the failed one + /// @return {@code true} for a transient failure within the attempt limit public boolean shouldRetry(Throwable t, int attemptsSoFar) { if (attemptsSoFar >= maxAttempts) { return false; @@ -96,6 +107,9 @@ public boolean shouldRetry(Throwable t, int attemptsSoFar) { /// Returns the delay to wait before the next attempt, honouring /// `Retry-After` from rate-limit exceptions when present. + /// @param t failure that triggered the retry + /// @param attemptIndex zero-based retry index + /// @return delay in milliseconds public long computeDelayMs(Throwable t, int attemptIndex /* 0-based */) { if (t instanceof LlmException && ((LlmException) t).getType() == LlmException.ErrorType.RATE_LIMIT) { @@ -120,6 +134,7 @@ public long computeDelayMs(Throwable t, int attemptIndex /* 0-based */) { return (long) delay; } + /// @return total allowed attempts including the initial call public int getMaxAttempts() { return maxAttempts; } diff --git a/CodenameOne/src/com/codename1/ai/SafetyFilter.java b/CodenameOne/src/com/codename1/ai/SafetyFilter.java index 00ce689ec35..17404b682ca 100644 --- a/CodenameOne/src/com/codename1/ai/SafetyFilter.java +++ b/CodenameOne/src/com/codename1/ai/SafetyFilter.java @@ -31,6 +31,8 @@ public interface SafetyFilter { /// Returns `null` to allow the call, or a human-readable reason /// string to block it. + /// @param messages immutable request conversation + /// @return {@code null} to allow submission, otherwise the rejection reason String check(List messages); /// A built-in filter that allows everything. Useful as a default diff --git a/CodenameOne/src/com/codename1/ai/StreamingListener.java b/CodenameOne/src/com/codename1/ai/StreamingListener.java index 5bb48067cf3..bb26acef231 100644 --- a/CodenameOne/src/com/codename1/ai/StreamingListener.java +++ b/CodenameOne/src/com/codename1/ai/StreamingListener.java @@ -35,6 +35,7 @@ public interface StreamingListener { /// A chunk of assistant text. Append it to whatever text buffer /// you're rendering. + /// @param textDelta next ordered text fragment void onContentDelta(String textDelta); /// A tool-call fragment. `index` lets you correlate fragments @@ -44,16 +45,22 @@ public interface StreamingListener { /// JSON; concatenate fragments for the same `index` to reassemble. /// `id` is the provider's tool-call id, present on the first /// fragment. + /// @param index zero-based tool call within the response + /// @param id provider tool-call id, normally present on the first fragment + /// @param name requested tool name, normally present on the first fragment + /// @param argumentsFragment next JSON argument fragment void onToolCallDelta(int index, String id, String name, String argumentsFragment); /// Token-accounting update. Most providers send this once at the /// end; some send incremental counts. + /// @param usage latest provider token counts void onUsage(Usage usage); /// Mid-stream error (e.g. connection reset). The `AsyncResource` /// returned by `chatStream` will also complete with this same /// exception, so listeners can typically ignore this and react to /// the resource. Implemented for parity with other SDKs. + /// @param t stream failure also delivered by the returned resource void onError(Throwable t); /// No-op default implementation. Subclass and override only what diff --git a/CodenameOne/src/com/codename1/ai/TextPart.java b/CodenameOne/src/com/codename1/ai/TextPart.java index 45ee958d1f7..9eb9a4d1a57 100644 --- a/CodenameOne/src/com/codename1/ai/TextPart.java +++ b/CodenameOne/src/com/codename1/ai/TextPart.java @@ -23,13 +23,17 @@ package com.codename1.ai; /// A plain-text fragment of a [ChatMessage]. +/// Text content within a multimodal {@link ChatMessage}. public final class TextPart extends MessagePart { private final String text; + /// Creates a text part. + /// @param text text sent to or received from the model public TextPart(String text) { this.text = text == null ? "" : text; } + /// @return text carried by this part public String getText() { return text; } diff --git a/CodenameOne/src/com/codename1/ai/Tool.java b/CodenameOne/src/com/codename1/ai/Tool.java index 720aae3d669..7d003b367db 100644 --- a/CodenameOne/src/com/codename1/ai/Tool.java +++ b/CodenameOne/src/com/codename1/ai/Tool.java @@ -65,10 +65,19 @@ public final class Tool { private final String parametersJsonSchema; private final ToolHandler handler; + /// Creates a description-only tool for manual dispatch. + /// @param name provider-visible function name + /// @param description guidance that helps the model choose the tool + /// @param parametersJsonSchema JSON Schema for accepted arguments public Tool(String name, String description, String parametersJsonSchema) { this(name, description, parametersJsonSchema, null); } + /// Creates a tool definition with an application executor. + /// @param name provider-visible function name + /// @param description guidance that helps the model choose the tool + /// @param parametersJsonSchema JSON Schema for accepted arguments + /// @param handler executor called by {@link #invoke(String)}, or {@code null} public Tool(String name, String description, String parametersJsonSchema, ToolHandler handler) { if (name == null || name.length() == 0) { @@ -82,26 +91,33 @@ public Tool(String name, String description, String parametersJsonSchema, this.handler = handler; } + /// @return provider-visible function name public String getName() { return name; } + /// @return description used by the model to decide when to call the tool public String getDescription() { return description; } + /// @return JSON Schema describing accepted function arguments public String getParametersJsonSchema() { return parametersJsonSchema; } /// The optional executor wired up by the constructor. Returns /// `null` for description-only tools. + /// @return registered executor, or {@code null} public ToolHandler getHandler() { return handler; } /// Invokes the handler with the given arguments JSON. Throws /// `IllegalStateException` when no handler was registered. + /// @param argumentsJson model-generated arguments + /// @return handler result encoded as JSON + /// @throws Exception when the handler rejects or cannot execute the call public String invoke(String argumentsJson) throws Exception { if (handler == null) { throw new IllegalStateException( diff --git a/CodenameOne/src/com/codename1/ai/ToolCall.java b/CodenameOne/src/com/codename1/ai/ToolCall.java index 5fddfa4394d..87038a8af25 100644 --- a/CodenameOne/src/com/codename1/ai/ToolCall.java +++ b/CodenameOne/src/com/codename1/ai/ToolCall.java @@ -38,20 +38,27 @@ public final class ToolCall { private final String name; private final String argumentsJson; + /// Creates a normalized model tool call. + /// @param id provider id used to associate the result + /// @param name requested tool name + /// @param argumentsJson model-generated JSON arguments; {@code null} becomes {@code {}} public ToolCall(String id, String name, String argumentsJson) { this.id = id; this.name = name; this.argumentsJson = argumentsJson == null ? "{}" : argumentsJson; } + /// @return provider id used to associate the eventual tool result public String getId() { return id; } + /// @return requested tool name public String getName() { return name; } + /// @return model-generated arguments encoded as JSON public String getArgumentsJson() { return argumentsJson; } @@ -65,6 +72,9 @@ public String getArgumentsJson() { /// Throws `IllegalArgumentException` when no tool in `tools` /// has a matching name, or `IllegalStateException` when the /// matching tool has no handler registered. + /// @param tools definitions offered with the request + /// @return handler result encoded as JSON + /// @throws Exception when lookup or application execution fails public String execute(List tools) throws Exception { Tool match = findTool(tools); if (match == null) { @@ -79,6 +89,8 @@ public String execute(List tools) throws Exception { /// Looks up the matching [Tool] without invoking it. Useful when /// the caller wants to dispatch by hand but still benefit from /// the name-matching plumbing. + /// @param tools candidate definitions, or {@code null} + /// @return matching definition, or {@code null} public Tool findTool(List tools) { if (tools == null) { return null; diff --git a/CodenameOne/src/com/codename1/ai/ToolChoice.java b/CodenameOne/src/com/codename1/ai/ToolChoice.java index 018e17c4546..0b705e757c2 100644 --- a/CodenameOne/src/com/codename1/ai/ToolChoice.java +++ b/CodenameOne/src/com/codename1/ai/ToolChoice.java @@ -52,10 +52,13 @@ public static ToolChoice named(String toolName) { return new ToolChoice("named", toolName); } + /// @return portable choice mode: {@code auto}, {@code none}, + /// {@code required}, or {@code named} public String getMode() { return mode; } + /// @return forced tool name for a named choice, otherwise {@code null} public String getForcedToolName() { return forcedToolName; } diff --git a/CodenameOne/src/com/codename1/ai/ToolResultPart.java b/CodenameOne/src/com/codename1/ai/ToolResultPart.java index 0ea7ea02a73..5b4cfff94a5 100644 --- a/CodenameOne/src/com/codename1/ai/ToolResultPart.java +++ b/CodenameOne/src/com/codename1/ai/ToolResultPart.java @@ -25,6 +25,8 @@ /// The result of a tool invocation, sent back to the model so it can /// continue reasoning. Pairs with the originating [ToolCall] via the /// `toolCallId`. The carrying [ChatMessage] should use [Role#TOOL]. +/// Result content sent back to a model after executing a requested tool call. +/// The result is JSON text so providers receive the original structured value. public final class ToolResultPart extends MessagePart { private final String toolCallId; private final String resultJson; @@ -32,15 +34,20 @@ public final class ToolResultPart extends MessagePart { /// `resultJson` is the literal JSON string the tool produced. If /// the tool result isn't valid JSON, wrap it like /// `"{\"text\":\"...\"}"` -- the providers expect JSON-shaped values. + /// Creates a tool result part. + /// @param toolCallId id of the {@link ToolCall} being answered + /// @param resultJson JSON value returned by the application tool public ToolResultPart(String toolCallId, String resultJson) { this.toolCallId = toolCallId; this.resultJson = resultJson == null ? "" : resultJson; } + /// @return provider tool-call id this result answers public String getToolCallId() { return toolCallId; } + /// @return application result encoded as JSON public String getResultJson() { return resultJson; } diff --git a/CodenameOne/src/com/codename1/ai/Usage.java b/CodenameOne/src/com/codename1/ai/Usage.java index e7cf089ea1a..96a3f0802fb 100644 --- a/CodenameOne/src/com/codename1/ai/Usage.java +++ b/CodenameOne/src/com/codename1/ai/Usage.java @@ -24,25 +24,35 @@ /// Token accounting returned by the provider. Any field that the /// provider didn't return is `-1`. +/// Token counts reported for one provider operation. Providers may count +/// tokens differently, so use these values for billing and diagnostics rather +/// than comparing tokenizers across services. public final class Usage { private final int promptTokens; private final int completionTokens; private final int totalTokens; + /// Creates a usage record from provider counters. + /// @param promptTokens tokens consumed by request input + /// @param completionTokens tokens generated in the response + /// @param totalTokens provider-reported total token count public Usage(int promptTokens, int completionTokens, int totalTokens) { this.promptTokens = promptTokens; this.completionTokens = completionTokens; this.totalTokens = totalTokens; } + /// @return tokens consumed by request input public int getPromptTokens() { return promptTokens; } + /// @return tokens generated in the response public int getCompletionTokens() { return completionTokens; } + /// @return provider-reported total token count public int getTotalTokens() { return totalTokens; } diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceException.java b/CodenameOne/src/com/codename1/ai/inference/InferenceException.java new file mode 100644 index 00000000000..959059567ba --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceException.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.inference; + +/// Reports model validation, delegate selection, allocation, or invocation +/// failures from the portable inference API. +public class InferenceException extends RuntimeException { + /// Creates an inference failure with a user-readable explanation. + /// @param message failure explanation + public InferenceException(String message) { + super(message); + } + + /// Creates an inference failure that preserves the native/port cause. + /// @param message failure explanation + /// @param cause originating error + public InferenceException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java new file mode 100644 index 00000000000..00d0485d2fa --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceOptions.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.inference; + +/// Configures how a reusable {@link InferenceSession} is created. +/// +/// Accelerator requests are portable preferences rather than promises. +/// With fallback enabled, a backend may execute on CPU when the requested +/// delegate is unavailable. With fallback disabled, opening the session +/// fails instead of silently changing the execution target. Android's NNAPI +/// runtime can mix NPU and CPU operations without reporting full delegation, +/// and the iOS Core ML delegate can schedule work across the Neural Engine, +/// GPU, and CPU. Both mobile backends therefore reject +/// {@link Accelerator#NPU} when fallback is disabled. iOS also rejects strict +/// {@link Accelerator#CORE_ML} sessions because the delegate does not report +/// whether unsupported model operations remained on LiteRT's CPU path. +public final class InferenceOptions { + /// Execution targets understood by the portable inference API. + public enum Accelerator { + AUTO, CPU, GPU, NPU, CORE_ML + } + + private Accelerator accelerator = Accelerator.AUTO; + private int threads; + private boolean allowFallback = true; + + /// Requests an execution target for the model. + /// + /// @param value requested target; {@code null} restores {@link Accelerator#AUTO} + /// @return this options object + public InferenceOptions accelerator(Accelerator value) { + accelerator = value == null ? Accelerator.AUTO : value; + return this; + } + + /// Sets the CPU worker count. Non-positive values let the native runtime + /// choose its default. + /// + /// @param value requested worker count + /// @return this options object + public InferenceOptions threads(int value) { + threads = Math.max(0, value); + return this; + } + + /// Controls whether opening may fall back from an unavailable accelerator + /// to CPU execution. + /// + /// On Android and iOS, setting this to {@code false} with + /// {@link Accelerator#NPU} rejects session creation. On iOS it also + /// rejects {@link Accelerator#CORE_ML}. Neither LiteRT's NNAPI delegate + /// nor its Core ML delegate can prove that every operation ran on the + /// requested accelerator instead of CPU or another processor. + /// + /// @param value {@code true} to permit CPU fallback + /// @return this options object + public InferenceOptions allowFallback(boolean value) { + allowFallback = value; + return this; + } + + /// @return the requested accelerator, never {@code null} + public Accelerator getAccelerator() { + return accelerator; + } + + /// @return the requested CPU worker count, or a non-positive runtime default + public int getThreads() { + return threads; + } + + /// @return whether an unavailable accelerator may fall back to CPU + public boolean isFallbackAllowed() { + return allowFallback; + } + + InferenceOptions snapshot() { + return new InferenceOptions() + .accelerator(accelerator) + .threads(threads) + .allowFallback(allowFallback); + } +} diff --git a/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java new file mode 100644 index 00000000000..33406176504 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/InferenceSession.java @@ -0,0 +1,391 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.inference; + +import com.codename1.impl.InferenceImpl; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; + +/// Reusable, native on-device session for a TensorFlow Lite model. +/// +/// Opening and execution are asynchronous because model allocation and +/// delegates can be expensive. Metadata and resize operations are synchronous. +/// A session is not usable after {@link #close()}; applications should retain +/// and reuse one session instead of reopening the model for every input. +public final class InferenceSession implements AutoCloseable { + private final InferenceImpl implementation; + private final Object handle; + private boolean closed; + private int activeRuns; + private boolean closePending; + + private InferenceSession(InferenceImpl implementation, Object handle) { + this.implementation = implementation; + this.handle = handle; + } + + /// Tests whether the current port includes a native LiteRT runtime. + /// + /// @return {@code true} when sessions can be opened on this target + public static boolean isSupported() { + InferenceImpl impl = Display.getInstance().getInferenceBackend(); + return impl != null && impl.isSupported(); + } + + /// Opens and allocates a model session off the EDT. + /// + /// The option values are copied before asynchronous backend work is + /// scheduled. Reusing or changing the supplied {@link InferenceOptions} + /// after this method returns therefore cannot alter the pending open. + /// Canceling the returned resource prevents session publication; if the + /// native backend finishes opening afterward, its handle is closed + /// automatically. + /// + /// @param source bytes, resource, or file containing a `.tflite` model + /// @param options execution options; {@code null} uses defaults + /// @return an asynchronous session, failed with {@link InferenceException} + /// when the model or requested accelerator cannot be used + public static AsyncResource open(ModelSource source, + InferenceOptions options) { + if (source == null) { + throw new NullPointerException("source"); + } + final SessionOpenResource out = new SessionOpenResource(); + final InferenceImpl impl = Display.getInstance().getInferenceBackend(); + if (impl == null || !impl.isSupported()) { + out.error(new InferenceException("LiteRT inference is not supported")); + return out; + } + InferenceOptions requested = options == null + ? new InferenceOptions() : options; + impl.open(source, requested.snapshot()) + .ready(new SuccessCallback() { + @Override + public void onSucess(Object value) { + out.publish(impl, value); + } + }).except(new SuccessCallback() { + @Override + public void onSucess(Throwable error) { + out.fail(error); + } + }); + return out; + } + + private static final class SessionOpenResource + extends AsyncResource { + @Override + public synchronized boolean cancel(boolean mayInterruptIfRunning) { + return super.cancel(mayInterruptIfRunning); + } + + synchronized void publish(InferenceImpl implementation, + Object nativeHandle) { + if (isCancelled()) { + implementation.close(nativeHandle); + return; + } + complete(new InferenceSession(implementation, nativeHandle)); + } + + synchronized void fail(Throwable error) { + if (!isCancelled()) { + error(error); + } + } + } + + /// Returns the model's current input metadata. Shapes reflect the most + /// recent successful {@link #resizeInput(String, int[])} call. Metadata + /// cannot be queried while {@link #run(Tensor[])} is pending because the + /// native interpreter is mutable and may be updating tensor state. + /// + /// @return a defensive copy of the input metadata array + /// @throws IllegalStateException if the session is closed or a run is pending + public TensorInfo[] getInputs() { + synchronized (this) { + ensureOpen(); + ensureNotRunning("read input metadata"); + return copyMetadata(implementation.getInputs(handle)); + } + } + + /// Returns the model's current output metadata. Backends refresh this + /// information after an invocation so models with dynamically resolved + /// output dimensions report the shape used to decode the returned tensor. + /// Metadata cannot be queried while {@link #run(Tensor[])} is pending + /// because the native interpreter is mutable and may be updating tensor + /// state. + /// + /// @return a defensive copy of the output metadata array + /// @throws IllegalStateException if the session is closed or a run is pending + public TensorInfo[] getOutputs() { + synchronized (this) { + ensureOpen(); + ensureNotRunning("read output metadata"); + return copyMetadata(implementation.getOutputs(handle)); + } + } + + private static TensorInfo[] copyMetadata(TensorInfo[] metadata) { + if (metadata == null || metadata.length == 0) { + return new TensorInfo[0]; + } + TensorInfo[] copy = new TensorInfo[metadata.length]; + System.arraycopy(metadata, 0, copy, 0, metadata.length); + return copy; + } + + /// Copies input tensors to native memory, invokes the model, and returns + /// every output tensor. Named tensors are matched by name; unnamed tensors + /// are matched by position. Calling {@link #close()} while this operation + /// is pending prevents new work immediately but defers native release until + /// the returned resource succeeds or fails. A session accepts one run at a + /// time because the underlying native interpreter is mutable. The array + /// container is defensively copied before it is handed to the asynchronous + /// backend, so replacing an element after this method returns cannot change + /// the pending invocation. Each {@link Tensor} is itself immutable. + /// Canceling the returned resource suppresses result publication but does + /// not interrupt an invocation that has already entered LiteRT. The + /// session remains busy, and a pending {@link #close()} remains deferred, + /// until the native operation actually succeeds or fails. + /// + /// @param inputs one tensor for each model input; {@code null} is treated + /// as an empty input array + /// @return asynchronous output tensors in model output order + /// @throws IllegalStateException if the session is closed or already running + /// @throws IllegalArgumentException if an input name, count, or shape does + /// not match the model's current input metadata + public AsyncResource run(Tensor[] inputs) { + final AsyncResource backendResult; + synchronized (this) { + ensureOpen(); + if (activeRuns > 0) { + throw new IllegalStateException( + "Inference session is already running"); + } + activeRuns++; + try { + Tensor[] inputSnapshot = inputs == null + ? new Tensor[0] : copyInputs(inputs); + validateInputShapes(inputSnapshot, + implementation.getInputs(handle)); + backendResult = implementation.run(handle, + inputSnapshot); + if (backendResult == null) { + throw new InferenceException( + "Inference backend returned no asynchronous result"); + } + } catch (RuntimeException error) { + activeRuns--; + throw error; + } catch (Error error) { + activeRuns--; + throw error; + } + } + final PendingRun pending = new PendingRun(this); + return new SessionRunResource(backendResult, pending); + } + + private static void validateInputShapes(Tensor[] inputs, + TensorInfo[] metadata) { + if (metadata == null || inputs.length != metadata.length) { + throw new IllegalArgumentException("Expected " + + (metadata == null ? 0 : metadata.length) + + " input tensors but received " + inputs.length); + } + boolean[] resolved = new boolean[metadata.length]; + for (int i = 0; i < inputs.length; i++) { + Tensor tensor = inputs[i]; + if (tensor == null) { + throw new IllegalArgumentException( + "Input tensor " + i + " is null"); + } + int metadataPosition = i; + if (tensor.getName() != null) { + metadataPosition = -1; + for (int candidate = 0; candidate < metadata.length; + candidate++) { + if (tensor.getName().equals( + metadata[candidate].getName())) { + metadataPosition = candidate; + break; + } + } + if (metadataPosition < 0) { + throw new IllegalArgumentException( + "Unknown model input " + tensor.getName()); + } + } + TensorInfo expected = metadata[metadataPosition]; + if (resolved[metadataPosition]) { + throw new IllegalArgumentException("Model input " + + expected.getName() + " was supplied more than once"); + } + resolved[metadataPosition] = true; + if (!sameShape(tensor.getShape(), expected.getShape())) { + throw new IllegalArgumentException("Input " + + expected.getName() + " shape does not match the " + + "model metadata; call resizeInput() before run()"); + } + } + } + + private static Tensor[] copyInputs(Tensor[] inputs) { + Tensor[] copy = new Tensor[inputs.length]; + for (int i = 0; i < inputs.length; i++) { + copy[i] = inputs[i]; + } + return copy; + } + + private static boolean sameShape(int[] supplied, int[] expected) { + if (supplied.length != expected.length) { + return false; + } + for (int i = 0; i < supplied.length; i++) { + if (supplied[i] != expected[i]) { + return false; + } + } + return true; + } + + /// Resizes an input and reallocates native tensors before the next run. + /// + /// This method throws while an asynchronous {@link #run(Tensor[])} is + /// pending because native runtimes cannot safely reallocate tensors during + /// an invocation. + /// + /// @param name model input name, or {@code null} for the first input + /// @param shape new non-negative dimensions + /// @throws IllegalStateException if the session is closed or a run is pending + public void resizeInput(String name, int[] shape) { + synchronized (this) { + ensureOpen(); + if (activeRuns > 0) { + throw new IllegalStateException( + "Cannot resize inputs while inference is running"); + } + implementation.resizeInput(handle, name, shape); + } + } + + /// Releases the interpreter, delegates, and any temporary staged model. + /// The original file supplied by {@link ModelSource#file(String)} is never + /// deleted. If a run is pending, release is deferred until that run + /// settles. Calling this method more than once has no effect. + @Override + public void close() { + boolean release = false; + synchronized (this) { + if (closed) { + return; + } + closed = true; + if (activeRuns == 0) { + release = true; + } else { + closePending = true; + } + } + if (release) { + implementation.close(handle); + } + } + + private void runFinished() { + boolean release = false; + synchronized (this) { + if (activeRuns > 0) { + activeRuns--; + } + if (activeRuns == 0 && closePending) { + closePending = false; + release = true; + } + } + if (release) { + implementation.close(handle); + } + } + + private void ensureOpen() { + if (closed) { + throw new IllegalStateException("Inference session is closed"); + } + } + + private void ensureNotRunning(String operation) { + if (activeRuns > 0) { + throw new IllegalStateException( + "Cannot " + operation + " while inference is running"); + } + } + + private static final class PendingRun { + private final InferenceSession session; + private boolean finished; + + PendingRun(InferenceSession session) { + this.session = session; + } + + synchronized void finish() { + if (!finished) { + finished = true; + session.runFinished(); + } + } + } + + private static final class SessionRunResource + extends AsyncResource { + private final PendingRun pending; + + SessionRunResource(AsyncResource backend, + PendingRun pending) { + this.pending = pending; + backend.ready(new SuccessCallback() { + @Override + public void onSucess(Tensor[] value) { + SessionRunResource.this.pending.finish(); + if (!isCancelled()) { + complete(value); + } + } + }).except(new SuccessCallback() { + @Override + public void onSucess(Throwable error) { + SessionRunResource.this.pending.finish(); + if (!isCancelled()) { + error(error); + } + } + }); + } + } +} diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelCache.java b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java new file mode 100644 index 00000000000..b97d9b5cb5a --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/ModelCache.java @@ -0,0 +1,520 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.inference; + +import com.codename1.io.ConnectionRequest; +import com.codename1.io.FileSystemStorage; +import com.codename1.io.NetworkEvent; +import com.codename1.io.NetworkManager; +import com.codename1.security.Hash; +import com.codename1.ui.Display; +import com.codename1.ui.events.ActionListener; +import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Hashtable; + +/// Downloads a large model into app-private storage and exposes it as a +/// file-backed {@link ModelSource}. The initial request requires HTTPS. +/// Ports that expose redirect responses reject any redirect to HTTP. Because +/// iOS follows redirects below the portable network layer, iOS downloads +/// require a SHA-256 digest so an unseen redirect cannot substitute the +/// executable model payload. Downloads use a temporary file and are promoted +/// only after optional digest verification. +/// +/// Supply a SHA-256 digest for third-party or remotely mutable models. Without +/// a digest HTTPS authenticates the connection but does not pin the executable +/// model payload. Small first-party models can instead be packaged with +/// {@link ModelSource#resource(String)}. +public final class ModelCache { + private static final int MAX_READABLE_CACHE_KEY_LENGTH = 160; + private static final Hashtable ACTIVE_FETCHES = + new Hashtable(); + + private ModelCache() { + } + + /// Fetches a model into the app-private {@code ai-models} directory. + /// A stale {@code .download} file is deleted and restarted rather than + /// resumed because the portable network layer cannot prove that a server's + /// partial response still represents the pinned model. + /// Concurrent requests for the same cache entry and content identity share + /// one underlying operation, but each caller receives an independent + /// resource. Canceling one caller's resource suppresses only that caller's + /// notification and does not cancel the shared download or other callers. + /// A different URL or digest using that cache key while the first operation + /// is active fails instead of racing on the temporary file. + /// + /// @param url initial HTTPS URL of the model; observable redirects must + /// also use HTTPS + /// @param cacheKey stable cache name, independent of the URL + /// @param sha256 lowercase or uppercase SHA-256 hex digest; optional on + /// ports that expose redirects and required on iOS + /// @return asynchronous cached model source + /// @throws IllegalArgumentException if the URL, cache key, or digest is + /// invalid, or if the digest is omitted on iOS + public static AsyncResource fetch( + final String url, final String cacheKey, final String sha256) { + if (!isHttpsUrl(url)) { + throw new IllegalArgumentException("Model URL must use HTTPS"); + } + if (cacheKey == null || cacheKey.length() == 0) { + throw new IllegalArgumentException("cacheKey must not be empty"); + } + if (sha256 != null && !isSha256(sha256)) { + throw new IllegalArgumentException("SHA-256 must contain 64 hex characters"); + } + if (sha256 == null && requiresPinnedModelDigest()) { + throw new IllegalArgumentException( + "iOS model downloads require a SHA-256 digest because " + + "redirects are followed below the portable network layer"); + } + + final String fileName = safeName(cacheKey) + ".tflite"; + final FetchRegistration registration = + registerFetch(fileName, url, sha256); + if (!registration.owner) { + return registration.resource; + } + final AsyncResource out = registration.resource; + final Completion completion = registration.completion; + Display.getInstance().scheduleBackgroundTask(new Runnable() { + @Override + public void run() { + final FileSystemStorage fs = FileSystemStorage.getInstance(); + final String directory = fs.getAppHomePath() + "ai-models/"; + fs.mkdir(directory); + final String target = directory + fileName; + try { + if (fs.exists(target) && verify(target, sha256)) { + completion.complete(ModelSource.file(target)); + return; + } + if (fs.exists(target)) { + fs.delete(target); + } + } catch (IOException error) { + completion.fail(error); + return; + } + Display.getInstance().callSerially(new Runnable() { + @Override + public void run() { + download(completion, url, sha256, target, fileName); + } + }); + } + }); + return out; + } + + /// Fetches a model without content pinning on ports that expose redirect + /// responses to the portable network layer. Prefer the three-argument + /// overload for any model that is not versioned by the app itself. iOS + /// rejects this overload because its native network stack follows + /// redirects before Codename One can validate their schemes. + /// Concurrent unpinned calls share an operation only when their URL and + /// cache key are identical. + /// + /// @param url HTTPS model URL + /// @param cacheKey stable cache name + /// @return asynchronous cached file source + /// @throws IllegalArgumentException if the URL or cache key is invalid, + /// or when called on iOS, where a digest is required + public static AsyncResource fetch(String url, String cacheKey) { + return fetch(url, cacheKey, null); + } + + private static void download(final Completion completion, + String url, final String sha256, + final String target, final String fileName) { + final FileSystemStorage fs = FileSystemStorage.getInstance(); + final String temporary = target + ".download"; + prepareTemporary(fs, temporary); + final ConnectionRequest request = new ModelDownloadRequest( + completion, fs, temporary); + request.setPost(false); + request.setFailSilently(true); + request.setReadResponseForErrors(false); + request.setDuplicateSupported(true); + request.setUrl(url); + request.setDestinationFile(temporary); + request.addResponseListener(new ActionListener() { + @Override + public void actionPerformed(NetworkEvent event) { + Display.getInstance().scheduleBackgroundTask(new Runnable() { + @Override + public void run() { + try { + verifyDownloaded(fs, temporary, sha256); + promoteDownloaded(fs, temporary, target, + fileName, sha256); + completion.complete(ModelSource.file(target)); + } catch (IOException error) { + completion.fail(error); + } + } + }); + } + }); + ActionListener failure = new ActionListener() { + @Override + public void actionPerformed(NetworkEvent event) { + if (fs.exists(temporary)) { + fs.delete(temporary); + } + Throwable error = event.getError(); + completion.fail(error == null + ? new IOException("Model download failed with HTTP " + + event.getResponseCode()) + : error); + } + }; + request.addExceptionListener(failure); + request.addResponseCodeListener(failure); + NetworkManager.getInstance().addToQueue(request); + } + + private static boolean verify(String path, String expected) throws IOException { + if (expected == null || expected.length() == 0) { + return true; + } + InputStream input = FileSystemStorage.getInstance().openInputStream(path); + try { + Hash hash = Hash.create(Hash.SHA256); + byte[] buffer = new byte[16384]; + int read; + while ((read = input.read(buffer)) >= 0) { + hash.update(buffer, 0, read); + } + return expected.equalsIgnoreCase(Hash.toHex(hash.digest())); + } finally { + input.close(); + } + } + + static void prepareTemporary(FileSystemStorage fs, String temporary) { + if (fs.exists(temporary)) { + fs.delete(temporary); + } + } + + static void verifyDownloaded(FileSystemStorage fs, String temporary, + String expected) throws IOException { + if (!verify(temporary, expected)) { + fs.delete(temporary); + throw new IOException("Downloaded model SHA-256 does not match"); + } + } + + static void promoteDownloaded(FileSystemStorage fs, String temporary, + String target, String fileName, + String expected) throws IOException { + try { + if (fs.exists(target)) { + fs.delete(target); + if (fs.exists(target)) { + throw new IOException( + "Could not replace existing cached model"); + } + } + fs.rename(temporary, fileName); + if (!fs.exists(target)) { + throw new IOException( + "Downloaded model could not be promoted to the cache"); + } + if (!verify(target, expected)) { + fs.delete(target); + throw new IOException( + "Promoted model SHA-256 does not match"); + } + if (fs.exists(temporary)) { + fs.delete(temporary); + } + } catch (IOException error) { + if (fs.exists(temporary)) { + fs.delete(temporary); + } + throw error; + } + } + + static String safeName(String value) { + StringBuilder out = new StringBuilder(); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + boolean safe = (c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') + || c == '-' || c == '_'; + out.append(safe ? c : '_'); + } + if (out.length() > MAX_READABLE_CACHE_KEY_LENGTH) { + out.setLength(MAX_READABLE_CACHE_KEY_LENGTH); + } + Hash hash = Hash.create(Hash.SHA256); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + hash.update((byte) (c >>> 8)); + hash.update((byte) c); + } + out.append('-').append(Hash.toHex(hash.digest()).substring(0, 32)); + return out.toString(); + } + + private static boolean isSha256(String value) { + if (value.length() != 64) { + return false; + } + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') + || (c >= 'A' && c <= 'F'))) { + return false; + } + } + return true; + } + + static boolean isHttpsUrl(String url) { + return url != null && url.regionMatches(true, 0, + "https://", 0, "https://".length()); + } + + static boolean requiresPinnedModelDigest() { + return "ios".equals(Display.getInstance().getPlatformName()); + } + + static FetchRegistration registerFetch( + String fileName, String url, String sha256) { + synchronized (ACTIVE_FETCHES) { + ActiveFetch active = ACTIVE_FETCHES.get(fileName); + if (active != null) { + if (active.matches(url, sha256)) { + return new FetchRegistration( + subscribe(active.resource), null, false); + } + AsyncResource conflict = + new AsyncResource(); + new Completion(conflict).fail( + new IOException("A different model download is " + + "already using cache key " + fileName)); + return new FetchRegistration(conflict, null, false); + } + AsyncResource resource = + new AsyncResource(); + Completion completion = + new Completion(resource, fileName); + ACTIVE_FETCHES.put(fileName, + new ActiveFetch(url, sha256, resource)); + return new FetchRegistration( + subscribe(resource), completion, true); + } + } + + private static AsyncResource subscribe( + AsyncResource operation) { + final SubscriberResource subscriber = + new SubscriberResource(); + operation.ready(new SuccessCallback() { + @Override + public void onSucess(T value) { + subscriber.publish(value); + } + }).except(new SuccessCallback() { + @Override + public void onSucess(Throwable error) { + subscriber.fail(error); + } + }); + return subscriber; + } + + private static void releaseFetch( + String fileName, AsyncResource resource) { + if (fileName == null) { + return; + } + synchronized (ACTIVE_FETCHES) { + ActiveFetch active = ACTIVE_FETCHES.get(fileName); + if (active != null && active.resource == resource) { + ACTIVE_FETCHES.remove(fileName); + } + } + } + + static final class ActiveFetch { + private final String url; + private final String sha256; + private final AsyncResource resource; + + ActiveFetch(String url, String sha256, + AsyncResource resource) { + this.url = url; + this.sha256 = sha256; + this.resource = resource; + } + + boolean matches(String otherUrl, String otherSha256) { + if (sha256 != null || otherSha256 != null) { + return sha256 != null && otherSha256 != null + && sha256.equalsIgnoreCase(otherSha256); + } + return url.equals(otherUrl); + } + } + + private static final class SubscriberResource + extends AsyncResource { + @Override + public synchronized boolean cancel(boolean mayInterruptIfRunning) { + return super.cancel(mayInterruptIfRunning); + } + + synchronized void publish(T value) { + if (!isCancelled()) { + complete(value); + } + } + + synchronized void fail(Throwable error) { + if (!isCancelled()) { + error(error); + } + } + } + + static final class FetchRegistration { + final AsyncResource resource; + final Completion completion; + final boolean owner; + + FetchRegistration(AsyncResource resource, + Completion completion, + boolean owner) { + this.resource = resource; + this.completion = completion; + this.owner = owner; + } + } + + static final class ModelDownloadRequest extends ConnectionRequest { + private final Completion completion; + private final FileSystemStorage fs; + private final String temporary; + + ModelDownloadRequest(Completion completion, + FileSystemStorage fs, String temporary) { + this.completion = completion; + this.fs = fs; + this.temporary = temporary; + } + + /// Compares the inherited URL and request arguments together with the + /// temporary destination path. Callback and storage-service instances + /// do not change the logical identity of a download. + /// + /// @param other request to compare + /// @return {@code true} when both requests download the same URL and + /// arguments to the same temporary path + @Override + public boolean equals(Object other) { + if (!super.equals(other)) { + return false; + } + ModelDownloadRequest request = (ModelDownloadRequest) other; + return temporary == null ? request.temporary == null + : temporary.equals(request.temporary); + } + + /// Produces a hash consistent with {@link #equals(Object)} from the + /// inherited request identity and temporary destination. + /// + /// @return hash of the logical download identity + @Override + public int hashCode() { + return 31 * super.hashCode() + + (temporary == null ? 0 : temporary.hashCode()); + } + + @Override + public boolean onRedirect(String url) { + if (isHttpsUrl(url)) { + return false; + } + setKilled(true); + if (fs.exists(temporary)) { + fs.delete(temporary); + } + completion.fail(new IOException( + "Model download redirect must use HTTPS")); + return true; + } + } + + static final class Completion { + private final AsyncResource resource; + private final String activeFileName; + private boolean done; + + Completion(AsyncResource resource) { + this(resource, null); + } + + Completion(AsyncResource resource, String activeFileName) { + this.resource = resource; + this.activeFileName = activeFileName; + } + + synchronized void complete(final T value) { + if (done) { + return; + } + done = true; + releaseFetch(activeFileName, resource); + Display.getInstance().callSerially(new Runnable() { + @Override + public void run() { + resource.complete(value); + } + }); + } + + synchronized void fail(final Throwable error) { + if (done) { + return; + } + done = true; + releaseFetch(activeFileName, resource); + Display.getInstance().callSerially(new Runnable() { + @Override + public void run() { + resource.error(new InferenceException( + "Could not cache LiteRT model", error)); + } + }); + } + } +} diff --git a/CodenameOne/src/com/codename1/ai/inference/ModelSource.java b/CodenameOne/src/com/codename1/ai/inference/ModelSource.java new file mode 100644 index 00000000000..cb270b10bce --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/ModelSource.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.inference; + +/// Describes where an {@link InferenceSession} should obtain a `.tflite` +/// model. Byte sources are defensively copied. File sources are especially +/// useful with {@link ModelCache} because native ports can open the cached +/// path without loading the full model into the Java heap. +public final class ModelSource { + public static final int BYTES = 1; + public static final int FILE = 2; + public static final int RESOURCE = 3; + + private final int kind; + private final byte[] bytes; + private final String path; + + private ModelSource(int kind, byte[] bytes, String path) { + this.kind = kind; + this.bytes = bytes; + this.path = path; + } + + /// Creates an in-memory model source. + /// @param value complete model bytes, copied by this method + /// @return a byte-backed source + public static ModelSource bytes(byte[] value) { + if (value == null || value.length == 0) { + throw new IllegalArgumentException("Model bytes must not be empty"); + } + byte[] copy = new byte[value.length]; + System.arraycopy(value, 0, copy, 0, value.length); + return new ModelSource(BYTES, copy, null); + } + + /// Creates a source for an existing private filesystem path. + /// @param path path understood by {@code FileSystemStorage} + /// @return a file-backed source; the session never owns or deletes the file + public static ModelSource file(String path) { + return named(FILE, path); + } + + /// Creates a source for a model packaged in application resources. + /// @param path absolute classpath-style resource path + /// @return a resource-backed source + public static ModelSource resource(String path) { + return named(RESOURCE, path); + } + + private static ModelSource named(int kind, String path) { + if (path == null || path.length() == 0) { + throw new IllegalArgumentException("Model path must not be empty"); + } + return new ModelSource(kind, null, path); + } + + /// @return {@link #BYTES}, {@link #FILE}, or {@link #RESOURCE} + public int getKind() { + return kind; + } + + /// @return a defensive copy of model bytes, or {@code null} for non-byte sources + public byte[] getBytes() { + if (bytes == null) { + return null; + } + byte[] out = new byte[bytes.length]; + System.arraycopy(bytes, 0, out, 0, bytes.length); + return out; + } + + /// Returns the internal model byte array without copying it. + /// + /// This escape hatch is intended for native backend handoff and + /// memory-sensitive applications. The returned array must be treated as + /// read-only: modifying it changes the model source and violates this + /// class's immutability contract. Prefer {@link #getBytes()} unless the + /// additional full-model copy is known to be unacceptable. + /// + /// @return internal model bytes, or {@code null} for non-byte sources + public byte[] getBytesUnsafe() { + return bytes; + } + + /// @return the file/resource path, or {@code null} for a byte source + public String getPath() { + return path; + } +} diff --git a/CodenameOne/src/com/codename1/ai/inference/Tensor.java b/CodenameOne/src/com/codename1/ai/inference/Tensor.java new file mode 100644 index 00000000000..bfe2c7e72cf --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/Tensor.java @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.inference; + +/// Immutable named tensor passed to or returned from an inference session. +/// The primitive data array and shape are copied on construction and by their +/// public getters, so callers can safely reuse or mutate their source arrays. +public final class Tensor { + private final String name; + private final TensorType type; + private final int[] shape; + private final Object data; + + /// Creates a tensor and validates that its type, shape, and primitive array + /// agree. Dynamic dimensions may be negative and skip the element-count + /// check; fixed shapes are overflow-checked. + /// + /// @param name model tensor name, or {@code null} for positional matching + /// @param type element type + /// @param shape tensor dimensions; an empty shape represents a scalar + /// @param data matching primitive array + public Tensor(String name, TensorType type, int[] shape, Object data) { + if (type == null || data == null) { + throw new NullPointerException("type and data are required"); + } + validateData(type, data); + int count = elementCount(shape); + if (count >= 0 && count != dataLength(data)) { + throw new IllegalArgumentException("Tensor shape does not match data length"); + } + this.name = name; + this.type = type; + this.shape = copy(shape); + this.data = copyData(data); + } + + /// @return a FLOAT32 tensor with defensively copied data + public static Tensor floats(String name, int[] shape, float[] data) { + return new Tensor(name, TensorType.FLOAT32, shape, data); + } + + /// @return an INT32 tensor with defensively copied data + public static Tensor ints(String name, int[] shape, int[] data) { + return new Tensor(name, TensorType.INT32, shape, data); + } + + /// Creates a byte-array tensor for UINT8, INT8, or BOOL data. + /// @return a tensor with defensively copied data + public static Tensor bytes(String name, TensorType type, int[] shape, byte[] data) { + return new Tensor(name, type, shape, data); + } + + /// @return the model tensor name, or {@code null} for an unnamed tensor + public String getName() { + return name; + } + + /// @return the element type + public TensorType getType() { + return type; + } + + /// @return a defensive copy of tensor dimensions + public int[] getShape() { + return copy(shape); + } + + /// @return a defensive copy of the matching primitive data array + public Object getData() { + return copyData(data); + } + + /// Returns the tensor's immutable backing primitive array to a port + /// implementation, avoiding an additional full-tensor copy at the native + /// boundary. Application code must use {@link #getData()}. + /// + /// @return the backing {@code float[]}, {@code int[]}, {@code long[]}, or + /// {@code byte[]} matching {@link #getType()} + /// @hidden + public Object getDataUnsafe() { + return data; + } + + private static void validateData(TensorType type, Object data) { + boolean valid; + switch (type) { + case FLOAT32: + valid = data instanceof float[]; + break; + case INT32: + valid = data instanceof int[]; + break; + case INT64: + valid = data instanceof long[]; + break; + case UINT8: + case INT8: + case BOOL: + valid = data instanceof byte[]; + break; + default: + valid = false; + } + if (!valid) { + throw new IllegalArgumentException("Data array does not match " + type); + } + } + + private static int elementCount(int[] shape) { + if (shape == null || shape.length == 0) { + return 1; + } + long out = 1; + for (int dimension : shape) { + if (dimension < 0) { + return -1; + } + out *= dimension; + if (out > Integer.MAX_VALUE) { + throw new IllegalArgumentException( + "Tensor shape contains more than 2147483647 elements"); + } + } + return (int) out; + } + + private static int dataLength(Object value) { + if (value instanceof float[]) { + return ((float[]) value).length; + } + if (value instanceof int[]) { + return ((int[]) value).length; + } + if (value instanceof long[]) { + return ((long[]) value).length; + } + if (value instanceof byte[]) { + return ((byte[]) value).length; + } + throw new IllegalArgumentException("Unsupported tensor data array"); + } + + private static int[] copy(int[] value) { + if (value == null) { + return new int[0]; + } + int[] out = new int[value.length]; + System.arraycopy(value, 0, out, 0, value.length); + return out; + } + + private static Object copyData(Object value) { + int length = dataLength(value); + if (value instanceof float[]) { + float[] out = new float[length]; + System.arraycopy(value, 0, out, 0, length); + return out; + } + if (value instanceof int[]) { + int[] out = new int[length]; + System.arraycopy(value, 0, out, 0, length); + return out; + } + if (value instanceof long[]) { + long[] out = new long[length]; + System.arraycopy(value, 0, out, 0, length); + return out; + } + if (value instanceof byte[]) { + byte[] out = new byte[length]; + System.arraycopy(value, 0, out, 0, length); + return out; + } + throw new IllegalArgumentException("Unsupported tensor data array"); + } +} diff --git a/CodenameOne/src/com/codename1/ai/inference/TensorInfo.java b/CodenameOne/src/com/codename1/ai/inference/TensorInfo.java new file mode 100644 index 00000000000..39e42c55005 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/TensorInfo.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.inference; + +/// Immutable metadata for one model input or output. Shapes may contain a +/// negative dynamic dimension until the input has been resized and tensors +/// reallocated. +public final class TensorInfo { + private final String name; + private final TensorType type; + private final int[] shape; + private final int index; + + /// Creates tensor metadata. + /// @param name runtime tensor name, possibly empty + /// @param type element type + /// @param shape current tensor dimensions + /// @param index native model index + public TensorInfo(String name, TensorType type, int[] shape, int index) { + this.name = name; + this.type = type; + this.shape = copy(shape); + this.index = index; + } + + /// @return the runtime tensor name + public String getName() { + return name; + } + + /// @return the tensor element type + public TensorType getType() { + return type; + } + + /// @return a defensive copy of current dimensions + public int[] getShape() { + return copy(shape); + } + + /// @return the zero-based model input or output index + public int getIndex() { + return index; + } + + private static int[] copy(int[] value) { + if (value == null) { + return new int[0]; + } + int[] out = new int[value.length]; + System.arraycopy(value, 0, out, 0, value.length); + return out; + } +} diff --git a/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetector.java b/CodenameOne/src/com/codename1/ai/inference/TensorType.java similarity index 78% rename from maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetector.java rename to CodenameOne/src/com/codename1/ai/inference/TensorType.java index 2e4d8e52eaf..962af3d6b39 100644 --- a/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetector.java +++ b/CodenameOne/src/com/codename1/ai/inference/TensorType.java @@ -20,13 +20,14 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ +package com.codename1.ai.inference; -package com.codename1.ai.mlkit.face; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [FaceDetector]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeFaceDetector extends NativeInterface { - int[] detect(byte[] imageBytes); +/// Tensor element types supported by the portable LiteRT contract. +public enum TensorType { + FLOAT32, + INT32, + INT64, + UINT8, + INT8, + BOOL } diff --git a/CodenameOne/src/com/codename1/ai/inference/package-info.java b/CodenameOne/src/com/codename1/ai/inference/package-info.java new file mode 100644 index 00000000000..1718b6140fd --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/inference/package-info.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +/// Reusable on-device inference sessions for {@code .tflite} models. +/// +///

Android uses LiteRT. iOS uses TensorFlow Lite Objective-C and may select +/// the Core ML delegate. The builder links the runtime only when an application +/// references {@link com.codename1.ai.inference.InferenceSession}. Other +/// targets expose the same API with an explicit unsupported fallback.

+package com.codename1.ai.inference; diff --git a/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScanner.java b/CodenameOne/src/com/codename1/ai/language/LanguageBackend.java similarity index 77% rename from maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScanner.java rename to CodenameOne/src/com/codename1/ai/language/LanguageBackend.java index 84cafb7deee..0801906c637 100644 --- a/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScanner.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageBackend.java @@ -20,13 +20,12 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ +package com.codename1.ai.language; -package com.codename1.ai.mlkit.barcode; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [BarcodeScanner]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeBarcodeScanner extends NativeInterface { - String[] scan(byte[] imageBytes); +/// Identifies a native language-service implementation. Obtain instances from +/// {@link LanguageBackends}; builder dependency selection relies on calls to +/// those selector methods. +public interface LanguageBackend { + /// @return stable backend identifier passed to the current platform port + String getId(); } diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java b/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java new file mode 100644 index 00000000000..00033d208c8 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/LanguageBackends.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.language; + +/// Selects native backends for language identification, translation, and +/// Smart Reply. Automatic selection uses ML Kit on Android, Apple Natural +/// Language for iOS identification. iOS translation and Smart Reply require +/// their feature-specific ML Kit selector methods. +/// +/// The feature-specific methods are also build-time dependency markers. They +/// intentionally remain distinct even though they return the same runtime +/// backend id, because the builder scans each call site to include only the +/// requested ML Kit component. +public final class LanguageBackends { + private static final LanguageBackend AUTO = new Named("auto"); + private static final LanguageBackend ML_KIT = new Named("ml-kit"); + private static final LanguageBackend APPLE_NATURAL_LANGUAGE = + new Named("apple-natural-language"); + + private LanguageBackends() { + } + + /// @return the platform-recommended dependency-minimal backend + public static LanguageBackend auto() { + return AUTO; + } + + /// Selects Apple's dependency-free Natural Language framework for + /// language identification. This backend is available on iOS 12 and + /// newer and is not available on Android. + /// + /// @return the Apple Natural Language backend selector + public static LanguageBackend appleNaturalLanguage() { + return APPLE_NATURAL_LANGUAGE; + } + + /// Selects ML Kit specifically for language identification. Calling this + /// method lets the builder add the Language ID pod only when the + /// application opts out of the Apple-native iOS default. + /// + /// @return the ML Kit language-identification selector + public static LanguageBackend mlKitLanguageIdentification() { + return ML_KIT; + } + + /// Selects ML Kit translation. Calling this method is the build-time + /// marker that adds the translation pod on iOS; merely referencing + /// {@link Translator} does not add it or disable the arm64 simulator. + /// + /// @return the ML Kit translation selector + public static LanguageBackend mlKitTranslation() { + return ML_KIT; + } + + /// Selects ML Kit Smart Reply. Calling this method is the build-time + /// marker that adds the Smart Reply pod on iOS; merely referencing + /// {@link SmartReply} does not add it or disable the arm64 simulator. + /// + /// @return the ML Kit Smart Reply selector + public static LanguageBackend mlKitSmartReply() { + return ML_KIT; + } + + private static final class Named implements LanguageBackend { + private final String id; + + private Named(String id) { + this.id = id; + } + + @Override + public String getId() { + return id; + } + } +} diff --git a/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenter.java b/CodenameOne/src/com/codename1/ai/language/LanguageCandidate.java similarity index 60% rename from maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenter.java rename to CodenameOne/src/com/codename1/ai/language/LanguageCandidate.java index 1fcb3fd96ec..bdfdb5b7036 100644 --- a/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenter.java +++ b/CodenameOne/src/com/codename1/ai/language/LanguageCandidate.java @@ -20,13 +20,28 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ +package com.codename1.ai.language; -package com.codename1.ai.mlkit.segmentation; +/// One ranked BCP-47 language candidate returned by +/// {@link LanguageIdentifier#identify(String, LanguageOptions)}. +public final class LanguageCandidate { + private final String languageTag; + private final float confidence; -import com.codename1.system.NativeInterface; + /// @param languageTag BCP-47 tag, or {@code und} when undetermined + /// @param confidence backend confidence in the range 0..1 + public LanguageCandidate(String languageTag, float confidence) { + this.languageTag = languageTag; + this.confidence = confidence; + } -/// Native bridge for [SelfieSegmenter]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeSelfieSegmenter extends NativeInterface { - byte[] segment(byte[] imageBytes); + /// @return BCP-47 language tag + public String getLanguageTag() { + return languageTag; + } + + /// @return backend confidence in the range 0..1 + public float getConfidence() { + return confidence; + } } diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java new file mode 100644 index 00000000000..e61e6ed7283 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/LanguageIdentifier.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.language; + +import com.codename1.impl.LanguageImpl; +import com.codename1.util.AsyncResource; + +/// Identifies possible languages entirely on device. Results are ranked by +/// descending backend confidence and filtered by +/// {@link LanguageOptions#getMinimumConfidence()}. +public final class LanguageIdentifier { + private LanguageIdentifier() { + } + + /// @return whether automatic language identification is available + public static boolean isSupported() { + return isSupported(new LanguageOptions()); + } + + /// @param options backend selection, or {@code null} for defaults + /// @return whether the selected backend is available on this target + public static boolean isSupported(LanguageOptions options) { + return LanguageSession.isSupported("language-id", options); + } + + /// Opens a reusable language-identification session. Reusing a session + /// avoids repeatedly creating the native recognizer when classifying many + /// strings. Close it when no more identifications are pending. + /// + /// @param options backend and confidence options, or {@code null} + /// @return a reusable session that owns one native language backend + /// @throws UnsupportedOperationException if the selected backend is absent + public static Session open(LanguageOptions options) { + return new Session(LanguageSession.open("language-id", options)); + } + + /// Identifies possible languages off the EDT without uploading text. The + /// option values are copied before the backend starts, so later mutations + /// of the supplied object cannot alter the pending identification. + /// Cancelling the returned resource suppresses late result callbacks; the + /// temporary backend is released after its pending native work settles. + /// @param text non-null text to classify + /// @param options backend and confidence options, or {@code null} + /// @return asynchronous ranked candidates; may be empty for undetermined text + public static AsyncResource identify( + String text, LanguageOptions options) { + final Session session; //NOPMD CloseResource - async result owns closure + try { + session = open(options); + } catch (RuntimeException error) { + AsyncResource out = + new AsyncResource(); + out.error(error); + return out; + } + return session.identify(text, true); + } + + /// Reusable owner of a native language-identification backend. + /// + /// A session accepts multiple asynchronous requests. Calling + /// {@link #close()} prevents new requests immediately and defers native + /// release until requests already in progress have completed. + public static final class Session implements AutoCloseable { + private final LanguageSession session; + + private Session(LanguageSession session) { + this.session = session; + } + + /// Identifies languages without recreating the native backend. + /// Cancelling the returned resource suppresses late callbacks without + /// closing this reusable session. + /// + /// @param text text to classify; {@code null} is treated as empty + /// @return asynchronous ranked candidates, possibly empty + /// @throws IllegalStateException if this session is closed + public AsyncResource identify(final String text) { + return identify(text, false); + } + + private AsyncResource identify( + final String text, boolean closeWhenFinished) { + return session.execute( + new LanguageSession.Operation() { + @Override + public AsyncResource run( + LanguageImpl implementation, + LanguageOptions options) { + return implementation.identify( + text == null ? "" : text, + options.getBackend().getId(), options); + } + }, closeWhenFinished); + } + + /// Closes the session. This method is idempotent; native release is + /// deferred until pending identifications finish. + @Override + public void close() { + session.close(); + } + } +} diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java b/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java new file mode 100644 index 00000000000..89ea53fe0e1 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/LanguageOptions.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.language; + +/// Reusable options shared by language identification, translation, and +/// Smart Reply operations. +public final class LanguageOptions { + private LanguageBackend backend = LanguageBackends.auto(); + private float minimumConfidence; + + /// @param value backend selector; {@code null} restores automatic selection + /// @return this options object + public LanguageOptions backend(LanguageBackend value) { + backend = value == null ? LanguageBackends.auto() : value; + return this; + } + + /// Sets the minimum language-identification confidence, clamped to 0..1. + /// NaN has no meaningful ordering and is rejected instead of being passed + /// to a platform language identifier. + /// @param value requested threshold + /// @return this options object + /// @throws IllegalArgumentException if {@code value} is NaN + public LanguageOptions minimumConfidence(float value) { + if (Float.isNaN(value)) { + throw new IllegalArgumentException( + "minimum confidence must be a number"); + } + minimumConfidence = Math.max(0, Math.min(1, value)); + return this; + } + + /// @return selected backend, never {@code null} + public LanguageBackend getBackend() { + return backend; + } + + /// @return language-identification threshold in the range 0..1 + public float getMinimumConfidence() { + return minimumConfidence; + } + + LanguageOptions snapshot() { + return new LanguageOptions() + .backend(backend) + .minimumConfidence(minimumConfidence); + } +} diff --git a/CodenameOne/src/com/codename1/ai/language/LanguageSession.java b/CodenameOne/src/com/codename1/ai/language/LanguageSession.java new file mode 100644 index 00000000000..0cb7d789a3a --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/LanguageSession.java @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.language; + +import com.codename1.impl.LanguageImpl; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; + +/// Shared lifecycle implementation for the public feature-specific language +/// sessions. This package-private type keeps native backends alive across +/// repeated operations and defers release while an operation is pending. +final class LanguageSession implements AutoCloseable { + interface Operation { + AsyncResource run(LanguageImpl implementation, + LanguageOptions options); + } + + private final LanguageImpl implementation; + private final LanguageOptions options; + private boolean closed; + private int activeOperations; + private boolean closePending; + + private LanguageSession(LanguageImpl implementation, + LanguageOptions options) { + this.implementation = implementation; + this.options = options; + } + + static boolean isSupported(String feature, LanguageOptions options) { + LanguageImpl implementation = + Display.getInstance().getLanguageBackend(); + if (implementation == null) { + return false; + } + try { + LanguageOptions actual = options == null + ? new LanguageOptions() : options; + return implementation.isSupported(feature, + actual.getBackend().getId()); + } finally { + implementation.close(); + } + } + + static LanguageSession open(String feature, LanguageOptions options) { + LanguageImpl implementation = + Display.getInstance().getLanguageBackend(); + LanguageOptions actual = (options == null + ? new LanguageOptions() : options).snapshot(); + if (implementation == null + || !implementation.isSupported(feature, + actual.getBackend().getId())) { + if (implementation != null) { + implementation.close(); + } + throw new UnsupportedOperationException( + feature + " is not supported"); + } + return new LanguageSession(implementation, actual); + } + + AsyncResource execute(Operation operation) { + return execute(operation, false); + } + + AsyncResource execute(Operation operation, + final boolean closeWhenFinished) { + final AsyncResource backendResult; + synchronized (this) { + if (closed) { + throw new IllegalStateException( + "Language session is closed"); + } + activeOperations++; + try { + backendResult = operation.run(implementation, options); + if (backendResult == null) { + throw new IllegalStateException( + "Language backend returned no asynchronous result"); + } + } catch (RuntimeException error) { + activeOperations--; + throw error; + } catch (Error error) { + activeOperations--; + throw error; + } + } + final OperationResource result = + new OperationResource(this, closeWhenFinished); + final Completion completion = new Completion(); + backendResult.ready(new SuccessCallback() { + @Override + public void onSucess(T value) { + if (completion.finish()) { + if (closeWhenFinished) { + close(); + } + operationFinished(); + result.publish(value); + } + } + }).except(new SuccessCallback() { + @Override + public void onSucess(Throwable error) { + if (completion.finish()) { + if (closeWhenFinished) { + close(); + } + operationFinished(); + result.fail(error); + } + } + }); + return result; + } + + private void operationFinished() { + LanguageImpl toClose = null; + synchronized (this) { + activeOperations--; + if (activeOperations == 0 && closePending) { + closePending = false; + toClose = implementation; + } + } + if (toClose != null) { + toClose.close(); + } + } + + @Override + public void close() { + LanguageImpl toClose = null; + synchronized (this) { + if (closed) { + return; + } + closed = true; + if (activeOperations == 0) { + toClose = implementation; + } else { + closePending = true; + } + } + if (toClose != null) { + toClose.close(); + } + } + + private static final class OperationResource + extends AsyncResource { + private final LanguageSession owner; + private final boolean closeWhenFinished; + + OperationResource(LanguageSession owner, + boolean closeWhenFinished) { + this.owner = owner; + this.closeWhenFinished = closeWhenFinished; + } + + @Override + public synchronized boolean cancel(boolean mayInterruptIfRunning) { + boolean cancelled = super.cancel(mayInterruptIfRunning); + if (cancelled && closeWhenFinished) { + owner.close(); + } + return cancelled; + } + + synchronized void publish(T value) { + if (!isCancelled()) { + complete(value); + } + } + + synchronized void fail(Throwable error) { + if (!isCancelled()) { + error(error); + } + } + } + + private static final class Completion { + private boolean finished; + + synchronized boolean finish() { + if (finished) { + return false; + } + finished = true; + return true; + } + } +} diff --git a/CodenameOne/src/com/codename1/ai/language/SmartReply.java b/CodenameOne/src/com/codename1/ai/language/SmartReply.java new file mode 100644 index 00000000000..3b363102ad8 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/SmartReply.java @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.language; + +import com.codename1.impl.LanguageImpl; +import com.codename1.util.AsyncResource; + +/// Produces short reply suggestions for a chronological conversation without +/// uploading its messages. ML Kit may return no suggestions when the language +/// or conversation context is unsupported. +public final class SmartReply { + private SmartReply() { + } + + /// @return whether automatic Smart Reply is available + public static boolean isSupported() { + return isSupported(new LanguageOptions()); + } + + /// Tests whether Smart Reply is available for a backend selection. + /// + /// This capability check creates and closes a temporary native backend. + /// Retain a {@link Session} from {@link #open(LanguageOptions)} when the + /// application will immediately request repeated suggestions. + /// + /// @param options backend selection, or {@code null} for defaults + /// @return whether the selected backend supports Smart Reply + public static boolean isSupported(LanguageOptions options) { + return LanguageSession.isSupported("smart-reply", options); + } + + /// Opens a reusable Smart Reply session. Reuse it for conversations that + /// need repeated suggestions to avoid recreating the native client. + /// + /// @param options backend options, or {@code null} + /// @return a reusable session that owns one native language backend + /// @throws UnsupportedOperationException if the selected backend is absent + public static Session open(LanguageOptions options) { + return new Session(LanguageSession.open("smart-reply", options)); + } + + /// The conversation array and option values are copied before backend work + /// begins. {@link SmartReplyMessage} values are immutable. + /// Cancelling the returned resource suppresses late result callbacks; the + /// temporary backend is released after its pending native work settles. + /// + /// @param conversation chronological messages, oldest first + /// @param options backend options, or {@code null} + /// @return asynchronous suggestions, possibly an empty array + public static AsyncResource suggest(SmartReplyMessage[] conversation, + LanguageOptions options) { + final Session session; //NOPMD CloseResource - async result owns closure + try { + session = open(options); + } catch (RuntimeException error) { + AsyncResource out = new AsyncResource(); + out.error(error); + return out; + } + return session.suggest(conversation, true); + } + + private static SmartReplyMessage[] copyConversation( + SmartReplyMessage[] conversation) { + if (conversation == null) { + return new SmartReplyMessage[0]; + } + SmartReplyMessage[] copy = + new SmartReplyMessage[conversation.length]; + for (int i = 0; i < conversation.length; i++) { + copy[i] = conversation[i]; + } + return copy; + } + + /// Reusable owner of a native Smart Reply backend. + /// + /// Calling {@link #close()} prevents new requests immediately and defers + /// native release until suggestions already in progress have completed. + public static final class Session implements AutoCloseable { + private final LanguageSession session; + + private Session(LanguageSession session) { + this.session = session; + } + + /// Generates reply suggestions without recreating the native backend. + /// Cancelling the returned resource suppresses late callbacks without + /// closing this reusable session. + /// + /// @param conversation chronological messages, oldest first; + /// {@code null} is treated as an empty conversation + /// @return asynchronous suggestions, possibly empty + /// @throws IllegalStateException if this session is closed + public AsyncResource suggest( + final SmartReplyMessage[] conversation) { + return suggest(conversation, false); + } + + private AsyncResource suggest( + final SmartReplyMessage[] conversation, + boolean closeWhenFinished) { + final SmartReplyMessage[] snapshot = + copyConversation(conversation); + return session.execute(new LanguageSession.Operation() { + @Override + public AsyncResource run( + LanguageImpl implementation, + LanguageOptions options) { + return implementation.suggestReplies(snapshot, + options.getBackend().getId(), options); + } + }, closeWhenFinished); + } + + /// Closes the session. This method is idempotent; native release is + /// deferred until pending suggestions finish. + @Override + public void close() { + session.close(); + } + } +} diff --git a/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java b/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java new file mode 100644 index 00000000000..2acbabb7bda --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/SmartReplyMessage.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.language; + +/// One message supplied to the on-device smart-reply model. +public final class SmartReplyMessage { + private final String text; + private final String participantId; + private final boolean localUser; + private final long timestampMillis; + + /// Creates one conversation turn used to generate reply suggestions. + /// @param text message body; {@code null} becomes an empty string + /// @param participantId stable speaker id used to group conversation turns; + /// {@code null} becomes {@code "remote"} + /// @param localUser whether this message was written by the current user + /// @param timestampMillis message time in Unix epoch milliseconds + public SmartReplyMessage(String text, String participantId, + boolean localUser, long timestampMillis) { + this.text = text == null ? "" : text; + this.participantId = participantId == null ? "remote" : participantId; + this.localUser = localUser; + this.timestampMillis = timestampMillis; + } + + /// @return message text supplied to the local model + public String getText() { + return text; + } + + /// @return stable participant identifier used to group remote speakers, + /// never {@code null} + public String getParticipantId() { + return participantId; + } + + /// @return whether this message was authored by the device user + public boolean isLocalUser() { + return localUser; + } + + /// @return conversation timestamp in milliseconds since the epoch + public long getTimestampMillis() { + return timestampMillis; + } +} diff --git a/CodenameOne/src/com/codename1/ai/language/Translator.java b/CodenameOne/src/com/codename1/ai/language/Translator.java new file mode 100644 index 00000000000..ee543a7fb8a --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/language/Translator.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.language; + +import com.codename1.impl.LanguageImpl; +import com.codename1.util.AsyncResource; + +/// Translates text on device with lazily installed language-pair models. +/// The first request for a pair may take longer while ML Kit downloads the +/// model; download failures are reported through the returned resource. +public final class Translator { + private Translator() { + } + + /// @return whether automatic on-device translation is available + public static boolean isSupported() { + return isSupported(new LanguageOptions()); + } + + /// Tests whether translation is available for a backend selection. + /// + /// This capability check creates and closes a temporary native backend. + /// Retain a {@link Session} from {@link #open(LanguageOptions)} when the + /// application will immediately perform repeated translations. + /// + /// @param options backend selection, or {@code null} for defaults + /// @return whether the selected backend supports translation + public static boolean isSupported(LanguageOptions options) { + return LanguageSession.isSupported("translation", options); + } + + /// Opens a reusable translation session. Reuse it for repeated requests + /// so native translation clients and downloaded model state are retained. + /// + /// @param options backend options, or {@code null} + /// @return a reusable session that owns one native language backend + /// @throws UnsupportedOperationException if the selected backend is absent + public static Session open(LanguageOptions options) { + return new Session(LanguageSession.open("translation", options)); + } + + /// Option values are copied before asynchronous backend work begins. + /// Current ML Kit backends accept a BCP-47 tag with an optional script or + /// region, such as {@code en-US}, and select the corresponding supported + /// base-language model ({@code en} in this example). The asynchronous + /// resource fails when either tag does not identify a supported model. + /// Cancelling the returned resource suppresses late result callbacks; the + /// temporary backend is released after its pending native work settles. + /// + /// @param text source text; {@code null} is treated as empty + /// @param sourceLanguage BCP-47 source language tag + /// @param targetLanguage BCP-47 target language tag + /// @param options backend options, or {@code null} + /// @return asynchronous translated text + public static AsyncResource translate(String text, String sourceLanguage, + String targetLanguage, + LanguageOptions options) { + final Session session; //NOPMD CloseResource - async result owns closure + try { + session = open(options); + } catch (RuntimeException error) { + AsyncResource out = new AsyncResource(); + out.error(error); + return out; + } + return session.translate(text, sourceLanguage, targetLanguage, true); + } + + /// Reusable owner of a native translation backend. + /// + /// A session can serve multiple language pairs and retains native clients + /// between calls. Calling {@link #close()} prevents new requests and + /// defers release until pending translations finish. + public static final class Session implements AutoCloseable { + private final LanguageSession session; + + private Session(LanguageSession session) { + this.session = session; + } + + /// Translates text without recreating the native backend. + /// Cancelling the returned resource suppresses late callbacks without + /// closing this reusable session. + /// + /// @param text source text; {@code null} is treated as empty + /// @param sourceLanguage supported BCP-47 source language tag + /// @param targetLanguage supported BCP-47 target language tag + /// @return asynchronous translated text + /// @throws IllegalStateException if this session is closed + public AsyncResource translate(final String text, + final String sourceLanguage, + final String targetLanguage) { + return translate(text, sourceLanguage, targetLanguage, false); + } + + private AsyncResource translate(final String text, + final String sourceLanguage, + final String targetLanguage, + boolean closeWhenFinished) { + return session.execute(new LanguageSession.Operation() { + @Override + public AsyncResource run( + LanguageImpl implementation, + LanguageOptions options) { + return implementation.translate( + text == null ? "" : text, sourceLanguage, + targetLanguage, options.getBackend().getId(), + options); + } + }, closeWhenFinished); + } + + /// Closes the session. This method is idempotent; native release is + /// deferred until pending translations finish. + @Override + public void close() { + session.close(); + } + } +} diff --git a/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/package-info.java b/CodenameOne/src/com/codename1/ai/language/package-info.java similarity index 58% rename from maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/package-info.java rename to CodenameOne/src/com/codename1/ai/language/package-info.java index ec5bd92c30c..e4510159efc 100644 --- a/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/package-info.java +++ b/CodenameOne/src/com/codename1/ai/language/package-info.java @@ -20,19 +20,17 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ - -/// ML Kit Language Identification. +/// Vendor-neutral on-device language identification, translation, and smart +/// reply. /// -/// Identifies the language of a given text string. -/// -/// The single public class in this package is [LanguageIdentifier], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeLanguageIdentifier` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `LanguageIdentifier.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.langid; +///

Android uses feature-scoped ML Kit components. On iOS, automatic +/// language identification uses Apple's Natural Language framework. An +/// application opts into each iOS ML Kit component with +/// {@link com.codename1.ai.language.LanguageBackends#mlKitLanguageIdentification()}, +/// {@link com.codename1.ai.language.LanguageBackends#mlKitTranslation()}, or +/// {@link com.codename1.ai.language.LanguageBackends#mlKitSmartReply()}. +/// Each entry point and optional selector is scanned independently, so using +/// one feature does not bundle the others. Translation model payloads are +/// installed lazily by ML Kit. Other targets expose the same API with an +/// explicit unsupported fallback.

+package com.codename1.ai.language; diff --git a/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java new file mode 100644 index 00000000000..04e46da8931 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/AbstractVisionAnalyzer.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +import com.codename1.impl.VisionImpl; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; + +abstract class AbstractVisionAnalyzer implements VisionAnalyzer { + private final VisionFeature feature; + private final VisionOptions options; + private VisionImpl implementation; + private boolean closed; + + AbstractVisionAnalyzer(VisionFeature feature, VisionOptions options) { + this.feature = feature; + this.options = (options == null + ? new VisionOptions() : options).snapshot(); + } + + /// @return whether this analyzer's feature/backend is available and open + @Override + public final synchronized boolean isSupported() { + VisionImpl impl = implementation(); + return impl != null && impl.isSupported(feature, options.getBackend().getId()); + } + + /// Starts one analysis using the retained native backend. + /// @param image immutable encoded or raw input + /// @return asynchronous typed result + @Override + public final synchronized AsyncResource process(VisionImage image) { + if (image == null) { + throw new NullPointerException("image"); + } + if (closed) { + throw new IllegalStateException("Analyzer is closed"); + } + VisionImpl impl = implementation(); + if (impl == null || !impl.isSupported(feature, options.getBackend().getId())) { + AsyncResource out = new AsyncResource(); + out.error(new VisionException(VisionException.UNSUPPORTED, + feature + " is not supported by " + options.getBackend().getId())); + return out; + } + return impl.analyze(feature, options.getBackend().getId(), image, options); + } + + /// Idempotently releases the retained native backend. + @Override + public final synchronized void close() { + closed = true; + if (implementation != null) { + implementation.close(); + implementation = null; + } + } + + private VisionImpl implementation() { + if (!closed && implementation == null) { + implementation = Display.getInstance().getVisionBackend(); + } + return implementation; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/Barcode.java b/CodenameOne/src/com/codename1/ai/vision/Barcode.java new file mode 100644 index 00000000000..802501ed8d4 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/Barcode.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Portable barcode observation with normalized geometry. Format names are +/// backend-neutral constants such as {@code QR_CODE}, {@code DATA_MATRIX}, +/// {@code CODE_128}, and {@code EAN_13}; unknown symbologies use +/// {@code UNKNOWN}. Corner points are ordered around the code when supplied +/// by the backend and may be empty when geometry is unavailable. +public final class Barcode { + private final String value; + private final String format; + private final byte[] rawBytes; + private final VisionRect bounds; + private final VisionPoint[] corners; + private final VisionMetadata metadata; + + /// Creates a barcode without backend metadata. + /// @param value decoded display value, or {@code null} + /// @param format normalized symbology name + /// @param rawBytes original payload bytes when available + /// @param bounds normalized top-left-origin bounds + /// @param corners normalized corner points + public Barcode(String value, String format, byte[] rawBytes, + VisionRect bounds, VisionPoint[] corners) { + this(value, format, rawBytes, bounds, corners, null); + } + + /// Creates a complete barcode observation. + /// @param value decoded display value, or {@code null} + /// @param format normalized symbology name + /// @param rawBytes original payload bytes when available + /// @param bounds normalized top-left-origin bounds + /// @param corners normalized corner points + /// @param metadata backend identity and optional diagnostic values + public Barcode(String value, String format, byte[] rawBytes, + VisionRect bounds, VisionPoint[] corners, + VisionMetadata metadata) { + this.value = value; + this.format = format; + this.rawBytes = copy(rawBytes); + this.bounds = bounds == null ? VisionRect.EMPTY : bounds; + if (corners == null) { + this.corners = new VisionPoint[0]; + } else { + this.corners = new VisionPoint[corners.length]; + System.arraycopy(corners, 0, this.corners, 0, corners.length); + } + this.metadata = metadata; + } + + /// @return decoded display value, or {@code null} when decoding failed + public String getValue() { + return value; + } + + /// @return normalized symbology name, never a vendor numeric identifier + public String getFormat() { + return format; + } + + /// @return a defensive copy of payload bytes, or {@code null} if unavailable + public byte[] getRawBytes() { + return copy(rawBytes); + } + + /// @return normalized top-left-origin bounds + public VisionRect getBounds() { + return bounds; + } + + /// @return defensive copy of normalized corners; possibly empty + public VisionPoint[] getCorners() { + VisionPoint[] out = new VisionPoint[corners.length]; + System.arraycopy(corners, 0, out, 0, corners.length); + return out; + } + + /// @return backend metadata, or {@code null} for manually created results + public VisionMetadata getMetadata() { + return metadata; + } + + private static byte[] copy(byte[] value) { + if (value == null) { + return null; + } + byte[] out = new byte[value.length]; + System.arraycopy(value, 0, out, 0, value.length); + return out; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java b/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java new file mode 100644 index 00000000000..d9ca1e9162c --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/BarcodeScanner.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Creates reusable still-image or live-frame barcode analyzers. +public final class BarcodeScanner extends AbstractVisionAnalyzer { + /// Creates an analyzer using the platform default backend and options. + /// @see VisionOptions + public BarcodeScanner() { + this(null); + } + + /// Creates a reusable analyzer with explicit backend and result options. + /// @param options configuration captured by this analyzer; {@code null} + /// uses defaults + public BarcodeScanner(VisionOptions options) { + super(VisionFeature.BARCODE_SCANNING, options); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/DocumentScanResult.java b/CodenameOne/src/com/codename1/ai/vision/DocumentScanResult.java new file mode 100644 index 00000000000..d74bbe3414e --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/DocumentScanResult.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Corrected document pages returned as encoded image data. Pages are ordered +/// as detected and both construction and access make defensive copies, so the +/// result can safely cross asynchronous boundaries. The still-image scanner is +/// currently Apple-only; Android's ML Kit document API owns an interactive +/// camera flow and does not implement this analyzer contract. +public final class DocumentScanResult { + private final byte[][] pages; + private final VisionMetadata metadata; + + /// Creates a corrected document scan without backend metadata. + /// @param pages encoded corrected page images, deeply defensively copied + public DocumentScanResult(byte[][] pages) { + this(pages, null); + } + + /// Creates a corrected document scan with backend diagnostics. + /// @param pages encoded corrected page images, deeply defensively copied + /// @param metadata backend details, or {@code null} + public DocumentScanResult(byte[][] pages, VisionMetadata metadata) { + if (pages == null) { + this.pages = new byte[0][]; + } else { + this.pages = new byte[pages.length][]; + for (int i = 0; i < pages.length; i++) { + byte[] page = pages[i]; + if (page == null) { + throw new NullPointerException("pages[" + i + "]"); + } + this.pages[i] = new byte[page.length]; + System.arraycopy(page, 0, this.pages[i], 0, page.length); + } + } + this.metadata = metadata; + } + + /// @return number of corrected pages + public int getPageCount() { + return pages.length; + } + + /// @param index zero-based page index + /// @return defensive copy of that page's encoded image + public byte[] getPage(int index) { + byte[] page = pages[index]; + byte[] out = new byte[page.length]; + System.arraycopy(page, 0, out, 0, page.length); + return out; + } + + /// @return backend metadata, or {@code null} for manually created results + public VisionMetadata getMetadata() { + return metadata; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java b/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java new file mode 100644 index 00000000000..94106796134 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/DocumentScanner.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Creates reusable document-boundary and perspective-correction analyzers. +public final class DocumentScanner extends AbstractVisionAnalyzer { + /// Creates an analyzer using the platform default backend and options. + /// @see VisionOptions + public DocumentScanner() { + this(null); + } + + /// Creates a reusable analyzer with explicit backend and result options. + /// @param options configuration captured by this analyzer; {@code null} + /// uses defaults + public DocumentScanner(VisionOptions options) { + super(VisionFeature.DOCUMENT_SCANNING, options); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/Face.java b/CodenameOne/src/com/codename1/ai/vision/Face.java new file mode 100644 index 00000000000..0d4ac975b2f --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/Face.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/// Portable face observation. +/// +///

Euler angles are expressed in degrees. Bounds and landmark points use +/// the normalized, top-left-origin coordinate space defined by +/// {@link VisionRect} and {@link VisionPoint}.

+public final class Face { + private final VisionRect bounds; + private final Map landmarks; + private final float yaw; + private final float pitch; + private final float roll; + private final float smilingProbability; + private final int trackingId; + private final VisionMetadata metadata; + + /// Creates a detected face without backend metadata. + /// @param bounds face bounds in the oriented image coordinate space + /// @param landmarks named feature points, defensively copied + /// @param yaw horizontal Euler angle in degrees + /// @param pitch vertical Euler angle in degrees + /// @param roll in-plane Euler angle in degrees + /// @param smilingProbability smile confidence, or a negative value when unavailable + /// @param trackingId stable streaming id, or a negative value when unavailable + public Face(VisionRect bounds, Map landmarks, + float yaw, float pitch, float roll, + float smilingProbability, int trackingId) { + this(bounds, landmarks, yaw, pitch, roll, smilingProbability, + trackingId, null); + } + + /// Creates a detected face with backend diagnostics. + /// @param bounds face bounds in the oriented image coordinate space + /// @param landmarks named feature points, defensively copied + /// @param yaw horizontal Euler angle in degrees + /// @param pitch vertical Euler angle in degrees + /// @param roll in-plane Euler angle in degrees + /// @param smilingProbability smile confidence, or a negative value when unavailable + /// @param trackingId stable streaming id, or a negative value when unavailable + /// @param metadata backend details, or {@code null} + public Face(VisionRect bounds, Map landmarks, + float yaw, float pitch, float roll, + float smilingProbability, int trackingId, + VisionMetadata metadata) { + this.bounds = bounds == null ? VisionRect.EMPTY : bounds; + this.landmarks = landmarks == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(new HashMap(landmarks)); + this.yaw = yaw; + this.pitch = pitch; + this.roll = roll; + this.smilingProbability = smilingProbability; + this.trackingId = trackingId; + this.metadata = metadata; + } + + /// @return normalized top-left-origin face bounds + public VisionRect getBounds() { + return bounds; + } + + /// @return immutable named landmark map; possibly empty + public Map getLandmarks() { + return landmarks; + } + + /// @return left/right head rotation in degrees, or 0 if unavailable + public float getYaw() { + return yaw; + } + + /// @return up/down head rotation in degrees, or 0 if unavailable + public float getPitch() { + return pitch; + } + + /// @return in-plane head rotation in degrees, or 0 if unavailable + public float getRoll() { + return roll; + } + + /// @return smile probability in 0..1, or -1 when unavailable + public float getSmilingProbability() { + return smilingProbability; + } + + /// @return stream tracking identifier, or -1 when unavailable + public int getTrackingId() { + return trackingId; + } + + /// @return backend metadata, or {@code null} when manually constructed + public VisionMetadata getMetadata() { + return metadata; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java b/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java new file mode 100644 index 00000000000..63a8d6b82c0 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/FaceDetector.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Creates reusable on-device face analyzers. +public final class FaceDetector extends AbstractVisionAnalyzer { + /// Creates an analyzer using the platform default backend and options. + /// @see VisionOptions + public FaceDetector() { + this(null); + } + + /// Creates a reusable analyzer with explicit backend and result options. + /// @param options configuration captured by this analyzer; {@code null} + /// uses defaults + public FaceDetector(VisionOptions options) { + super(VisionFeature.FACE_DETECTION, options); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java b/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java new file mode 100644 index 00000000000..1a7c004067a --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/ImageLabel.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Portable ranked image-classification label. Confidence is normalized to +/// 0..1. The numeric index is a backend/model class index and should not be +/// treated as stable across different backends; portable code should prefer +/// the text label. +public final class ImageLabel { + private final String text; + private final float confidence; + private final int index; + private final VisionMetadata metadata; + + /// Creates a label without backend metadata. + /// @param text normalized label text + /// @param confidence provider confidence, normally from zero to one + /// @param index provider label index, or a negative value when unavailable + public ImageLabel(String text, float confidence, int index) { + this(text, confidence, index, null); + } + + /// Creates a label with backend diagnostics. + /// @param text normalized label text + /// @param confidence provider confidence, normally from zero to one + /// @param index provider label index, or a negative value when unavailable + /// @param metadata backend details, or {@code null} + public ImageLabel(String text, float confidence, int index, + VisionMetadata metadata) { + this.text = text == null ? "" : text; + this.confidence = confidence; + this.index = index; + this.metadata = metadata; + } + + /// @return human-readable class label + public String getText() { + return text; + } + + /// @return classification confidence in the range 0..1 + public float getConfidence() { + return confidence; + } + + /// Returns the backend/model class index when the selected classifier + /// exposes one. The index identifies a class in that specific model; it is + /// not portable across models or backends. Apple Vision does not expose a + /// numeric class index and therefore returns {@code -1}. Prefer + /// {@link #getText()} when writing backend-independent application logic. + /// + /// @return backend/model class index, or {@code -1} when unavailable + public int getIndex() { + return index; + } + + /// @return backend metadata, or {@code null} when manually constructed + public VisionMetadata getMetadata() { + return metadata; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java b/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java new file mode 100644 index 00000000000..b607a7f03a8 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/ImageLabeler.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Creates reusable on-device image classifiers. +public final class ImageLabeler extends AbstractVisionAnalyzer { + /// Creates an analyzer using the platform default backend and options. + /// @see VisionOptions + public ImageLabeler() { + this(null); + } + + /// Creates a reusable analyzer with explicit backend and result options. + /// @param options configuration captured by this analyzer; {@code null} + /// uses defaults + public ImageLabeler(VisionOptions options) { + super(VisionFeature.IMAGE_LABELING, options); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/Pose.java b/CodenameOne/src/com/codename1/ai/vision/Pose.java new file mode 100644 index 00000000000..ae958ad0466 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/Pose.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Portable body-pose result. Landmark coordinates use the normalized +/// top-left-origin image space defined by {@link VisionPoint}; confidence is +/// in the range 0..1. +/// +/// Common backend-neutral names include `nose`, `leftEye`, `rightEye`, +/// `leftEar`, `rightEar`, and the `left`/`right` forms of `Shoulder`, `Elbow`, +/// `Wrist`, `Hip`, `Knee`, and `Ankle`. A backend can additionally report +/// finer eye, mouth, finger, heel, foot, `neck`, or `root` landmarks using the +/// same lower-camel-case convention. +public final class Pose { + private final Landmark[] landmarks; + private final VisionMetadata metadata; + + /// Creates a pose without backend metadata. + /// @param landmarks detected named body landmarks, defensively copied + public Pose(Landmark[] landmarks) { + this(landmarks, null); + } + + /// Creates a pose with backend diagnostics. + /// @param landmarks detected named body landmarks, defensively copied + /// @param metadata backend details, or {@code null} + public Pose(Landmark[] landmarks, VisionMetadata metadata) { + if (landmarks == null) { + this.landmarks = new Landmark[0]; + } else { + this.landmarks = new Landmark[landmarks.length]; + System.arraycopy(landmarks, 0, this.landmarks, 0, landmarks.length); + } + this.metadata = metadata; + } + + /// @return defensive copy of detected body landmarks + public Landmark[] getLandmarks() { + Landmark[] out = new Landmark[landmarks.length]; + System.arraycopy(landmarks, 0, out, 0, landmarks.length); + return out; + } + + /// @return backend metadata, or {@code null} when manually constructed + public VisionMetadata getMetadata() { + return metadata; + } + + /// One named body joint with normalized position and confidence. + public static final class Landmark { + private final String name; + private final VisionPoint position; + private final float confidence; + + /// Creates one detected body joint. + /// @param name joint name + /// @param position normalized joint position + /// @param confidence in-frame confidence in 0..1 + public Landmark(String name, VisionPoint position, float confidence) { + this.name = name; + this.position = position; + this.confidence = confidence; + } + + /// Returns the stable lower-camel-case joint name described by + /// {@link Pose}. Known native constants are normalized rather than + /// exposed as platform-specific numeric or symbolic identifiers. + /// + /// @return backend-neutral joint name, or `unknown` if unmapped + public String getName() { + return name; + } + + /// @return normalized top-left-origin joint position + public VisionPoint getPosition() { + return position; + } + + /// @return in-frame confidence in the range 0..1 + public float getConfidence() { + return confidence; + } + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java b/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java new file mode 100644 index 00000000000..8c50777180d --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/PoseDetector.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Creates reusable body-pose analyzers. +public final class PoseDetector extends AbstractVisionAnalyzer { + /// Creates an analyzer using the platform default backend and options. + /// @see VisionOptions + public PoseDetector() { + this(null); + } + + /// Creates a reusable analyzer with explicit backend and result options. + /// @param options configuration captured by this analyzer; {@code null} + /// uses defaults + public PoseDetector(VisionOptions options) { + super(VisionFeature.POSE_DETECTION, options); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java b/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java new file mode 100644 index 00000000000..86fd14d337f --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/SegmentationMask.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Dense per-pixel foreground confidence mask. The confidence array is +/// row-major with exactly {@code width * height} values in the range 0..1. +/// It is defensively copied on construction and access. +public final class SegmentationMask { + private final int width; + private final int height; + private final float[] confidence; + private final VisionMetadata metadata; + + /// Creates a row-major confidence mask without backend metadata. + /// @param width mask width in pixels + /// @param height mask height in pixels + /// @param confidence one foreground probability per pixel, defensively copied + /// @throws IllegalArgumentException if either dimension is negative, the + /// pixel count exceeds the maximum Java array length, or + /// {@code confidence} does not contain exactly one value per pixel + public SegmentationMask(int width, int height, float[] confidence) { + this(width, height, confidence, null); + } + + /// Creates a row-major confidence mask with backend diagnostics. + /// @param width mask width in pixels + /// @param height mask height in pixels + /// @param confidence one foreground probability per pixel, defensively copied + /// @param metadata backend details, or {@code null} + /// @throws IllegalArgumentException if either dimension is negative, the + /// pixel count exceeds the maximum Java array length, or + /// {@code confidence} does not contain exactly one value per pixel + public SegmentationMask(int width, int height, float[] confidence, + VisionMetadata metadata) { + long pixelCount = (long) width * (long) height; + if (width < 0 || height < 0 || pixelCount > Integer.MAX_VALUE + || confidence == null || confidence.length != pixelCount) { + throw new IllegalArgumentException("Mask dimensions do not match its data"); + } + this.width = width; + this.height = height; + this.confidence = new float[confidence.length]; + System.arraycopy(confidence, 0, this.confidence, 0, confidence.length); + this.metadata = metadata; + } + + /// @return mask width, which may differ from source image width + public int getWidth() { + return width; + } + + /// @return mask height, which may differ from source image height + public int getHeight() { + return height; + } + + /// @return defensive copy of row-major foreground confidences + public float[] getConfidence() { + float[] out = new float[confidence.length]; + System.arraycopy(confidence, 0, out, 0, confidence.length); + return out; + } + + /// @return backend metadata, or {@code null} when manually constructed + public VisionMetadata getMetadata() { + return metadata; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java b/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java new file mode 100644 index 00000000000..e31ab0a9b4c --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/SelfieSegmenter.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Creates reusable foreground/person segmentation analyzers. +public final class SelfieSegmenter extends AbstractVisionAnalyzer { + /// Creates an analyzer using the platform default backend and options. + /// @see VisionOptions + public SelfieSegmenter() { + this(null); + } + + /// Creates a reusable analyzer with explicit backend and result options. + /// @param options configuration captured by this analyzer; {@code null} + /// uses defaults + public SelfieSegmenter(VisionOptions options) { + super(VisionFeature.SELFIE_SEGMENTATION, options); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/TextRecognitionResult.java b/CodenameOne/src/com/codename1/ai/vision/TextRecognitionResult.java new file mode 100644 index 00000000000..244ec2e97dc --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/TextRecognitionResult.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// OCR output containing the complete recognized text and portable +/// block-level geometry. Block bounds use normalized top-left-origin +/// coordinates; a language tag may be absent when the backend does not expose +/// per-block language identification. +public final class TextRecognitionResult { + public static final TextRecognitionResult EMPTY = + new TextRecognitionResult("", new TextBlock[0]); + + private final String text; + private final TextBlock[] blocks; + private final VisionMetadata metadata; + + /// Creates an OCR result without backend metadata. + /// @param text full recognized text in reading order + /// @param blocks structured text regions, defensively copied + public TextRecognitionResult(String text, TextBlock[] blocks) { + this(text, blocks, null); + } + + /// Creates an OCR result with backend diagnostics. + /// @param text full recognized text in reading order + /// @param blocks structured text regions, defensively copied + /// @param metadata backend details, or {@code null} + public TextRecognitionResult(String text, TextBlock[] blocks, + VisionMetadata metadata) { + this.text = text == null ? "" : text; + if (blocks == null) { + this.blocks = new TextBlock[0]; + } else { + this.blocks = new TextBlock[blocks.length]; + System.arraycopy(blocks, 0, this.blocks, 0, blocks.length); + } + this.metadata = metadata; + } + + /// @return complete recognized text in reading order + public String getText() { + return text; + } + + /// @return defensive copy of recognized blocks + public TextBlock[] getBlocks() { + TextBlock[] out = new TextBlock[blocks.length]; + System.arraycopy(blocks, 0, out, 0, blocks.length); + return out; + } + + /// @return backend metadata, or {@code null} when manually constructed + public VisionMetadata getMetadata() { + return metadata; + } + + /// One recognized text block with confidence and normalized bounds. + public static final class TextBlock { + private final String text; + private final float confidence; + private final VisionRect bounds; + private final String languageTag; + + /// Creates one OCR block. + /// @param text recognized text + /// @param confidence recognition confidence in 0..1 + /// @param bounds normalized block bounds + /// @param languageTag BCP-47 tag, or {@code null} + public TextBlock(String text, float confidence, VisionRect bounds, String languageTag) { + this.text = text == null ? "" : text; + this.confidence = confidence; + this.bounds = bounds == null ? VisionRect.EMPTY : bounds; + this.languageTag = languageTag; + } + + /// @return recognized block text + public String getText() { + return text; + } + + /// @return recognition confidence in the range 0..1 + public float getConfidence() { + return confidence; + } + + /// @return normalized top-left-origin bounds + public VisionRect getBounds() { + return bounds; + } + + /// @return BCP-47 language tag, or {@code null} when unavailable + public String getLanguageTag() { + return languageTag; + } + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java b/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java new file mode 100644 index 00000000000..4bddef0788e --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/TextRecognizer.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Creates reusable on-device OCR analyzers. +public final class TextRecognizer extends AbstractVisionAnalyzer { + /// Creates an analyzer using the platform default backend and options. + /// @see VisionOptions + public TextRecognizer() { + this(null); + } + + /// Creates a reusable analyzer with explicit backend and result options. + /// @param options configuration captured by this analyzer; {@code null} + /// uses defaults + public TextRecognizer(VisionOptions options) { + super(VisionFeature.TEXT_RECOGNITION, options); + } +} diff --git a/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/package-info.java b/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java similarity index 53% rename from maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/package-info.java rename to CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java index e9d8c6920d9..84e5de95020 100644 --- a/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/package-info.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionAnalyzer.java @@ -20,19 +20,25 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ +package com.codename1.ai.vision; -/// ML Kit Pose Detection. -/// -/// Returns skeletal landmarks for human bodies detected in an image. -/// -/// The single public class in this package is [PoseDetector], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativePoseDetector` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `PoseDetector.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.pose; +import com.codename1.util.AsyncResource; + +/// Reusable, closable on-device analyzer for still images or camera frames. +/// Implementations may retain native detectors and models between calls, so +/// create one analyzer per stream/workflow and close it when finished. +/// Backend creation, request scheduling, capability checks, and close are +/// serialized so a concurrent close cannot leave a newly created backend +/// attached after the analyzer has closed. +public interface VisionAnalyzer extends AutoCloseable { + /// Tests the exact feature/backend pair configured for this analyzer. + /// @return {@code true} when the current target supports it + boolean isSupported(); + /// Starts one asynchronous analysis without uploading the image. + /// @param image encoded or raw input + /// @return asynchronous typed result delivered on the EDT + AsyncResource process(VisionImage image); + /// Releases native detector/model resources; further processing fails. + @Override + void close(); +} diff --git a/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScanner.java b/CodenameOne/src/com/codename1/ai/vision/VisionBackend.java similarity index 75% rename from maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScanner.java rename to CodenameOne/src/com/codename1/ai/vision/VisionBackend.java index 3426aa7e59e..00019a8de00 100644 --- a/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScanner.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionBackend.java @@ -20,13 +20,12 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ +package com.codename1.ai.vision; -package com.codename1.ai.mlkit.docscan; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [DocumentScanner]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeDocumentScanner extends NativeInterface { - String scanToFile(byte[] imageBytes); +/// Identifies a vision implementation. Use {@link VisionBackends} instead of +/// implementing this interface: builder scanning of those selector calls is +/// what adds an optional, feature-specific native dependency. +public interface VisionBackend { + /// @return stable identifier passed to the current platform implementation + String getId(); } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java b/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java new file mode 100644 index 00000000000..5828bd808b6 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionBackends.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Vision backend selectors. {@link #auto()} chooses Apple Vision on iOS and +/// ML Kit on Android. Feature-specific ML Kit methods are separate so one +/// selector never adds unrelated OCR, barcode, face, pose, labeling, or +/// segmentation pods. +/// +/// Each ML Kit selector call is also a build-time dependency marker. The +/// methods intentionally return the same runtime backend id but must remain +/// distinct so the builder can include only the selected feature. Pass the +/// selector that corresponds to the analyzer being constructed; for example, +/// the face-detection selector cannot satisfy a text recognizer and that +/// mismatched analyzer reports unsupported. +public final class VisionBackends { + private static final VisionBackend AUTO = new NamedBackend("auto"); + private static final VisionBackend APPLE = new NamedBackend("apple-vision"); + private static final VisionBackend ML_KIT = new NamedBackend("ml-kit"); + + private VisionBackends() { + } + + /// @return the dependency-minimal platform default + public static VisionBackend auto() { + return AUTO; + } + + /// Selects Apple Vision/Core Image without adding a third-party dependency. + /// @return Apple-native backend selector + public static VisionBackend appleVision() { + return APPLE; + } + + /// Selects ML Kit for text recognition on iOS. Android already uses ML + /// Kit for the automatic backend. + /// + /// @return the ML Kit backend selector + public static VisionBackend mlKitTextRecognition() { + return ML_KIT; + } + + /// Selects ML Kit for barcode scanning on iOS. + /// @return the ML Kit backend selector + public static VisionBackend mlKitBarcodeScanning() { + return ML_KIT; + } + + /// Selects ML Kit for face detection on iOS. + /// @return the ML Kit backend selector + public static VisionBackend mlKitFaceDetection() { + return ML_KIT; + } + + /// Selects ML Kit for image labeling on iOS. + /// @return the ML Kit backend selector + public static VisionBackend mlKitImageLabeling() { + return ML_KIT; + } + + /// Selects ML Kit for pose detection on iOS. + /// @return the ML Kit backend selector + public static VisionBackend mlKitPoseDetection() { + return ML_KIT; + } + + /// Selects ML Kit for selfie segmentation on iOS. + /// @return the ML Kit backend selector + public static VisionBackend mlKitSelfieSegmentation() { + return ML_KIT; + } + + private static final class NamedBackend implements VisionBackend { + private final String id; + + private NamedBackend(String id) { + this.id = id; + } + + @Override + public String getId() { + return id; + } + + @Override + public String toString() { + return id; + } + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionException.java b/CodenameOne/src/com/codename1/ai/vision/VisionException.java new file mode 100644 index 00000000000..570f0760d34 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionException.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Failure reported by an on-device vision backend. +public class VisionException extends RuntimeException { + public static final int UNSUPPORTED = 1; + public static final int INVALID_IMAGE = 2; + public static final int MODEL_UNAVAILABLE = 3; + public static final int CANCELLED = 4; + public static final int BACKEND_ERROR = 5; + + private final int code; + + /// Creates a classified vision failure. + /// @param code portable failure code defined by this class + /// @param message user-readable failure description + public VisionException(int code, String message) { + super(message); + this.code = code; + } + + /// Creates a classified vision failure with its native or port cause. + /// @param code portable failure code defined by this class + /// @param message user-readable failure description + /// @param cause originating detector or image-conversion error + public VisionException(int code, String message, Throwable cause) { + super(message, cause); + this.code = code; + } + + /// @return one of this class's portable failure-code constants + public int getCode() { + return code; + } +} diff --git a/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetector.java b/CodenameOne/src/com/codename1/ai/vision/VisionFeature.java similarity index 78% rename from maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetector.java rename to CodenameOne/src/com/codename1/ai/vision/VisionFeature.java index 2a636176311..d66d0ced60f 100644 --- a/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetector.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionFeature.java @@ -20,13 +20,15 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ +package com.codename1.ai.vision; -package com.codename1.ai.mlkit.pose; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [PoseDetector]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativePoseDetector extends NativeInterface { - float[] detect(byte[] imageBytes); +/// Features understood by the shared vision backend. +public enum VisionFeature { + TEXT_RECOGNITION, + BARCODE_SCANNING, + FACE_DETECTION, + IMAGE_LABELING, + POSE_DETECTION, + SELFIE_SEGMENTATION, + DOCUMENT_SCANNING } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionImage.java b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java new file mode 100644 index 00000000000..7a17290a6a2 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionImage.java @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +import com.codename1.camera.CameraFrame; +import com.codename1.camera.FrameFormat; + +/// Immutable input for encoded still images or raw camera pixels. Factory +/// methods defensively copy their arrays. In particular, +/// {@link #fromCameraFrame(CameraFrame)} detaches from callback-owned buffers +/// that a camera backend may recycle when the callback returns. +public final class VisionImage { + private final byte[] encodedBytes; + private final byte[] pixels; + private final int width; + private final int height; + private final int rotationDegrees; + private final long timestampNanos; + private final FrameFormat format; + + private VisionImage(byte[] encodedBytes, byte[] pixels, int width, int height, + int rotationDegrees, long timestampNanos, FrameFormat format) { + this(encodedBytes, pixels, width, height, rotationDegrees, + timestampNanos, format, true); + } + + private VisionImage(byte[] encodedBytes, byte[] pixels, int width, int height, + int rotationDegrees, long timestampNanos, + FrameFormat format, boolean copyData) { + this.encodedBytes = copyData ? copy(encodedBytes) : encodedBytes; + this.pixels = copyData ? copy(pixels) : pixels; + this.width = width; + this.height = height; + this.rotationDegrees = normalizeRotation(rotationDegrees); + this.timestampNanos = timestampNanos; + this.format = format == null ? FrameFormat.JPEG : format; + } + + static VisionImage detachedCameraData(byte[] data, boolean raw, + int width, int height, + int rotationDegrees, + long timestampNanos, + FrameFormat format) { + return new VisionImage(raw ? null : data, raw ? data : null, + width, height, rotationDegrees, timestampNanos, + raw ? format : FrameFormat.JPEG, false); + } + + /// Creates an encoded JPEG or PNG input whose stored pixels are already + /// upright. This method does not inspect EXIF orientation metadata. Use + /// {@link #encoded(byte[], int)} when the encoded image needs a clockwise + /// display rotation before analysis. + /// + /// @param bytes complete encoded image, copied by this method + /// @return immutable image input + /// @throws IllegalArgumentException if {@code bytes} is {@code null} or empty + public static VisionImage encoded(byte[] bytes) { + return encoded(bytes, 0); + } + + /// Creates an encoded JPEG or PNG input with display orientation metadata. + /// The rotation describes how the stored pixels must be rotated clockwise + /// to appear upright; for example, pass {@code 90} for a portrait JPEG + /// whose pixels are stored in landscape orientation. Decoders on Android + /// and Apple platforms receive this value directly, so detected bounds and + /// points use the displayed orientation. This method does not parse EXIF; + /// callers loading an image outside {@link #fromCameraFrame(CameraFrame)} + /// must supply the EXIF-derived rotation when it is relevant. + /// + /// @param bytes complete encoded image, copied by this method + /// @param rotationDegrees clockwise display rotation; only 0, 90, 180, + /// or 270 degrees are supported + /// @return immutable image input + /// @throws IllegalArgumentException if {@code bytes} is {@code null} or + /// empty, or if the rotation is not a quarter turn + public static VisionImage encoded(byte[] bytes, int rotationDegrees) { + if (bytes == null || bytes.length == 0) { + throw new IllegalArgumentException("Image bytes must not be empty"); + } + return new VisionImage(bytes, null, 0, 0, rotationDegrees, 0, + FrameFormat.JPEG); + } + + /// Creates raw NV21 or RGBA8888 input with display orientation metadata. + /// @param bytes pixel buffer in {@code format}, copied by this method + /// @param width unrotated pixel width + /// @param height unrotated pixel height + /// @param format supported raw frame format + /// @param rotationDegrees clockwise display rotation; only 0, 90, 180, + /// or 270 degrees are supported + /// @return immutable image input + /// @throws IllegalArgumentException if the data or dimensions are empty, + /// if {@code format} is not {@link FrameFormat#NV21} or + /// {@link FrameFormat#RGBA8888}, or if the rotation is not a quarter turn + public static VisionImage pixels(byte[] bytes, int width, int height, + FrameFormat format, int rotationDegrees) { + if (bytes == null || bytes.length == 0 || width <= 0 || height <= 0) { + throw new IllegalArgumentException("Pixels and positive dimensions are required"); + } + if (format != FrameFormat.NV21 + && format != FrameFormat.RGBA8888) { + throw new IllegalArgumentException( + "Raw pixels require NV21 or RGBA8888 format; " + + "use encoded() for JPEG or PNG data"); + } + return new VisionImage(null, bytes, width, height, rotationDegrees, 0, format); + } + + /// Copies the buffer selected by the camera frame's format, together with + /// its timestamp and orientation. JPEG frames copy only + /// {@link CameraFrame#getJpegBytes()}; NV21 and RGBA8888 frames copy only + /// {@link CameraFrame#getRawBytes()} when the port supplies that buffer. + /// If a port cannot supply the requested raw format, this method copies + /// the always-available JPEG fallback instead. This preserves the raw + /// pipeline without creating an empty image on JPEG-only camera ports. + /// + /// @param frame callback-owned frame + /// @return detached immutable input safe for asynchronous analysis + /// @throws IllegalArgumentException if the frame reports a rotation other + /// than 0, 90, 180, or 270 degrees + public static VisionImage fromCameraFrame(CameraFrame frame) { + if (frame == null) { + throw new NullPointerException("frame"); + } + FrameFormat format = frame.getFormat() == null + ? FrameFormat.JPEG : frame.getFormat(); + byte[] raw = frame.getRawBytes(); + boolean useRaw = format != FrameFormat.JPEG + && raw != null && raw.length > 0; + byte[] encoded = useRaw ? null : frame.getJpegBytes(); + byte[] pixels = useRaw ? raw : null; + if (!useRaw) { + format = FrameFormat.JPEG; + } + return new VisionImage(encoded, pixels, + frame.getWidth(), frame.getHeight(), frame.getRotationDegrees(), + frame.getTimestampNanos(), format); + } + + /// @return a defensive copy of encoded bytes, or {@code null} for raw input + public byte[] getEncodedBytes() { + return copy(encodedBytes); + } + + /// Returns the immutable object's backing encoded buffer to a port + /// implementation. Application code must use {@link #getEncodedBytes()}, + /// which preserves the class's defensive-copy contract. + /// + /// @return the backing encoded buffer, or {@code null} for a raw image + /// @hidden + public byte[] getEncodedBytesUnsafe() { + return encodedBytes; + } + + /// @return a defensive copy of raw pixels, or {@code null} for encoded input + public byte[] getPixels() { + return copy(pixels); + } + + /// Returns the immutable object's backing pixel buffer to a port + /// implementation. The returned array must never be modified or retained + /// beyond the native call that consumes it. + /// + /// @return the backing pixel buffer, or {@code null} for an encoded image + /// @hidden + public byte[] getPixelsUnsafe() { + return pixels; + } + + /// @return raw pixel width, or zero for encoded input + public int getWidth() { + return width; + } + + /// @return raw pixel height, or zero for encoded input + public int getHeight() { + return height; + } + + /// @return normalized clockwise rotation: 0, 90, 180, or 270 + public int getRotationDegrees() { + return rotationDegrees; + } + + /// @return capture timestamp in nanoseconds, or zero for manual input + public long getTimestampNanos() { + return timestampNanos; + } + + /// @return encoded/raw frame format + public FrameFormat getFormat() { + return format; + } + + private static byte[] copy(byte[] value) { + if (value == null) { + return null; + } + byte[] out = new byte[value.length]; + System.arraycopy(value, 0, out, 0, value.length); + return out; + } + + private static int normalizeRotation(int value) { + int out = value % 360; + if (out < 0) { + out += 360; + } + if (out % 90 != 0) { + throw new IllegalArgumentException( + "Rotation must be 0, 90, 180, or 270 degrees"); + } + return out; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionMetadata.java b/CodenameOne/src/com/codename1/ai/vision/VisionMetadata.java new file mode 100644 index 00000000000..47446f2fb9c --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionMetadata.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/// Optional backend identity and backend-specific diagnostic strings attached +/// to a vision result. Portable application logic should use typed result +/// fields first; metadata keys are intentionally not guaranteed across +/// backends. The values map is immutable. +public final class VisionMetadata { + private final String backendId; + private final Map values; + + /// Creates backend metadata with optional diagnostic values. + /// @param backendId stable portable backend id + /// @param values backend-specific string diagnostics, defensively copied + public VisionMetadata(String backendId, Map values) { + this.backendId = backendId; + this.values = values == null || values.isEmpty() + ? Collections.emptyMap() + : Collections.unmodifiableMap(new HashMap(values)); + } + + /// Creates metadata containing only the selected backend id. + /// @param backendId stable portable backend id + public VisionMetadata(String backendId) { + this(backendId, null); + } + + /// @return stable backend identifier such as {@code apple-vision} or {@code ml-kit} + public String getBackendId() { + return backendId; + } + + public Map getValues() { + return values; + } + + /// @param key backend-defined diagnostic key + /// @return associated value, or {@code null} + public String get(String key) { + return values.get(key); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java b/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java new file mode 100644 index 00000000000..ac6e3d689e7 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionOptions.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Common analyzer configuration. An analyzer captures the supplied options +/// when constructed; reuse it for frames needing the same backend, confidence +/// threshold, and result limit. +public final class VisionOptions { + private VisionBackend backend = VisionBackends.auto(); + private float minimumConfidence; + private int maximumResults; + + /// @param value selector, or {@code null} to restore automatic selection + /// @return this options object + public VisionOptions backend(VisionBackend value) { + backend = value == null ? VisionBackends.auto() : value; + return this; + } + + /// Sets the confidence threshold, clamped to 0..1. NaN has no meaningful + /// ordering and is rejected instead of being passed to platform detectors. + /// @param value requested threshold + /// @return this options object + /// @throws IllegalArgumentException if {@code value} is NaN + public VisionOptions minimumConfidence(float value) { + if (Float.isNaN(value)) { + throw new IllegalArgumentException( + "minimum confidence must be a number"); + } + minimumConfidence = Math.max(0, Math.min(1, value)); + return this; + } + + /// Sets a non-negative result limit; zero means backend default/unlimited. + /// @param value requested limit + /// @return this options object + public VisionOptions maximumResults(int value) { + maximumResults = Math.max(0, value); + return this; + } + + /// @return selected backend, never {@code null} + public VisionBackend getBackend() { + return backend; + } + + /// @return confidence threshold in the range 0..1 + public float getMinimumConfidence() { + return minimumConfidence; + } + + /// @return non-negative result limit; zero means backend default/unlimited + public int getMaximumResults() { + return maximumResults; + } + + VisionOptions snapshot() { + return new VisionOptions().backend(backend) + .minimumConfidence(minimumConfidence) + .maximumResults(maximumResults); + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java new file mode 100644 index 00000000000..ce4649a79df --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionPipeline.java @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +import com.codename1.camera.CameraFrame; +import com.codename1.camera.CameraSession; +import com.codename1.camera.FrameListener; +import com.codename1.camera.FrameFormat; +import com.codename1.ui.Display; +import com.codename1.util.SuccessCallback; + +/// Connects a camera frame stream to a reusable analyzer with keep-only-latest +/// backpressure. At most one analysis and one pending frame are retained, so a +/// slow model cannot build an unbounded queue. Results and errors arrive on +/// the EDT. The pipeline owns and closes the analyzer but not the camera +/// session. +public final class VisionPipeline implements AutoCloseable { + private final CameraSession session; + private final VisionAnalyzer analyzer; + private final VisionPipelineListener listener; + private final FrameListener frameListener; + private PendingFrame pending; + private boolean busy; + private boolean closed; + + /// Attaches immediately as the session's frame listener. + /// @param session active camera session whose frames should be analyzed + /// @param analyzer reusable analyzer owned by this pipeline + /// @param listener EDT result/error listener + public VisionPipeline(CameraSession session, VisionAnalyzer analyzer, + VisionPipelineListener listener) { + if (session == null || analyzer == null || listener == null) { + throw new NullPointerException("session, analyzer, and listener are required"); + } + this.session = session; + this.analyzer = analyzer; + this.listener = listener; + frameListener = new FrameListener() { + @Override + public void onFrame(CameraFrame frame) { + accept(frame); + } + }; + session.setFrameListener(frameListener); + } + + private void accept(CameraFrame frame) { + synchronized (this) { + if (closed) { + return; + } + if (busy) { + if (pending == null) { + pending = new PendingFrame(); + } + pending.copyFrom(frame); + return; + } + busy = true; + } + process(VisionImage.fromCameraFrame(frame)); + } + + private void process(final VisionImage image) { + final com.codename1.util.AsyncResource operation; + try { + synchronized (this) { + if (closed) { + busy = false; + pending = null; + return; + } + } + // Native analyzers may complete synchronously or re-enter app + // code. Do not invoke them while holding the pipeline monitor. + operation = analyzer.process(image); + if (operation == null) { + throw new IllegalStateException( + "Vision analyzer returned no operation"); + } + } catch (final Throwable error) { + onFinished(new Runnable() { + @Override + public void run() { + listener.error(error); + } + }); + return; + } + operation.ready(new SuccessCallback() { + @Override + public void onSucess(final T value) { + onFinished(new Runnable() { + @Override + public void run() { + listener.result(value, image); + } + }); + } + }).except(new SuccessCallback() { + @Override + public void onSucess(final Throwable error) { + onFinished(new Runnable() { + @Override + public void run() { + listener.error(error); + } + }); + } + }); + } + + private void onFinished(final Runnable notification) { + Display.getInstance().callSerially(new Runnable() { + @Override + public void run() { + PendingFrame pendingFrame; + synchronized (VisionPipeline.this) { + if (closed) { + busy = false; + pending = null; + return; + } + pendingFrame = pending; + pending = null; + busy = pendingFrame != null; + } + try { + notification.run(); + } finally { + if (pendingFrame != null) { + process(pendingFrame.toImage()); + } + } + } + }); + } + + /// Returns whether a frame is currently being analyzed. + /// + /// @return {@code true} while the open pipeline has an analysis in flight + public boolean isBusy() { + synchronized (this) { + return busy; + } + } + + /// Stops accepting frames, detaches from the camera session, discards the + /// pending frame, clears the busy state, and closes the analyzer. A pending + /// frame already selected for dispatch is rechecked and discarded before + /// it can reach the closed analyzer. Calling this method more than once + /// has no effect. + @Override + public void close() { + synchronized (this) { + if (closed) { + return; + } + closed = true; + busy = false; + pending = null; + } + session.removeFrameListener(frameListener); + analyzer.close(); + } + + private static final class PendingFrame { + private byte[] data; + private boolean raw; + private int width; + private int height; + private int rotationDegrees; + private long timestampNanos; + private FrameFormat format; + + void copyFrom(CameraFrame frame) { + FrameFormat requested = frame.getFormat() == null + ? FrameFormat.JPEG : frame.getFormat(); + byte[] rawBytes = frame.getRawBytes(); + raw = requested != FrameFormat.JPEG + && rawBytes != null && rawBytes.length > 0; + byte[] source = raw ? rawBytes : frame.getJpegBytes(); + if (source == null) { + source = new byte[0]; + } + if (data == null || data.length != source.length) { + data = new byte[source.length]; + } + System.arraycopy(source, 0, data, 0, source.length); + width = frame.getWidth(); + height = frame.getHeight(); + rotationDegrees = frame.getRotationDegrees(); + timestampNanos = frame.getTimestampNanos(); + format = raw ? requested : FrameFormat.JPEG; + } + + VisionImage toImage() { + return VisionImage.detachedCameraData(data, raw, width, height, + rotationDegrees, timestampNanos, format); + } + } +} diff --git a/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabeler.java b/CodenameOne/src/com/codename1/ai/vision/VisionPipelineListener.java similarity index 78% rename from maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabeler.java rename to CodenameOne/src/com/codename1/ai/vision/VisionPipelineListener.java index 6f0753f07f0..c2f41f95ce3 100644 --- a/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabeler.java +++ b/CodenameOne/src/com/codename1/ai/vision/VisionPipelineListener.java @@ -20,13 +20,10 @@ * Please contact Codename One through http://www.codenameone.com/ if you * need additional information or have any questions. */ +package com.codename1.ai.vision; -package com.codename1.ai.mlkit.labeling; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [ImageLabeler]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeImageLabeler extends NativeInterface { - String[] label(byte[] imageBytes); +/// Receives live vision results and recoverable analysis failures on the EDT. +public interface VisionPipelineListener { + void result(T value, VisionImage source); + void error(Throwable error); } diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionPoint.java b/CodenameOne/src/com/codename1/ai/vision/VisionPoint.java new file mode 100644 index 00000000000..2bb8f1f8d90 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionPoint.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Immutable point in a normalized, top-left-origin coordinate space. X grows +/// right and Y grows down; 0 and 1 correspond to image edges, independent of +/// source pixel dimensions. +public final class VisionPoint { + private final float x; + private final float y; + + /// Creates a point in the oriented input image's top-left coordinate space. + /// @param x horizontal coordinate + /// @param y vertical coordinate + public VisionPoint(float x, float y) { + this.x = x; + this.y = y; + } + + /// @return horizontal coordinate normalized to the oriented input width + public float getX() { + return x; + } + + /// @return downward vertical coordinate normalized to input height + public float getY() { + return y; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/VisionRect.java b/CodenameOne/src/com/codename1/ai/vision/VisionRect.java new file mode 100644 index 00000000000..c343eda5b37 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/VisionRect.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +/// Immutable normalized rectangle using a top-left origin. X/Y identify the +/// upper-left corner and width/height are fractions of the oriented input +/// dimensions. {@link #EMPTY} represents unavailable geometry. +public final class VisionRect { + public static final VisionRect EMPTY = new VisionRect(0, 0, 0, 0); + + private final float x; + private final float y; + private final float width; + private final float height; + + /// Creates a rectangle in the oriented input image's top-left coordinate space. + /// @param x left coordinate + /// @param y top coordinate + /// @param width non-negative rectangle width + /// @param height non-negative rectangle height + public VisionRect(float x, float y, float width, float height) { + this.x = x; + this.y = y; + this.width = width; + this.height = height; + } + + /// @return normalized left coordinate + public float getX() { + return x; + } + + /// @return normalized top coordinate + public float getY() { + return y; + } + + /// @return width as a fraction of oriented input width + public float getWidth() { + return width; + } + + /// @return height as a fraction of oriented input height + public float getHeight() { + return height; + } +} diff --git a/CodenameOne/src/com/codename1/ai/vision/package-info.java b/CodenameOne/src/com/codename1/ai/vision/package-info.java new file mode 100644 index 00000000000..e9726740490 --- /dev/null +++ b/CodenameOne/src/com/codename1/ai/vision/package-info.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +/// Vendor-neutral on-device vision APIs for still images and live camera frames. +/// +///

The automatic backend uses Apple Vision/Core Image on iOS and Mac +/// Catalyst and ML Kit on Android. Unsupported ports report that through +/// {@code isSupported()}. Optional backends are selected with +/// {@link com.codename1.ai.vision.VisionBackends}.

+/// +///

Each analyzer is a separate build-time feature. Referencing one analyzer +/// causes the builder to retain only its platform adapter and native +/// dependency. {@link com.codename1.ai.vision.VisionPipeline} safely copies +/// callback-owned camera frames and drops stale pending frames under load.

+package com.codename1.ai.vision; diff --git a/CodenameOne/src/com/codename1/camera/CameraFrame.java b/CodenameOne/src/com/codename1/camera/CameraFrame.java index 58424b7f82a..5a1c12f043b 100644 --- a/CodenameOne/src/com/codename1/camera/CameraFrame.java +++ b/CodenameOne/src/com/codename1/camera/CameraFrame.java @@ -52,15 +52,18 @@ public CameraFrame(byte[] jpegBytes, byte[] rawBytes, } /// JPEG-encoded bytes for this frame. Always non-null regardless of the - /// requested `FrameFormat`; the framework encodes raw frames into JPEG - /// on demand for AI module consumption. + /// requested `FrameFormat`; use this when encoded image data is needed. + /// `VisionImage#fromCameraFrame(CameraFrame)` instead selects the raw + /// buffer for NV21 and RGBA8888 frames. public byte[] getJpegBytes() { return jpegBytes; } /// Raw pixel bytes in the format requested via - /// `CameraSessionOptions#frameFormat(FrameFormat)`. `null` when the - /// requested format was `FrameFormat#JPEG`. + /// `CameraSessionOptions#frameFormat(FrameFormat)`. This is `null` for + /// JPEG requests and may also be `null` when the current port cannot + /// expose the requested raw format. In that case + /// {@link #getJpegBytes()} remains available as a fallback. public byte[] getRawBytes() { return rawBytes; } @@ -87,8 +90,11 @@ public long getTimestampNanos() { return timestampNanos; } - /// Pixel format actually used for `#getRawBytes()`. JPEG bytes returned by - /// `#getJpegBytes()` are always JPEG-encoded regardless of this value. + /// Pixel format requested for this frame. A non-JPEG value describes + /// {@link #getRawBytes()} when that buffer is available; ports that cannot + /// expose raw pixels may return {@code null} there while still reporting + /// the requested format. JPEG bytes returned by {@link #getJpegBytes()} + /// are always JPEG-encoded regardless of this value. public FrameFormat getFormat() { return format; } diff --git a/CodenameOne/src/com/codename1/camera/FrameFormat.java b/CodenameOne/src/com/codename1/camera/FrameFormat.java index 99e44ce824e..a7cefe267e0 100644 --- a/CodenameOne/src/com/codename1/camera/FrameFormat.java +++ b/CodenameOne/src/com/codename1/camera/FrameFormat.java @@ -24,14 +24,13 @@ /// Pixel format requested for `CameraFrame` data delivered to a `FrameListener`. /// -/// `JPEG` is the default and the right choice for feeding frames into the -/// `com.codename1.ai.*` modules, all of which accept JPEG `byte[]` directly. -/// Raw formats (`NV21`, `RGBA8888`) are useful when an application performs -/// its own pixel processing and wants to avoid the JPEG encode/decode round-trip. +/// `JPEG` is the default and is useful when an application needs a portable +/// encoded image. Built-in vision analyzers also accept raw `NV21` and +/// `RGBA8888` frames directly; select a raw format for live analysis to avoid +/// a JPEG encode/decode round-trip. public enum FrameFormat { /// JPEG-encoded bytes available via `CameraFrame#getJpegBytes()`. - /// Always available regardless of the requested format; this is the - /// universal format for AI/vision module integration. + /// Always available regardless of the requested format. JPEG, /// YUV 4:2:0 NV21 layout available via `CameraFrame#getRawBytes()`. diff --git a/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java b/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java index 329c97724ee..0b12a9d5e50 100644 --- a/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java +++ b/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java @@ -7385,8 +7385,6 @@ public void capturePhoto(ActionListener response) { /// Factory for the low-level `com.codename1.camera.Camera` API. Each call /// returns a fresh per-session backend, or `null` on platforms that do not /// implement the new API. Subclasses override to wire in their port. - /// - /// @hidden public CameraImpl createCameraImpl() { return null; } @@ -7394,12 +7392,25 @@ public CameraImpl createCameraImpl() { /// Factory for the `com.codename1.ar.AR` augmented reality API. Each call /// returns a fresh per-session backend, or `null` on platforms without AR /// support. Subclasses override to wire in their port. - /// - /// @hidden public ARImpl createARImpl() { return null; } + /// Factory for the built-in on-device vision API. + public VisionImpl createVisionImpl() { + return null; + } + + /// Factory for the built-in LiteRT inference API. + public InferenceImpl createInferenceImpl() { + return null; + } + + /// Factory for built-in on-device language services. + public LanguageImpl createLanguageImpl() { + return null; + } + /// Captures a screenshot of the screen. /// /// #### Returns diff --git a/CodenameOne/src/com/codename1/impl/InferenceImpl.java b/CodenameOne/src/com/codename1/impl/InferenceImpl.java new file mode 100644 index 00000000000..c79963e5ef2 --- /dev/null +++ b/CodenameOne/src/com/codename1/impl/InferenceImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl; + +import com.codename1.ai.inference.InferenceOptions; +import com.codename1.ai.inference.ModelSource; +import com.codename1.ai.inference.Tensor; +import com.codename1.ai.inference.TensorInfo; +import com.codename1.util.AsyncResource; + +/// Port contract behind the built-in LiteRT inference API. @hidden +public abstract class InferenceImpl { + public abstract boolean isSupported(); + public abstract AsyncResource open(ModelSource source, InferenceOptions options); + public abstract TensorInfo[] getInputs(Object handle); + public abstract TensorInfo[] getOutputs(Object handle); + public abstract AsyncResource run(Object handle, Tensor[] inputs); + public abstract void resizeInput(Object handle, String name, int[] shape); + public abstract void close(Object handle); +} diff --git a/CodenameOne/src/com/codename1/impl/LanguageImpl.java b/CodenameOne/src/com/codename1/impl/LanguageImpl.java new file mode 100644 index 00000000000..8bd9fe9995c --- /dev/null +++ b/CodenameOne/src/com/codename1/impl/LanguageImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl; + +import com.codename1.ai.language.LanguageCandidate; +import com.codename1.ai.language.LanguageOptions; +import com.codename1.ai.language.SmartReplyMessage; +import com.codename1.util.AsyncResource; + +/// Port contract behind the built-in on-device language APIs. @hidden +public abstract class LanguageImpl { + public abstract boolean isSupported(String feature, String backendId); + public abstract AsyncResource identify( + String text, String backendId, LanguageOptions options); + public abstract AsyncResource translate( + String text, String sourceLanguage, String targetLanguage, + String backendId, LanguageOptions options); + public abstract AsyncResource suggestReplies( + SmartReplyMessage[] conversation, String backendId, LanguageOptions options); + public abstract void close(); +} diff --git a/CodenameOne/src/com/codename1/impl/VisionImpl.java b/CodenameOne/src/com/codename1/impl/VisionImpl.java new file mode 100644 index 00000000000..cb80164a25d --- /dev/null +++ b/CodenameOne/src/com/codename1/impl/VisionImpl.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl; + +import com.codename1.ai.vision.VisionFeature; +import com.codename1.ai.vision.VisionImage; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.util.AsyncResource; + +/// Port contract behind {@code com.codename1.ai.vision}. @hidden +public abstract class VisionImpl { + public abstract boolean isSupported(VisionFeature feature, String backendId); + public abstract AsyncResource analyze(VisionFeature feature, String backendId, + VisionImage image, VisionOptions options); + public abstract void close(); +} diff --git a/CodenameOne/src/com/codename1/ui/Display.java b/CodenameOne/src/com/codename1/ui/Display.java index 83e61ccb8e3..fe149ef78c6 100644 --- a/CodenameOne/src/com/codename1/ui/Display.java +++ b/CodenameOne/src/com/codename1/ui/Display.java @@ -6237,6 +6237,27 @@ public com.codename1.impl.ARImpl getARBackend() { return impl.createARImpl(); } + /// Creates a fresh backend for `com.codename1.ai.vision`. + /// + /// @hidden + public com.codename1.impl.VisionImpl getVisionBackend() { + return impl.createVisionImpl(); + } + + /// Creates a fresh backend for `com.codename1.ai.inference`. + /// + /// @hidden + public com.codename1.impl.InferenceImpl getInferenceBackend() { + return impl.createInferenceImpl(); + } + + /// Creates a fresh backend for `com.codename1.ai.language`. + /// + /// @hidden + public com.codename1.impl.LanguageImpl getLanguageBackend() { + return impl.createLanguageImpl(); + } + /// Indicates whether the native picker dialog is supported for the given type /// which can include one of PICKER_TYPE_DATE_AND_TIME, PICKER_TYPE_TIME, PICKER_TYPE_DATE /// diff --git a/Ports/Android/build.xml b/Ports/Android/build.xml index 3ef7fedd75f..d9a9730f8ee 100644 --- a/Ports/Android/build.xml +++ b/Ports/Android/build.xml @@ -110,14 +110,14 @@ - + excludes="com/codename1/impl/android/ar/**,com/codename1/impl/android/ai/**"> diff --git a/Ports/Android/nbproject/project.properties b/Ports/Android/nbproject/project.properties index ae84c3cc82f..a8e7118aaa4 100644 --- a/Ports/Android/nbproject/project.properties +++ b/Ports/Android/nbproject/project.properties @@ -26,11 +26,11 @@ dist.dir=dist dist.jar=${dist.dir}/Android.jar dist.javadoc.dir=${dist.dir}/javadoc endorsed.classpath= -# The ARCore-backed AR impl compiles against com.google.ar.core which is not -# in cn1-binaries; it is compiled inside user app builds where the Android -# builder adds the ARCore gradle dependency (and deletes the package for -# non-AR apps). Mirrors the maven-compiler exclude in maven/android/pom.xml. -excludes=com/codename1/impl/android/ar/** +# Optional AR and AI implementations compile against dependencies which are +# not in cn1-binaries. They are compiled inside user app builds where the +# Android builder adds only the dependencies and sources used by the app. +# Mirrors the maven-compiler excludes in maven/android/pom.xml. +excludes=com/codename1/impl/android/ar/**,com/codename1/impl/android/ai/** file.reference.android-billing-4.0.0.jar=../../../cn1-binaries/android/android-billing-4.0.0.jar file.reference.android-support-v7-appcompat.jar=../../../cn1-binaries/android/android-support-v7-appcompat.jar file.reference.android-support-v7-cardview.jar=../../../cn1-binaries/android/android-support-v7-cardview.jar diff --git a/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java b/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java index be652ba5258..94209dead70 100644 --- a/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java +++ b/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java @@ -11247,6 +11247,32 @@ public com.codename1.impl.ARImpl createARImpl() { } } + @Override + public com.codename1.impl.VisionImpl createVisionImpl() { + return (com.codename1.impl.VisionImpl) createOptionalAiBackend( + "com.codename1.impl.android.ai.AndroidVisionImpl"); + } + + @Override + public com.codename1.impl.InferenceImpl createInferenceImpl() { + return (com.codename1.impl.InferenceImpl) createOptionalAiBackend( + "com.codename1.impl.android.ai.AndroidInferenceImpl"); + } + + @Override + public com.codename1.impl.LanguageImpl createLanguageImpl() { + return (com.codename1.impl.LanguageImpl) createOptionalAiBackend( + "com.codename1.impl.android.ai.AndroidLanguageImpl"); + } + + private Object createOptionalAiBackend(String className) { + try { + return Class.forName(className).newInstance(); + } catch (Throwable t) { + return null; + } + } + // Deeper-network connectivity platform factories. Each returns a small // platform-specific class living under // com.codename1.impl.android.connectivity. Those classes are loaded diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java new file mode 100644 index 00000000000..58b0175c3e5 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidBarcodeScanningAdapter.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ai; + +import android.graphics.Point; + +import com.codename1.ai.vision.Barcode; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.ai.vision.VisionPoint; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.vision.barcode.BarcodeScanner; +import com.google.mlkit.vision.barcode.BarcodeScanning; +import com.google.mlkit.vision.common.InputImage; + +import java.util.List; + +/** ML Kit barcode scanning; retained only for {@code BarcodeScanner} users. */ +final class AndroidBarcodeScanningAdapter extends AndroidVisionAdapter { + private final BarcodeScanner client = BarcodeScanning.getClient(); + @Override + @SuppressWarnings("unchecked") + void analyze(InputImage input, final int imageWidth, + final int imageHeight, VisionOptions options, + AsyncResource resource) { + final AsyncResource out = (AsyncResource) resource; + final int maximumResults = options.getMaximumResults(); + client.process(input).addOnSuccessListener( + new OnSuccessListener>() { + public void onSuccess( + List values) { + int count = maximumResults == 0 ? values.size() + : Math.min(maximumResults, values.size()); + Barcode[] result = new Barcode[count]; + for (int i = 0; i < result.length; i++) { + com.google.mlkit.vision.barcode.common.Barcode value = + values.get(i); + Point[] points = value.getCornerPoints(); + VisionPoint[] corners = points == null + ? new VisionPoint[0] : new VisionPoint[points.length]; + for (int p = 0; p < corners.length; p++) { + corners[p] = new VisionPoint( + points[p].x / (float) imageWidth, + points[p].y / (float) imageHeight); + } + result[i] = new Barcode(value.getRawValue(), + barcodeFormat(value.getFormat()), + value.getRawBytes(), + normalized(value.getBoundingBox(), + imageWidth, imageHeight), + corners, METADATA); + } + complete(out, result); + } + }).addOnFailureListener(failure(out)); + } + + @Override + void close() { + client.close(); + } + + private static String barcodeFormat(int format) { + switch (format) { + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_AZTEC: + return "AZTEC"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_CODABAR: + return "CODABAR"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_CODE_39: + return "CODE_39"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_CODE_93: + return "CODE_93"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_CODE_128: + return "CODE_128"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_DATA_MATRIX: + return "DATA_MATRIX"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_EAN_8: + return "EAN_8"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_EAN_13: + return "EAN_13"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_ITF: + return "ITF"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_PDF417: + return "PDF417"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_QR_CODE: + return "QR_CODE"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_UPC_A: + return "UPC_A"; + case com.google.mlkit.vision.barcode.common.Barcode.FORMAT_UPC_E: + return "UPC_E"; + default: + return "UNKNOWN"; + } + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java new file mode 100644 index 00000000000..bfa21a4c678 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidFaceDetectionAdapter.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.vision.Face; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.ai.vision.VisionPoint; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.face.FaceDetection; +import com.google.mlkit.vision.face.FaceDetector; +import com.google.mlkit.vision.face.FaceDetectorOptions; +import com.google.mlkit.vision.face.FaceLandmark; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** ML Kit face detection; retained only for {@code FaceDetector} users. */ +final class AndroidFaceDetectionAdapter extends AndroidVisionAdapter { + private final FaceDetector client = FaceDetection.getClient( + new FaceDetectorOptions.Builder() + .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_ALL) + .setClassificationMode( + FaceDetectorOptions.CLASSIFICATION_MODE_ALL) + .enableTracking().build()); + @Override + @SuppressWarnings("unchecked") + void analyze(InputImage input, final int imageWidth, + final int imageHeight, VisionOptions options, + AsyncResource resource) { + final AsyncResource out = (AsyncResource) resource; + final int maximumResults = options.getMaximumResults(); + client.process(input).addOnSuccessListener( + new OnSuccessListener>() { + public void onSuccess( + List values) { + int count = maximumResults == 0 ? values.size() + : Math.min(maximumResults, values.size()); + Face[] result = new Face[count]; + for (int i = 0; i < result.length; i++) { + com.google.mlkit.vision.face.Face value = values.get(i); + Map landmarks = + new HashMap(); + addLandmark(landmarks, "leftEye", + value.getLandmark(FaceLandmark.LEFT_EYE), + imageWidth, imageHeight); + addLandmark(landmarks, "rightEye", + value.getLandmark(FaceLandmark.RIGHT_EYE), + imageWidth, imageHeight); + addLandmark(landmarks, "noseBase", + value.getLandmark(FaceLandmark.NOSE_BASE), + imageWidth, imageHeight); + addLandmark(landmarks, "mouthLeft", + value.getLandmark(FaceLandmark.MOUTH_LEFT), + imageWidth, imageHeight); + addLandmark(landmarks, "mouthRight", + value.getLandmark(FaceLandmark.MOUTH_RIGHT), + imageWidth, imageHeight); + Float smile = value.getSmilingProbability(); + Integer tracking = value.getTrackingId(); + result[i] = new Face( + normalized(value.getBoundingBox(), + imageWidth, imageHeight), + landmarks, value.getHeadEulerAngleY(), + value.getHeadEulerAngleX(), + value.getHeadEulerAngleZ(), + smile == null ? -1 : smile, + tracking == null ? -1 : tracking, METADATA); + } + complete(out, result); + } + }).addOnFailureListener(failure(out)); + } + + @Override + void close() { + client.close(); + } + + private static void addLandmark(Map out, String name, + FaceLandmark landmark, int imageWidth, + int imageHeight) { + if (landmark != null) { + out.put(name, new VisionPoint( + landmark.getPosition().x / imageWidth, + landmark.getPosition().y / imageHeight)); + } + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java new file mode 100644 index 00000000000..3a23c1dc029 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidImageLabelingAdapter.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.vision.ImageLabel; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.label.ImageLabeler; +import com.google.mlkit.vision.label.ImageLabeling; +import com.google.mlkit.vision.label.defaults.ImageLabelerOptions; + +import java.util.List; + +/** ML Kit image labeling; retained only for {@code ImageLabeler} users. */ +final class AndroidImageLabelingAdapter extends AndroidVisionAdapter { + private ImageLabeler client; + @Override + @SuppressWarnings("unchecked") + void analyze(InputImage input, int imageWidth, int imageHeight, + VisionOptions options, AsyncResource resource) { + final AsyncResource out = + (AsyncResource) resource; + final int maximumResults = options.getMaximumResults(); + if (client == null) { + client = ImageLabeling.getClient(new ImageLabelerOptions.Builder() + .setConfidenceThreshold(options.getMinimumConfidence()) + .build()); + } + client.process(input).addOnSuccessListener( + new OnSuccessListener>() { + public void onSuccess( + List values) { + int count = maximumResults == 0 ? values.size() + : Math.min(maximumResults, values.size()); + ImageLabel[] result = new ImageLabel[count]; + for (int i = 0; i < result.length; i++) { + com.google.mlkit.vision.label.ImageLabel value = + values.get(i); + result[i] = new ImageLabel(value.getText(), + value.getConfidence(), value.getIndex(), METADATA); + } + complete(out, result); + } + }).addOnFailureListener(failure(out)); + } + + @Override + void close() { + if (client != null) { + client.close(); + } + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java new file mode 100644 index 00000000000..2b0bb2e416b --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidInferenceImpl.java @@ -0,0 +1,408 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.inference.InferenceException; +import com.codename1.ai.inference.InferenceOptions; +import com.codename1.ai.inference.ModelSource; +import com.codename1.ai.inference.Tensor; +import com.codename1.ai.inference.TensorInfo; +import com.codename1.ai.inference.TensorType; +import com.codename1.impl.InferenceImpl; +import com.codename1.io.FileSystemStorage; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; + +import org.tensorflow.lite.DataType; +import org.tensorflow.lite.Interpreter; + +import java.io.ByteArrayOutputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.util.HashMap; +import java.util.Map; + +/** Android LiteRT backend. */ +public final class AndroidInferenceImpl extends InferenceImpl { + private static volatile Method outputShapeRefreshMethod; + + private static final class Handle { + final Interpreter interpreter; + + Handle(Interpreter interpreter) { + this.interpreter = interpreter; + } + } + + @Override + public boolean isSupported() { + return true; + } + + @Override + public AsyncResource open(final ModelSource source, + final InferenceOptions options) { + final AsyncResource out = new AsyncResource(); + final InferenceOptions.Accelerator accelerator = + options.getAccelerator(); + final int threads = options.getThreads(); + final boolean allowFallback = options.isFallbackAllowed(); + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + try { + if ((accelerator == InferenceOptions.Accelerator.GPU + || accelerator == InferenceOptions.Accelerator.CORE_ML) + && !allowFallback) { + throw new InferenceException(accelerator + + " acceleration is unavailable on Android"); + } + if (accelerator == InferenceOptions.Accelerator.NPU + && !allowFallback) { + throw new InferenceException( + "Strict NPU execution cannot be verified on " + + "Android; NNAPI may leave unsupported " + + "operations on the CPU"); + } + ByteBuffer model = loadModel(source); + Interpreter.Options nativeOptions = new Interpreter.Options(); + if (threads > 0) { + nativeOptions.setNumThreads(threads); + } + if (accelerator == InferenceOptions.Accelerator.NPU) { + nativeOptions.setUseNNAPI(true); + } + Interpreter interpreter; + try { + interpreter = new Interpreter(model, nativeOptions); + } catch (Throwable acceleratedFailure) { + if (accelerator != InferenceOptions.Accelerator.NPU + || !allowFallback) { + throw acceleratedFailure; + } + Interpreter.Options cpuOptions = new Interpreter.Options(); + if (threads > 0) { + cpuOptions.setNumThreads(threads); + } + model.rewind(); + interpreter = new Interpreter(model, cpuOptions); + } + final Handle handle = new Handle(interpreter); + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(handle); + } + }); + } catch (final Throwable error) { + fail(out, "Could not open LiteRT model", error); + } + } + }); + return out; + } + + @Override + public TensorInfo[] getInputs(Object handle) { + Interpreter interpreter = checked(handle).interpreter; + TensorInfo[] result = new TensorInfo[interpreter.getInputTensorCount()]; + for (int i = 0; i < result.length; i++) { + org.tensorflow.lite.Tensor tensor = interpreter.getInputTensor(i); + result[i] = info(tensor, i); + } + return result; + } + + @Override + public TensorInfo[] getOutputs(Object handle) { + Interpreter interpreter = checked(handle).interpreter; + TensorInfo[] result = new TensorInfo[interpreter.getOutputTensorCount()]; + for (int i = 0; i < result.length; i++) { + org.tensorflow.lite.Tensor tensor = interpreter.getOutputTensor(i); + result[i] = info(tensor, i); + } + return result; + } + + @Override + public AsyncResource run(Object handle, final Tensor[] inputs) { + final AsyncResource out = new AsyncResource(); + final Interpreter interpreter = checked(handle).interpreter; + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + try { + int inputCount = interpreter.getInputTensorCount(); + if (inputs == null || inputs.length != inputCount) { + throw new IllegalArgumentException("Expected " + + inputCount + " input tensors but received " + + (inputs == null ? 0 : inputs.length)); + } + Object[] nativeInputs = new Object[inputCount]; + boolean[] resolved = new boolean[inputCount]; + for (int i = 0; i < inputs.length; i++) { + Tensor value = inputs[i]; + int index = value.getName() == null ? i + : inputIndex(interpreter, value.getName()); + org.tensorflow.lite.Tensor metadata = + interpreter.getInputTensor(index); + if (resolved[index]) { + throw new IllegalArgumentException( + "Model input " + metadata.name() + + " was supplied more than once"); + } + resolved[index] = true; + if (!sameShape(value.getShape(), + metadata.shape())) { + throw new IllegalArgumentException( + "Input " + metadata.name() + + " shape does not match the " + + "model metadata; call " + + "resizeInput() before run()"); + } + nativeInputs[index] = + toBuffer(value, metadata.dataType()); + } + Map nativeOutputs = + new HashMap(); + int outputCount = interpreter.getOutputTensorCount(); + for (int i = 0; i < outputCount; i++) { + // LiteRT 1.0.1 treats a null output as invocation-only: + // the native tensor remains available through + // Tensor.asReadOnlyBuffer() after the run. This avoids + // allocating from a pre-run size that may still contain + // unresolved, value-dependent output dimensions. + nativeOutputs.put(Integer.valueOf(i), null); + } + interpreter.runForMultipleInputsOutputs(nativeInputs, + nativeOutputs); + final Tensor[] result = new Tensor[outputCount]; + for (int i = 0; i < result.length; i++) { + org.tensorflow.lite.Tensor metadata = + interpreter.getOutputTensor(i); + refreshOutputShape(metadata); + result[i] = fromBuffer(metadata.name(), + metadata.shape(), metadata.dataType(), + metadata.asReadOnlyBuffer()); + } + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(result); + } + }); + } catch (final Throwable error) { + fail(out, "LiteRT inference failed", error); + } + } + }); + return out; + } + + @Override + public void resizeInput(Object handle, String name, int[] shape) { + Interpreter interpreter = checked(handle).interpreter; + int index = inputIndex(interpreter, name); + interpreter.resizeInput(index, shape); + interpreter.allocateTensors(); + } + + @Override + public void close(Object handle) { + checked(handle).interpreter.close(); + } + + private static Handle checked(Object value) { + if (!(value instanceof Handle)) { + throw new IllegalArgumentException("Invalid LiteRT session handle"); + } + return (Handle) value; + } + + private static boolean sameShape(int[] supplied, int[] expected) { + if (supplied.length != expected.length) { + return false; + } + for (int i = 0; i < supplied.length; i++) { + if (supplied[i] != expected[i]) { + return false; + } + } + return true; + } + + private static void refreshOutputShape( + org.tensorflow.lite.Tensor metadata) { + try { + // TensorImpl caches shape(), and LiteRT refreshes that cache only + // when the invocation also reallocates tensors. Value-dependent + // output shapes can change without input reallocation, so refresh + // the package-private cache after every run. The builders retain + // this one method name for R8 release builds. + Method refresh = outputShapeRefreshMethod; + if (refresh == null) { + synchronized (AndroidInferenceImpl.class) { + refresh = outputShapeRefreshMethod; + if (refresh == null) { + refresh = metadata.getClass() + .getDeclaredMethod("refreshShape"); + refresh.setAccessible(true); + outputShapeRefreshMethod = refresh; + } + } + } + refresh.invoke(metadata); + } catch (Throwable error) { + throw new InferenceException( + "Could not refresh the LiteRT output shape", error); + } + } + + private static int inputIndex(Interpreter interpreter, String name) { + for (int i = 0; i < interpreter.getInputTensorCount(); i++) { + if (name == null || name.equals(interpreter.getInputTensor(i).name())) { + return i; + } + } + throw new IllegalArgumentException("Unknown model input " + name); + } + + private static TensorInfo info(org.tensorflow.lite.Tensor value, int index) { + return new TensorInfo(value.name(), type(value.dataType()), value.shape(), index); + } + + private static TensorType type(DataType type) { + if (type == DataType.FLOAT32) return TensorType.FLOAT32; + if (type == DataType.INT32) return TensorType.INT32; + if (type == DataType.INT64) return TensorType.INT64; + if (type == DataType.UINT8) return TensorType.UINT8; + if (type == DataType.INT8) return TensorType.INT8; + if (type == DataType.BOOL) return TensorType.BOOL; + throw new InferenceException("Unsupported LiteRT tensor type " + type); + } + + private static ByteBuffer toBuffer(Tensor value, DataType nativeType) { + if (value.getType() != type(nativeType)) { + throw new IllegalArgumentException("Input " + value.getName() + + " type does not match the model"); + } + Object data = value.getDataUnsafe(); + int byteCount; + if (data instanceof float[]) byteCount = ((float[]) data).length * 4; + else if (data instanceof int[]) byteCount = ((int[]) data).length * 4; + else if (data instanceof long[]) byteCount = ((long[]) data).length * 8; + else if (data instanceof byte[]) byteCount = ((byte[]) data).length; + else throw new InferenceException("Unsupported input tensor data"); + ByteBuffer out = ByteBuffer.allocateDirect(byteCount).order(ByteOrder.nativeOrder()); + if (data instanceof float[]) out.asFloatBuffer().put((float[]) data); + else if (data instanceof int[]) out.asIntBuffer().put((int[]) data); + else if (data instanceof long[]) out.asLongBuffer().put((long[]) data); + else out.put((byte[]) data); + out.rewind(); + return out; + } + + private static Tensor fromBuffer(String name, int[] shape, DataType nativeType, + ByteBuffer value) { + value.rewind(); + value.order(ByteOrder.nativeOrder()); + TensorType type = type(nativeType); + int count = elementCount(shape); + if (type == TensorType.FLOAT32) { + float[] data = new float[count]; + value.asFloatBuffer().get(data); + return new Tensor(name, type, shape, data); + } + if (type == TensorType.INT32) { + int[] data = new int[count]; + value.asIntBuffer().get(data); + return new Tensor(name, type, shape, data); + } + if (type == TensorType.INT64) { + long[] data = new long[count]; + value.asLongBuffer().get(data); + return new Tensor(name, type, shape, data); + } + byte[] data = new byte[count]; + value.get(data); + return new Tensor(name, type, shape, data); + } + + private static int elementCount(int[] shape) { + int count = 1; + for (int i = 0; i < shape.length; i++) { + count *= shape[i]; + } + return count; + } + + private static ByteBuffer loadModel(ModelSource source) throws IOException { + if (source.getKind() == ModelSource.FILE) { + String nativePath = FileSystemStorage.getInstance().toNativePath( + source.getPath()); + FileInputStream file = new FileInputStream(nativePath); + try { + return file.getChannel().map(FileChannel.MapMode.READ_ONLY, + 0, file.getChannel().size()); + } finally { + file.close(); + } + } + byte[] bytes; + if (source.getKind() == ModelSource.BYTES) { + bytes = source.getBytesUnsafe(); + } else { + InputStream input = Display.getInstance().getResourceAsStream( + AndroidInferenceImpl.class, source.getPath()); + if (input == null) { + throw new IOException("Model resource not found: " + source.getPath()); + } + try { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[16384]; + int read; + while ((read = input.read(buffer)) >= 0) { + output.write(buffer, 0, read); + } + bytes = output.toByteArray(); + } finally { + input.close(); + } + } + ByteBuffer result = ByteBuffer.allocateDirect(bytes.length) + .order(ByteOrder.nativeOrder()); + result.put(bytes); + result.rewind(); + return result; + } + + private static void fail(final AsyncResource out, final String message, + final Throwable cause) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.error(new InferenceException(message, cause)); + } + }); + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageAdapter.java new file mode 100644 index 00000000000..f0f1e14edee --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageAdapter.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.language.LanguageCandidate; +import com.codename1.ai.language.LanguageOptions; +import com.codename1.ai.language.SmartReplyMessage; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnFailureListener; + +/** Shared contract and completion helpers for Android language adapters. */ +abstract class AndroidLanguageAdapter implements AutoCloseable { + AsyncResource identify( + String text, LanguageOptions options) { + return unsupported("Language identification is not supported"); + } + + AsyncResource translate( + String text, String sourceLanguage, String targetLanguage, + LanguageOptions options) { + return unsupported("Translation is not supported"); + } + + AsyncResource suggestReplies( + SmartReplyMessage[] conversation, LanguageOptions options) { + return unsupported("Smart Reply is not supported"); + } + + static AsyncResource unsupported(String message) { + AsyncResource out = new AsyncResource(); + out.error(new UnsupportedOperationException(message)); + return out; + } + + static void complete(final AsyncResource out, final T value) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(value); + } + }); + } + + static OnFailureListener failure(final AsyncResource out) { + return new OnFailureListener() { + public void onFailure(final Exception error) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.error(error); + } + }); + } + }; + } + + public void close() { + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageIdAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageIdAdapter.java new file mode 100644 index 00000000000..a53114bd7ec --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageIdAdapter.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.language.LanguageCandidate; +import com.codename1.ai.language.LanguageOptions; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.nl.languageid.IdentifiedLanguage; +import com.google.mlkit.nl.languageid.LanguageIdentification; +import com.google.mlkit.nl.languageid.LanguageIdentificationOptions; +import com.google.mlkit.nl.languageid.LanguageIdentifier; + +import java.util.List; + +/** ML Kit language ID; retained only for {@code LanguageIdentifier} users. */ +final class AndroidLanguageIdAdapter extends AndroidLanguageAdapter { + private LanguageIdentifier client; + + @Override + AsyncResource identify( + String text, LanguageOptions options) { + final AsyncResource out = + new AsyncResource(); + final LanguageIdentifier current = client(options); + current.identifyPossibleLanguages(text).addOnSuccessListener( + new OnSuccessListener>() { + public void onSuccess(List values) { + LanguageCandidate[] result = + new LanguageCandidate[values.size()]; + for (int i = 0; i < result.length; i++) { + IdentifiedLanguage value = values.get(i); + result[i] = new LanguageCandidate( + value.getLanguageTag(), value.getConfidence()); + } + complete(out, result); + } + }).addOnFailureListener(failure(out)); + return out; + } + + private synchronized LanguageIdentifier client(LanguageOptions options) { + if (client == null) { + client = LanguageIdentification.getClient( + new LanguageIdentificationOptions.Builder() + .setConfidenceThreshold( + options.getMinimumConfidence()) + .build()); + } + return client; + } + + public synchronized void close() { + if (client != null) { + client.close(); + client = null; + } + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageImpl.java new file mode 100644 index 00000000000..380cfdd7f37 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidLanguageImpl.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.language.LanguageCandidate; +import com.codename1.ai.language.LanguageOptions; +import com.codename1.ai.language.SmartReplyMessage; +import com.codename1.impl.LanguageImpl; +import com.codename1.util.AsyncResource; + +/** + * Android language-service dispatcher. The builder retains only the adapter + * source and ML Kit artifact for each language API referenced by the app. + */ +public final class AndroidLanguageImpl extends LanguageImpl { + private volatile boolean closed; + private AndroidLanguageAdapter languageIdAdapter; + private AndroidLanguageAdapter translationAdapter; + private AndroidLanguageAdapter smartReplyAdapter; + + @Override + public boolean isSupported(String feature, String backendId) { + return !closed + && ("auto".equals(backendId) || "ml-kit".equals(backendId)) + && adapter(feature) != null; + } + + @Override + public AsyncResource identify( + String text, String backendId, LanguageOptions options) { + AndroidLanguageAdapter adapter = adapter("language-id"); + return adapter == null + ? AndroidLanguageAdapter.unsupported( + "Language identification is not included in this build") + : adapter.identify(text, options); + } + + @Override + public AsyncResource translate( + String text, String sourceLanguage, String targetLanguage, + String backendId, LanguageOptions options) { + AndroidLanguageAdapter adapter = adapter("translation"); + return adapter == null + ? AndroidLanguageAdapter.unsupported( + "Translation is not included in this build") + : adapter.translate(text, sourceLanguage, targetLanguage, + options); + } + + @Override + public AsyncResource suggestReplies( + SmartReplyMessage[] conversation, String backendId, + LanguageOptions options) { + AndroidLanguageAdapter adapter = adapter("smart-reply"); + return adapter == null + ? AndroidLanguageAdapter.unsupported( + "Smart Reply is not included in this build") + : adapter.suggestReplies(conversation, options); + } + + private synchronized AndroidLanguageAdapter adapter(String feature) { + if (closed) { + return null; + } + AndroidLanguageAdapter cached; + String className; + if ("language-id".equals(feature)) { + cached = languageIdAdapter; + className = + "com.codename1.impl.android.ai.AndroidLanguageIdAdapter"; + } else if ("translation".equals(feature)) { + cached = translationAdapter; + className = + "com.codename1.impl.android.ai.AndroidTranslationAdapter"; + } else if ("smart-reply".equals(feature)) { + cached = smartReplyAdapter; + className = + "com.codename1.impl.android.ai.AndroidSmartReplyAdapter"; + } else { + return null; + } + if (cached != null) { + return cached; + } + try { + cached = (AndroidLanguageAdapter) + Class.forName(className).newInstance(); + if ("language-id".equals(feature)) { + languageIdAdapter = cached; + } else if ("translation".equals(feature)) { + translationAdapter = cached; + } else { + smartReplyAdapter = cached; + } + return cached; + } catch (Throwable ignored) { + return null; + } + } + + @Override + public synchronized void close() { + if (closed) { + return; + } + closed = true; + closeAdapter(languageIdAdapter); + closeAdapter(translationAdapter); + closeAdapter(smartReplyAdapter); + languageIdAdapter = null; + translationAdapter = null; + smartReplyAdapter = null; + } + + private static void closeAdapter(AndroidLanguageAdapter adapter) { + if (adapter != null) { + adapter.close(); + } + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java new file mode 100644 index 00000000000..41f02218a6b --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidPoseDetectionAdapter.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.vision.Pose; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.ai.vision.VisionPoint; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.pose.PoseDetection; +import com.google.mlkit.vision.pose.PoseDetector; +import com.google.mlkit.vision.pose.PoseLandmark; +import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions; + +import java.util.ArrayList; +import java.util.List; + +/** ML Kit pose detection; retained only for {@code PoseDetector} users. */ +final class AndroidPoseDetectionAdapter extends AndroidVisionAdapter { + private final PoseDetector client = PoseDetection.getClient( + new PoseDetectorOptions.Builder() + .setDetectorMode(PoseDetectorOptions.STREAM_MODE) + .build()); + @Override + @SuppressWarnings("unchecked") + void analyze(InputImage input, final int imageWidth, + final int imageHeight, VisionOptions options, + AsyncResource resource) { + final AsyncResource out = (AsyncResource) resource; + final float minimumConfidence = options.getMinimumConfidence(); + final int maximumResults = options.getMaximumResults(); + client.process(input).addOnSuccessListener( + new OnSuccessListener() { + public void onSuccess(com.google.mlkit.vision.pose.Pose value) { + List source = value.getAllPoseLandmarks(); + List result = new ArrayList(); + for (int i = 0; i < source.size(); i++) { + PoseLandmark point = source.get(i); + if (point.getInFrameLikelihood() < minimumConfidence) { + continue; + } + result.add(new Pose.Landmark( + landmarkName(point.getLandmarkType()), + new VisionPoint( + point.getPosition().x / imageWidth, + point.getPosition().y / imageHeight), + point.getInFrameLikelihood())); + if (maximumResults > 0 + && result.size() >= maximumResults) { + break; + } + } + complete(out, new Pose(result.toArray( + new Pose.Landmark[result.size()]), METADATA)); + } + }).addOnFailureListener(failure(out)); + } + + @Override + void close() { + client.close(); + } + + private static String landmarkName(int type) { + switch (type) { + case PoseLandmark.NOSE: return "nose"; + case PoseLandmark.LEFT_EYE_INNER: return "leftEyeInner"; + case PoseLandmark.LEFT_EYE: return "leftEye"; + case PoseLandmark.LEFT_EYE_OUTER: return "leftEyeOuter"; + case PoseLandmark.RIGHT_EYE_INNER: return "rightEyeInner"; + case PoseLandmark.RIGHT_EYE: return "rightEye"; + case PoseLandmark.RIGHT_EYE_OUTER: return "rightEyeOuter"; + case PoseLandmark.LEFT_EAR: return "leftEar"; + case PoseLandmark.RIGHT_EAR: return "rightEar"; + case PoseLandmark.LEFT_MOUTH: return "leftMouth"; + case PoseLandmark.RIGHT_MOUTH: return "rightMouth"; + case PoseLandmark.LEFT_SHOULDER: return "leftShoulder"; + case PoseLandmark.RIGHT_SHOULDER: return "rightShoulder"; + case PoseLandmark.LEFT_ELBOW: return "leftElbow"; + case PoseLandmark.RIGHT_ELBOW: return "rightElbow"; + case PoseLandmark.LEFT_WRIST: return "leftWrist"; + case PoseLandmark.RIGHT_WRIST: return "rightWrist"; + case PoseLandmark.LEFT_PINKY: return "leftPinky"; + case PoseLandmark.RIGHT_PINKY: return "rightPinky"; + case PoseLandmark.LEFT_INDEX: return "leftIndex"; + case PoseLandmark.RIGHT_INDEX: return "rightIndex"; + case PoseLandmark.LEFT_THUMB: return "leftThumb"; + case PoseLandmark.RIGHT_THUMB: return "rightThumb"; + case PoseLandmark.LEFT_HIP: return "leftHip"; + case PoseLandmark.RIGHT_HIP: return "rightHip"; + case PoseLandmark.LEFT_KNEE: return "leftKnee"; + case PoseLandmark.RIGHT_KNEE: return "rightKnee"; + case PoseLandmark.LEFT_ANKLE: return "leftAnkle"; + case PoseLandmark.RIGHT_ANKLE: return "rightAnkle"; + case PoseLandmark.LEFT_HEEL: return "leftHeel"; + case PoseLandmark.RIGHT_HEEL: return "rightHeel"; + case PoseLandmark.LEFT_FOOT_INDEX: return "leftFootIndex"; + case PoseLandmark.RIGHT_FOOT_INDEX: return "rightFootIndex"; + default: return "unknown"; + } + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidSelfieSegmentationAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSelfieSegmentationAdapter.java new file mode 100644 index 00000000000..a80c61f7100 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSelfieSegmentationAdapter.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.vision.SegmentationMask; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.segmentation.Segmentation; +import com.google.mlkit.vision.segmentation.Segmenter; +import com.google.mlkit.vision.segmentation.selfie.SelfieSegmenterOptions; + +import java.nio.ByteBuffer; + +/** + * ML Kit selfie segmentation; retained only for {@code SelfieSegmenter} users. + */ +final class AndroidSelfieSegmentationAdapter extends AndroidVisionAdapter { + private final Segmenter client = Segmentation.getClient( + new SelfieSegmenterOptions.Builder() + .setDetectorMode(SelfieSegmenterOptions.STREAM_MODE) + .build()); + @Override + @SuppressWarnings("unchecked") + void analyze(InputImage input, int imageWidth, int imageHeight, + VisionOptions options, AsyncResource resource) { + final AsyncResource out = + (AsyncResource) resource; + client.process(input).addOnSuccessListener( + new OnSuccessListener< + com.google.mlkit.vision.segmentation.SegmentationMask>() { + public void onSuccess( + com.google.mlkit.vision.segmentation.SegmentationMask value) { + ByteBuffer buffer = value.getBuffer(); + buffer.rewind(); + float[] confidence = + new float[value.getWidth() * value.getHeight()]; + buffer.asFloatBuffer().get(confidence); + complete(out, new SegmentationMask( + value.getWidth(), value.getHeight(), confidence, + METADATA)); + } + }).addOnFailureListener(failure(out)); + } + + @Override + void close() { + client.close(); + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java new file mode 100644 index 00000000000..2aa6754f07d --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidSmartReplyAdapter.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.language.LanguageOptions; +import com.codename1.ai.language.SmartReplyMessage; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.nl.smartreply.SmartReply; +import com.google.mlkit.nl.smartreply.SmartReplyGenerator; +import com.google.mlkit.nl.smartreply.SmartReplySuggestion; +import com.google.mlkit.nl.smartreply.SmartReplySuggestionResult; +import com.google.mlkit.nl.smartreply.TextMessage; + +import java.util.ArrayList; +import java.util.List; + +/** ML Kit Smart Reply; retained only for {@code SmartReply} users. */ +final class AndroidSmartReplyAdapter extends AndroidLanguageAdapter { + private SmartReplyGenerator client; + + @Override + AsyncResource suggestReplies( + SmartReplyMessage[] conversation, LanguageOptions options) { + final AsyncResource out = new AsyncResource(); + final SmartReplyGenerator current = client(); + List messages = new ArrayList(); + for (int i = 0; i < conversation.length; i++) { + SmartReplyMessage message = conversation[i]; + messages.add(message.isLocalUser() + ? TextMessage.createForLocalUser( + message.getText(), message.getTimestampMillis()) + : TextMessage.createForRemoteUser( + message.getText(), message.getTimestampMillis(), + message.getParticipantId() == null + ? "remote" + : message.getParticipantId())); + } + current.suggestReplies(messages).addOnSuccessListener( + new OnSuccessListener() { + public void onSuccess(SmartReplySuggestionResult value) { + List suggestions = + value.getSuggestions(); + String[] result = new String[suggestions.size()]; + for (int i = 0; i < result.length; i++) { + result[i] = suggestions.get(i).getText(); + } + complete(out, result); + } + }).addOnFailureListener(failure(out)); + return out; + } + + private synchronized SmartReplyGenerator client() { + if (client == null) { + client = SmartReply.getClient(); + } + return client; + } + + public synchronized void close() { + if (client != null) { + client.close(); + client = null; + } + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java new file mode 100644 index 00000000000..405e2bac4e8 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTextRecognitionAdapter.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.vision.TextRecognitionResult; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.text.Text; +import com.google.mlkit.vision.text.TextRecognition; +import com.google.mlkit.vision.text.TextRecognizer; +import com.google.mlkit.vision.text.latin.TextRecognizerOptions; + +import java.util.List; + +/** ML Kit text recognition; retained only for {@code TextRecognizer} users. */ +final class AndroidTextRecognitionAdapter extends AndroidVisionAdapter { + private final TextRecognizer client = TextRecognition.getClient( + TextRecognizerOptions.DEFAULT_OPTIONS); + @Override + @SuppressWarnings("unchecked") + void analyze(InputImage input, final int imageWidth, + final int imageHeight, VisionOptions options, + AsyncResource resource) { + final AsyncResource out = + (AsyncResource) resource; + final int maximumResults = options.getMaximumResults(); + client.process(input).addOnSuccessListener(new OnSuccessListener() { + public void onSuccess(Text text) { + List source = text.getTextBlocks(); + int count = maximumResults == 0 ? source.size() + : Math.min(maximumResults, source.size()); + TextRecognitionResult.TextBlock[] blocks = + new TextRecognitionResult.TextBlock[count]; + for (int i = 0; i < blocks.length; i++) { + Text.TextBlock block = source.get(i); + blocks[i] = new TextRecognitionResult.TextBlock( + block.getText(), 1f, + normalized(block.getBoundingBox(), + imageWidth, imageHeight), null); + } + complete(out, new TextRecognitionResult( + text.getText(), blocks, METADATA)); + } + }).addOnFailureListener(failure(out)); + } + + @Override + void close() { + client.close(); + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java new file mode 100644 index 00000000000..493491ee91a --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidTranslationAdapter.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ai; + +import com.codename1.ai.language.LanguageOptions; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.android.gms.tasks.SuccessContinuation; +import com.google.android.gms.tasks.Task; +import com.google.mlkit.nl.translate.Translation; +import com.google.mlkit.nl.translate.TranslateLanguage; +import com.google.mlkit.nl.translate.Translator; +import com.google.mlkit.nl.translate.TranslatorOptions; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +/** ML Kit translation; retained only for {@code Translator} users. */ +final class AndroidTranslationAdapter extends AndroidLanguageAdapter { + private final Map clients = + new HashMap(); + + @Override + AsyncResource translate( + final String text, String sourceLanguage, String targetLanguage, + LanguageOptions options) { + final AsyncResource out = new AsyncResource(); + String sourceCode = languageCode(sourceLanguage); + String targetCode = languageCode(targetLanguage); + if (sourceCode == null || targetCode == null) { + out.error(new IllegalArgumentException( + "Unsupported translation language")); + return out; + } + final Translator client = client(sourceCode, targetCode); + client.downloadModelIfNeeded().onSuccessTask( + new SuccessContinuation() { + public Task then(Void ignored) { + return client.translate(text); + } + }).addOnSuccessListener(new OnSuccessListener() { + public void onSuccess(String value) { + complete(out, value); + } + }).addOnFailureListener(failure(out)); + return out; + } + + private synchronized Translator client(String sourceCode, + String targetCode) { + String key = sourceCode + "\n" + targetCode; + Translator client = clients.get(key); + if (client == null) { + TranslatorOptions options = new TranslatorOptions.Builder() + .setSourceLanguage(sourceCode) + .setTargetLanguage(targetCode) + .build(); + client = Translation.getClient(options); + clients.put(key, client); + } + return client; + } + + public synchronized void close() { + for (Translator client : clients.values()) { + client.close(); + } + clients.clear(); + } + + private static String languageCode(String languageTag) { + if (languageTag == null) { + return null; + } + String language = Locale.forLanguageTag(languageTag).getLanguage(); + if (language.length() == 0) { + return null; + } + return TranslateLanguage.fromLanguageTag(language); + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionAdapter.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionAdapter.java new file mode 100644 index 00000000000..546e22156cd --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionAdapter.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ai; + +import android.graphics.Rect; + +import com.codename1.ai.vision.VisionException; +import com.codename1.ai.vision.VisionMetadata; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.ai.vision.VisionRect; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; +import com.google.android.gms.tasks.OnFailureListener; +import com.google.mlkit.vision.common.InputImage; + +/** Shared contract and result helpers for feature-scoped Android adapters. */ +abstract class AndroidVisionAdapter { + static final VisionMetadata METADATA = new VisionMetadata("ml-kit"); + + abstract void analyze(InputImage input, int imageWidth, int imageHeight, + VisionOptions options, AsyncResource out); + + abstract void close(); + + static VisionRect normalized(Rect rect, int imageWidth, int imageHeight) { + if (rect == null) { + return VisionRect.EMPTY; + } + return new VisionRect(rect.left / (float) imageWidth, + rect.top / (float) imageHeight, + rect.width() / (float) imageWidth, + rect.height() / (float) imageHeight); + } + + static void complete(final AsyncResource out, final T value) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(value); + } + }); + } + + static OnFailureListener failure(final AsyncResource out) { + return new OnFailureListener() { + public void onFailure(final Exception error) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.error(new VisionException( + VisionException.BACKEND_ERROR, + error.getMessage(), error)); + } + }); + } + }; + } +} diff --git a/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java new file mode 100644 index 00000000000..67a1c6281e0 --- /dev/null +++ b/Ports/Android/src/com/codename1/impl/android/ai/AndroidVisionImpl.java @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.android.ai; + +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; + +import com.codename1.ai.vision.VisionException; +import com.codename1.ai.vision.VisionFeature; +import com.codename1.ai.vision.VisionImage; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.camera.FrameFormat; +import com.codename1.impl.VisionImpl; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; +import com.google.mlkit.vision.common.InputImage; + +/** + * Android vision dispatcher. Feature implementations live in separate source + * files so the Android builder can retain one adapter and one ML Kit artifact + * per analyzer used by the application. + */ +public final class AndroidVisionImpl extends VisionImpl { + private volatile boolean closed; + private AndroidVisionAdapter adapter; + private VisionFeature adapterFeature; + + @Override + public boolean isSupported(VisionFeature feature, String backendId) { + return !closed + && ("auto".equals(backendId) || "ml-kit".equals(backendId)) + && adapterClass(feature) != null + && isClassPresent(adapterClass(feature)); + } + + @Override + @SuppressWarnings({"rawtypes", "unchecked"}) + public AsyncResource analyze(final VisionFeature feature, + String backendId, + final VisionImage image, + final VisionOptions options) { + final AsyncResource out = new AsyncResource(); + final VisionOptions optionSnapshot = new VisionOptions() + .backend(options.getBackend()) + .minimumConfidence(options.getMinimumConfidence()) + .maximumResults(options.getMaximumResults()); + if (!isSupported(feature, backendId)) { + out.error(new VisionException(VisionException.UNSUPPORTED, + feature + " is not included in this Android build")); + return out; + } + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + try { + DecodedInput decoded = decode(image); + if (decoded == null) { + error(out, new VisionException( + VisionException.INVALID_IMAGE, + "Vision input is not valid JPEG, PNG, NV21, " + + "or RGBA8888 data")); + return; + } + adapter(feature).analyze(decoded.input, decoded.width, + decoded.height, optionSnapshot, + (AsyncResource) out); + } catch (Throwable cause) { + error(out, new VisionException( + VisionException.BACKEND_ERROR, + "Could not start the Android " + feature + + " adapter", cause)); + } + } + }); + return out; + } + + private synchronized AndroidVisionAdapter adapter(VisionFeature feature) + throws Exception { + if (closed) { + throw new IllegalStateException("Vision backend is closed"); + } + if (adapter == null) { + adapter = (AndroidVisionAdapter) Class.forName( + adapterClass(feature)).newInstance(); + adapterFeature = feature; + } else if (adapterFeature != feature) { + throw new IllegalStateException( + "A vision backend instance cannot analyze multiple features"); + } + return adapter; + } + + private static DecodedInput decode(VisionImage image) { + byte[] encoded = image.getEncodedBytesUnsafe(); + if (encoded != null) { + Bitmap bitmap = BitmapFactory.decodeByteArray( + encoded, 0, encoded.length); + return bitmap == null ? null : new DecodedInput( + InputImage.fromBitmap(bitmap, image.getRotationDegrees()), + bitmap.getWidth(), bitmap.getHeight(), + image.getRotationDegrees()); + } + byte[] pixels = image.getPixelsUnsafe(); + int width = image.getWidth(); + int height = image.getHeight(); + if (pixels == null || width <= 0 || height <= 0) { + return null; + } + long pixelCount = (long) width * (long) height; + if (pixelCount > Integer.MAX_VALUE) { + return null; + } + if (image.getFormat() == FrameFormat.NV21) { + if ((width & 1) != 0 || (height & 1) != 0 + || pixels.length < pixelCount + pixelCount / 2) { + return null; + } + return new DecodedInput(InputImage.fromByteArray( + pixels, width, height, image.getRotationDegrees(), + InputImage.IMAGE_FORMAT_NV21), width, height, + image.getRotationDegrees()); + } + if (image.getFormat() == FrameFormat.RGBA8888) { + if (pixelCount * 4 > pixels.length) { + return null; + } + int[] argb = new int[(int) pixelCount]; + for (int i = 0, p = 0; i < argb.length; i++, p += 4) { + argb[i] = ((pixels[p + 3] & 255) << 24) + | ((pixels[p] & 255) << 16) + | ((pixels[p + 1] & 255) << 8) + | (pixels[p + 2] & 255); + } + Bitmap bitmap = Bitmap.createBitmap( + argb, width, height, Bitmap.Config.ARGB_8888); + return new DecodedInput(InputImage.fromBitmap( + bitmap, image.getRotationDegrees()), width, height, + image.getRotationDegrees()); + } + Bitmap bitmap = BitmapFactory.decodeByteArray( + pixels, 0, pixels.length); + return bitmap == null ? null : new DecodedInput( + InputImage.fromBitmap(bitmap, image.getRotationDegrees()), + bitmap.getWidth(), bitmap.getHeight(), + image.getRotationDegrees()); + } + + private static final class DecodedInput { + final InputImage input; + final int width; + final int height; + + DecodedInput(InputImage input, int width, int height, int rotation) { + this.input = input; + if (rotation == 90 || rotation == 270) { + this.width = height; + this.height = width; + } else { + this.width = width; + this.height = height; + } + } + } + + private static void error(final AsyncResource out, + final VisionException error) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.error(error); + } + }); + } + + private static boolean isClassPresent(String className) { + try { + Class.forName(className); + return true; + } catch (Throwable ignored) { + return false; + } + } + + private static String adapterClass(VisionFeature feature) { + if (feature == null) { + return null; + } + switch (feature) { + case TEXT_RECOGNITION: + return "com.codename1.impl.android.ai.AndroidTextRecognitionAdapter"; + case BARCODE_SCANNING: + return "com.codename1.impl.android.ai.AndroidBarcodeScanningAdapter"; + case FACE_DETECTION: + return "com.codename1.impl.android.ai.AndroidFaceDetectionAdapter"; + case IMAGE_LABELING: + return "com.codename1.impl.android.ai.AndroidImageLabelingAdapter"; + case POSE_DETECTION: + return "com.codename1.impl.android.ai.AndroidPoseDetectionAdapter"; + case SELFIE_SEGMENTATION: + return "com.codename1.impl.android.ai.AndroidSelfieSegmentationAdapter"; + default: + // ML Kit's document scanner owns an Activity camera flow; it + // cannot implement the still-image VisionAnalyzer contract. + return null; + } + } + + @Override + public synchronized void close() { + closed = true; + if (adapter != null) { + adapter.close(); + adapter = null; + } + } +} diff --git a/Ports/iOSPort/nativeSources/CN1Inference.m b/Ports/iOSPort/nativeSources/CN1Inference.m new file mode 100644 index 00000000000..fae51e072e8 --- /dev/null +++ b/Ports/iOSPort/nativeSources/CN1Inference.m @@ -0,0 +1,422 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +#import "CodenameOne_GLViewController.h" +#import "xmlvm.h" + +#if !__has_feature(objc_arc) +#error CN1Inference.m requires ARC (-fobjc-arc) +#endif + +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV +#import +#import "java_lang_String.h" + +#if __has_include() +#import +#define CN1_HAS_LITERT 1 +#endif + +#if defined(CN1_HAS_LITERT) +@interface CN1InferenceHandle : NSObject +@property(nonatomic, strong) TFLInterpreter *interpreter; +@property(nonatomic, strong) TFLDelegate *delegate; +@property(nonatomic, copy) NSString *modelPath; +@property(nonatomic) BOOL deleteModelOnClose; +@end + +@implementation CN1InferenceHandle +@end + +static NSMutableDictionary *cn1InferenceHandles; +static int cn1InferenceNextHandle = 1; + +static void cn1InferenceEnsureHandles(void) { + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + cn1InferenceHandles = [NSMutableDictionary dictionary]; + }); +} + +static NSString *cn1InferenceJSON(NSDictionary *value) { + NSError *error = nil; + NSData *data = [NSJSONSerialization dataWithJSONObject:value options:0 + error:&error]; + if (data == nil) { + return [NSString stringWithFormat:@"{\"error\":\"%@\"}", + error.localizedDescription ?: @"Could not encode inference result"]; + } + return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]; +} + +static NSString *cn1InferenceError(NSError *error, NSString *fallback) { + return cn1InferenceJSON(@{ + @"error": error.localizedDescription ?: fallback + }); +} + +static NSString *cn1InferenceType(TFLTensorDataType type) { + switch (type) { + case TFLTensorDataTypeFloat32: return @"FLOAT32"; + case TFLTensorDataTypeInt32: return @"INT32"; + case TFLTensorDataTypeUInt8: return @"UINT8"; + case TFLTensorDataTypeInt64: return @"INT64"; + case TFLTensorDataTypeBool: return @"BOOL"; + case TFLTensorDataTypeInt8: return @"INT8"; + default: return nil; + } +} + +static CN1InferenceHandle *cn1InferenceHandle(int handle) { + cn1InferenceEnsureHandles(); + @synchronized (cn1InferenceHandles) { + return cn1InferenceHandles[@(handle)]; + } +} + +static NSString *cn1InferenceOpenPath(NSString *path, BOOL deleteModelOnClose, + int threads, int accelerator, BOOL allowFallback) { + // TFLCoreMLDelegate does not report whether it delegated the whole graph. + // Unsupported operations may remain on LiteRT's CPU path, and delegated + // operations may use the Neural Engine, GPU, or CPU. Strict NPU and + // Core ML requests therefore cannot honor the no-fallback contract. + if ((accelerator == 3 || accelerator == 4) && !allowFallback) { + if (deleteModelOnClose) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + } + NSString *target = accelerator == 3 ? @"NPU" : @"Core ML"; + return cn1InferenceJSON(@{ + @"error": [NSString stringWithFormat: + @"Strict %@ execution cannot be verified on iOS; " + @"the Core ML delegate may leave unsupported operations " + @"on CPU", target] + }); + } + TFLInterpreterOptions *options = [[TFLInterpreterOptions alloc] init]; + if (threads > 0) options.numberOfThreads = (NSUInteger)threads; + NSMutableArray *delegates = [NSMutableArray array]; + TFLDelegate *delegate = nil; + + // InferenceOptions.Accelerator ordinal: + // AUTO=0, CPU=1, GPU=2, NPU=3, CORE_ML=4. + if (accelerator == 0 || accelerator == 3 || accelerator == 4) { +#if __has_include() + delegate = [[TFLCoreMLDelegate alloc] init]; + if (delegate != nil) [delegates addObject:delegate]; +#endif + if (delegate == nil && !allowFallback && accelerator != 0) { + if (deleteModelOnClose) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + } + return cn1InferenceJSON(@{ + @"error": @"Core ML delegate is unavailable on this device" + }); + } + } else if (accelerator == 2 && !allowFallback) { + if (deleteModelOnClose) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + } + return cn1InferenceJSON(@{ + @"error": @"The iOS backend does not provide a GPU delegate" + }); + } + + NSError *error = nil; + TFLInterpreter *interpreter = [[TFLInterpreter alloc] + initWithModelPath:path options:options delegates:delegates error:&error]; + BOOL interpreterCreated = interpreter != nil && error == nil; + BOOL tensorsAllocated = interpreterCreated + && [interpreter allocateTensorsWithError:&error]; + if ((!interpreterCreated || !tensorsAllocated) + && delegate != nil && allowFallback) { + delegate = nil; + error = nil; + interpreter = [[TFLInterpreter alloc] + initWithModelPath:path options:options delegates:@[] error:&error]; + interpreterCreated = interpreter != nil && error == nil; + tensorsAllocated = interpreterCreated + && [interpreter allocateTensorsWithError:&error]; + } + if (!interpreterCreated) { + if (deleteModelOnClose) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + } + return cn1InferenceError(error, @"Could not create LiteRT interpreter"); + } + if (!tensorsAllocated) { + if (deleteModelOnClose) { + [[NSFileManager defaultManager] removeItemAtPath:path error:nil]; + } + return cn1InferenceError(error, @"Could not allocate LiteRT tensors"); + } + + CN1InferenceHandle *value = [[CN1InferenceHandle alloc] init]; + value.interpreter = interpreter; + value.delegate = delegate; + value.modelPath = path; + value.deleteModelOnClose = deleteModelOnClose; + cn1InferenceEnsureHandles(); + int handle; + @synchronized (cn1InferenceHandles) { + handle = cn1InferenceNextHandle++; + cn1InferenceHandles[@(handle)] = value; + } + return cn1InferenceJSON(@{@"handle": @(handle)}); +} + +static NSString *cn1InferenceOpen(NSData *model, int threads, int accelerator, + BOOL allowFallback) { + NSString *path = [NSTemporaryDirectory() stringByAppendingPathComponent: + [NSString stringWithFormat:@"cn1-litert-%@.tflite", + NSUUID.UUID.UUIDString]]; + if (![model writeToFile:path atomically:YES]) { + return cn1InferenceJSON(@{@"error": @"Could not stage LiteRT model"}); + } + return cn1InferenceOpenPath(path, YES, threads, accelerator, allowFallback); +} + +static NSString *cn1InferenceMetadata(int handle, BOOL outputs) { + CN1InferenceHandle *value = cn1InferenceHandle(handle); + if (value == nil) { + return cn1InferenceJSON(@{@"error": @"Unknown LiteRT handle"}); + } + TFLInterpreter *interpreter = value.interpreter; + NSUInteger count = outputs ? interpreter.outputTensorCount + : interpreter.inputTensorCount; + NSMutableArray *items = [NSMutableArray array]; + for (NSUInteger i = 0; i < count; i++) { + NSError *error = nil; + TFLTensor *tensor = outputs + ? [interpreter outputTensorAtIndex:i error:&error] + : [interpreter inputTensorAtIndex:i error:&error]; + if (tensor == nil || error != nil) { + return cn1InferenceError(error, @"Could not read LiteRT tensor"); + } + NSString *type = cn1InferenceType(tensor.dataType); + if (type == nil) { + return cn1InferenceJSON(@{ + @"error": [NSString stringWithFormat: + @"Unsupported LiteRT tensor type %lu", + (unsigned long)tensor.dataType] + }); + } + NSArray *shape = [tensor shapeWithError:&error]; + if (shape == nil || error != nil) { + return cn1InferenceError(error, @"Could not read LiteRT tensor shape"); + } + [items addObject:@{ + @"index": @(i), + @"name": tensor.name ?: @"", + @"type": type, + @"shape": shape + }]; + } + return cn1InferenceJSON(@{@"items": items}); +} + +static NSString *cn1InferenceCopyInput(int handle, int index, NSData *data) { + CN1InferenceHandle *value = cn1InferenceHandle(handle); + if (value == nil) { + return cn1InferenceJSON(@{@"error": @"Unknown LiteRT handle"}); + } + NSError *error = nil; + TFLInterpreter *interpreter = value.interpreter; + if (index < 0 || (NSUInteger)index >= interpreter.inputTensorCount) { + return cn1InferenceJSON(@{@"error": @"Invalid LiteRT input index"}); + } + TFLTensor *tensor = [interpreter inputTensorAtIndex:(NSUInteger)index + error:&error]; + if (tensor == nil || error != nil || ![tensor copyData:data error:&error]) { + return cn1InferenceError(error, @"Could not copy LiteRT input"); + } + return cn1InferenceJSON(@{@"ok": @(YES)}); +} + +static NSString *cn1InferenceInvoke(int handle) { + CN1InferenceHandle *value = cn1InferenceHandle(handle); + if (value == nil) { + return cn1InferenceJSON(@{@"error": @"Unknown LiteRT handle"}); + } + NSError *error = nil; + TFLInterpreter *interpreter = value.interpreter; + if (![interpreter invokeWithError:&error]) { + return cn1InferenceError(error, @"LiteRT invocation failed"); + } + return cn1InferenceJSON(@{@"ok": @(YES)}); +} + +static NSData *cn1InferenceOutputData(int handle, int index) { + CN1InferenceHandle *value = cn1InferenceHandle(handle); + if (value == nil || index < 0 + || (NSUInteger)index >= value.interpreter.outputTensorCount) { + return nil; + } + NSError *error = nil; + TFLTensor *tensor = [value.interpreter outputTensorAtIndex:(NSUInteger)index + error:&error]; + return tensor == nil || error != nil ? nil : [tensor dataWithError:&error]; +} + +static NSString *cn1InferenceResize(int handle, int index, + NSArray *shape) { + CN1InferenceHandle *value = cn1InferenceHandle(handle); + if (value == nil) { + return cn1InferenceJSON(@{@"error": @"Unknown LiteRT handle"}); + } + NSError *error = nil; + if (![value.interpreter resizeInputTensorAtIndex:(NSUInteger)index + toShape:shape error:&error]) { + return cn1InferenceError(error, @"Could not resize LiteRT input"); + } + if (![value.interpreter allocateTensorsWithError:&error]) { + return cn1InferenceError(error, @"Could not reallocate LiteRT tensors"); + } + return cn1InferenceJSON(@{@"ok": @(YES)}); +} +#endif +#endif + +JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_cn1InferenceIsSupported___R_boolean( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + return 1; +#else + return 0; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceOpen___byte_1ARRAY_int_int_boolean_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT model, + JAVA_INT threads, JAVA_INT accelerator, JAVA_BOOLEAN allowFallback) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + if (model == JAVA_NULL) { + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + @"{\"error\":\"Model data is null\"}"); + } + JAVA_ARRAY bytes = (JAVA_ARRAY)model; + NSData *data = [NSData dataWithBytes:bytes->data + length:(NSUInteger)bytes->length]; + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1InferenceOpen(data, threads, accelerator, allowFallback)); +#else + return JAVA_NULL; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceMetadata___int_boolean_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT handle, + JAVA_BOOLEAN outputs) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1InferenceMetadata(handle, outputs)); +#else + return JAVA_NULL; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceOpenFile___java_lang_String_int_int_boolean_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT path, + JAVA_INT threads, JAVA_INT accelerator, JAVA_BOOLEAN allowFallback) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1InferenceOpenPath( + toNSString(CN1_THREAD_GET_STATE_PASS_ARG path), NO, + threads, accelerator, allowFallback)); +#else + return JAVA_NULL; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceCopyInput___int_int_byte_1ARRAY_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT handle, + JAVA_INT index, JAVA_OBJECT input) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + if (input == JAVA_NULL) { + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + @"{\"error\":\"Input data is null\"}"); + } + JAVA_ARRAY bytes = (JAVA_ARRAY)input; + NSData *data = [NSData dataWithBytes:bytes->data + length:(NSUInteger)bytes->length]; + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1InferenceCopyInput(handle, index, data)); +#else + return JAVA_NULL; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceInvoke___int_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT handle) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG cn1InferenceInvoke(handle)); +#else + return JAVA_NULL; +#endif +} + +JAVA_LONG com_codename1_impl_ios_IOSNative_cn1InferenceOutputData___int_int_R_long( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT handle, + JAVA_INT index) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + NSData *data = cn1InferenceOutputData(handle, index); + return data == nil ? 0 : (JAVA_LONG)CFBridgingRetain(data); +#else + return 0; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1InferenceResize___int_int_int_1ARRAY_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT handle, + JAVA_INT index, JAVA_OBJECT shape) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + if (shape == JAVA_NULL) { + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + @"{\"error\":\"Shape is null\"}"); + } + JAVA_ARRAY array = (JAVA_ARRAY)shape; + JAVA_ARRAY_INT *values = (JAVA_ARRAY_INT *)array->data; + NSMutableArray *nativeShape = + [NSMutableArray arrayWithCapacity:(NSUInteger)array->length]; + for (int i = 0; i < array->length; i++) { + [nativeShape addObject:@(values[i])]; + } + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1InferenceResize(handle, index, nativeShape)); +#else + return JAVA_NULL; +#endif +} + +void com_codename1_impl_ios_IOSNative_cn1InferenceClose___int( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT handle) { +#if defined(INCLUDE_CN1_INFERENCE) && !TARGET_OS_WATCH && !TARGET_OS_TV && defined(CN1_HAS_LITERT) + cn1InferenceEnsureHandles(); + CN1InferenceHandle *value; + @synchronized (cn1InferenceHandles) { + value = cn1InferenceHandles[@(handle)]; + [cn1InferenceHandles removeObjectForKey:@(handle)]; + } + if (value.deleteModelOnClose && value.modelPath != nil) { + [[NSFileManager defaultManager] removeItemAtPath:value.modelPath error:nil]; + } +#endif +} diff --git a/Ports/iOSPort/nativeSources/CN1Language.m b/Ports/iOSPort/nativeSources/CN1Language.m new file mode 100644 index 00000000000..981c9464d08 --- /dev/null +++ b/Ports/iOSPort/nativeSources/CN1Language.m @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +#import "CodenameOne_GLViewController.h" +#import "xmlvm.h" + +#if !__has_feature(objc_arc) +#error CN1Language.m requires ARC (-fobjc-arc) +#endif + +#if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV +#import +#import +#import "java_lang_String.h" + +#if __has_include() +#import +#define CN1_HAS_MLKIT_LANGUAGE_ID 1 +#endif +#if __has_include() +#import +#import +#define CN1_HAS_MLKIT_TRANSLATE 1 +#endif +#if __has_include() +#import +#define CN1_HAS_MLKIT_SMART_REPLY 1 +#endif + +static NSString *cn1LanguageJSON(NSDictionary *value) { + NSError *error = nil; + NSData *data = [NSJSONSerialization dataWithJSONObject:value options:0 + error:&error]; + if (data == nil) { + return [NSString stringWithFormat:@"{\"error\":\"%@\"}", + error.localizedDescription ?: @"Could not encode language result"]; + } + return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]; +} + +static NSString *cn1LanguageError(NSError *error, NSString *fallback) { + return cn1LanguageJSON(@{ + @"error": error.localizedDescription ?: fallback + }); +} + +static NSString *cn1IdentifyLanguage(NSString *text, float threshold, + BOOL useMLKit) { + if (!useMLKit) { + if (@available(iOS 12.0, *)) { + NLLanguageRecognizer *recognizer = + [[NLLanguageRecognizer alloc] init]; + [recognizer processString:text ?: @""]; + NSDictionary *hypotheses = + [recognizer languageHypothesesWithMaximum:20]; + NSMutableArray *items = [NSMutableArray array]; + [hypotheses enumerateKeysAndObjectsUsingBlock: + ^(NLLanguage language, NSNumber *confidence, BOOL *stop) { + if (confidence.floatValue >= threshold) { + [items addObject:@{ + @"language": language ?: @"und", + @"confidence": confidence + }]; + } + }]; + [items sortUsingComparator:^NSComparisonResult( + NSDictionary *left, NSDictionary *right) { + return [right[@"confidence"] compare:left[@"confidence"]]; + }]; + return cn1LanguageJSON(@{@"items": items}); + } + return cn1LanguageJSON(@{ + @"error": @"Apple Natural Language requires iOS 12 or newer" + }); + } +#if defined(CN1_HAS_MLKIT_LANGUAGE_ID) + MLKLanguageIdentificationOptions *options = + [[MLKLanguageIdentificationOptions alloc] + initWithConfidenceThreshold:threshold]; + MLKLanguageIdentification *identifier = + [MLKLanguageIdentification languageIdentificationWithOptions:options]; + __block NSArray *languages = nil; + __block NSError *requestError = nil; + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + [identifier identifyPossibleLanguagesForText:text ?: @"" + completion:^(NSArray *result, + NSError *error) { + languages = result; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) { + return cn1LanguageError(requestError, @"Language identification failed"); + } + NSMutableArray *items = [NSMutableArray array]; + for (MLKIdentifiedLanguage *language in languages ?: @[]) { + [items addObject:@{ + @"language": language.languageTag ?: @"und", + @"confidence": @(language.confidence) + }]; + } + return cn1LanguageJSON(@{@"items": items}); +#else + return cn1LanguageJSON(@{@"error": @"ML Kit Language ID is not linked"}); +#endif +} + +static NSString *cn1TranslateLanguageCode(NSString *languageTag) { + if (languageTag == nil) { + return nil; + } + NSString *normalized = [languageTag + stringByReplacingOccurrencesOfString:@"_" withString:@"-"]; + NSDictionary *components = + [NSLocale componentsFromLocaleIdentifier:normalized]; + NSString *language = components[NSLocaleLanguageCode]; + return language.length == 0 ? nil : language.lowercaseString; +} + +static NSString *cn1TranslateLanguage(NSString *text, NSString *source, + NSString *target) { +#if defined(CN1_HAS_MLKIT_TRANSLATE) + MLKTranslateLanguage sourceLanguage = cn1TranslateLanguageCode(source); + MLKTranslateLanguage targetLanguage = cn1TranslateLanguageCode(target); + NSSet *supported = MLKTranslateAllLanguages(); + if (sourceLanguage == nil || targetLanguage == nil || + ![supported containsObject:sourceLanguage] || + ![supported containsObject:targetLanguage]) { + return cn1LanguageJSON(@{@"error": @"Unsupported translation language"}); + } + MLKTranslatorOptions *options = [[MLKTranslatorOptions alloc] + initWithSourceLanguage:sourceLanguage targetLanguage:targetLanguage]; + MLKTranslator *translator = [MLKTranslator translatorWithOptions:options]; + MLKModelDownloadConditions *conditions = [[MLKModelDownloadConditions alloc] + initWithAllowsCellularAccess:YES allowsBackgroundDownloading:YES]; + __block NSString *translation = nil; + __block NSError *requestError = nil; + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + [translator downloadModelIfNeededWithConditions:conditions + completion:^(NSError *error) { + if (error != nil) { + requestError = error; + dispatch_semaphore_signal(semaphore); + return; + } + [translator translateText:text ?: @"" + completion:^(NSString *result, NSError *error) { + translation = result; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) { + return cn1LanguageError(requestError, @"Translation failed"); + } + return cn1LanguageJSON(@{@"text": translation ?: @""}); +#else + return cn1LanguageJSON(@{@"error": @"ML Kit Translate is not linked"}); +#endif +} + +static NSString *cn1SmartReply(NSString *conversationJSON) { +#if defined(CN1_HAS_MLKIT_SMART_REPLY) + NSError *jsonError = nil; + NSDictionary *root = [NSJSONSerialization JSONObjectWithData: + [conversationJSON dataUsingEncoding:NSUTF8StringEncoding] + options:0 error:&jsonError]; + if (jsonError != nil || ![root isKindOfClass:[NSDictionary class]]) { + return cn1LanguageError(jsonError, @"Invalid smart-reply conversation"); + } + NSMutableArray *messages = [NSMutableArray array]; + for (NSDictionary *value in root[@"items"] ?: @[]) { + if (![value isKindOfClass:[NSDictionary class]]) continue; + MLKTextMessage *message = [[MLKTextMessage alloc] + initWithText:value[@"text"] ?: @"" + timestamp:[value[@"timestamp"] doubleValue] / 1000.0 + userID:value[@"participant"] ?: @"remote" + isLocalUser:[value[@"local"] boolValue]]; + [messages addObject:message]; + } + __block MLKSmartReplySuggestionResult *suggestions = nil; + __block NSError *requestError = nil; + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + [[MLKSmartReply smartReply] suggestRepliesForMessages:messages + completion:^(MLKSmartReplySuggestionResult *result, + NSError *error) { + suggestions = result; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) { + return cn1LanguageError(requestError, @"Smart Reply failed"); + } + NSMutableArray *items = [NSMutableArray array]; + for (MLKSmartReplySuggestion *suggestion in suggestions.suggestions ?: @[]) { + if (suggestion.text != nil) [items addObject:suggestion.text]; + } + return cn1LanguageJSON(@{@"items": items}); +#else + return cn1LanguageJSON(@{@"error": @"ML Kit Smart Reply is not linked"}); +#endif +} +#endif + +JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_cn1LanguageIsSupported___int_boolean_R_boolean( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT feature, + JAVA_BOOLEAN useMLKit) { +#if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV + switch (feature) { + case 0: + if (!useMLKit) { + if (@available(iOS 12.0, *)) return 1; + return 0; + } +#if defined(CN1_HAS_MLKIT_LANGUAGE_ID) + return 1; +#else + return 0; +#endif + case 1: +#if defined(CN1_HAS_MLKIT_TRANSLATE) + return 1; +#else + return 0; +#endif + case 2: +#if defined(CN1_HAS_MLKIT_SMART_REPLY) + return 1; +#else + return 0; +#endif + default: + return 0; + } +#else + return 0; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1LanguageIdentify___java_lang_String_float_boolean_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT text, + JAVA_FLOAT minimumConfidence, JAVA_BOOLEAN useMLKit) { +#if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1IdentifyLanguage( + toNSString(CN1_THREAD_GET_STATE_PASS_ARG text), + minimumConfidence, useMLKit)); +#else + return JAVA_NULL; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1LanguageTranslate___java_lang_String_java_lang_String_java_lang_String_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT text, + JAVA_OBJECT sourceLanguage, JAVA_OBJECT targetLanguage) { +#if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1TranslateLanguage( + toNSString(CN1_THREAD_GET_STATE_PASS_ARG text), + toNSString(CN1_THREAD_GET_STATE_PASS_ARG sourceLanguage), + toNSString(CN1_THREAD_GET_STATE_PASS_ARG targetLanguage))); +#else + return JAVA_NULL; +#endif +} + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1LanguageSmartReply___java_lang_String_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, + JAVA_OBJECT conversationJSON) { +#if defined(INCLUDE_CN1_LANGUAGE) && !TARGET_OS_WATCH && !TARGET_OS_TV + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + cn1SmartReply(toNSString(CN1_THREAD_GET_STATE_PASS_ARG + conversationJSON))); +#else + return JAVA_NULL; +#endif +} diff --git a/Ports/iOSPort/nativeSources/CN1Vision.m b/Ports/iOSPort/nativeSources/CN1Vision.m new file mode 100644 index 00000000000..9411aaa5791 --- /dev/null +++ b/Ports/iOSPort/nativeSources/CN1Vision.m @@ -0,0 +1,915 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +#import "CodenameOne_GLViewController.h" +#import "xmlvm.h" + +#if !__has_feature(objc_arc) +#error CN1Vision.m requires ARC (-fobjc-arc) +#endif + +#if defined(INCLUDE_CN1_VISION) && !TARGET_OS_WATCH && !TARGET_OS_TV +#import +#import +#import +#import +#import +#import +#import +#import "java_lang_String.h" + +#if __has_include() +#import +#define CN1_HAS_MLKIT_VISION 1 +#endif +#if __has_include() +#import +#import +#define CN1_HAS_MLKIT_TEXT 1 +#endif +#if __has_include() +#import +#define CN1_HAS_MLKIT_BARCODE 1 +#endif +#if __has_include() +#import +#define CN1_HAS_MLKIT_FACE 1 +#endif +#if __has_include() +#import +#import +#define CN1_HAS_MLKIT_LABEL 1 +#endif +#if __has_include() +#import +#import +#define CN1_HAS_MLKIT_POSE 1 +#endif +#if __has_include() +#import +#import +#define CN1_HAS_MLKIT_SEGMENTATION 1 +#endif + +static NSDictionary *cn1VisionRect(CGRect rect) { + return @{ + @"x": @(rect.origin.x), + @"y": @(1.0 - CGRectGetMaxY(rect)), + @"width": @(rect.size.width), + @"height": @(rect.size.height) + }; +} + +static NSDictionary *cn1MLKitRect(CGRect rect, CGSize size) { + if (size.width <= 0 || size.height <= 0) { + return @{@"x": @0, @"y": @0, @"width": @0, @"height": @0}; + } + return @{ + @"x": @(rect.origin.x / size.width), + @"y": @(rect.origin.y / size.height), + @"width": @(rect.size.width / size.width), + @"height": @(rect.size.height / size.height) + }; +} + +#if defined(CN1_HAS_MLKIT_FACE) +static void cn1MLKitAddFaceLandmark(NSMutableDictionary *landmarks, + NSString *name, MLKFace *face, MLKFaceLandmarkType type, + CGSize imageSize) { + MLKFaceLandmark *landmark = [face landmarkOfType:type]; + if (landmark != nil && imageSize.width > 0 && imageSize.height > 0) { + landmarks[name] = @{ + @"x": @(landmark.position.x / imageSize.width), + @"y": @(landmark.position.y / imageSize.height) + }; + } +} +#endif + +#if defined(CN1_HAS_MLKIT_POSE) +static NSString *cn1MLKitPoseLandmarkName(MLKPoseLandmarkType type) { + if ([type isEqualToString:MLKPoseLandmarkTypeNose]) return @"nose"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftEyeInner]) return @"leftEyeInner"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftEye]) return @"leftEye"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftEyeOuter]) return @"leftEyeOuter"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightEyeInner]) return @"rightEyeInner"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightEye]) return @"rightEye"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightEyeOuter]) return @"rightEyeOuter"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftEar]) return @"leftEar"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightEar]) return @"rightEar"; + if ([type isEqualToString:MLKPoseLandmarkTypeMouthLeft]) return @"leftMouth"; + if ([type isEqualToString:MLKPoseLandmarkTypeMouthRight]) return @"rightMouth"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftShoulder]) return @"leftShoulder"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightShoulder]) return @"rightShoulder"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftElbow]) return @"leftElbow"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightElbow]) return @"rightElbow"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftWrist]) return @"leftWrist"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightWrist]) return @"rightWrist"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftPinkyFinger]) return @"leftPinky"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightPinkyFinger]) return @"rightPinky"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftIndexFinger]) return @"leftIndex"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightIndexFinger]) return @"rightIndex"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftThumb]) return @"leftThumb"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightThumb]) return @"rightThumb"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftHip]) return @"leftHip"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightHip]) return @"rightHip"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftKnee]) return @"leftKnee"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightKnee]) return @"rightKnee"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftAnkle]) return @"leftAnkle"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightAnkle]) return @"rightAnkle"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftHeel]) return @"leftHeel"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightHeel]) return @"rightHeel"; + if ([type isEqualToString:MLKPoseLandmarkTypeLeftToe]) return @"leftFootIndex"; + if ([type isEqualToString:MLKPoseLandmarkTypeRightToe]) return @"rightFootIndex"; + return @"unknown"; +} +#endif + +static NSDictionary *cn1VisionFaceLandmark( + VNFaceLandmarkRegion2D *region, CGRect faceBounds) { + if (region == nil || region.pointCount == 0 + || region.normalizedPoints == NULL) { + return nil; + } + double x = 0; + double y = 0; + for (NSUInteger i = 0; i < region.pointCount; i++) { + x += region.normalizedPoints[i].x; + y += region.normalizedPoints[i].y; + } + x = faceBounds.origin.x + + faceBounds.size.width * x / region.pointCount; + y = faceBounds.origin.y + + faceBounds.size.height * y / region.pointCount; + return @{@"x": @(x), @"y": @(1.0 - y)}; +} + +static NSDictionary *cn1VisionFaceLandmarkEdge( + VNFaceLandmarkRegion2D *region, CGRect faceBounds, BOOL left) { + if (region == nil || region.pointCount == 0 + || region.normalizedPoints == NULL) { + return nil; + } + CGPoint selected = region.normalizedPoints[0]; + for (NSUInteger i = 1; i < region.pointCount; i++) { + CGPoint candidate = region.normalizedPoints[i]; + if ((left && candidate.x < selected.x) + || (!left && candidate.x > selected.x)) { + selected = candidate; + } + } + double x = faceBounds.origin.x + + faceBounds.size.width * selected.x; + double y = faceBounds.origin.y + + faceBounds.size.height * selected.y; + return @{@"x": @(x), @"y": @(1.0 - y)}; +} + +static void cn1VisionAddFaceLandmark(NSMutableDictionary *landmarks, + NSString *name, VNFaceLandmarkRegion2D *region, CGRect faceBounds) { + NSDictionary *point = cn1VisionFaceLandmark(region, faceBounds); + if (point != nil) { + landmarks[name] = point; + } +} + +static NSString *cn1VisionJSON(NSDictionary *value) { + NSError *error = nil; + NSData *data = [NSJSONSerialization dataWithJSONObject:value options:0 error:&error]; + if (data == nil) { + return [NSString stringWithFormat:@"{\"error\":\"%@\"}", + error.localizedDescription ?: @"Could not encode result"]; + } + return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]; +} + +static NSString *cn1VisionError(NSError *error) { + return cn1VisionJSON(@{ + @"error": error.localizedDescription ?: @"Apple Vision request failed" + }); +} + +#if defined(CN1_HAS_MLKIT_VISION) +static UIImageOrientation cn1UIImageOrientation(int rotation) { + switch (rotation) { + case 90: return UIImageOrientationRight; + case 180: return UIImageOrientationDown; + case 270: return UIImageOrientationLeft; + default: return UIImageOrientationUp; + } +} + +static MLKVisionImage *cn1MLKitImage(UIImage *image, int rotation) { + MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; + vision.orientation = cn1UIImageOrientation(rotation); + return vision; +} +#endif + +#if defined(CN1_HAS_MLKIT_BARCODE) +static NSString *cn1MLKitBarcodeFormat(MLKBarcodeFormat format) { + switch (format) { + case MLKBarcodeFormatAztec: return @"AZTEC"; + case MLKBarcodeFormatCodaBar: return @"CODABAR"; + case MLKBarcodeFormatCode39: return @"CODE_39"; + case MLKBarcodeFormatCode93: return @"CODE_93"; + case MLKBarcodeFormatCode128: return @"CODE_128"; + case MLKBarcodeFormatDataMatrix: return @"DATA_MATRIX"; + case MLKBarcodeFormatEAN8: return @"EAN_8"; + case MLKBarcodeFormatEAN13: return @"EAN_13"; + case MLKBarcodeFormatITF: return @"ITF"; + case MLKBarcodeFormatPDF417: return @"PDF417"; + case MLKBarcodeFormatQRCode: return @"QR_CODE"; + case MLKBarcodeFormatUPCA: return @"UPC_A"; + case MLKBarcodeFormatUPCE: return @"UPC_E"; + default: return @"UNKNOWN"; + } +} +#endif + +static NSString *cn1MLKitVisionPerform(NSData *data, CGImageRef rawImage, + int feature, int rotation) { +#if defined(CN1_HAS_MLKIT_VISION) + UIImage *image = rawImage == NULL ? [UIImage imageWithData:data] + : [UIImage imageWithCGImage:rawImage]; + if (image == nil) { + return cn1VisionJSON(@{@"error": @"Could not decode ML Kit image"}); + } + MLKVisionImage *vision = cn1MLKitImage(image, rotation); + CGSize resultSize = (rotation == 90 || rotation == 270) + ? CGSizeMake(image.size.height, image.size.width) : image.size; + __block NSError *requestError = nil; + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + + if (feature == 0) { +#if defined(CN1_HAS_MLKIT_TEXT) + MLKTextRecognizer *recognizer = [MLKTextRecognizer textRecognizerWithOptions: + [[MLKTextRecognizerOptions alloc] init]]; + __block MLKText *result = nil; + [recognizer processImage:vision completion:^(MLKText *text, NSError *error) { + result = text; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) return cn1VisionError(requestError); + NSMutableArray *items = [NSMutableArray array]; + for (MLKTextBlock *block in result.blocks ?: @[]) { + NSMutableDictionary *item = [NSMutableDictionary + dictionaryWithDictionary:cn1MLKitRect(block.frame, resultSize)]; + item[@"text"] = block.text ?: @""; + item[@"confidence"] = @1; + [items addObject:item]; + } + return cn1VisionJSON(@{ + @"text": result.text ?: @"", + @"items": items + }); +#endif + } else if (feature == 1) { +#if defined(CN1_HAS_MLKIT_BARCODE) + MLKBarcodeScannerOptions *options = [[MLKBarcodeScannerOptions alloc] init]; + MLKBarcodeScanner *scanner = + [MLKBarcodeScanner barcodeScannerWithOptions:options]; + __block NSArray *result = nil; + [scanner processImage:vision completion:^(NSArray *barcodes, + NSError *error) { + result = barcodes; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) return cn1VisionError(requestError); + NSMutableArray *items = [NSMutableArray array]; + for (MLKBarcode *barcode in result ?: @[]) { + NSMutableDictionary *item = [NSMutableDictionary + dictionaryWithDictionary:cn1MLKitRect(barcode.frame, resultSize)]; + item[@"value"] = barcode.rawValue ?: [NSNull null]; + item[@"rawData"] = barcode.rawData == nil + ? [NSNull null] + : [barcode.rawData base64EncodedStringWithOptions:0]; + item[@"format"] = cn1MLKitBarcodeFormat(barcode.format); + NSMutableArray *corners = [NSMutableArray array]; + for (NSValue *pointValue in barcode.cornerPoints ?: @[]) { + CGPoint point = pointValue.CGPointValue; + [corners addObject:@{ + @"x": @(point.x / resultSize.width), + @"y": @(point.y / resultSize.height) + }]; + } + item[@"corners"] = corners; + [items addObject:item]; + } + return cn1VisionJSON(@{@"items": items}); +#endif + } else if (feature == 2) { +#if defined(CN1_HAS_MLKIT_FACE) + MLKFaceDetectorOptions *options = [[MLKFaceDetectorOptions alloc] init]; + options.landmarkMode = MLKFaceDetectorLandmarkModeAll; + options.classificationMode = MLKFaceDetectorClassificationModeAll; + options.trackingEnabled = YES; + MLKFaceDetector *detector = [MLKFaceDetector faceDetectorWithOptions:options]; + __block NSArray *result = nil; + [detector processImage:vision completion:^(NSArray *faces, + NSError *error) { + result = faces; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) return cn1VisionError(requestError); + NSMutableArray *items = [NSMutableArray array]; + for (MLKFace *face in result ?: @[]) { + NSMutableDictionary *item = [NSMutableDictionary + dictionaryWithDictionary:cn1MLKitRect(face.frame, resultSize)]; + NSMutableDictionary *landmarks = [NSMutableDictionary dictionary]; + cn1MLKitAddFaceLandmark(landmarks, @"leftEye", face, + MLKFaceLandmarkTypeLeftEye, resultSize); + cn1MLKitAddFaceLandmark(landmarks, @"rightEye", face, + MLKFaceLandmarkTypeRightEye, resultSize); + cn1MLKitAddFaceLandmark(landmarks, @"noseBase", face, + MLKFaceLandmarkTypeNoseBase, resultSize); + cn1MLKitAddFaceLandmark(landmarks, @"mouthLeft", face, + MLKFaceLandmarkTypeMouthLeft, resultSize); + cn1MLKitAddFaceLandmark(landmarks, @"mouthRight", face, + MLKFaceLandmarkTypeMouthRight, resultSize); + item[@"landmarks"] = landmarks; + item[@"yaw"] = @(face.headEulerAngleY); + item[@"pitch"] = @(face.headEulerAngleX); + item[@"roll"] = @(face.headEulerAngleZ); + item[@"smilingProbability"] = face.hasSmilingProbability + ? @(face.smilingProbability) : @(-1); + item[@"trackingId"] = face.hasTrackingID + ? @(face.trackingID) : @(-1); + [items addObject:item]; + } + return cn1VisionJSON(@{@"items": items}); +#endif + } else if (feature == 3) { +#if defined(CN1_HAS_MLKIT_LABEL) + MLKImageLabelerOptions *options = [[MLKImageLabelerOptions alloc] init]; + MLKImageLabeler *labeler = [MLKImageLabeler imageLabelerWithOptions:options]; + __block NSArray *result = nil; + [labeler processImage:vision completion:^(NSArray *labels, + NSError *error) { + result = labels; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) return cn1VisionError(requestError); + NSMutableArray *items = [NSMutableArray array]; + for (MLKImageLabel *label in result ?: @[]) { + [items addObject:@{ + @"text": label.text ?: @"", + @"confidence": @(label.confidence), + @"index": @(label.index) + }]; + } + return cn1VisionJSON(@{@"items": items}); +#endif + } else if (feature == 4) { +#if defined(CN1_HAS_MLKIT_POSE) + MLKPoseDetectorOptions *options = [[MLKPoseDetectorOptions alloc] init]; + options.detectorMode = MLKPoseDetectorModeSingleImage; + MLKPoseDetector *detector = [MLKPoseDetector poseDetectorWithOptions:options]; + __block NSArray *result = nil; + [detector processImage:vision completion:^(NSArray *poses, + NSError *error) { + result = poses; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) return cn1VisionError(requestError); + NSMutableArray *items = [NSMutableArray array]; + MLKPose *pose = result.firstObject; + for (MLKPoseLandmark *landmark in pose.landmarks ?: @[]) { + [items addObject:@{ + @"name": cn1MLKitPoseLandmarkName(landmark.type), + @"x": @(landmark.position.x / resultSize.width), + @"y": @(landmark.position.y / resultSize.height), + @"confidence": @(landmark.inFrameLikelihood) + }]; + } + return cn1VisionJSON(@{@"items": items}); +#endif + } else if (feature == 5) { +#if defined(CN1_HAS_MLKIT_SEGMENTATION) + MLKSelfieSegmenterOptions *options = + [[MLKSelfieSegmenterOptions alloc] init]; + options.segmenterMode = MLKSegmenterModeSingleImage; + MLKSegmenter *segmenter = [MLKSegmenter segmenterWithOptions:options]; + __block MLKSegmentationMask *result = nil; + [segmenter processImage:vision completion:^(MLKSegmentationMask *mask, + NSError *error) { + result = mask; + requestError = error; + dispatch_semaphore_signal(semaphore); + }]; + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + if (requestError != nil) return cn1VisionError(requestError); + CVPixelBufferRef buffer = result.buffer; + if (buffer == nil) { + return cn1VisionJSON(@{@"error": @"No segmentation mask returned"}); + } + CVPixelBufferLockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly); + size_t width = CVPixelBufferGetWidth(buffer); + size_t height = CVPixelBufferGetHeight(buffer); + size_t stride = CVPixelBufferGetBytesPerRow(buffer); + const uint8_t *base = CVPixelBufferGetBaseAddress(buffer); + NSMutableData *packed = [NSMutableData dataWithLength:width * height]; + uint8_t *dest = packed.mutableBytes; + for (size_t y = 0; y < height; y++) { + const float *row = (const float *)(base + y * stride); + for (size_t x = 0; x < width; x++) { + float confidence = fmaxf(0, fminf(1, row[x])); + dest[y * width + x] = (uint8_t)lrintf(confidence * 255); + } + } + CVPixelBufferUnlockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly); + return cn1VisionJSON(@{ + @"width": @(width), + @"height": @(height), + @"dataPeer": @((uint64_t)CFBridgingRetain(packed)) + }); +#endif + } + return cn1VisionJSON(@{ + @"error": @"Requested ML Kit vision component is not linked" + }); +#else + return cn1VisionJSON(@{@"error": @"ML Kit Vision is not linked"}); +#endif +} + +static CGImagePropertyOrientation cn1CGOrientation(int rotation) { + switch (rotation) { + case 90: return kCGImagePropertyOrientationRight; + case 180: return kCGImagePropertyOrientationDown; + case 270: return kCGImagePropertyOrientationLeft; + default: return kCGImagePropertyOrientationUp; + } +} + +static NSString *cn1AppleBarcodeFormat(NSString *symbology) { + NSString *value = symbology.uppercaseString; + if ([value containsString:@"QR"]) return @"QR_CODE"; + if ([value containsString:@"DATAMATRIX"]) return @"DATA_MATRIX"; + if ([value containsString:@"PDF417"]) return @"PDF417"; + if ([value containsString:@"CODE128"]) return @"CODE_128"; + if ([value containsString:@"CODE93"]) return @"CODE_93"; + if ([value containsString:@"CODE39"]) return @"CODE_39"; + if ([value containsString:@"EAN13"]) return @"EAN_13"; + if ([value containsString:@"EAN8"]) return @"EAN_8"; + if ([value containsString:@"UPCE"]) return @"UPC_E"; + if ([value containsString:@"ITF"]) return @"ITF"; + if ([value containsString:@"AZTEC"]) return @"AZTEC"; + return @"UNKNOWN"; +} + +static NSString *cn1ApplePoseLandmarkName( + VNHumanBodyPoseObservationJointName name) { + if ([name isEqual:VNHumanBodyPoseObservationJointNameNose]) return @"nose"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftEye]) return @"leftEye"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightEye]) return @"rightEye"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftEar]) return @"leftEar"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightEar]) return @"rightEar"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftShoulder]) return @"leftShoulder"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightShoulder]) return @"rightShoulder"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameNeck]) return @"neck"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftElbow]) return @"leftElbow"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightElbow]) return @"rightElbow"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftWrist]) return @"leftWrist"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightWrist]) return @"rightWrist"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftHip]) return @"leftHip"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightHip]) return @"rightHip"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRoot]) return @"root"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftKnee]) return @"leftKnee"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightKnee]) return @"rightKnee"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameLeftAnkle]) return @"leftAnkle"; + if ([name isEqual:VNHumanBodyPoseObservationJointNameRightAnkle]) return @"rightAnkle"; + return name ?: @"unknown"; +} + +static NSString *cn1VisionPerform(NSData *data, CGImageRef rawImage, + int feature, int rotation) { + NSError *error = nil; + VNImageRequestHandler *handler = rawImage == NULL + ? [[VNImageRequestHandler alloc] initWithData:data + orientation:cn1CGOrientation(rotation) options:@{}] + : [[VNImageRequestHandler alloc] initWithCGImage:rawImage + orientation:cn1CGOrientation(rotation) options:@{}]; + + if (feature == 0) { + if (@available(iOS 13.0, *)) { + VNRecognizeTextRequest *request = [[VNRecognizeTextRequest alloc] init]; + request.recognitionLevel = VNRequestTextRecognitionLevelAccurate; + if (![handler performRequests:@[request] error:&error]) { + return cn1VisionError(error); + } + NSMutableArray *items = [NSMutableArray array]; + NSMutableArray *lines = [NSMutableArray array]; + for (VNRecognizedTextObservation *observation in request.results) { + VNRecognizedText *candidate = [observation topCandidates:1].firstObject; + if (candidate == nil) continue; + NSMutableDictionary *item = + [NSMutableDictionary dictionaryWithDictionary: + cn1VisionRect(observation.boundingBox)]; + item[@"text"] = candidate.string ?: @""; + item[@"confidence"] = @(candidate.confidence); + [items addObject:item]; + [lines addObject:candidate.string ?: @""]; + } + return cn1VisionJSON(@{ + @"text": [lines componentsJoinedByString:@"\n"], + @"items": items + }); + } + } else if (feature == 1) { + VNDetectBarcodesRequest *request = [[VNDetectBarcodesRequest alloc] init]; +#if TARGET_OS_SIMULATOR + /* + * Recent iOS simulator runtimes can successfully perform the default + * barcode request while returning no observations for valid QR + * images. Revision 1 avoids that simulator-only Vision regression. + * Devices retain the latest revision and its additional symbologies. + */ + request.revision = VNDetectBarcodesRequestRevision1; +#endif + if (![handler performRequests:@[request] error:&error]) { + return cn1VisionError(error); + } + NSMutableArray *items = [NSMutableArray array]; + for (VNBarcodeObservation *observation in request.results) { + NSMutableDictionary *item = + [NSMutableDictionary dictionaryWithDictionary: + cn1VisionRect(observation.boundingBox)]; + item[@"value"] = observation.payloadStringValue ?: [NSNull null]; + item[@"format"] = cn1AppleBarcodeFormat(observation.symbology); + item[@"corners"] = @[ + @{@"x": @(observation.topLeft.x), + @"y": @(1.0 - observation.topLeft.y)}, + @{@"x": @(observation.topRight.x), + @"y": @(1.0 - observation.topRight.y)}, + @{@"x": @(observation.bottomRight.x), + @"y": @(1.0 - observation.bottomRight.y)}, + @{@"x": @(observation.bottomLeft.x), + @"y": @(1.0 - observation.bottomLeft.y)} + ]; + [items addObject:item]; + } + return cn1VisionJSON(@{@"items": items}); + } else if (feature == 2) { + VNDetectFaceLandmarksRequest *request = [[VNDetectFaceLandmarksRequest alloc] init]; + if (![handler performRequests:@[request] error:&error]) { + return cn1VisionError(error); + } + NSMutableArray *items = [NSMutableArray array]; + for (VNFaceObservation *observation in request.results) { + NSMutableDictionary *item = + [NSMutableDictionary dictionaryWithDictionary: + cn1VisionRect(observation.boundingBox)]; + NSMutableDictionary *landmarks = [NSMutableDictionary dictionary]; + VNFaceLandmarks2D *faceLandmarks = observation.landmarks; + cn1VisionAddFaceLandmark(landmarks, @"leftEye", + faceLandmarks.leftEye, observation.boundingBox); + cn1VisionAddFaceLandmark(landmarks, @"rightEye", + faceLandmarks.rightEye, observation.boundingBox); + cn1VisionAddFaceLandmark(landmarks, @"noseBase", + faceLandmarks.nose, observation.boundingBox); + NSDictionary *mouthLeft = cn1VisionFaceLandmarkEdge( + faceLandmarks.outerLips, observation.boundingBox, YES); + NSDictionary *mouthRight = cn1VisionFaceLandmarkEdge( + faceLandmarks.outerLips, observation.boundingBox, NO); + if (mouthLeft != nil) landmarks[@"mouthLeft"] = mouthLeft; + if (mouthRight != nil) landmarks[@"mouthRight"] = mouthRight; + item[@"landmarks"] = landmarks; + item[@"yaw"] = @((observation.yaw ?: @0).doubleValue + * 180.0 / M_PI); + if (@available(iOS 15.0, *)) { + item[@"pitch"] = @((observation.pitch ?: @0).doubleValue + * 180.0 / M_PI); + } else { + item[@"pitch"] = @0; + } + item[@"roll"] = @((observation.roll ?: @0).doubleValue + * 180.0 / M_PI); + item[@"smilingProbability"] = @(-1); + item[@"trackingId"] = @(-1); + [items addObject:item]; + } + return cn1VisionJSON(@{@"items": items}); + } else if (feature == 3) { + if (@available(iOS 15.0, *)) { + VNClassifyImageRequest *request = [[VNClassifyImageRequest alloc] init]; + if (![handler performRequests:@[request] error:&error]) { + return cn1VisionError(error); + } + NSMutableArray *items = [NSMutableArray array]; + for (VNClassificationObservation *observation in request.results) { + [items addObject:@{ + @"text": observation.identifier ?: @"", + @"confidence": @(observation.confidence), + @"index": @(-1) + }]; + } + return cn1VisionJSON(@{@"items": items}); + } + } else if (feature == 4) { + if (@available(iOS 14.0, *)) { + VNDetectHumanBodyPoseRequest *request = + [[VNDetectHumanBodyPoseRequest alloc] init]; + if (![handler performRequests:@[request] error:&error]) { + return cn1VisionError(error); + } + NSMutableArray *items = [NSMutableArray array]; + VNHumanBodyPoseObservation *pose = request.results.firstObject; + NSDictionary *points = + [pose recognizedPointsForGroupKey: + VNHumanBodyPoseObservationJointsGroupNameAll error:&error]; + if (error != nil) { + return cn1VisionError(error); + } + for (NSString *name in points) { + VNRecognizedPoint *point = points[name]; + [items addObject:@{ + @"name": cn1ApplePoseLandmarkName(name), + @"x": @(point.location.x), + @"y": @(1.0 - point.location.y), + @"confidence": @(point.confidence) + }]; + } + return cn1VisionJSON(@{@"items": items}); + } + } else if (feature == 5) { + if (@available(iOS 15.0, *)) { + VNGeneratePersonSegmentationRequest *request = + [[VNGeneratePersonSegmentationRequest alloc] init]; + request.qualityLevel = VNGeneratePersonSegmentationRequestQualityLevelBalanced; + request.outputPixelFormat = kCVPixelFormatType_OneComponent8; + if (![handler performRequests:@[request] error:&error]) { + return cn1VisionError(error); + } + VNPixelBufferObservation *observation = request.results.firstObject; + CVPixelBufferRef buffer = observation.pixelBuffer; + if (buffer == nil) { + return cn1VisionJSON(@{@"error": @"No segmentation mask returned"}); + } + CVPixelBufferLockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly); + size_t width = CVPixelBufferGetWidth(buffer); + size_t height = CVPixelBufferGetHeight(buffer); + size_t stride = CVPixelBufferGetBytesPerRow(buffer); + const uint8_t *base = CVPixelBufferGetBaseAddress(buffer); + NSMutableData *packed = [NSMutableData dataWithLength:width * height]; + uint8_t *dest = packed.mutableBytes; + for (size_t y = 0; y < height; y++) { + memcpy(dest + y * width, base + y * stride, width); + } + CVPixelBufferUnlockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly); + return cn1VisionJSON(@{ + @"width": @(width), + @"height": @(height), + @"dataPeer": @((uint64_t)CFBridgingRetain(packed)) + }); + } + } else if (feature == 6) { + UIImage *image = rawImage == NULL ? [UIImage imageWithData:data] + : [UIImage imageWithCGImage:rawImage]; + if (image == nil) { + return cn1VisionJSON(@{@"error": @"Could not decode document image"}); + } + CIImage *source = [CIImage imageWithCGImage:image.CGImage]; + if (rotation != 0) { + source = [source imageByApplyingOrientation: + cn1CGOrientation(rotation)]; + } + CIContext *context = [CIContext context]; + CIDetector *detector = [CIDetector detectorOfType:CIDetectorTypeRectangle + context:context + options:@{ + CIDetectorAccuracy: CIDetectorAccuracyHigh + }]; + NSArray *features = [detector featuresInImage:source]; + CIImage *corrected = source; + if (features.count > 0) { + CIRectangleFeature *rectangle = features.firstObject; + corrected = [source imageByApplyingFilter:@"CIPerspectiveCorrection" + withInputParameters:@{ + @"inputTopLeft": [CIVector vectorWithCGPoint:rectangle.topLeft], + @"inputTopRight": [CIVector vectorWithCGPoint:rectangle.topRight], + @"inputBottomLeft": [CIVector vectorWithCGPoint:rectangle.bottomLeft], + @"inputBottomRight": [CIVector vectorWithCGPoint:rectangle.bottomRight] + }]; + } + CGImageRef outputImage = [context createCGImage:corrected + fromRect:corrected.extent]; + UIImage *output = [UIImage imageWithCGImage:outputImage]; + CGImageRelease(outputImage); + NSData *jpeg = UIImageJPEGRepresentation(output, 0.92); + return cn1VisionJSON(@{ + @"dataPeer": @((uint64_t)CFBridgingRetain(jpeg)) + }); + } + return cn1VisionJSON(@{@"error": @"Vision feature is unavailable on this OS"}); +} +#endif + +JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_cn1VisionIsSupported___int_boolean_R_boolean( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT feature, + JAVA_BOOLEAN mlKit) { +#if defined(INCLUDE_CN1_VISION) && !TARGET_OS_WATCH && !TARGET_OS_TV + if (mlKit) { + switch (feature) { + case 0: +#if defined(CN1_HAS_MLKIT_TEXT) + return 1; +#else + return 0; +#endif + case 1: +#if defined(CN1_HAS_MLKIT_BARCODE) + return 1; +#else + return 0; +#endif + case 2: +#if defined(CN1_HAS_MLKIT_FACE) + return 1; +#else + return 0; +#endif + case 3: +#if defined(CN1_HAS_MLKIT_LABEL) + return 1; +#else + return 0; +#endif + case 4: +#if defined(CN1_HAS_MLKIT_POSE) + return 1; +#else + return 0; +#endif + case 5: +#if defined(CN1_HAS_MLKIT_SEGMENTATION) + return 1; +#else + return 0; +#endif + default: + return 0; + } + } + switch (feature) { + case 0: + if (@available(iOS 13.0, *)) return 1; + return 0; + case 3: + case 5: + if (@available(iOS 15.0, *)) return 1; + return 0; + case 4: + if (@available(iOS 14.0, *)) return 1; + return 0; + default: + return 1; + } +#else + return 0; +#endif +} + +#if defined(INCLUDE_CN1_VISION) && !TARGET_OS_WATCH && !TARGET_OS_TV +static uint8_t cn1VisionClamp(int value) { + return (uint8_t)(value < 0 ? 0 : (value > 255 ? 255 : value)); +} + +/* + * Converts the two raw CameraFrame formats to an uncompressed CGImage so + * Vision and ML Kit avoid a lossy JPEG encode/decode cycle on every frame. + * format mirrors FrameFormat: 0=JPEG/encoded, 1=NV21, 2=RGBA8888. + */ +static CGImageRef cn1VisionCreateRawImage(NSData *data, int width, int height, + int format) { + if (format == 0) { + return NULL; + } + if (width <= 0 || height <= 0) { + return NULL; + } + if (format != 1 && format != 2) { + return NULL; + } + if (format == 1 && ((width & 1) != 0 || (height & 1) != 0)) { + return NULL; + } + NSUInteger imageWidth = (NSUInteger)width; + NSUInteger imageHeight = (NSUInteger)height; + if (imageWidth > NSUIntegerMax / imageHeight) { + return NULL; + } + NSUInteger pixelCount = imageWidth * imageHeight; + NSUInteger requiredLength; + if (format == 2) { + if (pixelCount > NSUIntegerMax / 4) { + return NULL; + } + requiredLength = pixelCount * 4; + } else { + if (pixelCount > NSUIntegerMax - pixelCount / 2) { + return NULL; + } + requiredLength = pixelCount + pixelCount / 2; + } + if (data.length < requiredLength) { + return NULL; + } + const uint8_t *source = data.bytes; + NSMutableData *rgba = [NSMutableData dataWithLength:pixelCount * 4]; + uint8_t *dest = rgba.mutableBytes; + if (format == 2) { + memcpy(dest, source, pixelCount * 4); + } else { + for (int y = 0; y < height; y++) { + NSUInteger uvRow = pixelCount + + (NSUInteger)(y >> 1) * imageWidth; + for (int x = 0; x < width; x++) { + int yy = source[(NSUInteger)y * imageWidth + + (NSUInteger)x] & 255; + NSUInteger uv = uvRow + (NSUInteger)(x & ~1); + int v = (source[uv] & 255) - 128; + int u = (source[uv + 1] & 255) - 128; + int c = yy - 16; + if (c < 0) c = 0; + NSUInteger p = ((NSUInteger)y * imageWidth + + (NSUInteger)x) * 4; + dest[p] = cn1VisionClamp((298 * c + 409 * v + 128) >> 8); + dest[p + 1] = cn1VisionClamp( + (298 * c - 100 * u - 208 * v + 128) >> 8); + dest[p + 2] = cn1VisionClamp( + (298 * c + 516 * u + 128) >> 8); + dest[p + 3] = 255; + } + } + } + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + CGDataProviderRef provider = CGDataProviderCreateWithCFData( + (CFDataRef)rgba); + CGImageRef cgImage = CGImageCreate(imageWidth, imageHeight, 8, 32, + imageWidth * 4, + colorSpace, kCGBitmapByteOrderDefault | kCGImageAlphaLast, + provider, NULL, false, kCGRenderingIntentDefault); + CGDataProviderRelease(provider); + CGColorSpaceRelease(colorSpace); + return cgImage; +} +#endif + +JAVA_OBJECT com_codename1_impl_ios_IOSNative_cn1VisionAnalyze___byte_1ARRAY_int_boolean_int_int_int_int_R_java_lang_String( + CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, + JAVA_OBJECT encodedImage, JAVA_INT feature, JAVA_BOOLEAN mlKit, + JAVA_INT rotation, JAVA_INT width, JAVA_INT height, + JAVA_INT frameFormat) { +#if defined(INCLUDE_CN1_VISION) && !TARGET_OS_WATCH && !TARGET_OS_TV + if (encodedImage == JAVA_NULL) { + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + @"{\"error\":\"Image data is null\"}"); + } + JAVA_ARRAY bytes = (JAVA_ARRAY) encodedImage; + NSData *data = [NSData dataWithBytes:bytes->data length:(NSUInteger) bytes->length]; + CGImageRef rawImage = cn1VisionCreateRawImage( + data, width, height, frameFormat); + if (frameFormat != 0 && rawImage == NULL) { + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG + @"{\"error\":\"Invalid raw vision image\"}"); + } + NSString *result = mlKit + ? cn1MLKitVisionPerform(data, rawImage, feature, rotation) + : cn1VisionPerform(data, rawImage, feature, rotation); + if (rawImage != NULL) CGImageRelease(rawImage); + return fromNSString(CN1_THREAD_GET_STATE_PASS_ARG result); +#else + return JAVA_NULL; +#endif +} diff --git a/Ports/iOSPort/nativeSources/CodenameOne_GLViewController.h b/Ports/iOSPort/nativeSources/CodenameOne_GLViewController.h index b22fa97e66e..964a27f8cd9 100644 --- a/Ports/iOSPort/nativeSources/CodenameOne_GLViewController.h +++ b/Ports/iOSPort/nativeSources/CodenameOne_GLViewController.h @@ -122,6 +122,17 @@ void cn1RunSyncOnMainQueue(void (^block)(void)); #undef INCLUDE_CN1_AR #endif +// INCLUDE_CN1_VISION gates Apple Vision/Core Image analysis. It is enabled +// only when the app references com.codename1.ai.vision. +//#define INCLUDE_CN1_VISION +//#define INCLUDE_CN1_LANGUAGE +//#define INCLUDE_CN1_INFERENCE +#if TARGET_OS_WATCH || TARGET_OS_TV +#undef INCLUDE_CN1_VISION +#undef INCLUDE_CN1_LANGUAGE +#undef INCLUDE_CN1_INFERENCE +#endif + // CN1_USE_CARPLAY gates the Apple CarPlay native bridge // (CodenameOne_CarPlaySceneDelegate.{h,m} + the IOSNative carPlay* trampolines: // CarPlay.framework, the CPTemplate translation). IPhoneBuilder uncomments this diff --git a/Ports/iOSPort/nativeSources/IOSNative.m b/Ports/iOSPort/nativeSources/IOSNative.m index fdbb4cc31d5..1185eec534c 100644 --- a/Ports/iOSPort/nativeSources/IOSNative.m +++ b/Ports/iOSPort/nativeSources/IOSNative.m @@ -6727,6 +6727,14 @@ JAVA_OBJECT com_codename1_impl_ios_IOSNative_getDeviceName__(CN1_THREAD_STATE_MU #endif // !TARGET_OS_WATCH } +JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_isSimulator___R_boolean(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject) { +#if TARGET_OS_SIMULATOR + return JAVA_TRUE; +#else + return JAVA_FALSE; +#endif +} + JAVA_OBJECT com_codename1_impl_ios_IOSNative_getDeviceHardwareModel__(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject) { // hw.machine is the hardware/marketing model identifier (e.g. "iPhone15,2"). // On the simulator it is the host arch; we map that to a stable label so the @@ -8118,7 +8126,7 @@ JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_checkLocationUsage___R_boolean(CN1 } //native boolean checkMicrophoneUsage(); JAVA_BOOLEAN com_codename1_impl_ios_IOSNative_checkMicrophoneUsage___R_boolean(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT me) { -#ifdef INCLUDE_MICROPHONE_USAGE +#if defined(INCLUDE_MICROPHONE_USAGE) && !TARGET_OS_TV return JAVA_TRUE; #else return JAVA_FALSE; @@ -8584,7 +8592,7 @@ void com_codename1_impl_ios_IOSNative_nsDataToByteArray___long_byte_1ARRAY(CN1_T JAVA_LONG com_codename1_impl_ios_IOSNative_createAudioUnit___java_lang_String_int_float_float_1ARRAY_R_long(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT path, JAVA_INT audioChannels, JAVA_FLOAT sampleRate, JAVA_OBJECT sampleBuffer) { -#ifdef INCLUDE_MICROPHONE_USAGE +#if defined(INCLUDE_MICROPHONE_USAGE) && !TARGET_OS_TV && !TARGET_OS_WATCH __block CN1AudioUnit* recorder = nil; __block NSString *exStr = nil; @@ -8674,7 +8682,7 @@ void com_codename1_impl_ios_IOSNative_destroyAudioUnit___long(CN1_THREAD_STATE_M JAVA_LONG com_codename1_impl_ios_IOSNative_createAudioRecorder___java_lang_String_java_lang_String_int_int_int_int(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT destinationFile, JAVA_OBJECT mimeType, JAVA_INT sampleRate, JAVA_INT bitRate, JAVA_INT channels, JAVA_INT maxDuration) { -#ifdef INCLUDE_MICROPHONE_USAGE +#if defined(INCLUDE_MICROPHONE_USAGE) && !TARGET_OS_TV __block AVAudioRecorder* recorder = nil; __block NSString *exStr = nil; diff --git a/Ports/iOSPort/nativeSources/IOSSimd.m b/Ports/iOSPort/nativeSources/IOSSimd.m index a287ec9a562..23eca3c9836 100644 --- a/Ports/iOSPort/nativeSources/IOSSimd.m +++ b/Ports/iOSPort/nativeSources/IOSSimd.m @@ -1,9 +1,39 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + #include "xmlvm.h" #include #include #include + +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#define CN1_IOS_NEON 1 #include +#else +#define CN1_IOS_NEON 0 +#endif +#if CN1_IOS_NEON static JAVA_ARRAY_BYTE cn1_saturating_byte(int value) { if (value > 127) { return 127; @@ -37,6 +67,7 @@ static uint8x16_t cn1_load_ints_to_u8x16(JAVA_ARRAY_INT* s, int offset) { int8x16_t out = vcombine_s8(vmovn_s16(lo16), vmovn_s16(hi16)); return vreinterpretq_u8_s8(out); } +#endif JAVA_OBJECT com_codename1_impl_ios_IOSSimd_allocByteNative___int_R_byte_1ARRAY(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_INT size) { return allocArrayAligned(threadStateData, size, &class_array1__JAVA_BYTE, sizeof(JAVA_ARRAY_BYTE), 1, 16); @@ -50,6 +81,11 @@ JAVA_OBJECT com_codename1_impl_ios_IOSSimd_allocFloatNative___int_R_float_1ARRAY return allocArrayAligned(threadStateData, size, &class_array1__JAVA_FLOAT, sizeof(JAVA_ARRAY_FLOAT), 1, 16); } +JAVA_BOOLEAN com_codename1_impl_ios_IOSSimd_isNativeSupported___R_boolean(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject) { + return CN1_IOS_NEON ? JAVA_TRUE : JAVA_FALSE; +} + +#if CN1_IOS_NEON JAVA_VOID com_codename1_impl_ios_IOSSimd_lookupBytes___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int(CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, JAVA_OBJECT table, JAVA_OBJECT indices, JAVA_OBJECT dst, JAVA_INT offset, JAVA_INT length) { JAVA_ARRAY_BYTE* t = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)table)->data; JAVA_ARRAY_BYTE* idx = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)indices)->data; @@ -1777,3 +1813,117 @@ JAVA_VOID com_codename1_impl_ios_IOSSimd_replaceTopByteFromUnsignedBytes___int_1 d[dstOffset + i] = (JAVA_INT)((r[rgbSrcOffset + i] & 0x00ffffff) | ((a[alphaSrcOffset + i] & 0xff) << 24)); } } +#else +/* + * Google ML Kit still requires an x86_64 simulator build on Apple Silicon. + * IOSSimd is NEON-specific, so expose the generic Simd implementations under + * the native iOS symbol names on non-NEON architectures. Mach-O indirect + * symbols preserve the exact generated native ABI without duplicating the + * scalar implementations or adding a second native source file. + */ +#define CN1_IOS_SIMD_ALIAS(name) \ + __asm__(".globl _com_codename1_impl_ios_IOSSimd_" #name "\n" \ + ".set _com_codename1_impl_ios_IOSSimd_" #name \ + ", _com_codename1_util_Simd_" #name); + +CN1_IOS_SIMD_ALIAS(lookupBytes___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(lookupBytes___byte_1ARRAY_byte_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(add___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(sub___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(mul___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(min___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(max___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(abs___byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(clamp___byte_1ARRAY_byte_1ARRAY_byte_byte_int_int) +CN1_IOS_SIMD_ALIAS(add___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(sub___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(mul___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(min___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(max___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(abs___int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(clamp___int_1ARRAY_int_1ARRAY_int_int_int_int) +CN1_IOS_SIMD_ALIAS(sum___int_1ARRAY_int_int_R_int) +CN1_IOS_SIMD_ALIAS(dot___int_1ARRAY_int_1ARRAY_int_int_R_int) +CN1_IOS_SIMD_ALIAS(add___float_1ARRAY_float_1ARRAY_float_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(sub___float_1ARRAY_float_1ARRAY_float_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(mul___float_1ARRAY_float_1ARRAY_float_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(min___float_1ARRAY_float_1ARRAY_float_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(max___float_1ARRAY_float_1ARRAY_float_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(abs___float_1ARRAY_float_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(clamp___float_1ARRAY_float_1ARRAY_float_float_int_int) +CN1_IOS_SIMD_ALIAS(sum___float_1ARRAY_int_int_R_float) +CN1_IOS_SIMD_ALIAS(dot___float_1ARRAY_float_1ARRAY_int_int_R_float) +CN1_IOS_SIMD_ALIAS(and___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(and___byte_1ARRAY_int_byte_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(or___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(or___byte_1ARRAY_int_byte_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(xor___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(not___byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpEq___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpEq___byte_1ARRAY_byte_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpLt___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpLt___byte_1ARRAY_byte_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpGt___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpRange___byte_1ARRAY_byte_byte_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(select___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(unpackUnsignedByteToInt___byte_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packIntToByteSaturating___int_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packIntToByteTruncate___int_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packIntToByteTruncate___int_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packIntToByteTruncateInterleaved4___int_1ARRAY_int_int_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packBytesInterleaved3___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packBytesInterleaved3___byte_1ARRAY_int_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packBytesInterleaved4___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(packBytesInterleaved4___byte_1ARRAY_int_int_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(permuteBytes___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(and___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(and___int_1ARRAY_int_int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(or___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(or___int_1ARRAY_int_int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(xor___int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(not___int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shl___int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shl___int_1ARRAY_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shrLogical___int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shrLogical___int_1ARRAY_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shrArithmetic___int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpEq___int_1ARRAY_int_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpLt___int_1ARRAY_int_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpGt___int_1ARRAY_int_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(select___byte_1ARRAY_int_1ARRAY_int_1ARRAY_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shl___byte_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shl___byte_1ARRAY_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shrLogical___byte_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(shrLogical___byte_1ARRAY_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(addWrapping___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(addWrapping___byte_1ARRAY_byte_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(subWrapping___byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(subWrapping___byte_1ARRAY_byte_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(unpackUnsignedByteToInt___byte_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(unpackUnsignedByteToIntInterleaved3___byte_1ARRAY_int_int_1ARRAY_int_int_int_int) +CN1_IOS_SIMD_ALIAS(unpackBytesInterleaved3___byte_1ARRAY_int_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int) +CN1_IOS_SIMD_ALIAS(unpackBytesInterleaved3___byte_1ARRAY_int_byte_1ARRAY_int_int_int_int) +CN1_IOS_SIMD_ALIAS(unpackBytesInterleaved4___byte_1ARRAY_int_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int) +CN1_IOS_SIMD_ALIAS(unpackBytesInterleaved4___byte_1ARRAY_int_byte_1ARRAY_int_int_int_int_int) +CN1_IOS_SIMD_ALIAS(unpackLookupBytesInterleaved4___byte_1ARRAY_byte_1ARRAY_int_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_byte_1ARRAY_int_R_int) +CN1_IOS_SIMD_ALIAS(unpackLookupBytesInterleaved4___byte_1ARRAY_byte_1ARRAY_int_byte_1ARRAY_int_int_int_int_int_R_int) +CN1_IOS_SIMD_ALIAS(add___int_1ARRAY_int_int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpEq___int_1ARRAY_int_int_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpLt___int_1ARRAY_int_int_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(select___byte_1ARRAY_int_int_1ARRAY_int_int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(and___int_1ARRAY_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(or___int_1ARRAY_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(xor___int_1ARRAY_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpEq___int_1ARRAY_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpLt___int_1ARRAY_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(cmpGt___int_1ARRAY_int_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(not___byte_1ARRAY_int_byte_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(select___byte_1ARRAY_int_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(select___byte_1ARRAY_int_int_1ARRAY_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(select___byte_1ARRAY_int_int_int_1ARRAY_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(blendByMaskTestNonzero___int_1ARRAY_int_int_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(blendByMaskTestNonzeroSubstituteOnKeepEq___int_1ARRAY_int_int_int_int_int_int_int_1ARRAY_int_int) +CN1_IOS_SIMD_ALIAS(replaceTopByteFromUnsignedBytes___int_1ARRAY_int_byte_1ARRAY_int_int_1ARRAY_int_int) + +#undef CN1_IOS_SIMD_ALIAS +#endif diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java index d8033ec0b07..e40f91a957d 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSImplementation.java @@ -4764,6 +4764,33 @@ public com.codename1.impl.ARImpl createARImpl() { return new IOSARImpl(); } + @Override + public com.codename1.impl.VisionImpl createVisionImpl() { + for (int feature = 0; + feature < com.codename1.ai.vision.VisionFeature.values().length; + feature++) { + if (nativeInstance.cn1VisionIsSupported(feature, false) + || nativeInstance.cn1VisionIsSupported(feature, true)) { + return new IOSVisionImpl(); + } + } + return null; + } + + @Override + public com.codename1.impl.LanguageImpl createLanguageImpl() { + return nativeInstance.cn1LanguageIsSupported(0, false) + || nativeInstance.cn1LanguageIsSupported(1, false) + || nativeInstance.cn1LanguageIsSupported(2, false) + ? new IOSLanguageImpl() : null; + } + + @Override + public com.codename1.impl.InferenceImpl createInferenceImpl() { + return nativeInstance.cn1InferenceIsSupported() + ? new IOSInferenceImpl() : null; + } + @Override public String [] getAvailableRecordingMimeTypes() { // All of these amount to the same thing. @@ -7845,6 +7872,11 @@ else if(largest > 2000) { } return dDensity; } + + @Override + public boolean isSimulator() { + return nativeInstance.isSimulator(); + } double ppi = 0; private static final double PPI_458 = 18.031496062992126; diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java new file mode 100644 index 00000000000..8a8dc299615 --- /dev/null +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSInferenceImpl.java @@ -0,0 +1,473 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.ios; + +import com.codename1.ai.inference.InferenceException; +import com.codename1.ai.inference.InferenceOptions; +import com.codename1.ai.inference.ModelSource; +import com.codename1.ai.inference.Tensor; +import com.codename1.ai.inference.TensorInfo; +import com.codename1.ai.inference.TensorType; +import com.codename1.impl.InferenceImpl; +import com.codename1.io.FileSystemStorage; +import com.codename1.io.JSONParser; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringReader; +import java.util.List; +import java.util.Map; + +/** LiteRT Objective-C backend with optional Core ML delegation. */ +public final class IOSInferenceImpl extends InferenceImpl { + private static final class Handle { + final int id; + TensorInfo[] inputs; + TensorInfo[] outputs; + + Handle(int id, TensorInfo[] inputs, TensorInfo[] outputs) { + this.id = id; + this.inputs = inputs; + this.outputs = outputs; + } + } + + @Override + public boolean isSupported() { + return IOSImplementation.nativeInstance.cn1InferenceIsSupported(); + } + + @Override + public AsyncResource open(final ModelSource source, + final InferenceOptions options) { + final AsyncResource out = new AsyncResource(); + openInBackground(out, source, options.getThreads(), + options.getAccelerator().ordinal(), + options.isFallbackAllowed()); + return out; + } + + private static void openInBackground(final AsyncResource out, + final ModelSource source, + final int threads, + final int accelerator, + final boolean allowFallback) { + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + try { + String opened = source.getKind() == ModelSource.FILE + ? IOSImplementation.nativeInstance.cn1InferenceOpenFile( + FileSystemStorage.getInstance().toNativePath( + source.getPath()), + threads, accelerator, allowFallback) + : IOSImplementation.nativeInstance.cn1InferenceOpen( + loadModel(source), threads, accelerator, + allowFallback); + Map root = parse(opened); + int id = integer(root, "handle"); + final Handle handle; + try { + handle = new Handle(id, + metadata(id, false), metadata(id, true)); + } catch (Throwable metadataError) { + IOSImplementation.nativeInstance.cn1InferenceClose(id); + throw metadataError; + } + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(handle); + } + }); + } catch (final Throwable error) { + fail(out, "Could not open LiteRT model", error); + } + } + }); + } + + @Override + public TensorInfo[] getInputs(Object handle) { + return copy(checked(handle).inputs); + } + + @Override + public TensorInfo[] getOutputs(Object handle) { + return copy(checked(handle).outputs); + } + + @Override + public AsyncResource run(Object handle, final Tensor[] inputs) { + final AsyncResource out = new AsyncResource(); + final Handle checked = checked(handle); + runInBackground(out, checked, inputs); + return out; + } + + private static void runInBackground(final AsyncResource out, + final Handle checked, + final Tensor[] inputs) { + final int id = checked.id; + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + try { + TensorInfo[] metadata = checked.inputs; + if (inputs.length != metadata.length) { + throw new IllegalArgumentException("Expected " + + metadata.length + " input tensors but received " + + inputs.length); + } + int[] indices = new int[inputs.length]; + for (int i = 0; i < inputs.length; i++) { + Tensor tensor = inputs[i]; + int index = inputIndex(tensor, i, metadata); + TensorInfo expected = find(metadata, index); + for (int previous = 0; previous < i; previous++) { + if (indices[previous] == index) { + throw new IllegalArgumentException( + "Model input " + expected.getName() + + " was supplied more than once"); + } + } + indices[i] = index; + if (tensor.getType() != expected.getType()) { + throw new IllegalArgumentException("Input " + + expected.getName() + " expects " + + expected.getType() + " but received " + + tensor.getType()); + } + if (!sameShape(tensor.getShape(), + expected.getShape())) { + throw new IllegalArgumentException( + "Input " + expected.getName() + + " shape does not match the " + + "model metadata; call " + + "resizeInput() before run()"); + } + } + for (int i = 0; i < inputs.length; i++) { + Tensor tensor = inputs[i]; + parse(IOSImplementation.nativeInstance.cn1InferenceCopyInput( + id, indices[i], encodeData(tensor.getType(), + tensor.getDataUnsafe()))); + } + parse(IOSImplementation.nativeInstance.cn1InferenceInvoke(id)); + TensorInfo[] outputs = metadata(id, true); + checked.outputs = outputs; + final Tensor[] result = new Tensor[outputs.length]; + for (int i = 0; i < result.length; i++) { + TensorInfo output = outputs[i]; + long data = IOSImplementation.nativeInstance + .cn1InferenceOutputData(id, output.getIndex()); + if (data == 0) { + throw new InferenceException( + "Could not read LiteRT output " + + output.getName()); + } + byte[] bytes; + try { + bytes = new byte[IOSImplementation.nativeInstance + .getNSDataSize(data)]; + IOSImplementation.nativeInstance.nsDataToByteArray( + data, bytes); + } finally { + IOSImplementation.nativeInstance.releasePeer(data); + } + int[] shape = output.getShape(); + result[i] = new Tensor(output.getName(), + output.getType(), shape, + decodeData(output.getType(), bytes, + elementCount(shape))); + } + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(result); + } + }); + } catch (final Throwable error) { + fail(out, "LiteRT inference failed", error); + } + } + }); + } + + @Override + public void resizeInput(Object handle, String name, int[] shape) { + Handle checked = checked(handle); + TensorInfo[] inputs = checked.inputs; + int index = -1; + for (int i = 0; i < inputs.length; i++) { + if (name == null || name.equals(inputs[i].getName())) { + index = inputs[i].getIndex(); + break; + } + } + if (index < 0) { + throw new IllegalArgumentException("Unknown model input " + name); + } + try { + parse(IOSImplementation.nativeInstance.cn1InferenceResize( + checked.id, index, shape)); + checked.inputs = metadata(checked.id, false); + checked.outputs = metadata(checked.id, true); + } catch (Exception error) { + throw new InferenceException("Could not resize input " + name, error); + } + } + + @Override + public void close(Object handle) { + IOSImplementation.nativeInstance.cn1InferenceClose(checked(handle).id); + } + + private static TensorInfo[] metadata(int handle, boolean outputs) { + try { + Map root = parse(IOSImplementation.nativeInstance.cn1InferenceMetadata( + handle, outputs)); + List items = list(root, "items"); + TensorInfo[] result = new TensorInfo[items.size()]; + for (int i = 0; i < result.length; i++) { + Map value = (Map) items.get(i); + result[i] = new TensorInfo(string(value, "name"), + TensorType.valueOf(string(value, "type")), + intArray(list(value, "shape")), + integer(value, "index")); + } + return result; + } catch (Exception error) { + throw new InferenceException("Could not read LiteRT metadata", error); + } + } + + private static Handle checked(Object value) { + if (!(value instanceof Handle)) { + throw new IllegalArgumentException("Invalid LiteRT session handle"); + } + return (Handle) value; + } + + private static int inputIndex(Tensor tensor, int fallback, + TensorInfo[] metadata) { + if (tensor.getName() != null) { + for (int i = 0; i < metadata.length; i++) { + if (tensor.getName().equals(metadata[i].getName())) { + return metadata[i].getIndex(); + } + } + throw new IllegalArgumentException("Unknown model input " + + tensor.getName()); + } + return metadata[fallback].getIndex(); + } + + private static TensorInfo find(TensorInfo[] metadata, int index) { + for (int i = 0; i < metadata.length; i++) { + if (metadata[i].getIndex() == index) { + return metadata[i]; + } + } + throw new IllegalArgumentException("Unknown model input index " + index); + } + + private static byte[] encodeData(TensorType type, Object value) { + if (value instanceof byte[]) { + return (byte[]) value; + } + if (value instanceof float[]) { + float[] values = (float[]) value; + byte[] out = new byte[values.length * 4]; + for (int i = 0; i < values.length; i++) { + writeInt(out, i * 4, Float.floatToIntBits(values[i])); + } + return out; + } + if (value instanceof int[]) { + int[] values = (int[]) value; + byte[] out = new byte[values.length * 4]; + for (int i = 0; i < values.length; i++) { + writeInt(out, i * 4, values[i]); + } + return out; + } + if (value instanceof long[]) { + long[] values = (long[]) value; + byte[] out = new byte[values.length * 8]; + for (int i = 0; i < values.length; i++) { + writeLong(out, i * 8, values[i]); + } + return out; + } + throw new InferenceException("Unsupported input tensor data"); + } + + private static Object decodeData(TensorType type, byte[] bytes, int count) { + if (type == TensorType.FLOAT32) { + float[] out = new float[count]; + for (int i = 0; i < count; i++) { + out[i] = Float.intBitsToFloat(readInt(bytes, i * 4)); + } + return out; + } + if (type == TensorType.INT32) { + int[] out = new int[count]; + for (int i = 0; i < count; i++) { + out[i] = readInt(bytes, i * 4); + } + return out; + } + if (type == TensorType.INT64) { + long[] out = new long[count]; + for (int i = 0; i < count; i++) { + out[i] = readLong(bytes, i * 8); + } + return out; + } + if (bytes.length != count) { + throw new InferenceException("Unexpected output tensor byte count"); + } + return bytes; + } + + private static void writeInt(byte[] out, int offset, int value) { + out[offset] = (byte) value; + out[offset + 1] = (byte) (value >>> 8); + out[offset + 2] = (byte) (value >>> 16); + out[offset + 3] = (byte) (value >>> 24); + } + + private static int readInt(byte[] value, int offset) { + return (value[offset] & 255) + | ((value[offset + 1] & 255) << 8) + | ((value[offset + 2] & 255) << 16) + | ((value[offset + 3] & 255) << 24); + } + + private static void writeLong(byte[] out, int offset, long value) { + for (int i = 0; i < 8; i++) { + out[offset + i] = (byte) (value >>> (i * 8)); + } + } + + private static long readLong(byte[] value, int offset) { + long out = 0; + for (int i = 0; i < 8; i++) { + out |= (long) (value[offset + i] & 255) << (i * 8); + } + return out; + } + + private static int elementCount(int[] shape) { + int out = 1; + for (int i = 0; i < shape.length; i++) { + out *= shape[i]; + } + return out; + } + + private static boolean sameShape(int[] supplied, int[] expected) { + if (supplied.length != expected.length) { + return false; + } + for (int i = 0; i < supplied.length; i++) { + if (supplied[i] != expected[i]) { + return false; + } + } + return true; + } + + private static int[] intArray(List values) { + int[] out = new int[values.size()]; + for (int i = 0; i < out.length; i++) { + out[i] = ((Number) values.get(i)).intValue(); + } + return out; + } + + private static Map parse(String json) throws Exception { + if (json == null || json.length() == 0) { + throw new InferenceException("LiteRT returned no result"); + } + Map root = new JSONParser().parseJSON(new StringReader(json)); + Object error = root.get("error"); + if (error != null) { + throw new InferenceException(String.valueOf(error)); + } + return root; + } + + private static List list(Map value, String key) { + Object out = value.get(key); + return out instanceof List ? (List) out : java.util.Collections.EMPTY_LIST; + } + + private static String string(Map value, String key) { + Object out = value.get(key); + return out == null ? "" : String.valueOf(out); + } + + private static int integer(Map value, String key) { + Object out = value.get(key); + return out instanceof Number ? ((Number) out).intValue() : 0; + } + + private static byte[] loadModel(ModelSource source) throws IOException { + if (source.getKind() == ModelSource.BYTES) { + return source.getBytesUnsafe(); + } + InputStream input; + input = Display.getInstance().getResourceAsStream( + IOSInferenceImpl.class, source.getPath()); + if (input == null) { + throw new IOException("Model resource not found: " + source.getPath()); + } + try { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[16384]; + int read; + while ((read = input.read(buffer)) >= 0) { + output.write(buffer, 0, read); + } + return output.toByteArray(); + } finally { + input.close(); + } + } + + private static TensorInfo[] copy(TensorInfo[] value) { + TensorInfo[] out = new TensorInfo[value.length]; + System.arraycopy(value, 0, out, 0, value.length); + return out; + } + + private static void fail(final AsyncResource out, final String message, + final Throwable cause) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.error(new InferenceException(message, cause)); + } + }); + } +} diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java new file mode 100644 index 00000000000..37f2759e200 --- /dev/null +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSLanguageImpl.java @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.ios; + +import com.codename1.ai.language.LanguageCandidate; +import com.codename1.ai.language.LanguageOptions; +import com.codename1.ai.language.SmartReplyMessage; +import com.codename1.impl.LanguageImpl; +import com.codename1.io.JSONParser; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** iOS Natural Language and feature-scoped ML Kit language services. */ +public final class IOSLanguageImpl extends LanguageImpl { + private static final int LANGUAGE_ID = 0; + private static final int TRANSLATION = 1; + private static final int SMART_REPLY = 2; + + private volatile boolean closed; + + @Override + public boolean isSupported(String feature, String backendId) { + if (closed || (!"auto".equals(backendId) + && !"ml-kit".equals(backendId) + && !"apple-natural-language".equals(backendId))) { + return false; + } + if (!"language-id".equals(feature) + && "apple-natural-language".equals(backendId)) { + return false; + } + return IOSImplementation.nativeInstance.cn1LanguageIsSupported( + featureId(feature), "ml-kit".equals(backendId)); + } + + @Override + public AsyncResource identify( + final String text, String backendId, final LanguageOptions options) { + float minimumConfidence = options == null + ? 0 : options.getMinimumConfidence(); + return identifyInBackground(text, "ml-kit".equals(backendId), + minimumConfidence); + } + + private static AsyncResource identifyInBackground( + final String text, final boolean mlKit, + final float minimumConfidence) { + final AsyncResource out = + new AsyncResource(); + run(out, new NativeCall() { + public LanguageCandidate[] call() throws Exception { + String json = IOSImplementation.nativeInstance.cn1LanguageIdentify( + text, minimumConfidence, mlKit); + Map root = parse(json); + List values = list(root, "items"); + LanguageCandidate[] result = new LanguageCandidate[values.size()]; + for (int i = 0; i < result.length; i++) { + Map value = (Map) values.get(i); + result[i] = new LanguageCandidate( + string(value, "language"), + number(value, "confidence")); + } + return result; + } + }); + return out; + } + + @Override + public AsyncResource translate( + final String text, final String sourceLanguage, + final String targetLanguage, String backendId, + LanguageOptions options) { + return translateInBackground(text, sourceLanguage, targetLanguage); + } + + private static AsyncResource translateInBackground( + final String text, final String sourceLanguage, + final String targetLanguage) { + final AsyncResource out = new AsyncResource(); + run(out, new NativeCall() { + public String call() throws Exception { + Map root = parse(IOSImplementation.nativeInstance.cn1LanguageTranslate( + text, sourceLanguage, targetLanguage)); + return string(root, "text"); + } + }); + return out; + } + + @Override + public AsyncResource suggestReplies( + SmartReplyMessage[] conversation, String backendId, + LanguageOptions options) { + return suggestRepliesInBackground(conversationJson(conversation)); + } + + private static AsyncResource suggestRepliesInBackground( + final String json) { + final AsyncResource out = new AsyncResource(); + run(out, new NativeCall() { + public String[] call() throws Exception { + Map root = parse(IOSImplementation.nativeInstance + .cn1LanguageSmartReply(json)); + List values = list(root, "items"); + String[] result = new String[values.size()]; + for (int i = 0; i < result.length; i++) { + result[i] = String.valueOf(values.get(i)); + } + return result; + } + }); + return out; + } + + private static int featureId(String feature) { + if ("language-id".equals(feature)) return LANGUAGE_ID; + if ("translation".equals(feature)) return TRANSLATION; + if ("smart-reply".equals(feature)) return SMART_REPLY; + return -1; + } + + private static String conversationJson(SmartReplyMessage[] conversation) { + List> values = new ArrayList>(); + for (int i = 0; i < conversation.length; i++) { + SmartReplyMessage message = conversation[i]; + Map value = new HashMap(); + value.put("text", message.getText()); + value.put("participant", message.getParticipantId()); + value.put("local", Boolean.valueOf(message.isLocalUser())); + value.put("timestamp", Long.valueOf(message.getTimestampMillis())); + values.add(value); + } + Map root = new HashMap(); + root.put("items", values); + return JSONParser.mapToJson(root); + } + + private static Map parse(String json) throws Exception { + if (json == null || json.length() == 0) { + throw new IllegalStateException("ML Kit returned no result"); + } + Map root = new JSONParser().parseJSON(new StringReader(json)); + Object error = root.get("error"); + if (error != null) { + throw new IllegalStateException(String.valueOf(error)); + } + return root; + } + + private static List list(Map value, String key) { + Object out = value.get(key); + return out instanceof List ? (List) out : java.util.Collections.EMPTY_LIST; + } + + private static String string(Map value, String key) { + Object out = value.get(key); + return out == null ? "" : String.valueOf(out); + } + + private static float number(Map value, String key) { + Object out = value.get(key); + return out instanceof Number ? ((Number) out).floatValue() : 0; + } + + private static void run(final AsyncResource out, + final NativeCall call) { + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + try { + final T value = call.call(); + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(value); + } + }); + } catch (final Throwable error) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.error(error); + } + }); + } + } + }); + } + + private interface NativeCall { + T call() throws Exception; + } + + @Override + public void close() { + closed = true; + } +} diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java index 4e8afbc50ac..67640f80ed8 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java @@ -478,6 +478,7 @@ native void fillGradient(int kind, int stopCount, float[] positions, float[] pre native String getUDID(); native String getOSVersion(); native String getDeviceName(); + native boolean isSimulator(); // The hardware/marketing model identifier (e.g. "iPhone15,2"). Unlike // getDeviceName() -- which returns the user-assigned device name and is // therefore personally identifying -- this is safe to use for analytics @@ -538,6 +539,29 @@ native void fillGradient(int kind, int stopCount, float[] positions, float[] pre native void cn1CameraResume(long sessionPeer); native void cn1CameraClose(long sessionPeer); + // On-device image analysis backed by Apple Vision/Core Image. + native boolean cn1VisionIsSupported(int feature, boolean mlKit); + native String cn1VisionAnalyze(byte[] imageData, int feature, boolean mlKit, + int rotationDegrees, int width, int height, + int frameFormat); + native boolean cn1LanguageIsSupported(int feature, boolean mlKit); + native String cn1LanguageIdentify(String text, float minimumConfidence, + boolean mlKit); + native String cn1LanguageTranslate(String text, String sourceLanguage, + String targetLanguage); + native String cn1LanguageSmartReply(String conversationJson); + native boolean cn1InferenceIsSupported(); + native String cn1InferenceOpen(byte[] model, int threads, int accelerator, + boolean allowFallback); + native String cn1InferenceOpenFile(String path, int threads, int accelerator, + boolean allowFallback); + native String cn1InferenceMetadata(int handle, boolean outputs); + native String cn1InferenceCopyInput(int handle, int index, byte[] data); + native String cn1InferenceInvoke(int handle); + native long cn1InferenceOutputData(int handle, int index); + native String cn1InferenceResize(int handle, int index, int[] shape); + native void cn1InferenceClose(int handle); + // Augmented reality API (com.codename1.ar). Backed by CN1AR.m which wraps // an ARKit ARSession composited through an ARSCNView. The IOSARImpl class // on the Java side routes static callbacks delivered from the session and diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSSimd.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSSimd.java index 759376a6a90..9884d1adc30 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSSimd.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSSimd.java @@ -1,5 +1,24 @@ /* * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.ios; @@ -11,16 +30,18 @@ public class IOSSimd extends Simd { @Override public boolean isSupported() { - return true; + return isNativeSupported(); } /// NEON: the chained byte-shuffle codec (Base64) is ~75-83% faster than the /// autovectorized scalar even at -O2, so the codec should use the SIMD path. @Override public boolean isByteShuffleAccelerated() { - return true; + return isNativeSupported(); } + private native boolean isNativeSupported(); + @Override public byte[] allocByte(int size) { if (size < 16) { diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java new file mode 100644 index 00000000000..17a3eecddc5 --- /dev/null +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSVisionImpl.java @@ -0,0 +1,386 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.ios; + +import com.codename1.ai.vision.Barcode; +import com.codename1.ai.vision.DocumentScanResult; +import com.codename1.ai.vision.Face; +import com.codename1.ai.vision.ImageLabel; +import com.codename1.ai.vision.Pose; +import com.codename1.ai.vision.SegmentationMask; +import com.codename1.ai.vision.TextRecognitionResult; +import com.codename1.ai.vision.VisionException; +import com.codename1.ai.vision.VisionFeature; +import com.codename1.ai.vision.VisionImage; +import com.codename1.ai.vision.VisionMetadata; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.ai.vision.VisionPoint; +import com.codename1.ai.vision.VisionRect; +import com.codename1.camera.FrameFormat; +import com.codename1.impl.VisionImpl; +import com.codename1.io.JSONParser; +import com.codename1.ui.Display; +import com.codename1.util.AsyncResource; +import com.codename1.util.Base64; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Apple Vision/Core Image and optional ML Kit implementation. */ +public final class IOSVisionImpl extends VisionImpl { + private volatile boolean closed; + + @Override + public boolean isSupported(VisionFeature feature, String backendId) { + if (closed || (!"auto".equals(backendId) + && !"apple-vision".equals(backendId) + && !"ml-kit".equals(backendId))) { + return false; + } + return IOSImplementation.nativeInstance.cn1VisionIsSupported( + nativeFeatureId(feature), "ml-kit".equals(backendId)); + } + + @Override + public AsyncResource analyze(final VisionFeature feature, + String backendId, final VisionImage image, + VisionOptions options) { + final AsyncResource out = new AsyncResource(); + final boolean mlKit = "ml-kit".equals(backendId); + if (!isSupported(feature, backendId)) { + out.error(new VisionException(VisionException.UNSUPPORTED, + feature + " is unavailable in " + + (mlKit ? "ML Kit" : "Apple Vision"))); + return out; + } + byte[] encoded = image.getEncodedBytesUnsafe(); + final byte[] input = encoded == null + ? image.getPixelsUnsafe() : encoded; + final int width = encoded == null ? image.getWidth() : 0; + final int height = encoded == null ? image.getHeight() : 0; + final int frameFormat = encoded == null + ? nativeFrameFormatId(image.getFormat()) : 0; + if (input == null || input.length == 0) { + out.error(new VisionException(VisionException.INVALID_IMAGE, + "Apple Vision requires encoded, NV21, or RGBA8888 input")); + return out; + } + analyzeInBackground(out, feature, mlKit, input, + image.getRotationDegrees(), width, height, frameFormat, + options.getMinimumConfidence(), options.getMaximumResults()); + return out; + } + + private static void analyzeInBackground(final AsyncResource out, + final VisionFeature feature, + final boolean mlKit, + final byte[] input, + final int rotation, + final int width, + final int height, + final int frameFormat, + final float minimumConfidence, + final int maximumResults) { + Display.getInstance().scheduleBackgroundTask(new Runnable() { + public void run() { + try { + String json = IOSImplementation.nativeInstance.cn1VisionAnalyze( + input, nativeFeatureId(feature), mlKit, + rotation, width, height, frameFormat); + if (json == null || json.length() == 0) { + throw new VisionException(VisionException.BACKEND_ERROR, + "Apple Vision returned no result"); + } + final T value = parse(feature, json, + mlKit ? "ml-kit" : "apple-vision", + minimumConfidence, maximumResults); + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.complete(value); + } + }); + } catch (final Throwable error) { + Display.getInstance().callSerially(new Runnable() { + public void run() { + out.error(error instanceof VisionException ? error + : new VisionException(VisionException.BACKEND_ERROR, + error.getMessage(), error)); + } + }); + } + } + }); + } + + private static int nativeFeatureId(VisionFeature feature) { + switch (feature) { + case TEXT_RECOGNITION: + return 0; + case BARCODE_SCANNING: + return 1; + case FACE_DETECTION: + return 2; + case IMAGE_LABELING: + return 3; + case POSE_DETECTION: + return 4; + case SELFIE_SEGMENTATION: + return 5; + case DOCUMENT_SCANNING: + return 6; + default: + throw new IllegalArgumentException( + "Unsupported vision feature " + feature); + } + } + + private static int nativeFrameFormatId(FrameFormat format) { + switch (format) { + case JPEG: + return 0; + case NV21: + return 1; + case RGBA8888: + return 2; + default: + throw new IllegalArgumentException( + "Unsupported camera frame format " + format); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static T parse(VisionFeature feature, String json, + String backendId, float minimumConfidence, + int maximumResults) throws Exception { + Map root = new JSONParser().parseJSON(new StringReader(json)); + Object error = root.get("error"); + if (error != null) { + throw new VisionException(VisionException.BACKEND_ERROR, String.valueOf(error)); + } + List items = (List) root.get("items"); + if (items == null) { + items = java.util.Collections.EMPTY_LIST; + } + items = filterItems(items, minimumConfidence, maximumResults); + VisionMetadata metadata = new VisionMetadata(backendId); + switch (feature) { + case TEXT_RECOGNITION: { + TextRecognitionResult.TextBlock[] blocks = + new TextRecognitionResult.TextBlock[items.size()]; + for (int i = 0; i < blocks.length; i++) { + Map value = (Map) items.get(i); + blocks[i] = new TextRecognitionResult.TextBlock( + string(value, "text"), number(value, "confidence"), + rect(value), stringOrNull(value, "language")); + } + return (T) new TextRecognitionResult( + string(root, "text"), blocks, metadata); + } + case BARCODE_SCANNING: { + Barcode[] values = new Barcode[items.size()]; + for (int i = 0; i < values.length; i++) { + Map value = (Map) items.get(i); + List points = value.get("corners") instanceof List + ? (List) value.get("corners") + : java.util.Collections.EMPTY_LIST; + VisionPoint[] corners = new VisionPoint[points.size()]; + for (int p = 0; p < corners.length; p++) { + Map point = (Map) points.get(p); + corners[p] = new VisionPoint(number(point, "x"), + number(point, "y")); + } + values[i] = new Barcode(stringOrNull(value, "value"), + string(value, "format"), + decodeBase64OrNull(value, "rawData"), rect(value), + corners, metadata); + } + return (T) values; + } + case FACE_DETECTION: { + Face[] values = new Face[items.size()]; + for (int i = 0; i < values.length; i++) { + Map value = (Map) items.get(i); + Map landmarks = + parseLandmarks(value.get("landmarks")); + values[i] = new Face(rect(value), + landmarks, + number(value, "yaw"), number(value, "pitch"), + number(value, "roll"), + number(value, "smilingProbability", -1), + integer(value, "trackingId", -1), metadata); + } + return (T) values; + } + case IMAGE_LABELING: { + ImageLabel[] values = new ImageLabel[items.size()]; + for (int i = 0; i < values.length; i++) { + Map value = (Map) items.get(i); + values[i] = new ImageLabel(string(value, "text"), + number(value, "confidence"), + integer(value, "index", -1), metadata); + } + return (T) values; + } + case POSE_DETECTION: { + Pose.Landmark[] values = new Pose.Landmark[items.size()]; + for (int i = 0; i < values.length; i++) { + Map value = (Map) items.get(i); + values[i] = new Pose.Landmark(string(value, "name"), + new VisionPoint(number(value, "x"), number(value, "y")), + number(value, "confidence")); + } + return (T) new Pose(values, metadata); + } + case SELFIE_SEGMENTATION: { + byte[] bytes = nativeData(root); + float[] confidence = new float[bytes.length]; + for (int i = 0; i < bytes.length; i++) { + confidence[i] = (bytes[i] & 0xff) / 255f; + } + return (T) new SegmentationMask(integer(root, "width"), + integer(root, "height"), confidence, metadata); + } + case DOCUMENT_SCANNING: { + return (T) new DocumentScanResult(new byte[][] { + nativeData(root) + }, metadata); + } + default: + throw new VisionException(VisionException.UNSUPPORTED, + "Unknown Apple Vision feature " + feature); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static List filterItems(List items, float minimumConfidence, + int maximumResults) { + if (items.isEmpty() + || (minimumConfidence <= 0 && maximumResults == 0)) { + return items; + } + List filtered = new ArrayList(); + for (int i = 0; i < items.size(); i++) { + Object item = items.get(i); + if (minimumConfidence > 0 && item instanceof Map) { + Object confidence = ((Map) item).get("confidence"); + if (confidence instanceof Number + && ((Number) confidence).floatValue() + < minimumConfidence) { + continue; + } + } + filtered.add(item); + if (maximumResults > 0 && filtered.size() >= maximumResults) { + break; + } + } + return filtered; + } + + private static VisionRect rect(Map value) { + return new VisionRect(number(value, "x"), number(value, "y"), + number(value, "width"), number(value, "height")); + } + + @SuppressWarnings("rawtypes") + private static Map parseLandmarks(Object value) { + Map out = new HashMap(); + if (!(value instanceof Map)) { + return out; + } + Map source = (Map) value; + for (Object key : source.keySet()) { + Object point = source.get(key); + if (point instanceof Map) { + out.put(String.valueOf(key), new VisionPoint( + number((Map) point, "x"), + number((Map) point, "y"))); + } + } + return out; + } + + private static float number(Map value, String key) { + return number(value, key, 0); + } + + private static float number(Map value, String key, float fallback) { + Object out = value.get(key); + return out instanceof Number ? ((Number) out).floatValue() : fallback; + } + + private static int integer(Map value, String key) { + return integer(value, key, 0); + } + + private static int integer(Map value, String key, int fallback) { + Object out = value.get(key); + return out instanceof Number ? ((Number) out).intValue() : fallback; + } + + private static String string(Map value, String key) { + String out = stringOrNull(value, key); + return out == null ? "" : out; + } + + private static String stringOrNull(Map value, String key) { + Object out = value.get(key); + return out == null ? null : String.valueOf(out); + } + + private static byte[] nativeData(Map value) throws Exception { + Object peerValue = value.get("dataPeer"); + if (peerValue instanceof Number) { + long peer = ((Number) peerValue).longValue(); + if (peer == 0) { + throw new VisionException(VisionException.BACKEND_ERROR, + "Apple Vision returned no binary result"); + } + try { + byte[] out = new byte[IOSImplementation.nativeInstance + .getNSDataSize(peer)]; + IOSImplementation.nativeInstance.nsDataToByteArray(peer, out); + return out; + } finally { + IOSImplementation.nativeInstance.releasePeer(peer); + } + } + String base64 = string(value, "data"); + return Base64.decode(base64.getBytes("UTF-8")); + } + + private static byte[] decodeBase64OrNull(Map value, String key) + throws Exception { + String base64 = stringOrNull(value, key); + return base64 == null ? null + : Base64.decode(base64.getBytes("UTF-8")); + } + + @Override + public void close() { + closed = true; + } +} diff --git a/docs/ai-on-device-architecture.md b/docs/ai-on-device-architecture.md new file mode 100644 index 00000000000..ea1280d7f96 --- /dev/null +++ b/docs/ai-on-device-architecture.md @@ -0,0 +1,261 @@ +# Built-in on-device AI architecture + +This document is the maintainer reference for the built-in vision, +language, and LiteRT APIs. User-facing examples live in +`docs/developer-guide/Ai-And-Speech.asciidoc`. + +## Scope + +The public surface is in: + +- `CodenameOne/src/com/codename1/ai/vision` +- `CodenameOne/src/com/codename1/ai/language` +- `CodenameOne/src/com/codename1/ai/inference` + +The API is part of `codenameone-core` on every target. The default +`CodenameOneImplementation` factories return `null`, so an unsupported +target has deterministic `isSupported() == false` behavior and never falls +back to a cloud service. + +Native backends are: + +| Target | Vision | Language | `.tflite` inference | +| --- | --- | --- | --- | +| Android | ML Kit | ML Kit | LiteRT | +| iOS | Apple Vision/Core Image; optional ML Kit | Apple Natural Language for identification; ML Kit for translation, Smart Reply, and optional identification | TensorFlow Lite Objective-C; optional Core ML delegate | +| Mac native | Apple Vision/Core Image through Mac Catalyst | Unsupported fallback | Unsupported fallback | +| JavaSE, JavaScript, native Windows/Linux | Unsupported fallback | Unsupported fallback | Unsupported fallback | +| watchOS, tvOS | Unsupported fallback | Unsupported fallback | Unsupported fallback | + +Document correction is intentionally Apple-only. The Google ML Kit document +scanner is an interactive Activity camera flow, while the core +`DocumentScanner` contract analyzes an existing `VisionImage`. + +## Build-time selection + +`maven/platform-feature-catalog` is the authoritative dependency registry. +`PlatformFeatureCatalog.Accumulator` consumes exact class and method +references from the existing bytecode scanners. It rejects unrelated symbols +at consume time and memoizes its immutable hit set behind a dirty flag, so +memory and repeated builder queries scale with catalog matches rather than +total application bytecode. + +The local Maven plugin and BuildDaemon both consume the same source contract: + +- `maven/codenameone-maven-plugin/.../AndroidGradleBuilder.java` +- `maven/codenameone-maven-plugin/.../IPhoneBuilder.java` +- `BuildDaemon/src/com/codename1/build/daemon/AndroidGradleBuilder.java` +- `BuildDaemon/src/com/codename1/build/daemon/IPhoneBuilder.java` + +BuildDaemon carries a source copy of `PlatformFeatureCatalog.java` because it +is built and deployed separately. Keep that copy byte-for-byte equal to the +Maven module source and run `cmp` as part of validation. + +Local source builds replace the generated iOS/Mac project directory before +copying the new result. This is required for granular removal: a plain +directory merge would retain an old Podfile, workspace, Pods directory, or +optional native source after the application stops referencing a feature. + +Catalog-selected CocoaPods are deduplicated by pod name. An existing +user-declared spec appears first and wins, preserving constraints such as +`GoogleMLKit/TextRecognition ~> 7.0`. Explicit `ios.dependencyManager=spm` +and `none` modes reject incompatible declared or catalog-selected +dependencies instead of silently omitting them. + +The companion archive extraction hardening is intentionally broader than AI: +every Android/build archive entry is canonicalized and checked for traversal +before extraction. Trusted extraction filters may still route a validated +entry to a deliberate sibling such as the project settings file. + +### Android source granularity + +The optional Android sources are excluded from `maven/android` compilation +and compiled in the generated application after dependencies are selected. +The port bundle contains: + +- one reflection-loaded group dispatcher (`AndroidVisionImpl` or + `AndroidLanguageImpl`); +- one dependency-neutral group adapter base; +- one source file per concrete ML Kit feature; +- one `AndroidInferenceImpl`, because LiteRT is already a single dependency. + +`AndroidGradleBuilder.androidAiAdapterSource()` maps each exact public entry +point to one source. `pruneOptionalAiSources()` deletes every unselected +source before Gradle compiles the generated application. R8 keeps only the +remaining `com.codename1.impl.android.ai` classes so reflection cannot rename +the selected dispatcher or adapter. + +| Core entry point | Retained Android source | Gradle dependency | +| --- | --- | --- | +| `TextRecognizer` | `AndroidTextRecognitionAdapter.java` | `com.google.mlkit:text-recognition:16.0.0` | +| `BarcodeScanner` | `AndroidBarcodeScanningAdapter.java` | `com.google.mlkit:barcode-scanning:17.2.0` | +| `FaceDetector` | `AndroidFaceDetectionAdapter.java` | `com.google.mlkit:face-detection:16.1.5` | +| `ImageLabeler` | `AndroidImageLabelingAdapter.java` | `com.google.mlkit:image-labeling:17.0.7` | +| `PoseDetector` | `AndroidPoseDetectionAdapter.java` | `com.google.mlkit:pose-detection:18.0.0-beta3` | +| `SelfieSegmenter` | `AndroidSelfieSegmentationAdapter.java` | `com.google.mlkit:segmentation-selfie:16.0.0-beta5` | +| `LanguageIdentifier` | `AndroidLanguageIdAdapter.java` | `com.google.mlkit:language-id:17.0.6` | +| `Translator` | `AndroidTranslationAdapter.java` | `com.google.mlkit:translate:17.0.3` | +| `SmartReply` | `AndroidSmartReplyAdapter.java` | `com.google.mlkit:smart-reply:17.0.4` | +| `InferenceSession` | `AndroidInferenceImpl.java` | `com.google.ai.edge.litert:litert:1.0.1` | + +### Apple method granularity + +`CN1Vision.m` and `CN1Language.m` use `__has_include` around each ML Kit +component. The class scanner always links the small Apple system frameworks +needed by a referenced analyzer. It adds a Google ML Kit vision pod only +when it observes both: + +1. the concrete analyzer class; and +2. a call to the matching feature-specific selector, such as + `VisionBackends.mlKitTextRecognition()`. + +On iOS, language identification defaults to the system Natural Language +framework. Calling `LanguageBackends.mlKitLanguageIdentification()` opts +into `GoogleMLKit/LanguageID`. Translation and Smart Reply pods are selected +only by calls to `LanguageBackends.mlKitTranslation()` and +`LanguageBackends.mlKitSmartReply()`. Merely referencing their API classes +does not change simulator architectures. `InferenceSession` selects +`TensorFlowLiteObjC/CoreML`. + +The native implementation is disabled for watchOS and tvOS. Mac native uses +the Catalyst slice of the framework-only Apple Vision implementation. The +official Google ML Kit and TensorFlow Lite Objective-C packages do not ship +Mac Catalyst slices, so their catalog entries are marked incompatible with +Catalyst. The builders omit those package dependencies from a Mac-native +build; language and inference consequently use their explicit unsupported +stubs there. + +Google ML Kit's iOS binary frameworks contain device `arm64` and simulator +`x86_64` slices, but no `arm64` simulator slice. Catalog entries declare that +constraint independently from Catalyst support. When one of those entries is +selected, both builders set +`EXCLUDED_ARCHS[sdk=iphonesimulator*]=arm64` on the generated application and +Pods projects and emit a diagnostic identifying the selected dependency. +TensorFlow Lite's XCFramework does include an `arm64` simulator slice and +does not trigger this fallback. +The same Google ML Kit catalog entries declare an iOS 15.5 deployment floor. +The iOS builder folds that value into its existing maximum-target calculation +before generating the app target and Podfile, preventing a lower application +hint from producing an unsatisfiable CocoaPods resolution. +The iOS NEON implementation reports SIMD as unsupported in that x86_64 +configuration and aliases its native entry points to the generic scalar +implementation; device and arm64-simulator builds retain the NEON path. + +Apple Vision barcode requests use revision 1 only when compiling for an iOS +simulator. Recent simulator runtimes can otherwise complete the default +request without returning observations for valid QR images. Physical devices +retain the current OS revision and its additional symbologies. + +## Image and camera contract + +`VisionImage` owns defensive copies of its input. It accepts encoded JPEG or +PNG, NV21, and RGBA8888. Android feeds raw NV21 directly to ML Kit and +converts RGBA to a bitmap. Apple creates a `CGImage` directly from NV21 or +RGBA memory and passes it to Vision or ML Kit without an intermediate JPEG +encode/decode. `VisionImage.encoded(bytes, rotationDegrees)` carries the +clockwise display rotation for encoded gallery or file images; the +one-argument overload deliberately assumes upright stored pixels rather than +silently guessing or discarding EXIF orientation. + +`VisionImage.fromCameraFrame()` is safe beyond the +`FrameListener.onFrame()` callback because it copies the callback-owned +buffer selected by the requested `FrameFormat`. Raw frames do not retain the +always-available JPEG fallback, so mobile backends consume NV21 or RGBA8888 +without accidentally selecting the encoded path. When a camera port cannot +supply the requested raw buffer, `fromCameraFrame()` retains the JPEG fallback +and reports JPEG as the image format instead of creating an empty image. +`VisionPipeline` allows one request to run and retains only the newest pending +frame. Superseding same-sized pending frames reuse one detached buffer, and +analyzer invocation occurs outside the pipeline monitor. This bounds memory, +latency, and lock re-entry for live OCR, barcode, face, pose, labeling, or +segmentation. + +Native results are converted to stable Codename One value types. Geometry is +normalized to the top-left coordinate system. `VisionMetadata` carries the +actual backend id without exposing platform classes. + +## Inference contract + +`InferenceSession` supports named typed tensors, multiple inputs and outputs, +input resizing, and reusable native sessions. Model sources are bytes, +resources, private files, or the HTTPS-only `ModelCache`. The cache rejects +redirects that downgrade a request to HTTP on ports where redirects are +observable. iOS follows redirects below the portable network layer, so the +cache requires a SHA-256 digest for every iOS download and rejects its unpinned +overload. Identical concurrent fetches for one cache entry coalesce, while +conflicting in-flight content identities fail instead of sharing a temporary +path. Each coalesced caller has an independent resource, so cancellation +suppresses only that caller's notification and never cancels the shared +download or another subscriber. File sources are opened by path without +copying the model through the Java heap. Cache promotion verifies that the +final file exists and, when supplied, still matches the requested digest +before publishing its path. +Android invokes LiteRT with null output destinations, then reads each result +from its native tensor buffer. This allows value-dependent output dimensions +to resolve before Codename One allocates or copies the result. LiteRT caches +output shapes when no input reallocation occurred, so the Android port +explicitly refreshes each output shape after invocation; both Android builders +retain that package-private LiteRT method name through R8. NPU selection is +best effort when fallback is enabled. Strict NPU requests are rejected on +Android because LiteRT cannot verify that NNAPI delegated every operation, +and on iOS because the Core ML delegate may schedule work on CPU or GPU. + +All native sessions and analyzers must be closed. Language APIs expose +reusable feature sessions in addition to static one-shot methods; Android +sessions cache their ML Kit client or translation clients by language pair. +Expensive open, analysis, and inference work runs off the EDT; completion and +error delivery return to the EDT. An inference session rejects metadata +queries, resizing, and another invocation while a run is pending so no two API +calls can touch the mutable native interpreter concurrently. + +## Permanent cross-platform coverage + +`scripts/hellocodenameone` registers three non-screenshot conformance tests: + +- `VisionOnDeviceApiTest` covers `VisionImage.fromCameraFrame()` ownership, + option normalization, capability queries for every analyzer, close + semantics, and—when a native barcode backend is available—a deterministic + QR decode through image marshalling, native detection, JSON parsing, format + normalization, and corner geometry. +- `LanguageOnDeviceApiTest` covers language value/options contracts, + capability queries for identification, translation, and smart reply, and + immediate unsupported resources. +- `InferenceOnDeviceApiTest` covers immutable tensors and model sources, + validation and options, runtime capability reporting, and the unsupported + session-open contract. + +The tests avoid permission prompts, mutable model downloads, and large bundled +models. Their concrete API references are nevertheless part of the application +bytecode, so the Android and Apple source-build jobs exercise the same granular +builder selection used by applications. The tests map to independent +`on-device-vision`, `on-device-language`, and `on-device-inference` features in +`docs/website/data/port_status.json`; fresh per-port reports replace the +bootstrap `not-run` results after the updated suite runs on `master`. + +## Adding a feature + +1. Add the vendor-neutral API and result types to core. +2. Extend the appropriate implementation SPI under + `CodenameOne/src/com/codename1/impl`. +3. Add one Android adapter source and one exact + `androidAiAdapterSource()` mapping. +4. Add the smallest Android dependency to `PlatformFeatureCatalog`. +5. Add the Apple implementation behind a feature-specific compile guard. +6. Add the system framework and optional pod mapping to the catalog. +7. Copy the catalog into BuildDaemon and update both builders if source + selection changes. +8. Add core API tests, catalog tests, builder-selection tests, Java 8 Android + compilation, Objective-C syntax compilation, CocoaPods integration, and + Xcode device/Catalyst builds as applicable. Some Google ML Kit releases + cannot link an arm64 simulator even though the sources compile there. + +Use an isolated Maven repository for validation: + +```sh +mvn -Dmaven.repo.local=/private/tmp/cn1-ai-validation-m2 ... +``` + +Large model families and native runtimes such as Whisper and Stable Diffusion +remain opt-in cn1libs. The core/builder approach is intended for APIs whose +code can be selected cheaply and whose model payloads are supplied by the OS, +downloaded lazily by the native SDK, or supplied by the application. diff --git a/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java b/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java new file mode 100644 index 00000000000..67926897934 --- /dev/null +++ b/docs/demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.snippets.generated; + +import com.codename1.ai.inference.InferenceOptions; +import com.codename1.ai.inference.InferenceSession; +import com.codename1.ai.inference.ModelSource; +import com.codename1.ai.inference.Tensor; +import com.codename1.ai.language.LanguageBackends; +import com.codename1.ai.language.LanguageOptions; +import com.codename1.ai.language.Translator; +import com.codename1.ai.vision.TextRecognizer; +import com.codename1.ai.vision.VisionBackends; +import com.codename1.ai.vision.VisionImage; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.io.Log; + +class AiAndSpeechOnDeviceSnippet { + private byte[] jpegBytes; + private float[] pixels; + + void vision() { + // tag::ai-and-speech-on-device-vision[] + VisionOptions options = new VisionOptions() + .backend(VisionBackends.mlKitTextRecognition()) + .minimumConfidence(0.5f); + + new TextRecognizer(options) + .process(VisionImage.encoded(jpegBytes)) + .ready(result -> Log.p(result.getText())) + .except(error -> Log.e(error)); + // end::ai-and-speech-on-device-vision[] + } + + void inference() { + // tag::ai-and-speech-on-device-inference[] + InferenceSession.open( + ModelSource.resource("/models/classifier.tflite"), + new InferenceOptions().accelerator( + InferenceOptions.Accelerator.AUTO)) + .ready(session -> { + Tensor input = Tensor.floats("serving_default_input", + new int[] {1, 224, 224, 3}, pixels); + session.run(new Tensor[] {input}) + .ready(outputs -> { + try { + consume(outputs[0]); + } finally { + session.close(); + } + }) + .except(error -> { + try { + Log.e(error); + } finally { + session.close(); + } + }); + }) + .except(error -> Log.e(error)); + // end::ai-and-speech-on-device-inference[] + } + + void language() { + // tag::ai-and-speech-on-device-language[] + LanguageOptions options = new LanguageOptions() + .backend(LanguageBackends.mlKitTranslation()); + if (Translator.isSupported(options)) { + Translator.translate("Where is the station?", "en", "fr", + options) + .ready(value -> Log.p(value)) + .except(error -> Log.e(error)); + } + // end::ai-and-speech-on-device-language[] + } + + private void consume(Tensor output) { + } +} diff --git a/docs/demos/common/src/main/snippets/developer-guide/ai-and-speech.xml b/docs/demos/common/src/main/snippets/developer-guide/ai-and-speech.xml deleted file mode 100644 index 7098b1402ee..00000000000 --- a/docs/demos/common/src/main/snippets/developer-guide/ai-and-speech.xml +++ /dev/null @@ -1,10 +0,0 @@ -// Generated from docs/developer-guide source blocks. Edit the guide snippets here, not inline. - -// tag::ai-and-speech-xml-001[] - - com.codenameone - cn1-ai-mlkit-barcode-lib - ${cn1.version} - pom - -// end::ai-and-speech-xml-001[] diff --git a/docs/developer-guide/Ai-And-Speech.asciidoc b/docs/developer-guide/Ai-And-Speech.asciidoc index 0d5ffb29561..13ea3d402c8 100644 --- a/docs/developer-guide/Ai-And-Speech.asciidoc +++ b/docs/developer-guide/Ai-And-Speech.asciidoc @@ -2,10 +2,10 @@ [[ai-and-speech-section,AI And Speech Section]] Codename One ships a portable LLM client, a streaming chat component, -speech-to-text and text-to-speech APIs, and a family of cn1lib bridges -that wire ML Kit into iOS and Android builds. All the public surface -lives next to the rest of the framework: the simulator, the cloud -builder, and your CI pipeline all run the same code. +speech-to-text and text-to-speech APIs, built-in vision and language +services, and a portable LiteRT inference API. The builders add native +frameworks and dependencies when application bytecode references the +corresponding entry point. This chapter introduces every piece in turn: @@ -19,8 +19,12 @@ This chapter introduces every piece in turn: * `com.codename1.security.SecureStorage` non-prompting overloads -- silent reads for secrets the network layer needs on every call, such as LLM API keys. -* The `cn1-ai-mlkit-*` cn1libs -- ML Kit barcode, document, and face - detection bridges with the native build-time scanner already wired up. +* `com.codename1.ai.vision` -- OCR, barcode, face, labeling, pose, + segmentation, document correction, and camera-frame pipelines. +* `com.codename1.ai.inference` -- reusable LiteRT sessions for + `.tflite` models. +* `com.codename1.ai.language` -- language identification, translation, + and smart reply. NOTE: Each async call in this chapter returns an `com.codename1.util.AsyncResource`. Use `.ready(...)` for the success @@ -267,49 +271,222 @@ when the OS exposes them; `setVoiceId(...)` accepts any of those strings, or `null` to use the default voice for the configured language. -=== ML Kit cn1libs +=== On-device vision -Three cn1libs ship with the framework today, each backed by ML Kit on -both platforms. The build-time scanner picks up the `com.codename1.ai.*` -class references in your code and injects the matching Pods, Swift -Packages, Gradle dependencies, `Info.plist` usage strings, and Android -permissions; you don't edit build hints by hand. +The vision API accepts JPEG, PNG, NV21, or RGBA8888 data through +`VisionImage`. `VisionImage.fromCameraFrame(frame)` is normally the +best way to connect a camera because it copies the frame's +callback-owned buffers and preserves the frame rotation before analysis +becomes asynchronous. For an encoded gallery or file image whose pixels +need an EXIF display rotation, use +`VisionImage.encoded(bytes, rotationDegrees)`; the one-argument overload +assumes the stored pixels are already upright and doesn't inspect EXIF +metadata. Each analyzer returns stable Codename One result types. Bounds +and points use normalized coordinates in the range 0 through 1. +`VisionOptions.minimumConfidence(...)` clamps finite and infinite values to +that range and rejects NaN. -[cols="1,1,2", options="header"] -|=== -| cn1lib | Public API | Provides - -| `cn1-ai-mlkit-barcode` | `BarcodeScanner.scan(byte[])` | Decode QR, -EAN, Code 128, and the other ML Kit-supported barcode formats from an -image. -| `cn1-ai-mlkit-docscan` | `DocumentScanner.scanToFile(byte[])` | -Capture and crop document photos. On iOS this routes through `VisionKit` -and on Android through the Google Play Services document scanner. -| `cn1-ai-mlkit-face` | `FaceDetector.detect(byte[])` | Detect faces and -return packed `int[]` bounding rectangles (four ints per face: x, y, -width, height). +Android uses ML Kit. iOS and Mac native builds default to Apple Vision +and Core Image. iOS can select ML Kit explicitly: + +[source,java] +---- +include::../demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java[tag=ai-and-speech-on-device-vision,indent=0] +---- + +On an iOS simulator, the Apple barcode backend uses Vision barcode +request revision 1. Newer simulator runtimes can otherwise complete a +barcode request without reporting valid QR codes. Physical devices use +the latest revision supplied by the OS. Test newer symbologies such as +Micro QR on a device, or select the matching ML Kit backend. + +Create and reuse an analyzer when processing a sequence. Call `close()` +when the analyzer is no longer needed. Before showing a feature, call +its `isSupported()` method: availability can depend on the target, +linked backend, and OS version. + +`VisionPipeline` attaches an analyzer to a `CameraSession` frame stream +and keeps only the newest pending frame. Live OCR and pose detection +therefore can't build an unbounded queue when inference is slower than +the camera. While an analysis is busy, the pipeline reuses the pending +buffer when newer same-sized frames replace it. Results and errors are +delivered on the Codename One EDT. + +`CameraFrame.getRawBytes()` can be `null` even after requesting NV21 or +RGBA8888 when a port can't expose that format. JPEG bytes remain available; +`VisionImage.fromCameraFrame(...)` automatically selects that fallback. + +`DocumentScanner` currently performs still-image rectangle correction on +Apple platforms. Google's Android document scanner is an interactive +camera flow rather than an analyzer for an existing `VisionImage`, so the +Android backend reports it as unsupported rather than launching a UI flow. + +==== Vision feature and dependency selection + +The Java package is always part of core. The builders scan the concrete +analyzer classes and retain only the native adapter and dependency for +each class the application actually uses: + +[cols="2,2,3",options="header"] |=== +| API | Android | Apple default / optional backend -==== Adding a cn1lib +| `TextRecognizer` +| ML Kit text recognition +| Vision / `GoogleMLKit/TextRecognition` -Add the dependency to `common/pom.xml` with `pom` so Maven -pulls in the per-platform classifier jars: +| `BarcodeScanner` +| ML Kit barcode scanning +| Vision / `GoogleMLKit/BarcodeScanning` -[source,xml] ----- -include::../demos/common/src/main/snippets/developer-guide/ai-and-speech.xml[tag=ai-and-speech-xml-001,indent=0] ----- +| `FaceDetector` +| ML Kit face detection +| Vision / `GoogleMLKit/FaceDetection` -==== Example: Scanning a barcode +| `ImageLabeler` +| ML Kit image labeling +| Vision / `GoogleMLKit/ImageLabeling` +| `PoseDetector` +| ML Kit pose detection +| Vision / `GoogleMLKit/PoseDetection` -==== Example: Face detection +| `SelfieSegmenter` +| ML Kit selfie segmentation +| Vision / `GoogleMLKit/SegmentationSelfie` + +| `DocumentScanner` +| Unsupported for still images +| Core Image perspective correction +|=== +The optional iOS ML Kit pod is added only when both the analyzer and its +feature-specific selector are referenced. For example, +`VisionBackends.mlKitBarcodeScanning()` adds only the barcode pod. +Current Google ML Kit iOS pods require iOS 15.5. Selecting one of these +optional backends automatically raises the effective application and +CocoaPods deployment target to 15.5 when the configured target is lower. +Referencing one Android analyzer doesn't compile or package the other +five adapters. + +=== Portable LiteRT inference + +`InferenceSession` loads `.tflite` data from bytes, a bundled resource, +or a private file. Android uses LiteRT and can request NNAPI. iOS uses +TensorFlow Lite Objective-C and may request the Core ML delegate. Both +mobile implementations fall back to CPU unless +`allowFallback(false)` is set. Android and iOS reject strict +`Accelerator.NPU` sessions: NNAPI can mix delegated and CPU operations, +and the Core ML delegate can schedule work across the Neural Engine, GPU, +and CPU. Neither reports that the entire graph ran on the requested NPU. +With fallback enabled, NPU selection is a best-effort accelerator +preference. +On iOS, strict `Accelerator.CORE_ML` sessions are also rejected because +the delegate doesn't report whether unsupported model operations remained +on LiteRT's CPU path. +The JavaSE simulator and targets without +a native LiteRT runtime, including Mac native, return `false` from +`InferenceSession.isSupported()`. + +[source,java] +---- +include::../demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java[tag=ai-and-speech-on-device-inference,indent=0] +---- + +Open is asynchronous because model initialization can allocate native +resources. A session is reusable and supports multiple named inputs, +multiple outputs, and `resizeInput(...)`. A session serializes its mutable +native interpreter: while `run(...)` is pending, another run, input resize, +or input/output metadata query throws `IllegalStateException`. Tensor data is +copied across the port boundary. Always close the session. + +Bundle small models as resources. For larger models use +`ModelCache.fetch(url, cacheKey, sha256)`, which downloads over HTTPS, +verifies the supplied SHA-256 digest, and reuses the app-private cached +file. The initial URL must use HTTPS. Ports that expose redirect responses +reject a redirect to HTTP and discard the temporary download. iOS follows +redirects below the portable network layer, so iOS requires the SHA-256 +argument and rejects the unpinned two-argument overload. Supply the digest +on every platform for every third-party or remotely mutable model. +Interrupted downloads restart through a temporary `.download` file and are +never promoted to the cache until verification succeeds. Concurrent calls for +the same cache key and content identity share one download but receive +independent asynchronous resources. Canceling one caller suppresses only that +caller's notification; it doesn't cancel the shared download or the other +callers. When a cache key already has an active download, a request with a +conflicting URL or digest fails instead of sharing the temporary file. + +=== On-device language services + +`LanguageIdentifier`, `Translator`, and `SmartReply` share +`LanguageOptions`. Its minimum-confidence threshold clamps to 0 through 1 and +rejects NaN. Translation models are downloaded and cached by ML Kit when a +language pair is first used. Android and iOS builders +select `language-id`, `translate`, and `smart-reply` independently, so +using language identification alone doesn't package the other two +SDKs. + +[source,java] +---- +include::../demos/common/src/main/java/com/codenameone/developerguide/snippets/generated/AiAndSpeechOnDeviceSnippet.java[tag=ai-and-speech-on-device-language,indent=0] +---- -NOTE: ML Kit barcode and face detection ship as small Pods or Gradle -dependencies that add a few megabytes to the binary. The document -scanner depends on Google Play Services on Android; on devices without -Play Services the call returns an error through `AsyncResource.except`. +`LanguageBackends.auto()` selects ML Kit on Android and Apple's Natural +Language framework for language identification on iOS. The Apple default +keeps arm64 simulator builds native and adds no third-party dependency. +`LanguageBackends.mlKitLanguageIdentification()` opts identification into +the ML Kit iOS pod. iOS translation and Smart Reply are explicit opt-ins: +use `LanguageBackends.mlKitTranslation()` or +`LanguageBackends.mlKitSmartReply()` respectively. Merely referencing +`Translator` or `SmartReply` doesn't add a pod or exclude the arm64 +simulator. Selecting one of those ML Kit backends requires the x86_64 +simulator slice and raises an iOS deployment target below 15.5. + +The static language methods are convenient for one operation and close their +temporary backend after either success or failure. For repeated work, call +`LanguageIdentifier.open(...)`, `Translator.open(...)`, or +`SmartReply.open(...)`, reuse the returned feature session, and close it when +the last pending operation completes. Reusable sessions retain backend state +and, on Android, cache ML Kit clients and translation clients per language +pair. +Mac native and the other unsupported targets return `false` from the +language service `isSupported()` checks. Completion and failure callbacks +arrive on the EDT. + +=== Platform availability + +The public API and the unsupported fallback are present on every +Codename One target. Vision has native implementations on Android, iOS, +and Mac native (Mac Catalyst); Apple Vision is the default on Apple +targets and Android uses ML Kit. Language services and LiteRT inference +have native implementations on Android and iOS. JavaSE, JavaScript, +native Windows, native Linux, watchOS, and tvOS report unsupported for +these new APIs; Mac native reports unsupported for language and LiteRT. +No target sends data to a cloud inference service. Large +portable runtimes or model payloads for the remaining targets remain +appropriate opt-in libraries. + +No build hints or cn1lib coordinates are required for the built-in +mobile implementations. The local Maven plugin and BuildDaemon use the +same `PlatformFeatureCatalog`, so local and cloud builds make the same +dependency decisions. + +=== Migrating from the AI cn1libs + +The old `com.codename1.ai.mlkit.*` and `com.codename1.ai.tflite` +packages aren't compatibility aliases. Remove those dependencies and +move directly to `com.codename1.ai.vision`, +`com.codename1.ai.language`, or `com.codename1.ai.inference`. Result +coordinates are normalized floats rather than packed +platform-specific integer arrays. `cn1-ai-whisper` and +`cn1-ai-stablediffusion` remain cn1libs because their native payloads +and models are intentionally large. + +Remove the retired cn1lib dependency before adding the built-in class. +During migration, the builders deduplicate identical CocoaPods names and +preserve an explicitly declared user version constraint. Removing the old +library is still required because compatibility aliases would keep its old +native-interface and packaging contracts alive. ==== Putting it all together diff --git a/docs/developer-guide/Working-With-iOS.asciidoc b/docs/developer-guide/Working-With-iOS.asciidoc index 71889dac989..ff327851b20 100644 --- a/docs/developer-guide/Working-With-iOS.asciidoc +++ b/docs/developer-guide/Working-With-iOS.asciidoc @@ -276,6 +276,14 @@ Supported need formats are `from:`, `exact:`, `branch:`, `revision:`, and `range `ios.dependencyManager=auto` preserves backward compatibility. Existing projects with `ios.pods` continue to use CocoaPods. Projects with `ios.spm.*` use SPM. If both hint families are present, both are applied. +Explicit dependency-manager modes are validated rather than dropping +dependencies. `ios.dependencyManager=spm` fails when the project or a built-in +API requires a CocoaPod; use `auto` or `both`. Likewise, +`ios.dependencyManager=none` fails while pod or SPM hints are present; remove +the dependency hints only if suppression is intentional, or switch to +`auto`. This is a migration-visible change for older projects that used +`spm` or `none` to ignore declared native dependencies. + === Including dynamic frameworks If you need to use a dynamic framework (for example: SomeThirdPartySDK.framework), and it isn't available through CocoaPods, then you can add it to your project by zipping up the framework and copying it to your native/ios directory. diff --git a/docs/website/content/blog/platform-apis-in-the-core.md b/docs/website/content/blog/platform-apis-in-the-core.md index e7f83d76982..324c79ac73e 100644 --- a/docs/website/content/blog/platform-apis-in-the-core.md +++ b/docs/website/content/blog/platform-apis-in-the-core.md @@ -244,194 +244,64 @@ view.setInputListener(userText -> { `appendToLastMessage(...)` is the streaming entry point; it marshals through `callSerially` so deltas land on the EDT in order. `ConversationStore` persists the thread (the default backing is `Storage`; pluggable via a custom implementation if you would rather keep it in SQLite or push it to your server). -### The AI cn1libs +### On-device vision, language, and LiteRT -The core LLM stack is paired with a set of opt-in cn1libs that wrap specific on-device capabilities: Google ML Kit features, the TensorFlow Lite runtime, a local Whisper transcription engine, and an on-device Stable Diffusion model. Thirteen new cn1libs ship this release. +**Update (July 2026):** On-device vision, language services, and LiteRT +inference now ship as framework APIs. Do not add separate ML Kit or LiteRT +cn1lib dependencies. Import the APIs from `com.codename1.ai.vision`, +`com.codename1.ai.language`, and `com.codename1.ai.inference`. -These cn1libs are not yet listed in the Codename One Preferences cn1lib picker, so for the moment they are added by hand. Drop the matching dependency block into your project's `common/pom.xml` and rebuild. The build-time scanner does the rest: the iOS pod or Swift Package, the Android Gradle dependency, the plist usage strings (`NSCameraUsageDescription` for the vision libraries, `NSSpeechRecognitionUsageDescription` for Whisper, etc.), and the Android permissions (`android.permission.RECORD_AUDIO` for audio capture) are all injected automatically the first time the scanner sees the matching class on the classpath. - -For each cn1lib below, the dependency block is identical in shape; only the `` changes. The shared pattern is: - -```xml - - com.codenameone - - ${cn1.version} - -``` - -#### `cn1-ai-mlkit-text`: text recognition (OCR) - -**TL;DR.** Pull printed or handwritten text out of an image (a photo of a page, a sign, a receipt) entirely on-device. - -**Platforms.** iOS bridges to `GoogleMLKit/TextRecognition`. Android bridges to `com.google.mlkit:text-recognition`. The JavaSE simulator returns an unsupported error. - -**Use cases.** Receipt scanning, sign translation pipelines (combine with `cn1-ai-mlkit-translate`), accessibility tools that read printed text aloud, automated form ingestion. - -```java -byte[] jpeg = capturePhotoBytes(); -TextRecognizer.recognize(jpeg).onResult((text, err) -> { - if (err == null) Log.p("OCR: " + text); -}); -``` - -#### `cn1-ai-mlkit-barcode`: barcode and QR scanning - -**TL;DR.** Decodes QR, EAN, UPC, Data Matrix, PDF417, and the rest of the common 1D / 2D code families from a captured image. - -**Platforms.** iOS bridges to `MLKitBarcodeScanning`. Android bridges to `com.google.mlkit:barcode-scanning`. The JavaSE simulator returns an unsupported error. - -**Use cases.** Inventory scanning, ticket / boarding-pass readers, QR-driven onboarding flows, retail loyalty cards. - -```java -byte[] jpeg = capturePhotoBytes(); -BarcodeScanner.scan(jpeg).onResult((codes, err) -> { - if (err == null) { - for (String code : codes) Log.p("Found: " + code); - } -}); -``` - -#### `cn1-ai-mlkit-face`: face detection - -**TL;DR.** Returns bounding boxes for human faces detected in an image. Each face is reported as a packed `int[4]` (`x`, `y`, `width`, `height`). - -**Platforms.** iOS bridges to `MLKitFaceDetection`. Android bridges to `com.google.mlkit:face-detection`. - -**Use cases.** Auto-crop a contact photo, mosaic / blur bystanders in a group shot, drive a face-tracked overlay for AR-lite filters. +The builders inspect the feature classes and backend-selector methods that an +application actually uses. They retain only the matching native adapters and +add the required Android library, Apple framework, or optional iOS ML Kit pod. +For example, Android text recognition uses ML Kit automatically. Apple targets +default to Vision, while this explicit selector opts iOS into the text-only ML +Kit pod: ```java -FaceDetector.detect(jpeg).onResult((boxes, err) -> { - if (err != null) return; - for (int i = 0; i < boxes.length; i += 4) { - Log.p("face at " + boxes[i] + "," + boxes[i + 1] + " " - + boxes[i + 2] + "x" + boxes[i + 3]); - } -}); -``` +VisionOptions options = new VisionOptions() + .backend(VisionBackends.mlKitTextRecognition()); -#### `cn1-ai-mlkit-labeling`: image labeling - -**TL;DR.** "What is in this picture." Returns a list of descriptive labels for the image content. - -**Platforms.** iOS bridges to `MLKitImageLabeling`. Android bridges to `com.google.mlkit:image-labeling`. - -**Use cases.** Auto-tagging uploaded photos, content moderation pre-filters, content-based image search. - -```java -ImageLabeler.label(jpeg).onResult((labels, err) -> { - if (err == null) Log.p("labels: " + String.join(", ", labels)); -}); +new TextRecognizer(options) + .process(VisionImage.encoded(jpeg)) + .ready(result -> Log.p(result.getText())) + .except(error -> Log.e(error)); ``` -#### `cn1-ai-mlkit-translate`: on-device translation - -**TL;DR.** Translate short text between supported language pairs entirely on-device; no server round-trip, no API key, works offline. - -**Platforms.** iOS bridges to `MLKitTranslate`. Android bridges to `com.google.mlkit:translate`. Languages are identified by their ISO 639-1 codes (`en`, `fr`, `es`, ...). - -**Use cases.** Offline travel assistants, chat translation, accessibility readers for foreign signage (combine with `cn1-ai-mlkit-text`). - -```java -Translator.translate("Where is the train station?", "en", "fr") - .onResult((fr, err) -> { - if (err == null) Log.p(fr); // "Où est la gare ?" - }); -``` - -#### `cn1-ai-mlkit-smartreply`: short reply suggestions - -**TL;DR.** Generates short suggested replies for chat conversations, similar to Gmail's Smart Reply chips. - -**Platforms.** iOS bridges to `MLKitSmartReply`. Android bridges to `com.google.mlkit:smart-reply`. The input is a JSON array of `{role, message, timestamp, userId}` objects. - -**Use cases.** A "quick reply" row above the keyboard in your in-app chat, response suggestions in a CRM inbox. +`InferenceSession` owns native interpreter and delegate resources. Reuse a +session for repeated runs and close it when the owner is disposed. A one-shot +operation must close the session in both completion paths: ```java -String thread = "[{\"role\":\"remote\",\"message\":\"See you at 6?\"," - + "\"timestamp\":" + System.currentTimeMillis() + "," - + "\"userId\":\"u42\"}]"; - -SmartReply.suggest(thread).onResult((suggestions, err) -> { - if (err == null) { - for (String s : suggestions) Log.p("suggestion: " + s); - } -}); -``` - -#### `cn1-ai-mlkit-langid`: language identification - -**TL;DR.** Returns the most likely ISO 639-1 code for a given text, or `und` (undetermined) when the input is too short or ambiguous. - -**Platforms.** iOS bridges to `MLKitLanguageID`. Android bridges to `com.google.mlkit:language-id`. - -**Use cases.** Auto-route a customer-support message to the right team, pick the correct TTS voice for an arbitrary string, pre-screen input before running an expensive translate. - -```java -LanguageIdentifier.identify("Bonjour le monde").onResult((code, err) -> { - if (err == null) Log.p(code); // "fr" -}); -``` - -#### `cn1-ai-mlkit-pose`: pose detection - -**TL;DR.** Returns 33 skeletal landmarks per detected pose as a packed `float[3 * 33]` (`x`, `y`, `confidence` triples). - -**Platforms.** iOS bridges to `MLKitPoseDetection`. Android bridges to `com.google.mlkit:pose-detection`. - -**Use cases.** Fitness apps with form correction, dance / yoga timing analysis, gesture-driven controls. - -```java -PoseDetector.detect(jpeg).onResult((landmarks, err) -> { - if (err != null || landmarks.length < 99) return; - float noseX = landmarks[0], noseY = landmarks[1], noseConf = landmarks[2]; - Log.p("nose at (" + noseX + ", " + noseY + ") conf=" + noseConf); -}); -``` - -#### `cn1-ai-mlkit-segmentation`: selfie segmentation - -**TL;DR.** Returns a per-pixel mask separating the person in the foreground from the background as `byte[width * height]` (`0` = background, `255` = foreground). - -**Platforms.** iOS bridges to `MLKitSegmentationSelfie`. Android bridges to `com.google.mlkit:segmentation-selfie`. - -**Use cases.** Background replacement for video calls, sticker / portrait-mode effects, blur-the-background privacy filters. - -```java -SelfieSegmenter.segment(jpeg).onResult((mask, err) -> { - if (err == null) applyBackgroundReplacement(mask); -}); -``` - -#### `cn1-ai-mlkit-docscan`: document scanner - -**TL;DR.** Detects a rectangular document in a photo, perspective-corrects it, and writes the cropped JPEG to a temporary file. Returns the file path. - -**Platforms.** iOS uses Apple's VisionKit + Core Image rectangle detection (no extra pod). Android uses `com.google.android.gms:play-services-mlkit-document-scanner`. - -**Use cases.** "Scan to PDF" flows, expense apps that capture receipts, contract signing flows, ID-document capture. - -```java -DocumentScanner.scanToFile(jpeg).onResult((path, err) -> { - if (err == null) uploadDocument(path); -}); -``` - -#### `cn1-ai-tflite`: TensorFlow Lite interpreter - -**TL;DR.** A general-purpose on-device inference engine. Bring your own `.tflite` model and run it against a float32 input tensor. - -**Platforms.** iOS uses `TensorFlowLiteSwift` (Pods or Swift Package). Android uses `org.tensorflow:tensorflow-lite` + `tensorflow-lite-support`. - -**Use cases.** Any custom on-device ML model your team trains or pulls from TF Hub. Image classification, simple regression, recommendation pre-filters. - -```java -byte[] modelBytes = Util.readFully(Display.getInstance().getResourceAsStream(null, "/model.tflite")); -float[] input = featureVector(); -Interpreter.run(modelBytes, input).onResult((output, err) -> { - if (err == null) Log.p("model returned " + output.length + " values"); -}); -``` +InferenceSession.open(ModelSource.resource("/model.tflite"), + new InferenceOptions()) + .ready(session -> session.run(new Tensor[] {input}) + .ready(outputs -> { + try { + consume(outputs); + } finally { + session.close(); + } + }) + .except(error -> { + try { + Log.e(error); + } finally { + session.close(); + } + })) + .except(error -> Log.e(error)); +``` + +See [AI, Chat UI, and Speech](https://www.codenameone.com/developer-guide/#_ai_chat_ui_and_speech) +for the current API reference, feature matrix, camera pipeline, model cache, +and backend-selection examples. + +### Large optional AI cn1libs + +Vision, language, and LiteRT fit in core because their native dependencies can +be selected granularly by the builders. Runtimes with very large bundled data +or native libraries remain explicit dependencies. #### `cn1-ai-whisper`: speech-to-text via whisper.cpp @@ -466,15 +336,20 @@ StableDiffusion.generate("a teal hot-air balloon over Lisbon, watercolour", }); ``` -### Why these are cn1libs and not part of the core - -The core gets the AI plumbing every app that adopts AI at all wants: the LLM client, streaming, the chat UI, the secure storage primitive for credentials, the simulator Ollama redirect for offline iteration. +### Why these remain cn1libs -The cn1libs above are specialized verticals. Barcode scanning, document scanning, face detection, smart reply, pose detection, on-device translation, transcription, on-device image generation, each is genuinely useful, but only for some apps. They also each bring a non-trivial native dependency. The Google ML Kit Android frameworks are large; the iOS pods carry their own weight; the bundled `libwhisper.a` and the Stable Diffusion model are big. Pulling all of them into core would tax every app whether the feature is used or not. +The core provides the portable APIs and lets the builders retain native vision, +language, and LiteRT dependencies at feature or selector granularity. Whisper +and Stable Diffusion are different: they bundle unusually large native +libraries or model data that should remain an explicit application choice. The Stable Diffusion cn1lib in particular is large enough that the cloud build server cannot accept the upload at all (it trips the 2 GB pre-upload guard). That kind of opt-in does not belong in a dependency every app inherits. -The corresponding chapter, including the full `LlmClient` API table, the `ChatView` reference, the `SecureStorage` overloads, the simulator Ollama redirect, and the full cn1lib coverage, is at [AI, Chat UI, and Speech](https://www.codenameone.com/developer-guide/#_ai_chat_ui_and_speech) in the developer guide. +The corresponding chapter, including the full `LlmClient` API table, the +`ChatView` reference, the `SecureStorage` overloads, the simulator Ollama +redirect, the built-in on-device APIs, and the remaining cn1lib coverage, is +at [AI, Chat UI, and Speech](https://www.codenameone.com/developer-guide/#_ai_chat_ui_and_speech) +in the developer guide. ## OAuth and OIDC: the modern identity stack diff --git a/docs/website/data/port_status.json b/docs/website/data/port_status.json index 65efde54555..5da97844775 100644 --- a/docs/website/data/port_status.json +++ b/docs/website/data/port_status.json @@ -377,6 +377,27 @@ "description": "Checks camera discovery and capture where hardware and runtime permissions are available.", "tests": ["CameraApiTest"] }, + { + "id": "on-device-vision", + "category": "On-device AI", + "name": "Vision analysis", + "description": "Checks camera-frame image ownership, backend capability reporting, and analyzer lifecycle for OCR, barcodes, faces, image labels, pose, segmentation, and document scanning.", + "tests": ["VisionOnDeviceApiTest"] + }, + { + "id": "on-device-language", + "category": "On-device AI", + "name": "Language services", + "description": "Checks language identification, translation, and smart-reply capability reporting plus deterministic unsupported fallbacks.", + "tests": ["LanguageOnDeviceApiTest"] + }, + { + "id": "on-device-inference", + "category": "On-device AI", + "name": "Model inference", + "description": "Checks immutable tensor and model-source contracts, inference options, runtime capability reporting, and deterministic unsupported fallback behavior.", + "tests": ["InferenceOnDeviceApiTest"] + }, { "id": "calendar-integration", "category": "Platform-dependent APIs", diff --git a/docs/website/data/port_status_reports/android.json b/docs/website/data/port_status_reports/android.json index 8da69261ae6..fb044632f8f 100644 --- a/docs/website/data/port_status_reports/android.json +++ b/docs/website/data/port_status_reports/android.json @@ -7,8 +7,8 @@ "suite_finished": true, "summary": { "fail": 0, - "not-run": 0, - "pass": 163, + "not-run": 5, + "pass": 164, "skip": 1 }, "tests": { @@ -682,6 +682,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/ios-gl.json b/docs/website/data/port_status_reports/ios-gl.json index a17b7de0cbf..56dceeb7689 100644 --- a/docs/website/data/port_status_reports/ios-gl.json +++ b/docs/website/data/port_status_reports/ios-gl.json @@ -7,8 +7,8 @@ "suite_finished": true, "summary": { "fail": 0, - "not-run": 0, - "pass": 162, + "not-run": 5, + "pass": 163, "skip": 2 }, "tests": { @@ -685,6 +685,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/ios-metal.json b/docs/website/data/port_status_reports/ios-metal.json index dd8910cd5e8..41e6d8bb21c 100644 --- a/docs/website/data/port_status_reports/ios-metal.json +++ b/docs/website/data/port_status_reports/ios-metal.json @@ -7,8 +7,8 @@ "suite_finished": true, "summary": { "fail": 0, - "not-run": 0, - "pass": 162, + "not-run": 5, + "pass": 163, "skip": 2 }, "tests": { @@ -685,6 +685,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/javascript.json b/docs/website/data/port_status_reports/javascript.json index c48cde4c8c1..a31881d039b 100644 --- a/docs/website/data/port_status_reports/javascript.json +++ b/docs/website/data/port_status_reports/javascript.json @@ -7,8 +7,8 @@ "suite_finished": true, "summary": { "fail": 0, - "not-run": 0, - "pass": 163, + "not-run": 5, + "pass": 164, "skip": 1 }, "tests": { @@ -682,6 +682,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/linux-arm64.json b/docs/website/data/port_status_reports/linux-arm64.json index b7e1e9e0acf..e2858afe2d2 100644 --- a/docs/website/data/port_status_reports/linux-arm64.json +++ b/docs/website/data/port_status_reports/linux-arm64.json @@ -7,8 +7,8 @@ "suite_finished": false, "summary": { "fail": 0, - "not-run": 0, - "pass": 164, + "not-run": 5, + "pass": 165, "skip": 0 }, "tests": { @@ -679,6 +679,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/linux-x64.json b/docs/website/data/port_status_reports/linux-x64.json index 782e551b720..91949b47702 100644 --- a/docs/website/data/port_status_reports/linux-x64.json +++ b/docs/website/data/port_status_reports/linux-x64.json @@ -7,8 +7,8 @@ "suite_finished": false, "summary": { "fail": 0, - "not-run": 0, - "pass": 164, + "not-run": 5, + "pass": 165, "skip": 0 }, "tests": { @@ -679,6 +679,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/mac-native.json b/docs/website/data/port_status_reports/mac-native.json index 0ff436a2839..d7792f7aa4b 100644 --- a/docs/website/data/port_status_reports/mac-native.json +++ b/docs/website/data/port_status_reports/mac-native.json @@ -7,8 +7,8 @@ "suite_finished": true, "summary": { "fail": 0, - "not-run": 0, - "pass": 162, + "not-run": 5, + "pass": 163, "skip": 2 }, "tests": { @@ -685,6 +685,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/tvos.json b/docs/website/data/port_status_reports/tvos.json index 5a608484997..d9c49c0ec83 100644 --- a/docs/website/data/port_status_reports/tvos.json +++ b/docs/website/data/port_status_reports/tvos.json @@ -7,8 +7,8 @@ "suite_finished": true, "summary": { "fail": 0, - "not-run": 0, - "pass": 162, + "not-run": 5, + "pass": 163, "skip": 2 }, "tests": { @@ -685,6 +685,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/watchos.json b/docs/website/data/port_status_reports/watchos.json index d0a5fc21f7a..b2bdbfad26b 100644 --- a/docs/website/data/port_status_reports/watchos.json +++ b/docs/website/data/port_status_reports/watchos.json @@ -7,8 +7,8 @@ "suite_finished": true, "summary": { "fail": 0, - "not-run": 0, - "pass": 162, + "not-run": 5, + "pass": 163, "skip": 2 }, "tests": { @@ -685,6 +685,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/windows-arm64.json b/docs/website/data/port_status_reports/windows-arm64.json index 636a57e5cbb..591628f26f5 100644 --- a/docs/website/data/port_status_reports/windows-arm64.json +++ b/docs/website/data/port_status_reports/windows-arm64.json @@ -7,8 +7,8 @@ "suite_finished": false, "summary": { "fail": 0, - "not-run": 0, - "pass": 164, + "not-run": 5, + "pass": 165, "skip": 0 }, "tests": { @@ -679,6 +679,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/docs/website/data/port_status_reports/windows-x64.json b/docs/website/data/port_status_reports/windows-x64.json index b0025155457..1530b2cc8e0 100644 --- a/docs/website/data/port_status_reports/windows-x64.json +++ b/docs/website/data/port_status_reports/windows-x64.json @@ -7,8 +7,8 @@ "suite_finished": false, "summary": { "fail": 0, - "not-run": 0, - "pass": 164, + "not-run": 5, + "pass": 165, "skip": 0 }, "tests": { @@ -679,6 +679,18 @@ "ClipboardRoundTripTest": { "feature": "clipboard", "status": "not-run" + }, + "InferenceOnDeviceApiTest": { + "feature": "on-device-inference", + "status": "not-run" + }, + "LanguageOnDeviceApiTest": { + "feature": "on-device-language", + "status": "not-run" + }, + "VisionOnDeviceApiTest": { + "feature": "on-device-vision", + "status": "not-run" } }, "bootstrap_source": "successful-master-workflow", diff --git a/maven/android/pom.xml b/maven/android/pom.xml index 2afe8497b10..c56214a4553 100644 --- a/maven/android/pom.xml +++ b/maven/android/pom.xml @@ -89,6 +89,10 @@ below because that zip takes the whole source dir. --> com/codename1/impl/android/ar/** + + com/codename1/impl/android/ai/** diff --git a/maven/cn1-ai-mlkit-barcode/android/pom.xml b/maven/cn1-ai-mlkit-barcode/android/pom.xml deleted file mode 100644 index cdd53c44ed4..00000000000 --- a/maven/cn1-ai-mlkit-barcode/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-barcode - 8.0-SNAPSHOT - - - cn1-ai-mlkit-barcode-android - jar - Codename One AI: cn1-ai-mlkit-barcode-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-barcode-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-barcode/android/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScannerImpl.java b/maven/cn1-ai-mlkit-barcode/android/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScannerImpl.java deleted file mode 100644 index b9143cd3b2a..00000000000 --- a/maven/cn1-ai-mlkit-barcode/android/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScannerImpl.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.codename1.ai.mlkit.barcode; - - -public class NativeBarcodeScannerImpl { - public String[] scan(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new String[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.barcode.BarcodeScannerOptions o = - new com.google.mlkit.vision.barcode.BarcodeScannerOptions.Builder().build(); - com.google.mlkit.vision.barcode.BarcodeScanner scanner = - com.google.mlkit.vision.barcode.BarcodeScanning.getClient(o); - final java.util.List out = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - scanner.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - java.util.List>() { - public void onSuccess(java.util.List rs) { - for (com.google.mlkit.vision.barcode.common.Barcode b : rs) { - String v = b.getRawValue(); - if (v != null) out.add(v); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.toArray(new String[0]); - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-barcode/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-barcode/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-barcode/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-barcode/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-barcode/common/codenameone_library_required.properties deleted file mode 100644 index 62c8e803db1..00000000000 --- a/maven/cn1-ai-mlkit-barcode/common/codenameone_library_required.properties +++ /dev/null @@ -1,7 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-barcode. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/BarcodeScanning -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:barcode-scanning:17.2.0' -codename1.arg.android.xpermissions= diff --git a/maven/cn1-ai-mlkit-barcode/common/pom.xml b/maven/cn1-ai-mlkit-barcode/common/pom.xml deleted file mode 100644 index fd6174a399c..00000000000 --- a/maven/cn1-ai-mlkit-barcode/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-barcode - 8.0-SNAPSHOT - - - cn1-ai-mlkit-barcode-common - jar - Codename One AI: cn1-ai-mlkit-barcode-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/BarcodeScanner.java b/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/BarcodeScanner.java deleted file mode 100644 index 64724c6cc01..00000000000 --- a/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/BarcodeScanner.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.barcode; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Barcode Scanning. -/// -/// Decodes barcodes (QR, EAN, UPC, Data Matrix, PDF417, etc.) from images. -/// Bridges to `MLKitBarcodeScanning` on iOS and -/// `com.google.mlkit:barcode-scanning` on Android. -/// -public final class BarcodeScanner { - private BarcodeScanner() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeBarcodeScanner bridge = NativeLookup.create(NativeBarcodeScanner.class); - return bridge != null && bridge.isSupported(); - } - - public static AsyncResource scan(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - if (imageBytes == null || imageBytes.length == 0) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(new String[0]); } - }); - return out; - } - final NativeBarcodeScanner bridge = NativeLookup.create(NativeBarcodeScanner.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("BarcodeScanner.scan is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String[] r = bridge.scan(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new String[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("BarcodeScanner.scan failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/package-info.java b/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/package-info.java deleted file mode 100644 index 573acffadf5..00000000000 --- a/maven/cn1-ai-mlkit-barcode/common/src/main/java/com/codename1/ai/mlkit/barcode/package-info.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit Barcode Scanning. -/// -/// Decodes barcodes (QR, EAN, UPC, Data Matrix, PDF417, etc.) from images. -/// Bridges to `MLKitBarcodeScanning` on iOS and -/// `com.google.mlkit:barcode-scanning` on Android. -/// -/// The single public class in this package is [BarcodeScanner], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeBarcodeScanner` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `BarcodeScanner.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.barcode; diff --git a/maven/cn1-ai-mlkit-barcode/common/src/test/java/com/codename1/ai/mlkit/barcode/BarcodeScannerTest.java b/maven/cn1-ai-mlkit-barcode/common/src/test/java/com/codename1/ai/mlkit/barcode/BarcodeScannerTest.java deleted file mode 100644 index aad5a6bf8d1..00000000000 --- a/maven/cn1-ai-mlkit-barcode/common/src/test/java/com/codename1/ai/mlkit/barcode/BarcodeScannerTest.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.codename1.ai.mlkit.barcode; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class BarcodeScannerTest { - - /** Mock implementation of NativeBarcodeScanner for headless JVM tests. */ - static class MockBridge implements NativeBarcodeScanner { - boolean supported = true; - public boolean isSupported() { return supported; } - public String[] scan(byte[] imageBytes) { - return new String[]{"x", "y"}; - } - } - - @Test - void mock_bridge_returns_two_codes() { - MockBridge b = new MockBridge(); - String[] r = b.scan(new byte[]{1, 2, 3}); - assertEquals(2, r.length); - assertEquals("x", r[0]); - } - - @Test - void bridge_reports_supported() { - assertTrue(new MockBridge().isSupported()); - } -} diff --git a/maven/cn1-ai-mlkit-barcode/ios/pom.xml b/maven/cn1-ai-mlkit-barcode/ios/pom.xml deleted file mode 100644 index d949a9c1c35..00000000000 --- a/maven/cn1-ai-mlkit-barcode/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-barcode - 8.0-SNAPSHOT - - - cn1-ai-mlkit-barcode-ios - jar - Codename One AI: cn1-ai-mlkit-barcode-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-barcode-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-barcode/ios/src/main/objectivec/com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.h b/maven/cn1-ai-mlkit-barcode/ios/src/main/objectivec/com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.h deleted file mode 100644 index aa55f016d55..00000000000 --- a/maven/cn1-ai-mlkit-barcode/ios/src/main/objectivec/com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl : NSObject { -} - --(NSData*)scan:(NSData*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-barcode/ios/src/main/objectivec/com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.m b/maven/cn1-ai-mlkit-barcode/ios/src/main/objectivec/com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.m deleted file mode 100644 index 2c93b7eb6e1..00000000000 --- a/maven/cn1-ai-mlkit-barcode/ios/src/main/objectivec/com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.m +++ /dev/null @@ -1,48 +0,0 @@ -#import "com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl.h" -#import -#import -#import -#import - -@implementation com_codename1_ai_mlkit_barcode_NativeBarcodeScannerImpl - --(NSData*)scan:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [self packStrings:@[]]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKBarcodeScannerOptions *opts = [[MLKBarcodeScannerOptions alloc] init]; - MLKBarcodeScanner *scanner = [MLKBarcodeScanner barcodeScannerWithOptions:opts]; - __block NSArray *values = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [scanner processImage:vision completion:^(NSArray * _Nullable barcodes, - NSError * _Nullable error) { - NSMutableArray *m = [NSMutableArray array]; - for (MLKBarcode *b in barcodes ?: @[]) { - if (b.rawValue) [m addObject:b.rawValue]; - } - values = m; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return [self packStrings:values]; -} - --(NSData*)packStrings:(NSArray *)strings { - // Encode as length-prefixed UTF-8 (network byte order int + bytes). - NSMutableData *out = [NSMutableData data]; - uint32_t count = htonl((uint32_t)strings.count); - [out appendBytes:&count length:sizeof(count)]; - for (NSString *s in strings) { - NSData *u = [s dataUsingEncoding:NSUTF8StringEncoding]; - uint32_t len = htonl((uint32_t)u.length); - [out appendBytes:&len length:sizeof(len)]; - [out appendData:u]; - } - return out; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-barcode/javascript/pom.xml b/maven/cn1-ai-mlkit-barcode/javascript/pom.xml deleted file mode 100644 index 9b47c753f93..00000000000 --- a/maven/cn1-ai-mlkit-barcode/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-barcode - 8.0-SNAPSHOT - - - cn1-ai-mlkit-barcode-javascript - jar - Codename One AI: cn1-ai-mlkit-barcode-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-barcode-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-barcode/javascript/src/main/javascript/com_codename1_ai_mlkit_barcode_NativeBarcodeScanner.js b/maven/cn1-ai-mlkit-barcode/javascript/src/main/javascript/com_codename1_ai_mlkit_barcode_NativeBarcodeScanner.js deleted file mode 100644 index 0c74196d599..00000000000 --- a/maven/cn1-ai-mlkit-barcode/javascript/src/main/javascript/com_codename1_ai_mlkit_barcode_NativeBarcodeScanner.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.scan__byte_1ARRAY = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_barcode_NativeBarcodeScanner= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-barcode/javase/pom.xml b/maven/cn1-ai-mlkit-barcode/javase/pom.xml deleted file mode 100644 index c398acd1b44..00000000000 --- a/maven/cn1-ai-mlkit-barcode/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-barcode - 8.0-SNAPSHOT - - - cn1-ai-mlkit-barcode-javase - jar - Codename One AI: cn1-ai-mlkit-barcode-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-barcode-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-barcode/javase/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScannerImpl.java b/maven/cn1-ai-mlkit-barcode/javase/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScannerImpl.java deleted file mode 100644 index 55e1343f022..00000000000 --- a/maven/cn1-ai-mlkit-barcode/javase/src/main/java/com/codename1/ai/mlkit/barcode/NativeBarcodeScannerImpl.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.codename1.ai.mlkit.barcode; - -public class NativeBarcodeScannerImpl implements NativeBarcodeScanner { - - private static boolean hintsEnsured; - private static synchronized void ensureSimulatorHints() { - if (hintsEnsured) return; - hintsEnsured = true; - java.util.Map hints = - com.codename1.ui.Display.getInstance().getProjectBuildHints(); - if (hints == null) return; // not running in the simulator - if (!hints.containsKey("ios.NSCameraUsageDescription")) { - com.codename1.ui.Display.getInstance() - .setProjectBuildHint("ios.NSCameraUsageDescription", "This app uses the camera to scan barcodes."); - } - } - - public NativeBarcodeScannerImpl() { - ensureSimulatorHints(); - } - public String[] scan(byte[] imageBytes) { - if (imageBytes == null || imageBytes.length == 0) return new String[0]; - // Deterministic stub for simulator runs. - return new String[]{"SIMULATOR_BARCODE_" + imageBytes.length}; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-barcode/lib/pom.xml b/maven/cn1-ai-mlkit-barcode/lib/pom.xml deleted file mode 100644 index 5cb4277dbcb..00000000000 --- a/maven/cn1-ai-mlkit-barcode/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-barcode - 8.0-SNAPSHOT - - - cn1-ai-mlkit-barcode-lib - pom - Codename One AI: cn1-ai-mlkit-barcode-lib - - - - com.codenameone - cn1-ai-mlkit-barcode-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-barcode-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-barcode-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-barcode-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-barcode-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-barcode/pom.xml b/maven/cn1-ai-mlkit-barcode/pom.xml deleted file mode 100644 index e1f34b84bc1..00000000000 --- a/maven/cn1-ai-mlkit-barcode/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-barcode - pom - Codename One AI: cn1-ai-mlkit-barcode - ML Kit Barcode Scanning - - - cn1-ai-mlkit-barcode - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-docscan/android/pom.xml b/maven/cn1-ai-mlkit-docscan/android/pom.xml deleted file mode 100644 index d13a13c6b03..00000000000 --- a/maven/cn1-ai-mlkit-docscan/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-docscan - 8.0-SNAPSHOT - - - cn1-ai-mlkit-docscan-android - jar - Codename One AI: cn1-ai-mlkit-docscan-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-docscan-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-docscan/android/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScannerImpl.java b/maven/cn1-ai-mlkit-docscan/android/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScannerImpl.java deleted file mode 100644 index c29139f4819..00000000000 --- a/maven/cn1-ai-mlkit-docscan/android/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScannerImpl.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.mlkit.docscan; - - -public class NativeDocumentScannerImpl { - public String scanToFile(byte[] imageBytes) { - try { - java.io.File f = java.io.File.createTempFile("docscan-", ".jpg"); - java.io.FileOutputStream fos = new java.io.FileOutputStream(f); - fos.write(imageBytes); - fos.close(); - return f.getAbsolutePath(); - } catch (java.io.IOException ioe) { - return ""; - } - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-docscan/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-docscan/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-docscan/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-docscan/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-docscan/common/codenameone_library_required.properties deleted file mode 100644 index 59ed1294531..00000000000 --- a/maven/cn1-ai-mlkit-docscan/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-docscan. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.add_frameworks=VisionKit -codename1.arg.android.gradleDep=implementation 'com.google.android.gms:play-services-mlkit-document-scanner:16.0.0-beta1' diff --git a/maven/cn1-ai-mlkit-docscan/common/pom.xml b/maven/cn1-ai-mlkit-docscan/common/pom.xml deleted file mode 100644 index 23d18917299..00000000000 --- a/maven/cn1-ai-mlkit-docscan/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-docscan - 8.0-SNAPSHOT - - - cn1-ai-mlkit-docscan-common - jar - Codename One AI: cn1-ai-mlkit-docscan-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/DocumentScanner.java b/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/DocumentScanner.java deleted file mode 100644 index 2eb94018e20..00000000000 --- a/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/DocumentScanner.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.docscan; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit / VisionKit Document Scanner. -/// -/// Captures and crops document photos. On iOS uses Apple's VisionKit + Core Image rectangle detection (no extra pod). On Android uses the Google Play services document-scanner module. -/// -public final class DocumentScanner { - private DocumentScanner() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeDocumentScanner bridge = NativeLookup.create(NativeDocumentScanner.class); - return bridge != null && bridge.isSupported(); - } - - public static AsyncResource scanToFile(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativeDocumentScanner bridge = NativeLookup.create(NativeDocumentScanner.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("DocumentScanner.scanToFile is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String r = bridge.scanToFile(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? "" : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("DocumentScanner.scanToFile failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/package-info.java b/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/package-info.java deleted file mode 100644 index 1dd15b5334f..00000000000 --- a/maven/cn1-ai-mlkit-docscan/common/src/main/java/com/codename1/ai/mlkit/docscan/package-info.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit / VisionKit Document Scanner. -/// -/// Captures and crops document photos. On iOS uses Apple's VisionKit + Core Image rectangle detection (no extra pod). On Android uses the Google Play services document-scanner module. -/// -/// The single public class in this package is [DocumentScanner], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeDocumentScanner` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `DocumentScanner.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.docscan; diff --git a/maven/cn1-ai-mlkit-docscan/common/src/test/java/com/codename1/ai/mlkit/docscan/DocumentScannerTest.java b/maven/cn1-ai-mlkit-docscan/common/src/test/java/com/codename1/ai/mlkit/docscan/DocumentScannerTest.java deleted file mode 100644 index d348880c08a..00000000000 --- a/maven/cn1-ai-mlkit-docscan/common/src/test/java/com/codename1/ai/mlkit/docscan/DocumentScannerTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.mlkit.docscan; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class DocumentScannerTest { - - /** Mock implementation of NativeDocumentScanner for headless JVM tests. */ - static class MockBridge implements NativeDocumentScanner { - boolean supported = true; - public boolean isSupported() { return supported; } - public String scanToFile(byte[] imageBytes) { return "/tmp/x.jpg"; } - } - - @Test - void mock_returns_path() { - MockBridge b = new MockBridge(); - assertEquals("/tmp/x.jpg", b.scanToFile(new byte[]{1})); - } -} diff --git a/maven/cn1-ai-mlkit-docscan/ios/pom.xml b/maven/cn1-ai-mlkit-docscan/ios/pom.xml deleted file mode 100644 index f416f905a37..00000000000 --- a/maven/cn1-ai-mlkit-docscan/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-docscan - 8.0-SNAPSHOT - - - cn1-ai-mlkit-docscan-ios - jar - Codename One AI: cn1-ai-mlkit-docscan-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-docscan-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-docscan/ios/src/main/objectivec/com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.h b/maven/cn1-ai-mlkit-docscan/ios/src/main/objectivec/com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.h deleted file mode 100644 index c68dd8802ee..00000000000 --- a/maven/cn1-ai-mlkit-docscan/ios/src/main/objectivec/com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl : NSObject { -} - --(NSString*)scanToFile:(NSData*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-docscan/ios/src/main/objectivec/com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.m b/maven/cn1-ai-mlkit-docscan/ios/src/main/objectivec/com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.m deleted file mode 100644 index 23ca8e045d0..00000000000 --- a/maven/cn1-ai-mlkit-docscan/ios/src/main/objectivec/com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.m +++ /dev/null @@ -1,42 +0,0 @@ -#import "com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl.h" -#import -#import - -@implementation com_codename1_ai_mlkit_docscan_NativeDocumentScannerImpl - -// VisionKit-based fallback: Apple's VNDocumentCameraViewController is -// interactive; this bridge accepts a pre-captured image and returns its -// cropped JPEG path. On iOS 13+ VisionKit handles the live UI flow; the -// sample app drives that flow and feeds the bytes into the cn1lib. --(NSString*)scanToFile:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return @""; - CIImage *ci = [CIImage imageWithCGImage:image.CGImage]; - CIContext *ctx = [CIContext context]; - CIDetector *det = [CIDetector detectorOfType:CIDetectorTypeRectangle context:ctx - options:@{CIDetectorAccuracy: CIDetectorAccuracyHigh}]; - NSArray *features = [det featuresInImage:ci]; - UIImage *cropped = image; - if (features.count > 0) { - CIRectangleFeature *rf = (CIRectangleFeature *)features.firstObject; - CIImage *flat = [ci imageByApplyingFilter:@"CIPerspectiveCorrection" withInputParameters:@{ - @"inputTopLeft": [CIVector vectorWithCGPoint:rf.topLeft], - @"inputTopRight": [CIVector vectorWithCGPoint:rf.topRight], - @"inputBottomLeft": [CIVector vectorWithCGPoint:rf.bottomLeft], - @"inputBottomRight": [CIVector vectorWithCGPoint:rf.bottomRight] - }]; - CGImageRef cg = [ctx createCGImage:flat fromRect:flat.extent]; - cropped = [UIImage imageWithCGImage:cg]; - CGImageRelease(cg); - } - NSString *path = [NSString stringWithFormat:@"%@/docscan-%@.jpg", - NSTemporaryDirectory(), [[NSUUID UUID] UUIDString]]; - [UIImageJPEGRepresentation(cropped, 0.92) writeToFile:path atomically:YES]; - return path; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-docscan/javascript/pom.xml b/maven/cn1-ai-mlkit-docscan/javascript/pom.xml deleted file mode 100644 index 459f61c8265..00000000000 --- a/maven/cn1-ai-mlkit-docscan/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-docscan - 8.0-SNAPSHOT - - - cn1-ai-mlkit-docscan-javascript - jar - Codename One AI: cn1-ai-mlkit-docscan-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-docscan-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-docscan/javascript/src/main/javascript/com_codename1_ai_mlkit_docscan_NativeDocumentScanner.js b/maven/cn1-ai-mlkit-docscan/javascript/src/main/javascript/com_codename1_ai_mlkit_docscan_NativeDocumentScanner.js deleted file mode 100644 index a57708226b0..00000000000 --- a/maven/cn1-ai-mlkit-docscan/javascript/src/main/javascript/com_codename1_ai_mlkit_docscan_NativeDocumentScanner.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.scanToFile__byte_1ARRAY = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_docscan_NativeDocumentScanner= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-docscan/javase/pom.xml b/maven/cn1-ai-mlkit-docscan/javase/pom.xml deleted file mode 100644 index f85a17983ea..00000000000 --- a/maven/cn1-ai-mlkit-docscan/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-docscan - 8.0-SNAPSHOT - - - cn1-ai-mlkit-docscan-javase - jar - Codename One AI: cn1-ai-mlkit-docscan-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-docscan-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-docscan/javase/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScannerImpl.java b/maven/cn1-ai-mlkit-docscan/javase/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScannerImpl.java deleted file mode 100644 index 573928cc411..00000000000 --- a/maven/cn1-ai-mlkit-docscan/javase/src/main/java/com/codename1/ai/mlkit/docscan/NativeDocumentScannerImpl.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.codename1.ai.mlkit.docscan; - -public class NativeDocumentScannerImpl implements NativeDocumentScanner { - public String scanToFile(byte[] imageBytes) { - try { - java.io.File f = java.io.File.createTempFile("docscan-stub-", ".jpg"); - java.nio.file.Files.write(f.toPath(), imageBytes); - return f.getAbsolutePath(); - } catch (java.io.IOException ioe) { - return ""; - } - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-docscan/lib/pom.xml b/maven/cn1-ai-mlkit-docscan/lib/pom.xml deleted file mode 100644 index 42e7ecbeb82..00000000000 --- a/maven/cn1-ai-mlkit-docscan/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-docscan - 8.0-SNAPSHOT - - - cn1-ai-mlkit-docscan-lib - pom - Codename One AI: cn1-ai-mlkit-docscan-lib - - - - com.codenameone - cn1-ai-mlkit-docscan-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-docscan-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-docscan-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-docscan-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-docscan-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-docscan/pom.xml b/maven/cn1-ai-mlkit-docscan/pom.xml deleted file mode 100644 index 4fc73d70b6f..00000000000 --- a/maven/cn1-ai-mlkit-docscan/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-docscan - pom - Codename One AI: cn1-ai-mlkit-docscan - ML Kit / VisionKit Document Scanner - - - cn1-ai-mlkit-docscan - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-face/android/pom.xml b/maven/cn1-ai-mlkit-face/android/pom.xml deleted file mode 100644 index e6d31b07afb..00000000000 --- a/maven/cn1-ai-mlkit-face/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-face - 8.0-SNAPSHOT - - - cn1-ai-mlkit-face-android - jar - Codename One AI: cn1-ai-mlkit-face-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-face-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-face/android/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetectorImpl.java b/maven/cn1-ai-mlkit-face/android/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetectorImpl.java deleted file mode 100644 index 959577903bc..00000000000 --- a/maven/cn1-ai-mlkit-face/android/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetectorImpl.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.codename1.ai.mlkit.face; - - -public class NativeFaceDetectorImpl { - public int[] detect(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new int[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.face.FaceDetector det = - com.google.mlkit.vision.face.FaceDetection.getClient( - new com.google.mlkit.vision.face.FaceDetectorOptions.Builder().build()); - final java.util.List rs = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - det.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - java.util.List>() { - public void onSuccess(java.util.List faces) { - for (com.google.mlkit.vision.face.Face f : faces) { - android.graphics.Rect r = f.getBoundingBox(); - rs.add(new int[]{r.left, r.top, r.width(), r.height()}); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - int[] flat = new int[rs.size() * 4]; - int i = 0; - for (int[] r : rs) { System.arraycopy(r, 0, flat, i, 4); i += 4; } - return flat; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-face/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-face/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-face/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-face/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-face/common/codenameone_library_required.properties deleted file mode 100644 index 7232c084e6a..00000000000 --- a/maven/cn1-ai-mlkit-face/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-face. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/FaceDetection -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:face-detection:16.1.5' diff --git a/maven/cn1-ai-mlkit-face/common/pom.xml b/maven/cn1-ai-mlkit-face/common/pom.xml deleted file mode 100644 index 987f71784b5..00000000000 --- a/maven/cn1-ai-mlkit-face/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-face - 8.0-SNAPSHOT - - - cn1-ai-mlkit-face-common - jar - Codename One AI: cn1-ai-mlkit-face-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/FaceDetector.java b/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/FaceDetector.java deleted file mode 100644 index a73e47dc29e..00000000000 --- a/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/FaceDetector.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.face; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Face Detection. -/// -/// Detects faces in images and returns bounding boxes. -/// Bridges to `MLKitFaceDetection` on iOS and -/// `com.google.mlkit:face-detection` on Android. -/// -public final class FaceDetector { - private FaceDetector() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeFaceDetector bridge = NativeLookup.create(NativeFaceDetector.class); - return bridge != null && bridge.isSupported(); - } - - /// Returns an array of face bounding-box quadruples - /// (x, y, width, height) packed as `int[4 * n]`. - public static AsyncResource detect(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativeFaceDetector bridge = NativeLookup.create(NativeFaceDetector.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("FaceDetector.detect is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final int[] r = bridge.detect(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new int[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("FaceDetector.detect failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/package-info.java b/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/package-info.java deleted file mode 100644 index b6ce855ece6..00000000000 --- a/maven/cn1-ai-mlkit-face/common/src/main/java/com/codename1/ai/mlkit/face/package-info.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit Face Detection. -/// -/// Detects faces in images and returns bounding boxes. -/// Bridges to `MLKitFaceDetection` on iOS and -/// `com.google.mlkit:face-detection` on Android. -/// -/// The single public class in this package is [FaceDetector], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeFaceDetector` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `FaceDetector.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.face; diff --git a/maven/cn1-ai-mlkit-face/common/src/test/java/com/codename1/ai/mlkit/face/FaceDetectorTest.java b/maven/cn1-ai-mlkit-face/common/src/test/java/com/codename1/ai/mlkit/face/FaceDetectorTest.java deleted file mode 100644 index 718c8fdf837..00000000000 --- a/maven/cn1-ai-mlkit-face/common/src/test/java/com/codename1/ai/mlkit/face/FaceDetectorTest.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.codename1.ai.mlkit.face; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class FaceDetectorTest { - - /** Mock implementation of NativeFaceDetector for headless JVM tests. */ - static class MockBridge implements NativeFaceDetector { - boolean supported = true; - public boolean isSupported() { return supported; } - public int[] detect(byte[] imageBytes) { - return new int[]{1, 2, 3, 4, 5, 6, 7, 8}; - } - } - - @Test - void mock_bridge_returns_two_faces() { - MockBridge b = new MockBridge(); - int[] r = b.detect(new byte[]{1}); - assertEquals(8, r.length); - } -} diff --git a/maven/cn1-ai-mlkit-face/ios/pom.xml b/maven/cn1-ai-mlkit-face/ios/pom.xml deleted file mode 100644 index f7187d44e74..00000000000 --- a/maven/cn1-ai-mlkit-face/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-face - 8.0-SNAPSHOT - - - cn1-ai-mlkit-face-ios - jar - Codename One AI: cn1-ai-mlkit-face-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-face-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-face/ios/src/main/objectivec/com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.h b/maven/cn1-ai-mlkit-face/ios/src/main/objectivec/com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.h deleted file mode 100644 index 9fa4ad92523..00000000000 --- a/maven/cn1-ai-mlkit-face/ios/src/main/objectivec/com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_face_NativeFaceDetectorImpl : NSObject { -} - --(NSData*)detect:(NSData*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-face/ios/src/main/objectivec/com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.m b/maven/cn1-ai-mlkit-face/ios/src/main/objectivec/com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.m deleted file mode 100644 index df6c511fdb2..00000000000 --- a/maven/cn1-ai-mlkit-face/ios/src/main/objectivec/com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.m +++ /dev/null @@ -1,36 +0,0 @@ -#import "com_codename1_ai_mlkit_face_NativeFaceDetectorImpl.h" -#import -#import -#import -#import - -@implementation com_codename1_ai_mlkit_face_NativeFaceDetectorImpl - --(NSData*)detect:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [NSData data]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKFaceDetectorOptions *opts = [[MLKFaceDetectorOptions alloc] init]; - MLKFaceDetector *det = [MLKFaceDetector faceDetectorWithOptions:opts]; - __block NSArray *faces = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [det processImage:vision completion:^(NSArray * _Nullable f, NSError * _Nullable e) { - faces = f ?: @[]; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - NSMutableData *out = [NSMutableData data]; - for (MLKFace *face in faces) { - CGRect r = face.frame; - int32_t v[4] = { htonl((int32_t)r.origin.x), htonl((int32_t)r.origin.y), - htonl((int32_t)r.size.width), htonl((int32_t)r.size.height) }; - [out appendBytes:v length:sizeof(v)]; - } - return out; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-face/javascript/pom.xml b/maven/cn1-ai-mlkit-face/javascript/pom.xml deleted file mode 100644 index b0517a4dd53..00000000000 --- a/maven/cn1-ai-mlkit-face/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-face - 8.0-SNAPSHOT - - - cn1-ai-mlkit-face-javascript - jar - Codename One AI: cn1-ai-mlkit-face-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-face-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-face/javascript/src/main/javascript/com_codename1_ai_mlkit_face_NativeFaceDetector.js b/maven/cn1-ai-mlkit-face/javascript/src/main/javascript/com_codename1_ai_mlkit_face_NativeFaceDetector.js deleted file mode 100644 index 26de1134e7a..00000000000 --- a/maven/cn1-ai-mlkit-face/javascript/src/main/javascript/com_codename1_ai_mlkit_face_NativeFaceDetector.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.detect__byte_1ARRAY = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_face_NativeFaceDetector= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-face/javase/pom.xml b/maven/cn1-ai-mlkit-face/javase/pom.xml deleted file mode 100644 index 425750cadfe..00000000000 --- a/maven/cn1-ai-mlkit-face/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-face - 8.0-SNAPSHOT - - - cn1-ai-mlkit-face-javase - jar - Codename One AI: cn1-ai-mlkit-face-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-face-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-face/javase/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetectorImpl.java b/maven/cn1-ai-mlkit-face/javase/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetectorImpl.java deleted file mode 100644 index f9ab7c094cb..00000000000 --- a/maven/cn1-ai-mlkit-face/javase/src/main/java/com/codename1/ai/mlkit/face/NativeFaceDetectorImpl.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.codename1.ai.mlkit.face; - -public class NativeFaceDetectorImpl implements NativeFaceDetector { - - private static boolean hintsEnsured; - private static synchronized void ensureSimulatorHints() { - if (hintsEnsured) return; - hintsEnsured = true; - java.util.Map hints = - com.codename1.ui.Display.getInstance().getProjectBuildHints(); - if (hints == null) return; // not running in the simulator - if (!hints.containsKey("ios.NSCameraUsageDescription")) { - com.codename1.ui.Display.getInstance() - .setProjectBuildHint("ios.NSCameraUsageDescription", "This app uses the camera to detect faces."); - } - } - - public NativeFaceDetectorImpl() { - ensureSimulatorHints(); - } - public int[] detect(byte[] imageBytes) { - // Deterministic 1-face stub for simulator runs. - if (imageBytes == null || imageBytes.length == 0) return new int[0]; - return new int[]{10, 20, 100, 120}; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-face/lib/pom.xml b/maven/cn1-ai-mlkit-face/lib/pom.xml deleted file mode 100644 index 19f8679b8c4..00000000000 --- a/maven/cn1-ai-mlkit-face/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-face - 8.0-SNAPSHOT - - - cn1-ai-mlkit-face-lib - pom - Codename One AI: cn1-ai-mlkit-face-lib - - - - com.codenameone - cn1-ai-mlkit-face-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-face-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-face-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-face-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-face-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-face/pom.xml b/maven/cn1-ai-mlkit-face/pom.xml deleted file mode 100644 index cfab4a1dc37..00000000000 --- a/maven/cn1-ai-mlkit-face/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-face - pom - Codename One AI: cn1-ai-mlkit-face - ML Kit Face Detection - - - cn1-ai-mlkit-face - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-labeling/android/pom.xml b/maven/cn1-ai-mlkit-labeling/android/pom.xml deleted file mode 100644 index 259b93fbdfe..00000000000 --- a/maven/cn1-ai-mlkit-labeling/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-labeling - 8.0-SNAPSHOT - - - cn1-ai-mlkit-labeling-android - jar - Codename One AI: cn1-ai-mlkit-labeling-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-labeling-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-labeling/android/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabelerImpl.java b/maven/cn1-ai-mlkit-labeling/android/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabelerImpl.java deleted file mode 100644 index 484bd451305..00000000000 --- a/maven/cn1-ai-mlkit-labeling/android/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabelerImpl.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.codename1.ai.mlkit.labeling; - - -public class NativeImageLabelerImpl { - public String[] label(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new String[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.label.ImageLabeler labeler = - com.google.mlkit.vision.label.ImageLabeling.getClient( - com.google.mlkit.vision.label.defaults.ImageLabelerOptions.DEFAULT_OPTIONS); - final java.util.List out = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - labeler.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - java.util.List>() { - public void onSuccess(java.util.List rs) { - for (com.google.mlkit.vision.label.ImageLabel l : rs) out.add(l.getText()); - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.toArray(new String[0]); - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-labeling/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-labeling/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-labeling/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-labeling/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-labeling/common/codenameone_library_required.properties deleted file mode 100644 index e70e9c425cd..00000000000 --- a/maven/cn1-ai-mlkit-labeling/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-labeling. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/ImageLabeling -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:image-labeling:17.0.7' diff --git a/maven/cn1-ai-mlkit-labeling/common/pom.xml b/maven/cn1-ai-mlkit-labeling/common/pom.xml deleted file mode 100644 index 1d9a1b6adb5..00000000000 --- a/maven/cn1-ai-mlkit-labeling/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-labeling - 8.0-SNAPSHOT - - - cn1-ai-mlkit-labeling-common - jar - Codename One AI: cn1-ai-mlkit-labeling-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/ImageLabeler.java b/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/ImageLabeler.java deleted file mode 100644 index 7a634b32411..00000000000 --- a/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/ImageLabeler.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.labeling; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Image Labeling. -/// -/// Returns descriptive labels for the contents of an image. -/// Bridges to `MLKitImageLabeling` on iOS and -/// `com.google.mlkit:image-labeling` on Android. -/// -public final class ImageLabeler { - private ImageLabeler() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeImageLabeler bridge = NativeLookup.create(NativeImageLabeler.class); - return bridge != null && bridge.isSupported(); - } - - public static AsyncResource label(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativeImageLabeler bridge = NativeLookup.create(NativeImageLabeler.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("ImageLabeler.label is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String[] r = bridge.label(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new String[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("ImageLabeler.label failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/package-info.java b/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/package-info.java deleted file mode 100644 index da8f0a06a14..00000000000 --- a/maven/cn1-ai-mlkit-labeling/common/src/main/java/com/codename1/ai/mlkit/labeling/package-info.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit Image Labeling. -/// -/// Returns descriptive labels for the contents of an image. -/// Bridges to `MLKitImageLabeling` on iOS and -/// `com.google.mlkit:image-labeling` on Android. -/// -/// The single public class in this package is [ImageLabeler], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeImageLabeler` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `ImageLabeler.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.labeling; diff --git a/maven/cn1-ai-mlkit-labeling/common/src/test/java/com/codename1/ai/mlkit/labeling/ImageLabelerTest.java b/maven/cn1-ai-mlkit-labeling/common/src/test/java/com/codename1/ai/mlkit/labeling/ImageLabelerTest.java deleted file mode 100644 index 900ceb7acdb..00000000000 --- a/maven/cn1-ai-mlkit-labeling/common/src/test/java/com/codename1/ai/mlkit/labeling/ImageLabelerTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.mlkit.labeling; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class ImageLabelerTest { - - /** Mock implementation of NativeImageLabeler for headless JVM tests. */ - static class MockBridge implements NativeImageLabeler { - boolean supported = true; - public boolean isSupported() { return supported; } - public String[] label(byte[] imageBytes) { return new String[]{"a", "b"}; } - } - - @Test - void mock_bridge_returns_labels() { - MockBridge b = new MockBridge(); - assertArrayEquals(new String[]{"a", "b"}, b.label(new byte[]{1})); - } -} diff --git a/maven/cn1-ai-mlkit-labeling/ios/pom.xml b/maven/cn1-ai-mlkit-labeling/ios/pom.xml deleted file mode 100644 index b943abc66bd..00000000000 --- a/maven/cn1-ai-mlkit-labeling/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-labeling - 8.0-SNAPSHOT - - - cn1-ai-mlkit-labeling-ios - jar - Codename One AI: cn1-ai-mlkit-labeling-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-labeling-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-labeling/ios/src/main/objectivec/com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.h b/maven/cn1-ai-mlkit-labeling/ios/src/main/objectivec/com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.h deleted file mode 100644 index a0cb28db0b2..00000000000 --- a/maven/cn1-ai-mlkit-labeling/ios/src/main/objectivec/com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl : NSObject { -} - --(NSData*)label:(NSData*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-labeling/ios/src/main/objectivec/com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.m b/maven/cn1-ai-mlkit-labeling/ios/src/main/objectivec/com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.m deleted file mode 100644 index 3b686a66c0a..00000000000 --- a/maven/cn1-ai-mlkit-labeling/ios/src/main/objectivec/com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.m +++ /dev/null @@ -1,45 +0,0 @@ -#import "com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl.h" -#import -#import -#import -#import -#import - -@implementation com_codename1_ai_mlkit_labeling_NativeImageLabelerImpl - --(NSData*)label:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [self packStrings:@[]]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKImageLabelerOptions *opts = [[MLKImageLabelerOptions alloc] init]; - MLKImageLabeler *labeler = [MLKImageLabeler imageLabelerWithOptions:opts]; - __block NSArray *labels = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [labeler processImage:vision completion:^(NSArray * _Nullable r, NSError * _Nullable e) { - labels = r ?: @[]; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - NSMutableArray *m = [NSMutableArray array]; - for (MLKImageLabel *l in labels) { if (l.text) [m addObject:l.text]; } - return [self packStrings:m]; -} - --(NSData*)packStrings:(NSArray *)strings { - NSMutableData *out = [NSMutableData data]; - uint32_t count = htonl((uint32_t)strings.count); - [out appendBytes:&count length:sizeof(count)]; - for (NSString *s in strings) { - NSData *u = [s dataUsingEncoding:NSUTF8StringEncoding]; - uint32_t len = htonl((uint32_t)u.length); - [out appendBytes:&len length:sizeof(len)]; - [out appendData:u]; - } - return out; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-labeling/javascript/pom.xml b/maven/cn1-ai-mlkit-labeling/javascript/pom.xml deleted file mode 100644 index 51273cbb35f..00000000000 --- a/maven/cn1-ai-mlkit-labeling/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-labeling - 8.0-SNAPSHOT - - - cn1-ai-mlkit-labeling-javascript - jar - Codename One AI: cn1-ai-mlkit-labeling-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-labeling-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-labeling/javascript/src/main/javascript/com_codename1_ai_mlkit_labeling_NativeImageLabeler.js b/maven/cn1-ai-mlkit-labeling/javascript/src/main/javascript/com_codename1_ai_mlkit_labeling_NativeImageLabeler.js deleted file mode 100644 index a3fab1d15ba..00000000000 --- a/maven/cn1-ai-mlkit-labeling/javascript/src/main/javascript/com_codename1_ai_mlkit_labeling_NativeImageLabeler.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.label__byte_1ARRAY = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_labeling_NativeImageLabeler= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-labeling/javase/pom.xml b/maven/cn1-ai-mlkit-labeling/javase/pom.xml deleted file mode 100644 index a240f1b8d07..00000000000 --- a/maven/cn1-ai-mlkit-labeling/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-labeling - 8.0-SNAPSHOT - - - cn1-ai-mlkit-labeling-javase - jar - Codename One AI: cn1-ai-mlkit-labeling-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-labeling-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-labeling/javase/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabelerImpl.java b/maven/cn1-ai-mlkit-labeling/javase/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabelerImpl.java deleted file mode 100644 index de44d817bc8..00000000000 --- a/maven/cn1-ai-mlkit-labeling/javase/src/main/java/com/codename1/ai/mlkit/labeling/NativeImageLabelerImpl.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.codename1.ai.mlkit.labeling; - -public class NativeImageLabelerImpl implements NativeImageLabeler { - public String[] label(byte[] imageBytes) { - if (imageBytes == null || imageBytes.length == 0) return new String[0]; - return new String[]{"object", "stub", "simulator"}; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-labeling/lib/pom.xml b/maven/cn1-ai-mlkit-labeling/lib/pom.xml deleted file mode 100644 index cf99170089d..00000000000 --- a/maven/cn1-ai-mlkit-labeling/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-labeling - 8.0-SNAPSHOT - - - cn1-ai-mlkit-labeling-lib - pom - Codename One AI: cn1-ai-mlkit-labeling-lib - - - - com.codenameone - cn1-ai-mlkit-labeling-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-labeling-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-labeling-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-labeling-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-labeling-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-labeling/pom.xml b/maven/cn1-ai-mlkit-labeling/pom.xml deleted file mode 100644 index f9842d84290..00000000000 --- a/maven/cn1-ai-mlkit-labeling/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-labeling - pom - Codename One AI: cn1-ai-mlkit-labeling - ML Kit Image Labeling - - - cn1-ai-mlkit-labeling - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-langid/android/pom.xml b/maven/cn1-ai-mlkit-langid/android/pom.xml deleted file mode 100644 index 7b6b78367b6..00000000000 --- a/maven/cn1-ai-mlkit-langid/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-langid - 8.0-SNAPSHOT - - - cn1-ai-mlkit-langid-android - jar - Codename One AI: cn1-ai-mlkit-langid-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-langid-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-langid/android/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifierImpl.java b/maven/cn1-ai-mlkit-langid/android/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifierImpl.java deleted file mode 100644 index 56a88ba9106..00000000000 --- a/maven/cn1-ai-mlkit-langid/android/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifierImpl.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.codename1.ai.mlkit.langid; - - -public class NativeLanguageIdentifierImpl { - public String identify(String input) { - com.google.mlkit.nl.languageid.LanguageIdentifier id = - com.google.mlkit.nl.languageid.LanguageIdentification.getClient(); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference("und"); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - id.identifyLanguage(input) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener() { - public void onSuccess(String s) { if (s != null) out.set(s); latch.countDown(); } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-langid/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-langid/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-langid/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-langid/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-langid/common/codenameone_library_required.properties deleted file mode 100644 index 049e4fa23ec..00000000000 --- a/maven/cn1-ai-mlkit-langid/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-langid. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/LanguageID -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:language-id:17.0.6' diff --git a/maven/cn1-ai-mlkit-langid/common/pom.xml b/maven/cn1-ai-mlkit-langid/common/pom.xml deleted file mode 100644 index f6097b7208c..00000000000 --- a/maven/cn1-ai-mlkit-langid/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-langid - 8.0-SNAPSHOT - - - cn1-ai-mlkit-langid-common - jar - Codename One AI: cn1-ai-mlkit-langid-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/LanguageIdentifier.java b/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/LanguageIdentifier.java deleted file mode 100644 index d0ad135a207..00000000000 --- a/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/LanguageIdentifier.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.langid; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Language Identification. -/// -/// Identifies the language of a given text string. -/// -public final class LanguageIdentifier { - private LanguageIdentifier() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeLanguageIdentifier bridge = NativeLookup.create(NativeLanguageIdentifier.class); - return bridge != null && bridge.isSupported(); - } - - public static AsyncResource identify(final String input) { - final AsyncResource out = new AsyncResource(); - final NativeLanguageIdentifier bridge = NativeLookup.create(NativeLanguageIdentifier.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("LanguageIdentifier.identify is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String r = bridge.identify(input); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? "" : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("LanguageIdentifier.identify failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifier.java b/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifier.java deleted file mode 100644 index 21cf9a5a350..00000000000 --- a/maven/cn1-ai-mlkit-langid/common/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifier.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.langid; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [LanguageIdentifier]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeLanguageIdentifier extends NativeInterface { - String identify(String input); -} diff --git a/maven/cn1-ai-mlkit-langid/common/src/test/java/com/codename1/ai/mlkit/langid/LanguageIdentifierTest.java b/maven/cn1-ai-mlkit-langid/common/src/test/java/com/codename1/ai/mlkit/langid/LanguageIdentifierTest.java deleted file mode 100644 index 571211dec60..00000000000 --- a/maven/cn1-ai-mlkit-langid/common/src/test/java/com/codename1/ai/mlkit/langid/LanguageIdentifierTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.mlkit.langid; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class LanguageIdentifierTest { - - /** Mock implementation of NativeLanguageIdentifier for headless JVM tests. */ - static class MockBridge implements NativeLanguageIdentifier { - boolean supported = true; - public boolean isSupported() { return supported; } - public String identify(String input) { return "en"; } - } - - @Test - void mock_identifies_english() { - MockBridge b = new MockBridge(); - assertEquals("en", b.identify("hello")); - } -} diff --git a/maven/cn1-ai-mlkit-langid/ios/pom.xml b/maven/cn1-ai-mlkit-langid/ios/pom.xml deleted file mode 100644 index 1209768a42b..00000000000 --- a/maven/cn1-ai-mlkit-langid/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-langid - 8.0-SNAPSHOT - - - cn1-ai-mlkit-langid-ios - jar - Codename One AI: cn1-ai-mlkit-langid-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-langid-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-langid/ios/src/main/objectivec/com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.h b/maven/cn1-ai-mlkit-langid/ios/src/main/objectivec/com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.h deleted file mode 100644 index 921b3831354..00000000000 --- a/maven/cn1-ai-mlkit-langid/ios/src/main/objectivec/com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl : NSObject { -} - --(NSString*)identify:(NSString*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-langid/ios/src/main/objectivec/com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.m b/maven/cn1-ai-mlkit-langid/ios/src/main/objectivec/com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.m deleted file mode 100644 index 88c6de4cf81..00000000000 --- a/maven/cn1-ai-mlkit-langid/ios/src/main/objectivec/com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.m +++ /dev/null @@ -1,23 +0,0 @@ -#import "com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl.h" -#import -#import - -@implementation com_codename1_ai_mlkit_langid_NativeLanguageIdentifierImpl - --(NSString*)identify:(NSString*)param { - MLKLanguageIdentification *id = [MLKLanguageIdentification languageIdentification]; - __block NSString *result = @"und"; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [id identifyLanguageForText:param completion:^(NSString * _Nullable lang, NSError * _Nullable e) { - if (lang) result = lang; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-langid/javascript/pom.xml b/maven/cn1-ai-mlkit-langid/javascript/pom.xml deleted file mode 100644 index 5a976e413c8..00000000000 --- a/maven/cn1-ai-mlkit-langid/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-langid - 8.0-SNAPSHOT - - - cn1-ai-mlkit-langid-javascript - jar - Codename One AI: cn1-ai-mlkit-langid-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-langid-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-langid/javascript/src/main/javascript/com_codename1_ai_mlkit_langid_NativeLanguageIdentifier.js b/maven/cn1-ai-mlkit-langid/javascript/src/main/javascript/com_codename1_ai_mlkit_langid_NativeLanguageIdentifier.js deleted file mode 100644 index 76582a049e0..00000000000 --- a/maven/cn1-ai-mlkit-langid/javascript/src/main/javascript/com_codename1_ai_mlkit_langid_NativeLanguageIdentifier.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.identify__java_lang_String = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_langid_NativeLanguageIdentifier= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-langid/javase/pom.xml b/maven/cn1-ai-mlkit-langid/javase/pom.xml deleted file mode 100644 index b0b91e3b73e..00000000000 --- a/maven/cn1-ai-mlkit-langid/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-langid - 8.0-SNAPSHOT - - - cn1-ai-mlkit-langid-javase - jar - Codename One AI: cn1-ai-mlkit-langid-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-langid-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-langid/javase/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifierImpl.java b/maven/cn1-ai-mlkit-langid/javase/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifierImpl.java deleted file mode 100644 index d0b384aa78d..00000000000 --- a/maven/cn1-ai-mlkit-langid/javase/src/main/java/com/codename1/ai/mlkit/langid/NativeLanguageIdentifierImpl.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.codename1.ai.mlkit.langid; - -public class NativeLanguageIdentifierImpl implements NativeLanguageIdentifier { - public String identify(String input) { - // Crude language ID stub for simulator. - if (input == null || input.isEmpty()) return "und"; - return "en"; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-langid/lib/pom.xml b/maven/cn1-ai-mlkit-langid/lib/pom.xml deleted file mode 100644 index 40d01b6dfcd..00000000000 --- a/maven/cn1-ai-mlkit-langid/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-langid - 8.0-SNAPSHOT - - - cn1-ai-mlkit-langid-lib - pom - Codename One AI: cn1-ai-mlkit-langid-lib - - - - com.codenameone - cn1-ai-mlkit-langid-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-langid-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-langid-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-langid-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-langid-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-langid/pom.xml b/maven/cn1-ai-mlkit-langid/pom.xml deleted file mode 100644 index f4d79b963a2..00000000000 --- a/maven/cn1-ai-mlkit-langid/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-langid - pom - Codename One AI: cn1-ai-mlkit-langid - ML Kit Language Identification - - - cn1-ai-mlkit-langid - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-pose/android/pom.xml b/maven/cn1-ai-mlkit-pose/android/pom.xml deleted file mode 100644 index b86a5bc62bf..00000000000 --- a/maven/cn1-ai-mlkit-pose/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-pose - 8.0-SNAPSHOT - - - cn1-ai-mlkit-pose-android - jar - Codename One AI: cn1-ai-mlkit-pose-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-pose-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-pose/android/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetectorImpl.java b/maven/cn1-ai-mlkit-pose/android/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetectorImpl.java deleted file mode 100644 index 1e5fc6b2ab8..00000000000 --- a/maven/cn1-ai-mlkit-pose/android/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetectorImpl.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.codename1.ai.mlkit.pose; - - -public class NativePoseDetectorImpl { - public float[] detect(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new float[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.pose.PoseDetector det = - com.google.mlkit.vision.pose.PoseDetection.getClient( - new com.google.mlkit.vision.pose.defaults.PoseDetectorOptions.Builder().build()); - final float[] out = new float[33 * 3]; - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - det.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.vision.pose.Pose>() { - public void onSuccess(com.google.mlkit.vision.pose.Pose p) { - java.util.List lms = p.getAllPoseLandmarks(); - for (int i = 0; i < 33 && i < lms.size(); i++) { - com.google.mlkit.vision.pose.PoseLandmark lm = lms.get(i); - out[i * 3] = lm.getPosition().x; - out[i * 3 + 1] = lm.getPosition().y; - out[i * 3 + 2] = lm.getInFrameLikelihood(); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-pose/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-pose/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-pose/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-pose/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-pose/common/codenameone_library_required.properties deleted file mode 100644 index 3f7f45a924c..00000000000 --- a/maven/cn1-ai-mlkit-pose/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-pose. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/PoseDetection -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:pose-detection:18.0.0-beta3' diff --git a/maven/cn1-ai-mlkit-pose/common/pom.xml b/maven/cn1-ai-mlkit-pose/common/pom.xml deleted file mode 100644 index 8defd866f57..00000000000 --- a/maven/cn1-ai-mlkit-pose/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-pose - 8.0-SNAPSHOT - - - cn1-ai-mlkit-pose-common - jar - Codename One AI: cn1-ai-mlkit-pose-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/PoseDetector.java b/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/PoseDetector.java deleted file mode 100644 index a273d87f561..00000000000 --- a/maven/cn1-ai-mlkit-pose/common/src/main/java/com/codename1/ai/mlkit/pose/PoseDetector.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.pose; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Pose Detection. -/// -/// Returns skeletal landmarks for human bodies detected in an image. -/// -public final class PoseDetector { - private PoseDetector() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativePoseDetector bridge = NativeLookup.create(NativePoseDetector.class); - return bridge != null && bridge.isSupported(); - } - - /// Returns 33 landmark triples (x,y,confidence) per detected pose - /// packed as `float[3 * 33]`. - public static AsyncResource detect(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativePoseDetector bridge = NativeLookup.create(NativePoseDetector.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("PoseDetector.detect is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final float[] r = bridge.detect(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new float[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("PoseDetector.detect failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-pose/common/src/test/java/com/codename1/ai/mlkit/pose/PoseDetectorTest.java b/maven/cn1-ai-mlkit-pose/common/src/test/java/com/codename1/ai/mlkit/pose/PoseDetectorTest.java deleted file mode 100644 index ba4f4a8f63a..00000000000 --- a/maven/cn1-ai-mlkit-pose/common/src/test/java/com/codename1/ai/mlkit/pose/PoseDetectorTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.mlkit.pose; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class PoseDetectorTest { - - /** Mock implementation of NativePoseDetector for headless JVM tests. */ - static class MockBridge implements NativePoseDetector { - boolean supported = true; - public boolean isSupported() { return supported; } - public float[] detect(byte[] imageBytes) { return new float[99]; } - } - - @Test - void mock_returns_33_landmarks() { - MockBridge b = new MockBridge(); - assertEquals(99, b.detect(new byte[]{1}).length); - } -} diff --git a/maven/cn1-ai-mlkit-pose/ios/pom.xml b/maven/cn1-ai-mlkit-pose/ios/pom.xml deleted file mode 100644 index 58cfd6232b1..00000000000 --- a/maven/cn1-ai-mlkit-pose/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-pose - 8.0-SNAPSHOT - - - cn1-ai-mlkit-pose-ios - jar - Codename One AI: cn1-ai-mlkit-pose-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-pose-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-pose/ios/src/main/objectivec/com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.h b/maven/cn1-ai-mlkit-pose/ios/src/main/objectivec/com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.h deleted file mode 100644 index 551df98fe57..00000000000 --- a/maven/cn1-ai-mlkit-pose/ios/src/main/objectivec/com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_pose_NativePoseDetectorImpl : NSObject { -} - --(NSData*)detect:(NSData*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-pose/ios/src/main/objectivec/com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.m b/maven/cn1-ai-mlkit-pose/ios/src/main/objectivec/com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.m deleted file mode 100644 index e6682ff6b38..00000000000 --- a/maven/cn1-ai-mlkit-pose/ios/src/main/objectivec/com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.m +++ /dev/null @@ -1,39 +0,0 @@ -#import "com_codename1_ai_mlkit_pose_NativePoseDetectorImpl.h" -#import -#import -#import -#import - -@implementation com_codename1_ai_mlkit_pose_NativePoseDetectorImpl - --(NSData*)detect:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [NSData data]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKPoseDetectorOptions *opts = [[MLKPoseDetectorOptions alloc] init]; - MLKPoseDetector *det = [MLKPoseDetector poseDetectorWithOptions:opts]; - __block MLKPose *pose = nil; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [det processImage:vision completion:^(NSArray * _Nullable r, NSError * _Nullable e) { - if (r.count > 0) pose = r[0]; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - float buf[99] = {0}; - if (pose) { - for (NSInteger i = 0; i < 33 && i < pose.landmarks.count; i++) { - MLKPoseLandmark *lm = pose.landmarks[i]; - buf[i * 3] = (float)lm.position.x; - buf[i * 3 + 1] = (float)lm.position.y; - buf[i * 3 + 2] = (float)lm.inFrameLikelihood; - } - } - // Pack as big-endian float bytes (matches JAVA_ARRAY_FLOAT on iOS port). - return [NSData dataWithBytes:buf length:sizeof(buf)]; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-pose/javascript/pom.xml b/maven/cn1-ai-mlkit-pose/javascript/pom.xml deleted file mode 100644 index 12d00e75017..00000000000 --- a/maven/cn1-ai-mlkit-pose/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-pose - 8.0-SNAPSHOT - - - cn1-ai-mlkit-pose-javascript - jar - Codename One AI: cn1-ai-mlkit-pose-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-pose-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-pose/javascript/src/main/javascript/com_codename1_ai_mlkit_pose_NativePoseDetector.js b/maven/cn1-ai-mlkit-pose/javascript/src/main/javascript/com_codename1_ai_mlkit_pose_NativePoseDetector.js deleted file mode 100644 index f04d324c889..00000000000 --- a/maven/cn1-ai-mlkit-pose/javascript/src/main/javascript/com_codename1_ai_mlkit_pose_NativePoseDetector.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.detect__byte_1ARRAY = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_pose_NativePoseDetector= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-pose/javase/pom.xml b/maven/cn1-ai-mlkit-pose/javase/pom.xml deleted file mode 100644 index 93bdaa9a5fc..00000000000 --- a/maven/cn1-ai-mlkit-pose/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-pose - 8.0-SNAPSHOT - - - cn1-ai-mlkit-pose-javase - jar - Codename One AI: cn1-ai-mlkit-pose-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-pose-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-pose/javase/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetectorImpl.java b/maven/cn1-ai-mlkit-pose/javase/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetectorImpl.java deleted file mode 100644 index ac05c406fbb..00000000000 --- a/maven/cn1-ai-mlkit-pose/javase/src/main/java/com/codename1/ai/mlkit/pose/NativePoseDetectorImpl.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.codename1.ai.mlkit.pose; - -public class NativePoseDetectorImpl implements NativePoseDetector { - public float[] detect(byte[] imageBytes) { - float[] out = new float[99]; - for (int i = 0; i < 33; i++) { out[i * 3] = i; out[i * 3 + 2] = 0.5f; } - return out; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-pose/lib/pom.xml b/maven/cn1-ai-mlkit-pose/lib/pom.xml deleted file mode 100644 index dce8cf9e244..00000000000 --- a/maven/cn1-ai-mlkit-pose/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-pose - 8.0-SNAPSHOT - - - cn1-ai-mlkit-pose-lib - pom - Codename One AI: cn1-ai-mlkit-pose-lib - - - - com.codenameone - cn1-ai-mlkit-pose-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-pose-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-pose-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-pose-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-pose-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-pose/pom.xml b/maven/cn1-ai-mlkit-pose/pom.xml deleted file mode 100644 index 59b1266fa79..00000000000 --- a/maven/cn1-ai-mlkit-pose/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-pose - pom - Codename One AI: cn1-ai-mlkit-pose - ML Kit Pose Detection - - - cn1-ai-mlkit-pose - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-segmentation/android/pom.xml b/maven/cn1-ai-mlkit-segmentation/android/pom.xml deleted file mode 100644 index 3c1eab0d7a5..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-segmentation - 8.0-SNAPSHOT - - - cn1-ai-mlkit-segmentation-android - jar - Codename One AI: cn1-ai-mlkit-segmentation-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-segmentation-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-segmentation/android/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenterImpl.java b/maven/cn1-ai-mlkit-segmentation/android/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenterImpl.java deleted file mode 100644 index a832c4c323d..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/android/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenterImpl.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.codename1.ai.mlkit.segmentation; - - -public class NativeSelfieSegmenterImpl { - public byte[] segment(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new byte[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.segmentation.Segmenter seg = - com.google.mlkit.vision.segmentation.Segmentation.getClient( - new com.google.mlkit.vision.segmentation.SelfieSegmenterOptions.Builder() - .setDetectorMode( - com.google.mlkit.vision.segmentation.SelfieSegmenterOptions.SINGLE_IMAGE_MODE) - .build()); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference(new byte[0]); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - seg.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.vision.segmentation.SegmentationMask>() { - public void onSuccess(com.google.mlkit.vision.segmentation.SegmentationMask mask) { - int w = mask.getWidth(), h = mask.getHeight(); - java.nio.ByteBuffer buf = mask.getBuffer(); - buf.rewind(); - byte[] outb = new byte[w * h]; - for (int i = 0; i < w * h; i++) { - float v = buf.getFloat(); - outb[i] = (byte)(int)(v * 255); - } - out.set(outb); - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-segmentation/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-segmentation/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-segmentation/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-segmentation/common/codenameone_library_required.properties deleted file mode 100644 index 29e4a86796d..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-segmentation. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/SegmentationSelfie -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:segmentation-selfie:16.0.0-beta5' diff --git a/maven/cn1-ai-mlkit-segmentation/common/pom.xml b/maven/cn1-ai-mlkit-segmentation/common/pom.xml deleted file mode 100644 index 42a389f1738..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-segmentation - 8.0-SNAPSHOT - - - cn1-ai-mlkit-segmentation-common - jar - Codename One AI: cn1-ai-mlkit-segmentation-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/SelfieSegmenter.java b/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/SelfieSegmenter.java deleted file mode 100644 index 5af60615dd5..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/SelfieSegmenter.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.segmentation; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Selfie Segmentation. -/// -/// Returns a per-pixel mask separating a person in the foreground from the background. -/// -public final class SelfieSegmenter { - private SelfieSegmenter() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeSelfieSegmenter bridge = NativeLookup.create(NativeSelfieSegmenter.class); - return bridge != null && bridge.isSupported(); - } - - /// Returns a per-pixel mask separating foreground (person) from - /// background as `byte[width * height]` (0=background, 255=foreground). - public static AsyncResource segment(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativeSelfieSegmenter bridge = NativeLookup.create(NativeSelfieSegmenter.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("SelfieSegmenter.segment is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final byte[] r = bridge.segment(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new byte[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("SelfieSegmenter.segment failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/package-info.java b/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/package-info.java deleted file mode 100644 index 3f12b066b68..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/common/src/main/java/com/codename1/ai/mlkit/segmentation/package-info.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit Selfie Segmentation. -/// -/// Returns a per-pixel mask separating a person in the foreground from the background. -/// -/// The single public class in this package is [SelfieSegmenter], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeSelfieSegmenter` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `SelfieSegmenter.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.segmentation; diff --git a/maven/cn1-ai-mlkit-segmentation/common/src/test/java/com/codename1/ai/mlkit/segmentation/SelfieSegmenterTest.java b/maven/cn1-ai-mlkit-segmentation/common/src/test/java/com/codename1/ai/mlkit/segmentation/SelfieSegmenterTest.java deleted file mode 100644 index 6ff2b587824..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/common/src/test/java/com/codename1/ai/mlkit/segmentation/SelfieSegmenterTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.mlkit.segmentation; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class SelfieSegmenterTest { - - /** Mock implementation of NativeSelfieSegmenter for headless JVM tests. */ - static class MockBridge implements NativeSelfieSegmenter { - boolean supported = true; - public boolean isSupported() { return supported; } - public byte[] segment(byte[] imageBytes) { return new byte[16]; } - } - - @Test - void mock_returns_mask_bytes() { - MockBridge b = new MockBridge(); - assertEquals(16, b.segment(new byte[]{1}).length); - } -} diff --git a/maven/cn1-ai-mlkit-segmentation/ios/pom.xml b/maven/cn1-ai-mlkit-segmentation/ios/pom.xml deleted file mode 100644 index 38d5951a239..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-segmentation - 8.0-SNAPSHOT - - - cn1-ai-mlkit-segmentation-ios - jar - Codename One AI: cn1-ai-mlkit-segmentation-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-segmentation-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-segmentation/ios/src/main/objectivec/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.h b/maven/cn1-ai-mlkit-segmentation/ios/src/main/objectivec/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.h deleted file mode 100644 index d9abad43f4e..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/ios/src/main/objectivec/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl : NSObject { -} - --(NSData*)segment:(NSData*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-segmentation/ios/src/main/objectivec/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.m b/maven/cn1-ai-mlkit-segmentation/ios/src/main/objectivec/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.m deleted file mode 100644 index 0274648353d..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/ios/src/main/objectivec/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.m +++ /dev/null @@ -1,48 +0,0 @@ -#import "com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl.h" -#import -#import -#import -#import -#import - -@implementation com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenterImpl - --(NSData*)segment:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [NSData data]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKSelfieSegmenterOptions *opts = [[MLKSelfieSegmenterOptions alloc] init]; - opts.segmenterMode = MLKSegmenterModeSingleImage; - MLKSegmenter *seg = [MLKSegmenter segmenterWithOptions:opts]; - __block NSData *result = [NSData data]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [seg processImage:vision completion:^(MLKSegmentationMask * _Nullable mask, NSError * _Nullable e) { - if (mask) { - // MLKSegmentationMask exposes only `buffer` (a - // CVPixelBuffer); dimensions come from CVPixelBufferGet*. - CVPixelBufferRef buf = mask.buffer; - size_t w = CVPixelBufferGetWidth(buf); - size_t h = CVPixelBufferGetHeight(buf); - CVPixelBufferLockBaseAddress(buf, kCVPixelBufferLock_ReadOnly); - void *base = CVPixelBufferGetBaseAddress(buf); - NSMutableData *m = [NSMutableData dataWithLength:w * h]; - uint8_t *out = m.mutableBytes; - float *src = (float *)base; - for (size_t i = 0; i < w * h; i++) { - float v = src[i]; - out[i] = (uint8_t)(v * 255.0f); - } - CVPixelBufferUnlockBaseAddress(buf, kCVPixelBufferLock_ReadOnly); - result = m; - } - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-segmentation/javascript/pom.xml b/maven/cn1-ai-mlkit-segmentation/javascript/pom.xml deleted file mode 100644 index 797cc8c41c6..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-segmentation - 8.0-SNAPSHOT - - - cn1-ai-mlkit-segmentation-javascript - jar - Codename One AI: cn1-ai-mlkit-segmentation-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-segmentation-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-segmentation/javascript/src/main/javascript/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenter.js b/maven/cn1-ai-mlkit-segmentation/javascript/src/main/javascript/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenter.js deleted file mode 100644 index 2da08051d23..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/javascript/src/main/javascript/com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenter.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.segment__byte_1ARRAY = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_segmentation_NativeSelfieSegmenter= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-segmentation/javase/pom.xml b/maven/cn1-ai-mlkit-segmentation/javase/pom.xml deleted file mode 100644 index 933039325a9..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-segmentation - 8.0-SNAPSHOT - - - cn1-ai-mlkit-segmentation-javase - jar - Codename One AI: cn1-ai-mlkit-segmentation-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-segmentation-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-segmentation/javase/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenterImpl.java b/maven/cn1-ai-mlkit-segmentation/javase/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenterImpl.java deleted file mode 100644 index edd293eed28..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/javase/src/main/java/com/codename1/ai/mlkit/segmentation/NativeSelfieSegmenterImpl.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.codename1.ai.mlkit.segmentation; - -public class NativeSelfieSegmenterImpl implements NativeSelfieSegmenter { - public byte[] segment(byte[] imageBytes) { - // 8x8 checkerboard stub. - byte[] out = new byte[64]; - for (int i = 0; i < 64; i++) out[i] = (byte)(((i / 8) + (i % 8)) % 2 == 0 ? 255 : 0); - return out; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-segmentation/lib/pom.xml b/maven/cn1-ai-mlkit-segmentation/lib/pom.xml deleted file mode 100644 index bbe5e0bd225..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-segmentation - 8.0-SNAPSHOT - - - cn1-ai-mlkit-segmentation-lib - pom - Codename One AI: cn1-ai-mlkit-segmentation-lib - - - - com.codenameone - cn1-ai-mlkit-segmentation-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-segmentation-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-segmentation-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-segmentation-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-segmentation-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-segmentation/pom.xml b/maven/cn1-ai-mlkit-segmentation/pom.xml deleted file mode 100644 index 25dcd7294fe..00000000000 --- a/maven/cn1-ai-mlkit-segmentation/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-segmentation - pom - Codename One AI: cn1-ai-mlkit-segmentation - ML Kit Selfie Segmentation - - - cn1-ai-mlkit-segmentation - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-smartreply/android/pom.xml b/maven/cn1-ai-mlkit-smartreply/android/pom.xml deleted file mode 100644 index 2006c1b9b4a..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-smartreply - 8.0-SNAPSHOT - - - cn1-ai-mlkit-smartreply-android - jar - Codename One AI: cn1-ai-mlkit-smartreply-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-smartreply-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-smartreply/android/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReplyImpl.java b/maven/cn1-ai-mlkit-smartreply/android/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReplyImpl.java deleted file mode 100644 index f94269fe1e4..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/android/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReplyImpl.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.codename1.ai.mlkit.smartreply; - - -public class NativeSmartReplyImpl { - public String[] suggest(String conversationJson) { - java.util.List msgs = - new java.util.ArrayList(); - try { - org.json.JSONArray a = new org.json.JSONArray(conversationJson); - for (int i = 0; i < a.length(); i++) { - org.json.JSONObject o = a.getJSONObject(i); - String role = o.optString("role", "user"); - long ts = o.optLong("timestamp", 0); - String text = o.optString("message", ""); - String userId = o.optString("userId", "u"); - if ("user".equals(role)) { - msgs.add(com.google.mlkit.nl.smartreply.TextMessage.createForLocalUser(text, ts)); - } else { - msgs.add(com.google.mlkit.nl.smartreply.TextMessage.createForRemoteUser(text, ts, userId)); - } - } - } catch (org.json.JSONException jex) { - return new String[0]; - } - final java.util.List out = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - com.google.mlkit.nl.smartreply.SmartReplyGenerator gen = - com.google.mlkit.nl.smartreply.SmartReply.getClient(); - gen.suggestReplies(msgs) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.nl.smartreply.SmartReplySuggestionResult>() { - public void onSuccess(com.google.mlkit.nl.smartreply.SmartReplySuggestionResult r) { - for (com.google.mlkit.nl.smartreply.SmartReplySuggestion s : r.getSuggestions()) { - out.add(s.getText()); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.toArray(new String[0]); - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-smartreply/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-smartreply/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-smartreply/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-smartreply/common/codenameone_library_required.properties deleted file mode 100644 index c02fc2c10be..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-smartreply. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/SmartReply -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:smart-reply:17.0.4' diff --git a/maven/cn1-ai-mlkit-smartreply/common/pom.xml b/maven/cn1-ai-mlkit-smartreply/common/pom.xml deleted file mode 100644 index cfa4154d85f..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-smartreply - 8.0-SNAPSHOT - - - cn1-ai-mlkit-smartreply-common - jar - Codename One AI: cn1-ai-mlkit-smartreply-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReply.java b/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReply.java deleted file mode 100644 index 2161f2610eb..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReply.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.smartreply; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [SmartReply]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeSmartReply extends NativeInterface { - String[] suggest(String conversationJson); -} diff --git a/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/SmartReply.java b/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/SmartReply.java deleted file mode 100644 index e6330b1f3f5..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/SmartReply.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.smartreply; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Smart Reply. -/// -/// Generates short reply suggestions for chat conversations on-device. -/// -public final class SmartReply { - private SmartReply() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeSmartReply bridge = NativeLookup.create(NativeSmartReply.class); - return bridge != null && bridge.isSupported(); - } - - public static AsyncResource suggest(final String input) { - final AsyncResource out = new AsyncResource(); - final NativeSmartReply bridge = NativeLookup.create(NativeSmartReply.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("SmartReply.suggest is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String[] r = bridge.suggest(input); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new String[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("SmartReply.suggest failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/package-info.java b/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/package-info.java deleted file mode 100644 index e31db21b55d..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/common/src/main/java/com/codename1/ai/mlkit/smartreply/package-info.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit Smart Reply. -/// -/// Generates short reply suggestions for chat conversations on-device. -/// -/// The single public class in this package is [SmartReply], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeSmartReply` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `SmartReply.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.smartreply; diff --git a/maven/cn1-ai-mlkit-smartreply/common/src/test/java/com/codename1/ai/mlkit/smartreply/SmartReplyTest.java b/maven/cn1-ai-mlkit-smartreply/common/src/test/java/com/codename1/ai/mlkit/smartreply/SmartReplyTest.java deleted file mode 100644 index 59f881fd758..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/common/src/test/java/com/codename1/ai/mlkit/smartreply/SmartReplyTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.mlkit.smartreply; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class SmartReplyTest { - - /** Mock implementation of NativeSmartReply for headless JVM tests. */ - static class MockBridge implements NativeSmartReply { - boolean supported = true; - public boolean isSupported() { return supported; } - public String[] suggest(String c) { return new String[]{"ok"}; } - } - - @Test - void mock_returns_single_suggestion() { - MockBridge b = new MockBridge(); - assertEquals(1, b.suggest("[]").length); - } -} diff --git a/maven/cn1-ai-mlkit-smartreply/ios/pom.xml b/maven/cn1-ai-mlkit-smartreply/ios/pom.xml deleted file mode 100644 index 29ecb190185..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-smartreply - 8.0-SNAPSHOT - - - cn1-ai-mlkit-smartreply-ios - jar - Codename One AI: cn1-ai-mlkit-smartreply-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-smartreply-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-smartreply/ios/src/main/objectivec/com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.h b/maven/cn1-ai-mlkit-smartreply/ios/src/main/objectivec/com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.h deleted file mode 100644 index 5ac5c16fea2..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/ios/src/main/objectivec/com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl : NSObject { -} - --(NSData*)suggest:(NSString*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-smartreply/ios/src/main/objectivec/com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.m b/maven/cn1-ai-mlkit-smartreply/ios/src/main/objectivec/com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.m deleted file mode 100644 index b5dd8ba2115..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/ios/src/main/objectivec/com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.m +++ /dev/null @@ -1,61 +0,0 @@ -#import "com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl.h" -#import -#import -#import - -@implementation com_codename1_ai_mlkit_smartreply_NativeSmartReplyImpl - --(NSData*)suggest:(NSString*)param { - // param is a JSON array of {role,message,timestamp,userId}. - NSError *err = nil; - NSArray *items = [NSJSONSerialization JSONObjectWithData: - [param dataUsingEncoding:NSUTF8StringEncoding] - options:0 error:&err]; - NSMutableArray *messages = [NSMutableArray array]; - if ([items isKindOfClass:[NSArray class]]) { - for (NSDictionary *d in items) { - if (![d isKindOfClass:[NSDictionary class]]) continue; - NSString *role = d[@"role"] ?: @"user"; - BOOL isLocalUser = [role isEqualToString:@"user"]; - NSString *text = d[@"message"] ?: @""; - NSNumber *ts = d[@"timestamp"] ?: @0; - MLKTextMessage *m = [[MLKTextMessage alloc] - initWithText:text timestamp:[ts doubleValue] - userID:(d[@"userId"] ?: @"u") - isLocalUser:isLocalUser]; - [messages addObject:m]; - } - } - __block NSArray *out = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [[MLKSmartReply smartReply] suggestRepliesForMessages:messages - completion:^(MLKSmartReplySuggestionResult * _Nullable result, NSError * _Nullable e) { - NSMutableArray *m = [NSMutableArray array]; - for (MLKSmartReplySuggestion *s in result.suggestions ?: @[]) { - if (s.text) [m addObject:s.text]; - } - out = m; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return [self packStrings:out]; -} - --(NSData*)packStrings:(NSArray *)strings { - NSMutableData *out = [NSMutableData data]; - uint32_t count = htonl((uint32_t)strings.count); - [out appendBytes:&count length:sizeof(count)]; - for (NSString *s in strings) { - NSData *u = [s dataUsingEncoding:NSUTF8StringEncoding]; - uint32_t len = htonl((uint32_t)u.length); - [out appendBytes:&len length:sizeof(len)]; - [out appendData:u]; - } - return out; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-smartreply/javascript/pom.xml b/maven/cn1-ai-mlkit-smartreply/javascript/pom.xml deleted file mode 100644 index 7c4de49b0f3..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-smartreply - 8.0-SNAPSHOT - - - cn1-ai-mlkit-smartreply-javascript - jar - Codename One AI: cn1-ai-mlkit-smartreply-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-smartreply-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-smartreply/javascript/src/main/javascript/com_codename1_ai_mlkit_smartreply_NativeSmartReply.js b/maven/cn1-ai-mlkit-smartreply/javascript/src/main/javascript/com_codename1_ai_mlkit_smartreply_NativeSmartReply.js deleted file mode 100644 index 9d0a9c0a594..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/javascript/src/main/javascript/com_codename1_ai_mlkit_smartreply_NativeSmartReply.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.suggest__java_lang_String = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_smartreply_NativeSmartReply= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-smartreply/javase/pom.xml b/maven/cn1-ai-mlkit-smartreply/javase/pom.xml deleted file mode 100644 index de53fe91c38..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-smartreply - 8.0-SNAPSHOT - - - cn1-ai-mlkit-smartreply-javase - jar - Codename One AI: cn1-ai-mlkit-smartreply-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-smartreply-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-smartreply/javase/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReplyImpl.java b/maven/cn1-ai-mlkit-smartreply/javase/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReplyImpl.java deleted file mode 100644 index 0190707fdad..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/javase/src/main/java/com/codename1/ai/mlkit/smartreply/NativeSmartReplyImpl.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.codename1.ai.mlkit.smartreply; - -public class NativeSmartReplyImpl implements NativeSmartReply { - public String[] suggest(String conversationJson) { - return new String[]{"Sounds good", "Thanks!", "Got it"}; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-smartreply/lib/pom.xml b/maven/cn1-ai-mlkit-smartreply/lib/pom.xml deleted file mode 100644 index b5d8bf097c6..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-smartreply - 8.0-SNAPSHOT - - - cn1-ai-mlkit-smartreply-lib - pom - Codename One AI: cn1-ai-mlkit-smartreply-lib - - - - com.codenameone - cn1-ai-mlkit-smartreply-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-smartreply-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-smartreply-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-smartreply-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-smartreply-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-smartreply/pom.xml b/maven/cn1-ai-mlkit-smartreply/pom.xml deleted file mode 100644 index 1bd869f426e..00000000000 --- a/maven/cn1-ai-mlkit-smartreply/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-smartreply - pom - Codename One AI: cn1-ai-mlkit-smartreply - ML Kit Smart Reply - - - cn1-ai-mlkit-smartreply - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-text/android/pom.xml b/maven/cn1-ai-mlkit-text/android/pom.xml deleted file mode 100644 index 4997f4164f7..00000000000 --- a/maven/cn1-ai-mlkit-text/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-text - 8.0-SNAPSHOT - - - cn1-ai-mlkit-text-android - jar - Codename One AI: cn1-ai-mlkit-text-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-text-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-text/android/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizerImpl.java b/maven/cn1-ai-mlkit-text/android/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizerImpl.java deleted file mode 100644 index b3fa0212158..00000000000 --- a/maven/cn1-ai-mlkit-text/android/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizerImpl.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.codename1.ai.mlkit.text; - - -public class NativeTextRecognizerImpl { - public String recognize(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return ""; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.text.TextRecognizer rec = - com.google.mlkit.vision.text.TextRecognition.getClient( - com.google.mlkit.vision.text.latin.TextRecognizerOptions.DEFAULT_OPTIONS); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference(""); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - rec.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.vision.text.Text>() { - public void onSuccess(com.google.mlkit.vision.text.Text t) { - out.set(t.getText() == null ? "" : t.getText()); - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-text/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-text/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-text/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-text/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-text/common/codenameone_library_required.properties deleted file mode 100644 index ba65d8437d0..00000000000 --- a/maven/cn1-ai-mlkit-text/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-text. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/TextRecognition -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:text-recognition:16.0.0' diff --git a/maven/cn1-ai-mlkit-text/common/pom.xml b/maven/cn1-ai-mlkit-text/common/pom.xml deleted file mode 100644 index cbe95b4cb73..00000000000 --- a/maven/cn1-ai-mlkit-text/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-text - 8.0-SNAPSHOT - - - cn1-ai-mlkit-text-common - jar - Codename One AI: cn1-ai-mlkit-text-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizer.java b/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizer.java deleted file mode 100644 index 80203101333..00000000000 --- a/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizer.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.text; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [TextRecognizer]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeTextRecognizer extends NativeInterface { - String recognize(byte[] imageBytes); -} diff --git a/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/TextRecognizer.java b/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/TextRecognizer.java deleted file mode 100644 index ac689727410..00000000000 --- a/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/TextRecognizer.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.text; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit Text Recognition (OCR). -/// -/// Extracts text strings from images entirely on-device via Google's ML Kit. -/// Bridges to `GoogleMLKit/TextRecognition` on iOS and -/// `com.google.mlkit:text-recognition` on Android. -/// -public final class TextRecognizer { - private TextRecognizer() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeTextRecognizer bridge = NativeLookup.create(NativeTextRecognizer.class); - return bridge != null && bridge.isSupported(); - } - - /// Runs OCR on the supplied image bytes (JPEG or PNG). Completes with - /// the recognised text. Empty image -> empty string. No text -> empty - /// string. Hard errors fire `AsyncResource.error(...)`. - public static AsyncResource recognize(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - if (imageBytes == null || imageBytes.length == 0) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(""); } - }); - return out; - } - final NativeTextRecognizer bridge = NativeLookup.create(NativeTextRecognizer.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException( - "TextRecognizer.recognize is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String result = bridge.recognize(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(result == null ? "" : result); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("TextRecognizer.recognize failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/package-info.java b/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/package-info.java deleted file mode 100644 index 06da696570b..00000000000 --- a/maven/cn1-ai-mlkit-text/common/src/main/java/com/codename1/ai/mlkit/text/package-info.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit Text Recognition (OCR). -/// -/// Extracts text strings from images entirely on-device via Google's ML Kit. -/// Bridges to `GoogleMLKit/TextRecognition` on iOS and -/// `com.google.mlkit:text-recognition` on Android. -/// -/// The single public class in this package is [TextRecognizer], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeTextRecognizer` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `TextRecognizer.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.text; diff --git a/maven/cn1-ai-mlkit-text/common/src/test/java/com/codename1/ai/mlkit/text/TextRecognizerTest.java b/maven/cn1-ai-mlkit-text/common/src/test/java/com/codename1/ai/mlkit/text/TextRecognizerTest.java deleted file mode 100644 index d5e7cbde6f2..00000000000 --- a/maven/cn1-ai-mlkit-text/common/src/test/java/com/codename1/ai/mlkit/text/TextRecognizerTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.codename1.ai.mlkit.text; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class TextRecognizerTest { - - /** Mock implementation of NativeTextRecognizer for headless JVM tests. */ - static class MockBridge implements NativeTextRecognizer { - boolean supported = true; - public boolean isSupported() { return supported; } - String response = "hello"; - public String recognize(byte[] imageBytes) { - if (imageBytes == null) throw new NullPointerException(); - return response; - } - } - - @Test - void bridge_returns_canned_string() { - MockBridge b = new MockBridge(); - assertEquals("hello", b.recognize(new byte[]{1, 2, 3})); - } - - @Test - void bridge_reports_supported() { - MockBridge b = new MockBridge(); - assertTrue(b.isSupported()); - } - - @Test - void bridge_rejects_null_input() { - MockBridge b = new MockBridge(); - assertThrows(NullPointerException.class, () -> b.recognize(null)); - } -} diff --git a/maven/cn1-ai-mlkit-text/ios/pom.xml b/maven/cn1-ai-mlkit-text/ios/pom.xml deleted file mode 100644 index ad02c249737..00000000000 --- a/maven/cn1-ai-mlkit-text/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-text - 8.0-SNAPSHOT - - - cn1-ai-mlkit-text-ios - jar - Codename One AI: cn1-ai-mlkit-text-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-text-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-text/ios/src/main/objectivec/com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.h b/maven/cn1-ai-mlkit-text/ios/src/main/objectivec/com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.h deleted file mode 100644 index 70bce8ba280..00000000000 --- a/maven/cn1-ai-mlkit-text/ios/src/main/objectivec/com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_text_NativeTextRecognizerImpl : NSObject { -} - --(NSString*)recognize:(NSData*)param; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-text/ios/src/main/objectivec/com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.m b/maven/cn1-ai-mlkit-text/ios/src/main/objectivec/com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.m deleted file mode 100644 index b3bac461d66..00000000000 --- a/maven/cn1-ai-mlkit-text/ios/src/main/objectivec/com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.m +++ /dev/null @@ -1,33 +0,0 @@ -#import "com_codename1_ai_mlkit_text_NativeTextRecognizerImpl.h" -#import -#import -#import -#import - -@implementation com_codename1_ai_mlkit_text_NativeTextRecognizerImpl - --(NSString*)recognize:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return @""; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKTextRecognizerOptions *opts = [[MLKTextRecognizerOptions alloc] init]; - MLKTextRecognizer *recognizer = [MLKTextRecognizer textRecognizerWithOptions:opts]; - __block NSString *result = @""; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [recognizer processImage:vision completion:^(MLKText * _Nullable text, NSError * _Nullable err) { - if (text && !err) { - result = text.text ?: @""; - } else if (err) { - result = @""; - } - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-text/javascript/pom.xml b/maven/cn1-ai-mlkit-text/javascript/pom.xml deleted file mode 100644 index 03df3541787..00000000000 --- a/maven/cn1-ai-mlkit-text/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-text - 8.0-SNAPSHOT - - - cn1-ai-mlkit-text-javascript - jar - Codename One AI: cn1-ai-mlkit-text-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-text-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-text/javascript/src/main/javascript/com_codename1_ai_mlkit_text_NativeTextRecognizer.js b/maven/cn1-ai-mlkit-text/javascript/src/main/javascript/com_codename1_ai_mlkit_text_NativeTextRecognizer.js deleted file mode 100644 index de4cb7d3e12..00000000000 --- a/maven/cn1-ai-mlkit-text/javascript/src/main/javascript/com_codename1_ai_mlkit_text_NativeTextRecognizer.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.recognize__byte_1ARRAY = function(param1, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_text_NativeTextRecognizer= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-text/javase/pom.xml b/maven/cn1-ai-mlkit-text/javase/pom.xml deleted file mode 100644 index 1b2dc7c3783..00000000000 --- a/maven/cn1-ai-mlkit-text/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-text - 8.0-SNAPSHOT - - - cn1-ai-mlkit-text-javase - jar - Codename One AI: cn1-ai-mlkit-text-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-text-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-text/javase/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizerImpl.java b/maven/cn1-ai-mlkit-text/javase/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizerImpl.java deleted file mode 100644 index 97802c2f919..00000000000 --- a/maven/cn1-ai-mlkit-text/javase/src/main/java/com/codename1/ai/mlkit/text/NativeTextRecognizerImpl.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.codename1.ai.mlkit.text; - -public class NativeTextRecognizerImpl implements NativeTextRecognizer { - - private static boolean hintsEnsured; - private static synchronized void ensureSimulatorHints() { - if (hintsEnsured) return; - hintsEnsured = true; - java.util.Map hints = - com.codename1.ui.Display.getInstance().getProjectBuildHints(); - if (hints == null) return; // not running in the simulator - if (!hints.containsKey("ios.NSCameraUsageDescription")) { - com.codename1.ui.Display.getInstance() - .setProjectBuildHint("ios.NSCameraUsageDescription", "This app uses the camera to recognise text."); - } - } - - public NativeTextRecognizerImpl() { - ensureSimulatorHints(); - } - public String recognize(byte[] imageBytes) { - if (imageBytes == null || imageBytes.length == 0) return ""; - return "[mlkit-text simulator stub] " + imageBytes.length + " bytes"; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-text/lib/pom.xml b/maven/cn1-ai-mlkit-text/lib/pom.xml deleted file mode 100644 index f5d94e33e9c..00000000000 --- a/maven/cn1-ai-mlkit-text/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-text - 8.0-SNAPSHOT - - - cn1-ai-mlkit-text-lib - pom - Codename One AI: cn1-ai-mlkit-text-lib - - - - com.codenameone - cn1-ai-mlkit-text-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-text-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-text-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-text-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-text-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-text/pom.xml b/maven/cn1-ai-mlkit-text/pom.xml deleted file mode 100644 index 6552a73bba3..00000000000 --- a/maven/cn1-ai-mlkit-text/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-text - pom - Codename One AI: cn1-ai-mlkit-text - ML Kit Text Recognition (OCR) - - - cn1-ai-mlkit-text - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-mlkit-translate/android/pom.xml b/maven/cn1-ai-mlkit-translate/android/pom.xml deleted file mode 100644 index 3e9154f5b2d..00000000000 --- a/maven/cn1-ai-mlkit-translate/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-translate - 8.0-SNAPSHOT - - - cn1-ai-mlkit-translate-android - jar - Codename One AI: cn1-ai-mlkit-translate-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-translate-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-translate/android/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslatorImpl.java b/maven/cn1-ai-mlkit-translate/android/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslatorImpl.java deleted file mode 100644 index 8ee87b28c7a..00000000000 --- a/maven/cn1-ai-mlkit-translate/android/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslatorImpl.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.codename1.ai.mlkit.translate; - - -public class NativeTranslatorImpl { - public String translate(String text, String sourceLang, String targetLang) { - com.google.mlkit.nl.translate.TranslatorOptions opts = - new com.google.mlkit.nl.translate.TranslatorOptions.Builder() - .setSourceLanguage(sourceLang) - .setTargetLanguage(targetLang) - .build(); - com.google.mlkit.nl.translate.Translator t = - com.google.mlkit.nl.translate.Translation.getClient(opts); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference(""); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - t.downloadModelIfNeeded() - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener() { - public void onSuccess(Void v) { - t.translate(text) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener() { - public void onSuccess(String r) { out.set(r); latch.countDown(); } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-translate/common/codenameone_library_appended.properties b/maven/cn1-ai-mlkit-translate/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-mlkit-translate/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-mlkit-translate/common/codenameone_library_required.properties b/maven/cn1-ai-mlkit-translate/common/codenameone_library_required.properties deleted file mode 100644 index 758e2eca244..00000000000 --- a/maven/cn1-ai-mlkit-translate/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-mlkit-translate. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=GoogleMLKit/Translate -codename1.arg.android.gradleDep=implementation 'com.google.mlkit:translate:17.0.3' diff --git a/maven/cn1-ai-mlkit-translate/common/pom.xml b/maven/cn1-ai-mlkit-translate/common/pom.xml deleted file mode 100644 index 04aa7fecf2b..00000000000 --- a/maven/cn1-ai-mlkit-translate/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-translate - 8.0-SNAPSHOT - - - cn1-ai-mlkit-translate-common - jar - Codename One AI: cn1-ai-mlkit-translate-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslator.java b/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslator.java deleted file mode 100644 index 84c232bd8fe..00000000000 --- a/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslator.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.translate; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [Translator]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeTranslator extends NativeInterface { - String translate(String text, String sourceLang, String targetLang); -} diff --git a/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/Translator.java b/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/Translator.java deleted file mode 100644 index 0a8fc8aac05..00000000000 --- a/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/Translator.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.mlkit.translate; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// ML Kit on-device Translation. -/// -/// Translates short text between language pairs entirely on-device. -/// -public final class Translator { - private Translator() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeTranslator bridge = NativeLookup.create(NativeTranslator.class); - return bridge != null && bridge.isSupported(); - } - - public static AsyncResource translate(final String text, - final String sourceLang, - final String targetLang) { - final AsyncResource out = new AsyncResource(); - final NativeTranslator bridge = NativeLookup.create(NativeTranslator.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("Translator.translate is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String r = bridge.translate(text, sourceLang, targetLang); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? "" : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("Translator.translate failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/package-info.java b/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/package-info.java deleted file mode 100644 index f0509240c9a..00000000000 --- a/maven/cn1-ai-mlkit-translate/common/src/main/java/com/codename1/ai/mlkit/translate/package-info.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// ML Kit on-device Translation. -/// -/// Translates short text between language pairs entirely on-device. -/// -/// The single public class in this package is [Translator], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeTranslator` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `Translator.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.mlkit.translate; diff --git a/maven/cn1-ai-mlkit-translate/common/src/test/java/com/codename1/ai/mlkit/translate/TranslatorTest.java b/maven/cn1-ai-mlkit-translate/common/src/test/java/com/codename1/ai/mlkit/translate/TranslatorTest.java deleted file mode 100644 index 4299d695667..00000000000 --- a/maven/cn1-ai-mlkit-translate/common/src/test/java/com/codename1/ai/mlkit/translate/TranslatorTest.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.codename1.ai.mlkit.translate; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class TranslatorTest { - - /** Mock implementation of NativeTranslator for headless JVM tests. */ - static class MockBridge implements NativeTranslator { - boolean supported = true; - public boolean isSupported() { return supported; } - public String translate(String text, String sourceLang, String targetLang) { - return text + "@" + sourceLang + "->" + targetLang; - } - } - - @Test - void mock_translate_round_trip() { - MockBridge b = new MockBridge(); - assertEquals("hi@en->fr", b.translate("hi", "en", "fr")); - } -} diff --git a/maven/cn1-ai-mlkit-translate/ios/pom.xml b/maven/cn1-ai-mlkit-translate/ios/pom.xml deleted file mode 100644 index 6fbc9ae87cb..00000000000 --- a/maven/cn1-ai-mlkit-translate/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-translate - 8.0-SNAPSHOT - - - cn1-ai-mlkit-translate-ios - jar - Codename One AI: cn1-ai-mlkit-translate-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-translate-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-translate/ios/src/main/objectivec/com_codename1_ai_mlkit_translate_NativeTranslatorImpl.h b/maven/cn1-ai-mlkit-translate/ios/src/main/objectivec/com_codename1_ai_mlkit_translate_NativeTranslatorImpl.h deleted file mode 100644 index 437112805f1..00000000000 --- a/maven/cn1-ai-mlkit-translate/ios/src/main/objectivec/com_codename1_ai_mlkit_translate_NativeTranslatorImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_mlkit_translate_NativeTranslatorImpl : NSObject { -} - --(NSString*)translate:(NSString*)param param1:(NSString*)param1 param2:(NSString*)param2; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-mlkit-translate/ios/src/main/objectivec/com_codename1_ai_mlkit_translate_NativeTranslatorImpl.m b/maven/cn1-ai-mlkit-translate/ios/src/main/objectivec/com_codename1_ai_mlkit_translate_NativeTranslatorImpl.m deleted file mode 100644 index 56aadf63d22..00000000000 --- a/maven/cn1-ai-mlkit-translate/ios/src/main/objectivec/com_codename1_ai_mlkit_translate_NativeTranslatorImpl.m +++ /dev/null @@ -1,33 +0,0 @@ -#import "com_codename1_ai_mlkit_translate_NativeTranslatorImpl.h" -#import -#import -#import - -@implementation com_codename1_ai_mlkit_translate_NativeTranslatorImpl - --(NSString*)translate:(NSString*)param param1:(NSString*)param1 param2:(NSString*)param2 { - MLKTranslatorOptions *opts = [[MLKTranslatorOptions alloc] - initWithSourceLanguage:param1 - targetLanguage:param2]; - MLKTranslator *t = [MLKTranslator translatorWithOptions:opts]; - MLKModelDownloadConditions *cond = [[MLKModelDownloadConditions alloc] - initWithAllowsCellularAccess:YES - allowsBackgroundDownloading:YES]; - __block NSString *result = @""; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [t downloadModelIfNeededWithConditions:cond completion:^(NSError * _Nullable err) { - if (err) { dispatch_semaphore_signal(sem); return; } - [t translateText:param completion:^(NSString * _Nullable r, NSError * _Nullable e) { - if (r && !e) result = r; - dispatch_semaphore_signal(sem); - }]; - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-mlkit-translate/javascript/pom.xml b/maven/cn1-ai-mlkit-translate/javascript/pom.xml deleted file mode 100644 index 526a14678fc..00000000000 --- a/maven/cn1-ai-mlkit-translate/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-translate - 8.0-SNAPSHOT - - - cn1-ai-mlkit-translate-javascript - jar - Codename One AI: cn1-ai-mlkit-translate-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-translate-common - ${project.version} - - - diff --git a/maven/cn1-ai-mlkit-translate/javascript/src/main/javascript/com_codename1_ai_mlkit_translate_NativeTranslator.js b/maven/cn1-ai-mlkit-translate/javascript/src/main/javascript/com_codename1_ai_mlkit_translate_NativeTranslator.js deleted file mode 100644 index 3b00f5db46c..00000000000 --- a/maven/cn1-ai-mlkit-translate/javascript/src/main/javascript/com_codename1_ai_mlkit_translate_NativeTranslator.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.translate__java_lang_String_java_lang_String_java_lang_String = function(param1, param2, param3, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_mlkit_translate_NativeTranslator= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-mlkit-translate/javase/pom.xml b/maven/cn1-ai-mlkit-translate/javase/pom.xml deleted file mode 100644 index fd867181408..00000000000 --- a/maven/cn1-ai-mlkit-translate/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-translate - 8.0-SNAPSHOT - - - cn1-ai-mlkit-translate-javase - jar - Codename One AI: cn1-ai-mlkit-translate-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-mlkit-translate-common - ${project.version} - - - - diff --git a/maven/cn1-ai-mlkit-translate/javase/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslatorImpl.java b/maven/cn1-ai-mlkit-translate/javase/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslatorImpl.java deleted file mode 100644 index eef89f81388..00000000000 --- a/maven/cn1-ai-mlkit-translate/javase/src/main/java/com/codename1/ai/mlkit/translate/NativeTranslatorImpl.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.codename1.ai.mlkit.translate; - -public class NativeTranslatorImpl implements NativeTranslator { - public String translate(String text, String sourceLang, String targetLang) { - if (text == null) return ""; - return "[" + sourceLang + "->" + targetLang + "] " + text; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-mlkit-translate/lib/pom.xml b/maven/cn1-ai-mlkit-translate/lib/pom.xml deleted file mode 100644 index 37784d38cf6..00000000000 --- a/maven/cn1-ai-mlkit-translate/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-mlkit-translate - 8.0-SNAPSHOT - - - cn1-ai-mlkit-translate-lib - pom - Codename One AI: cn1-ai-mlkit-translate-lib - - - - com.codenameone - cn1-ai-mlkit-translate-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-mlkit-translate-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-mlkit-translate-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-mlkit-translate-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-mlkit-translate-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-mlkit-translate/pom.xml b/maven/cn1-ai-mlkit-translate/pom.xml deleted file mode 100644 index e7dc9267bde..00000000000 --- a/maven/cn1-ai-mlkit-translate/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-mlkit-translate - pom - Codename One AI: cn1-ai-mlkit-translate - ML Kit on-device Translation - - - cn1-ai-mlkit-translate - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/cn1-ai-tflite/android/pom.xml b/maven/cn1-ai-tflite/android/pom.xml deleted file mode 100644 index 3e33e0aec67..00000000000 --- a/maven/cn1-ai-tflite/android/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-tflite - 8.0-SNAPSHOT - - - cn1-ai-tflite-android - jar - Codename One AI: cn1-ai-tflite-android - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-tflite-common - ${project.version} - - - - diff --git a/maven/cn1-ai-tflite/android/src/main/java/com/codename1/ai/tflite/NativeInterpreterImpl.java b/maven/cn1-ai-tflite/android/src/main/java/com/codename1/ai/tflite/NativeInterpreterImpl.java deleted file mode 100644 index 18e20fa4699..00000000000 --- a/maven/cn1-ai-tflite/android/src/main/java/com/codename1/ai/tflite/NativeInterpreterImpl.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.codename1.ai.tflite; - - -public class NativeInterpreterImpl { - public float[] run(byte[] modelBytes, float[] input, int outputLength) { - java.nio.ByteBuffer bb = java.nio.ByteBuffer.allocateDirect(modelBytes.length); - bb.order(java.nio.ByteOrder.nativeOrder()); - bb.put(modelBytes); - bb.rewind(); - org.tensorflow.lite.Interpreter interp = new org.tensorflow.lite.Interpreter(bb); - float[][] out = new float[1][outputLength]; - interp.run(new float[][]{input}, out); - interp.close(); - return out[0]; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-tflite/common/codenameone_library_appended.properties b/maven/cn1-ai-tflite/common/codenameone_library_appended.properties deleted file mode 100644 index f47bb32ba04..00000000000 --- a/maven/cn1-ai-tflite/common/codenameone_library_appended.properties +++ /dev/null @@ -1 +0,0 @@ -# Reserved for build hints appended to the consuming app's properties. diff --git a/maven/cn1-ai-tflite/common/codenameone_library_required.properties b/maven/cn1-ai-tflite/common/codenameone_library_required.properties deleted file mode 100644 index f6e175a63fe..00000000000 --- a/maven/cn1-ai-tflite/common/codenameone_library_required.properties +++ /dev/null @@ -1,6 +0,0 @@ -# Auto-installed build hints for cn1-ai-tflite. -# Loaded by the Codename One build server when this cn1lib is in the -# project classpath. The build-time AiDependencyTable scanner adds -# further per-class entries as needed. -codename1.arg.ios.pods=TensorFlowLiteObjC -codename1.arg.android.gradleDep=implementation 'org.tensorflow:tensorflow-lite:2.14.0' diff --git a/maven/cn1-ai-tflite/common/pom.xml b/maven/cn1-ai-tflite/common/pom.xml deleted file mode 100644 index 1a2412f35d8..00000000000 --- a/maven/cn1-ai-tflite/common/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-tflite - 8.0-SNAPSHOT - - - cn1-ai-tflite-common - jar - Codename One AI: cn1-ai-tflite-common - - - UTF-8 - 1.8 - 1.8 - - - - - com.codenameone - codenameone-core - ${project.version} - provided - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - - - com.codenameone - codenameone-maven-plugin - ${project.version} - - - build-legacy-cn1lib - package - - cn1lib - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - copy-library-required-properties - process-resources - - - - - - - - run - - - - - - - diff --git a/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/Interpreter.java b/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/Interpreter.java deleted file mode 100644 index 8fcaa4a8ddc..00000000000 --- a/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/Interpreter.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.tflite; - -import com.codename1.ai.LlmException; -import com.codename1.system.NativeLookup; -import com.codename1.ui.Display; -import com.codename1.util.AsyncResource; - -/// TensorFlow Lite on-device inference. -/// -/// Loads a `.tflite` model and runs inference against `float[]` inputs. -/// Bridges to `TensorFlowLiteObjC` on iOS and `org.tensorflow:tensorflow-lite` -/// on Android. -/// -public final class Interpreter { - private Interpreter() { } - - /// True only when the running platform has a native bridge wired up. - public static boolean isSupported() { - NativeInterpreter bridge = NativeLookup.create(NativeInterpreter.class); - return bridge != null && bridge.isSupported(); - } - - /// Loads a TensorFlow Lite model from the supplied bytes and runs - /// inference against a float32 input tensor. Returns the output as - /// `float[]`. The model file is held in a native handle keyed by - /// the SHA-1 of the input bytes; repeated calls reuse the loaded - /// model. - public static AsyncResource run(final byte[] modelBytes, - final float[] input, - final int outputLength) { - final AsyncResource out = new AsyncResource(); - final NativeInterpreter bridge = NativeLookup.create(NativeInterpreter.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("Interpreter.run is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final float[] r = bridge.run(modelBytes, input, outputLength); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new float[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("Interpreter.run failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } -} diff --git a/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/NativeInterpreter.java b/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/NativeInterpreter.java deleted file mode 100644 index 75bfac35172..00000000000 --- a/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/NativeInterpreter.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -package com.codename1.ai.tflite; - -import com.codename1.system.NativeInterface; - -/// Native bridge for [Interpreter]. iOS, Android, and JavaSE implementations -/// live in their respective port modules under this cn1lib. -public interface NativeInterpreter extends NativeInterface { - float[] run(byte[] modelBytes, float[] input, int outputLength); -} diff --git a/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/package-info.java b/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/package-info.java deleted file mode 100644 index 7bf52a592d0..00000000000 --- a/maven/cn1-ai-tflite/common/src/main/java/com/codename1/ai/tflite/package-info.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ - -/// TensorFlow Lite on-device inference. -/// -/// Loads a `.tflite` model and runs inference against `float[]` inputs. -/// Bridges to `TensorFlowLiteObjC` on iOS and `org.tensorflow:tensorflow-lite` -/// on Android. -/// -/// The single public class in this package is [Interpreter], which exposes -/// the feature via static methods returning -/// [com.codename1.util.AsyncResource]. A package-private -/// `NativeInterpreter` interface holds the platform contract; iOS Obj-C and -/// Android Java implementations live in `nativeios.zip` / `nativeand.zip` -/// inside the cn1lib bundle. References to `Interpreter.*` are recognised -/// by the Codename One build server's `AiDependencyTable`, which -/// auto-injects the matching CocoaPod / Swift Package / Android Gradle -/// dep / `Info.plist` usage strings / Android permissions on every -/// build -- no manual build hints required. -package com.codename1.ai.tflite; diff --git a/maven/cn1-ai-tflite/common/src/test/java/com/codename1/ai/tflite/InterpreterTest.java b/maven/cn1-ai-tflite/common/src/test/java/com/codename1/ai/tflite/InterpreterTest.java deleted file mode 100644 index 246690c5a9f..00000000000 --- a/maven/cn1-ai-tflite/common/src/test/java/com/codename1/ai/tflite/InterpreterTest.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.codename1.ai.tflite; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -public class InterpreterTest { - - /** Mock implementation of NativeInterpreter for headless JVM tests. */ - static class MockBridge implements NativeInterpreter { - boolean supported = true; - public boolean isSupported() { return supported; } - public float[] run(byte[] modelBytes, float[] input, int outputLength) { - float[] r = new float[outputLength]; - for (int i = 0; i < r.length; i++) r[i] = i; - return r; - } - } - - @Test - void mock_returns_increasing_vector() { - MockBridge b = new MockBridge(); - float[] r = b.run(new byte[0], new float[]{1f, 2f}, 4); - assertEquals(4, r.length); - assertEquals(3.0f, r[3], 1e-6); - } -} diff --git a/maven/cn1-ai-tflite/ios/pom.xml b/maven/cn1-ai-tflite/ios/pom.xml deleted file mode 100644 index a00bcd00cc4..00000000000 --- a/maven/cn1-ai-tflite/ios/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-tflite - 8.0-SNAPSHOT - - - cn1-ai-tflite-ios - jar - Codename One AI: cn1-ai-tflite-ios - - - src/main/dummy - - src/main/objectivec - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-tflite-common - ${project.version} - - - diff --git a/maven/cn1-ai-tflite/ios/src/main/objectivec/com_codename1_ai_tflite_NativeInterpreterImpl.h b/maven/cn1-ai-tflite/ios/src/main/objectivec/com_codename1_ai_tflite_NativeInterpreterImpl.h deleted file mode 100644 index f76a7e48606..00000000000 --- a/maven/cn1-ai-tflite/ios/src/main/objectivec/com_codename1_ai_tflite_NativeInterpreterImpl.h +++ /dev/null @@ -1,8 +0,0 @@ -#import - -@interface com_codename1_ai_tflite_NativeInterpreterImpl : NSObject { -} - --(NSData*)run:(NSData*)param param1:(NSData*)param1 param2:(int)param2; --(BOOL)isSupported; -@end diff --git a/maven/cn1-ai-tflite/ios/src/main/objectivec/com_codename1_ai_tflite_NativeInterpreterImpl.m b/maven/cn1-ai-tflite/ios/src/main/objectivec/com_codename1_ai_tflite_NativeInterpreterImpl.m deleted file mode 100644 index f6be4c4d80c..00000000000 --- a/maven/cn1-ai-tflite/ios/src/main/objectivec/com_codename1_ai_tflite_NativeInterpreterImpl.m +++ /dev/null @@ -1,28 +0,0 @@ -#import "com_codename1_ai_tflite_NativeInterpreterImpl.h" -#import -#import - -@implementation com_codename1_ai_tflite_NativeInterpreterImpl - --(NSData*)run:(NSData*)param param1:(NSData*)param1 param2:(int)param2 { - NSError *err = nil; - NSString *modelPath = [NSString stringWithFormat:@"%@/tflite-%@.tflite", - NSTemporaryDirectory(), [[NSUUID UUID] UUIDString]]; - [param writeToFile:modelPath atomically:YES]; - TFLInterpreter *interp = [[TFLInterpreter alloc] initWithModelPath:modelPath error:&err]; - if (err) return [NSData data]; - [interp allocateTensorsWithError:&err]; - if (err) return [NSData data]; - TFLTensor *in0 = [interp inputTensorAtIndex:0 error:&err]; - [in0 copyData:param1 error:&err]; - [interp invokeWithError:&err]; - TFLTensor *out0 = [interp outputTensorAtIndex:0 error:&err]; - NSData *outBytes = [out0 dataWithError:&err]; - return outBytes ?: [NSData data]; -} - --(BOOL)isSupported{ - return YES; -} - -@end diff --git a/maven/cn1-ai-tflite/javascript/pom.xml b/maven/cn1-ai-tflite/javascript/pom.xml deleted file mode 100644 index 89211e280d4..00000000000 --- a/maven/cn1-ai-tflite/javascript/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-tflite - 8.0-SNAPSHOT - - - cn1-ai-tflite-javascript - jar - Codename One AI: cn1-ai-tflite-javascript - - - src/main/dummy - - src/main/javascript - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-tflite-common - ${project.version} - - - diff --git a/maven/cn1-ai-tflite/javascript/src/main/javascript/com_codename1_ai_tflite_NativeInterpreter.js b/maven/cn1-ai-tflite/javascript/src/main/javascript/com_codename1_ai_tflite_NativeInterpreter.js deleted file mode 100644 index 141325d49cc..00000000000 --- a/maven/cn1-ai-tflite/javascript/src/main/javascript/com_codename1_ai_tflite_NativeInterpreter.js +++ /dev/null @@ -1,15 +0,0 @@ -(function(exports){ - -var o = {}; - - o.run__byte_1ARRAY_float_1ARRAY_int = function(param1, param2, param3, callback) { - callback.error(new Error("Not implemented yet")); - }; - - o.isSupported_ = function(callback) { - callback.complete(false); - }; - -exports.com_codename1_ai_tflite_NativeInterpreter= o; - -})(cn1_get_native_interfaces()); diff --git a/maven/cn1-ai-tflite/javase/pom.xml b/maven/cn1-ai-tflite/javase/pom.xml deleted file mode 100644 index 5f90217b0f1..00000000000 --- a/maven/cn1-ai-tflite/javase/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-tflite - 8.0-SNAPSHOT - - - cn1-ai-tflite-javase - jar - Codename One AI: cn1-ai-tflite-javase - - - 1.8 - 1.8 - - - - src/main/dummy - - src/main/java - - - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - package - - - - - - - - - - run - - - - - - - - - com.codenameone - cn1-ai-tflite-common - ${project.version} - - - - diff --git a/maven/cn1-ai-tflite/javase/src/main/java/com/codename1/ai/tflite/NativeInterpreterImpl.java b/maven/cn1-ai-tflite/javase/src/main/java/com/codename1/ai/tflite/NativeInterpreterImpl.java deleted file mode 100644 index 03f151c0868..00000000000 --- a/maven/cn1-ai-tflite/javase/src/main/java/com/codename1/ai/tflite/NativeInterpreterImpl.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.codename1.ai.tflite; - -public class NativeInterpreterImpl implements NativeInterpreter { - public float[] run(byte[] modelBytes, float[] input, int outputLength) { - // Identity stub: returns first outputLength entries of input - // (or zero-padded if input shorter). Lets simulator test plumbing. - float[] out = new float[outputLength]; - int n = Math.min(input.length, outputLength); - System.arraycopy(input, 0, out, 0, n); - return out; - } - - public boolean isSupported() { - return true; - } -} diff --git a/maven/cn1-ai-tflite/lib/pom.xml b/maven/cn1-ai-tflite/lib/pom.xml deleted file mode 100644 index 5ecb3fab5cd..00000000000 --- a/maven/cn1-ai-tflite/lib/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - 4.0.0 - - - com.codenameone - cn1-ai-tflite - 8.0-SNAPSHOT - - - cn1-ai-tflite-lib - pom - Codename One AI: cn1-ai-tflite-lib - - - - com.codenameone - cn1-ai-tflite-common - ${project.version} - - - - - - ios - - codename1.platformios - - - - com.codenameone - cn1-ai-tflite-ios - ${project.version} - - - - - android - - codename1.platformandroid - - - - com.codenameone - cn1-ai-tflite-android - ${project.version} - - - - - javase - - codename1.platformjavase - - - - com.codenameone - cn1-ai-tflite-javase - ${project.version} - - - - - javascript - - codename1.platformjavascript - - - - com.codenameone - cn1-ai-tflite-javascript - ${project.version} - - - - - diff --git a/maven/cn1-ai-tflite/pom.xml b/maven/cn1-ai-tflite/pom.xml deleted file mode 100644 index 699c6ac8d75..00000000000 --- a/maven/cn1-ai-tflite/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - com.codenameone - codenameone - 8.0-SNAPSHOT - - - cn1-ai-tflite - pom - Codename One AI: cn1-ai-tflite - TensorFlow Lite on-device inference - - - cn1-ai-tflite - 1.8 - 1.8 - 1.8 - - - - common - ios - android - javase - javascript - lib - - diff --git a/maven/codenameone-maven-plugin/pom.xml b/maven/codenameone-maven-plugin/pom.xml index 21e1f626816..4aabedc6fd1 100644 --- a/maven/codenameone-maven-plugin/pom.xml +++ b/maven/codenameone-maven-plugin/pom.xml @@ -34,6 +34,11 @@ + + ${project.groupId} + codenameone-platform-feature-catalog + ${project.version} + org.jdom jdom2 diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AiDependencyTable.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AiDependencyTable.java deleted file mode 100644 index 589034db84d..00000000000 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AiDependencyTable.java +++ /dev/null @@ -1,437 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ -package com.codename1.builders; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; - -/** - * Central registry that maps class-name prefixes used by the - * {@code com.codename1.ai.*} family (and the speech/TTS sister APIs - * in {@code com.codename1.media}) to the native dependencies and - * permissions each one requires. - * - *

The build server's class scanners ({@link IPhoneBuilder} and - * {@link AndroidGradleBuilder}) call into this table from inside their - * existing {@link Executor.ClassScanner#usesClass(String)} blocks; the - * resulting set of {@link Entry} records is then applied just before - * iOS pods / SPM are resolved and just before the Android Gradle - * dependencies / manifest fragments are written.

- * - *

Keep this table small and declarative: any class prefix whose - * needs change (different pod version, additional plist entry) should - * be edited here, not in the builder hot loop.

- */ -public final class AiDependencyTable { - - private static final List ENTRIES; - - static { - List e = new ArrayList(); - - // LLM clients: pure HTTPS. INTERNET is on by default on - // Android so no permission needed; we still register the - // entry so the scanner has a positive hit for diagnostics. - e.add(new Entry("com/codename1/ai/LlmClient") - .description("LLM client (OpenAI / Anthropic / Gemini / Ollama)")); - e.add(new Entry("com/codename1/ai/OpenAiClient").description("OpenAI client")); - e.add(new Entry("com/codename1/ai/AnthropicClient").description("Anthropic client")); - e.add(new Entry("com/codename1/ai/GeminiClient").description("Gemini client")); - - // Core speech recognition: iOS Speech framework + mic & speech plist - // strings; Android record-audio permission. The TTS API has no - // plist requirement (AVSpeech is unrestricted) and no Android - // permission (built-in). - e.add(new Entry("com/codename1/media/SpeechRecognizer") - .iosFrameworks("Speech", "AVFoundation") - .iosPlist("NSSpeechRecognitionUsageDescription", - "Used to transcribe your voice into text.") - .iosPlist("NSMicrophoneUsageDescription", - "Required to capture audio for speech recognition.") - .androidPermissions("android.permission.RECORD_AUDIO") - .description("On-device speech-to-text")); - - e.add(new Entry("com/codename1/media/TextToSpeech") - .iosFrameworks("AVFAudio") - .description("Text-to-speech")); - - // ML Kit feature submodules. Class prefix matches the - // (forward-referenced) cn1libs' package layout. - e.add(new Entry("com/codename1/ai/mlkit/text/") - .iosPod("GoogleMLKit/TextRecognition") - .androidGradle("com.google.mlkit:text-recognition:16.0.0") - .iosPlist("NSCameraUsageDescription", - "Used to recognise text from your camera.") - .description("ML Kit Text Recognition")); - - e.add(new Entry("com/codename1/ai/mlkit/barcode/") - .iosPod("GoogleMLKit/BarcodeScanning") - .androidGradle("com.google.mlkit:barcode-scanning:17.2.0") - .iosPlist("NSCameraUsageDescription", - "Used to scan barcodes with your camera.") - .androidFeatures("android.hardware.camera") - .description("ML Kit Barcode Scanning")); - - e.add(new Entry("com/codename1/ai/mlkit/face/") - .iosPod("GoogleMLKit/FaceDetection") - .androidGradle("com.google.mlkit:face-detection:16.1.5") - .iosPlist("NSCameraUsageDescription", - "Used to detect faces in images.") - .description("ML Kit Face Detection")); - - e.add(new Entry("com/codename1/ai/mlkit/labeling/") - .iosPod("GoogleMLKit/ImageLabeling") - .androidGradle("com.google.mlkit:image-labeling:17.0.7") - .description("ML Kit Image Labeling")); - - e.add(new Entry("com/codename1/ai/mlkit/translate/") - .iosPod("GoogleMLKit/Translate") - .androidGradle("com.google.mlkit:translate:17.0.1") - .description("ML Kit Translation")); - - e.add(new Entry("com/codename1/ai/mlkit/smartreply/") - .iosPod("GoogleMLKit/SmartReply") - .androidGradle("com.google.mlkit:smart-reply:17.0.2") - .description("ML Kit Smart Reply")); - - e.add(new Entry("com/codename1/ai/mlkit/langid/") - .iosPod("GoogleMLKit/LanguageID") - .androidGradle("com.google.mlkit:language-id:17.0.4") - .description("ML Kit Language ID")); - - e.add(new Entry("com/codename1/ai/mlkit/pose/") - .iosPod("GoogleMLKit/PoseDetection") - .androidGradle("com.google.mlkit:pose-detection:18.0.0-beta3") - .description("ML Kit Pose Detection")); - - e.add(new Entry("com/codename1/ai/mlkit/segmentation/") - .iosPod("GoogleMLKit/SegmentationSelfie") - .androidGradle("com.google.mlkit:segmentation-selfie:16.0.0-beta4") - .description("ML Kit Selfie Segmentation")); - - e.add(new Entry("com/codename1/ai/mlkit/docscan/") - .iosPod("GoogleMLKit/DocumentScanner") - .iosFrameworks("VisionKit") - .androidGradle("com.google.android.gms:play-services-mlkit-document-scanner:16.0.0-beta1") - .description("ML Kit Document Scanner")); - - // TFLite has both Pods and SPM publishers. We register both - // so IOSDependencyManager.resolve() can route to whichever - // the project uses. - e.add(new Entry("com/codename1/ai/tflite/") - .iosPod("TensorFlowLiteSwift") - .iosSpm("TensorFlowLiteSwift", - "https://github.com/tensorflow/tensorflow.git", - "from:2.13.0", - "TensorFlowLite") - .androidGradle("org.tensorflow:tensorflow-lite:2.13.0") - .androidGradle("org.tensorflow:tensorflow-lite-support:0.4.4") - .description("TensorFlow Lite interpreter")); - - e.add(new Entry("com/codename1/ai/whisper/") - .iosFrameworks("Accelerate") - .description("On-device Whisper transcription (libwhisper.a ships with the cn1lib)")); - - // Low-level cross-platform camera API: live preview + frame - // stream + photo + video. iOS uses AVFoundation (framework - // only, no pod); Android uses CameraX (androidx.camera) which - // is added as Gradle deps below. Just referencing classes in - // com.codename1.camera causes the build to inject the right - // permissions and plist strings; developers may still override - // the plist text via the ios.NSCameraUsageDescription build - // hint. - e.add(new Entry("com/codename1/camera/") - .iosFrameworks("AVFoundation", "CoreMedia", "CoreVideo") - .iosPlist("NSCameraUsageDescription", - "Used to capture photos and video.") - .iosPlist("NSMicrophoneUsageDescription", - "Used to capture audio for video recording.") - .androidPermissions("android.permission.CAMERA", - "android.permission.RECORD_AUDIO") - .androidFeatures("android.hardware.camera", - "android.hardware.camera.autofocus") - .androidGradle("androidx.camera:camera-core:1.3.4") - .androidGradle("androidx.camera:camera-camera2:1.3.4") - .androidGradle("androidx.camera:camera-lifecycle:1.3.4") - .androidGradle("androidx.camera:camera-view:1.3.4") - .androidGradle("androidx.camera:camera-video:1.3.4") - .description("Cross-platform camera (preview + frames + photo + video)")); - - // First-class Bluetooth (com.codename1.bluetooth.*): CoreBluetooth - // on iOS with the two privacy strings defaulted only-if-unset via - // the standard entry application. Android permissions are - // deliberately NOT listed here -- the Android 12 permission split - // needs maxSdkVersion / usesPermissionFlags attributes this table - // cannot express, so AndroidGradleBuilder injects nuanced manifest - // fragments through BluetoothManifestFragments instead. The - // framework linking + CN1_INCLUDE_BLUETOOTH define flip likewise - // happen in IPhoneBuilder (iosFrameworks is documentation-only). - e.add(new Entry("com/codename1/bluetooth/") - .iosFrameworks("CoreBluetooth") - .iosPlist("NSBluetoothAlwaysUsageDescription", - "Communicates with nearby Bluetooth accessories.") - .iosPlist("NSBluetoothPeripheralUsageDescription", - "Communicates with nearby Bluetooth accessories.") - .description("Cross-platform Bluetooth (BLE central/peripheral, L2CAP, classic RFCOMM)")); - - // On-device Stable Diffusion: bundled Core ML model on iOS, - // ONNX runtime on Android. Flag the >2 GB upload concern so - // the cloud build server can abort early with a helpful - // message. - e.add(new Entry("com/codename1/ai/imagegen/StableDiffusion") - .iosFrameworks("CoreML", "Vision") - .androidGradle("com.microsoft.onnxruntime:onnxruntime-android:1.16.3") - .markBigUpload() - .description("On-device Stable Diffusion (local-build only)")); - - // Cross-platform augmented reality (com.codename1.ar): ARKit + - // SceneKit on iOS (linked explicitly by IPhoneBuilder, gated by - // the INCLUDE_CN1_AR define since neither is default-linked), - // Google Play Services for AR (ARCore) on Android. The camera - // permission and plist string ride along because AR always - // drives the camera. The com.google.ar.core meta-data marks - // ARCore optional so the app still installs on non-AR devices; - // the android.ar.required=true build hint flips it to required. - e.add(new Entry("com/codename1/ar/") - .iosFrameworks("ARKit", "SceneKit") - .iosPlist("NSCameraUsageDescription", - "Used to display augmented reality content.") - .androidGradle("com.google.ar:core:1.44.0") - .androidPermissions("android.permission.CAMERA") - .androidFeatures("android.hardware.camera.ar") - .androidMetaData("com.google.ar.core", "optional") - .description("Cross-platform augmented reality (world/image/face tracking)")); - - ENTRIES = Collections.unmodifiableList(e); - } - - private AiDependencyTable() { - } - - /** All registered entries. Mostly useful for tests and tooling. */ - public static List entries() { - return ENTRIES; - } - - /** - * Returns every entry whose {@link Entry#classPrefix} matches the - * given internal-form class name (slashes, not dots). When the - * prefix ends with a slash, package-prefix matching is used; - * otherwise an exact class match is required. - */ - public static List matchesFor(String internalClassName) { - if (internalClassName == null) { - return Collections.emptyList(); - } - List out = new ArrayList(); - for (Entry e : ENTRIES) { - if (e.matches(internalClassName)) { - out.add(e); - } - } - return out; - } - - /** - * Builder/scanner output: the de-duplicated union of every entry - * fired by a class scan. Use {@link Accumulator#consume(String)} - * from inside an {@link Executor.ClassScanner#usesClass(String)} - * implementation. - */ - public static final class Accumulator { - private final Set hits = new LinkedHashSet(); - - public void consume(String internalClassName) { - hits.addAll(matchesFor(internalClassName)); - } - - public Set hits() { - return hits; - } - - public boolean anyRequiresBigUpload() { - for (Entry e : hits) { - if (e.requiresBigUpload) { - return true; - } - } - return false; - } - } - - /** - * A single registry record. Mutable while the table is being - * built (the fluent setters); semantically immutable once exposed - * via {@link #entries()}. - */ - public static final class Entry { - private final String classPrefix; - private final List iosPods = new ArrayList(); - private final List iosSpm = new ArrayList(); - private final List iosFrameworks = new ArrayList(); - private final List iosPlist = new ArrayList(); - private final List androidGradle = new ArrayList(); - private final List androidPermissions = new ArrayList(); - private final List androidFeatures = new ArrayList(); - private final List androidMetaData = new ArrayList(); - private boolean requiresBigUpload; - private String description = ""; - - Entry(String classPrefix) { - this.classPrefix = classPrefix; - } - - boolean matches(String internalClassName) { - if (classPrefix.endsWith("/")) { - return internalClassName.startsWith(classPrefix); - } - return internalClassName.equals(classPrefix); - } - - Entry iosPod(String pod) { - iosPods.add(pod); - return this; - } - - Entry iosSpm(String identity, String url, String requirement, String... products) { - iosSpm.add(new IosSpm(identity, url, requirement, - Arrays.asList(products))); - return this; - } - - Entry iosFrameworks(String... fws) { - for (String f : fws) { - iosFrameworks.add(f); - } - return this; - } - - Entry iosPlist(String key, String defaultValue) { - iosPlist.add(new String[]{key, defaultValue}); - return this; - } - - Entry androidGradle(String gav) { - androidGradle.add(gav); - return this; - } - - Entry androidPermissions(String... perms) { - for (String p : perms) { - androidPermissions.add(p); - } - return this; - } - - Entry androidFeatures(String... feats) { - for (String f : feats) { - androidFeatures.add(f); - } - return this; - } - - Entry androidMetaData(String name, String value) { - androidMetaData.add(new String[]{name, value}); - return this; - } - - Entry markBigUpload() { - this.requiresBigUpload = true; - return this; - } - - Entry description(String d) { - this.description = d; - return this; - } - - public String classPrefix() { - return classPrefix; - } - - public List iosPods() { - return Collections.unmodifiableList(iosPods); - } - - public List iosSpmSpecs() { - return Collections.unmodifiableList(iosSpm); - } - - public List iosFrameworks() { - return Collections.unmodifiableList(iosFrameworks); - } - - /** Each entry is {key, defaultValue}. The builder injects the - * value only if the app hasn't already declared one for the - * same key in its build hints. */ - public List iosPlistEntries() { - return Collections.unmodifiableList(iosPlist); - } - - public List androidGradleDeps() { - return Collections.unmodifiableList(androidGradle); - } - - public List androidPermissions() { - return Collections.unmodifiableList(androidPermissions); - } - - public List androidFeatures() { - return Collections.unmodifiableList(androidFeatures); - } - - /** Each entry is {name, value}: an application-level manifest - * <meta-data> element the Android builder injects unless the - * app already declares the same name. */ - public List androidMetaDataEntries() { - return Collections.unmodifiableList(androidMetaData); - } - - public boolean requiresBigUpload() { - return requiresBigUpload; - } - - public String description() { - return description; - } - } - - /** Swift Package Manager dependency descriptor. */ - public static final class IosSpm { - public final String identity; - public final String url; - public final String requirement; - public final List products; - - IosSpm(String identity, String url, String requirement, List products) { - this.identity = identity; - this.url = url; - this.requirement = requirement; - this.products = Collections.unmodifiableList(new ArrayList(products)); - } - } -} diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java index 1c6d1f58071..51b09c27519 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/AndroidGradleBuilder.java @@ -22,7 +22,7 @@ */ package com.codename1.builders; - +import com.codename1.build.shared.PlatformFeatureCatalog; import static com.codename1.maven.PathUtil.path; @@ -467,6 +467,11 @@ public File getGradleProjectDirectory() { private boolean migrateToAndroidX; private boolean shouldIncludeGoogleImpl; private boolean arSupport; + private boolean visionSupport; + private boolean inferenceSupport; + private boolean languageSupport; + private final Set includedAiAdapterSources = + new HashSet(); static { isMac = System.getProperty("os.name").toLowerCase().contains("mac"); @@ -1313,13 +1318,20 @@ public boolean build(File sourceZip, final BuildRequest request) throws BuildExc wakeLock = true; } mediaPlaybackPermission = false; + visionSupport = false; + inferenceSupport = false; + languageSupport = false; + includedAiAdapterSources.clear(); // Accumulator for AI/ML class hits. After the scan we apply - // every matched AiDependencyTable.Entry -- appending Gradle + // every matched PlatformFeatureCatalog.Entry -- appending Gradle // deps to additionalDependencies (later) and permissions/ // features to xPermissions right now. - final AiDependencyTable.Accumulator aiAcc = new AiDependencyTable.Accumulator(); + final PlatformFeatureCatalog.Accumulator aiAcc = new PlatformFeatureCatalog.Accumulator(); + // scanClassesForPermissions invokes callbacks serially on this build + // thread. The mutable feature flags and source set are intentionally + // unsynchronized; parallelizing the scanner requires revisiting them. try { scanClassesForPermissions(dummyClassesDir, new Executor.ClassScanner() { @@ -1327,6 +1339,21 @@ public boolean build(File sourceZip, final BuildRequest request) throws BuildExc @Override public void usesClass(String cls) { aiAcc.consume(cls); + String aiAdapter = androidAiAdapterSource(cls); + if (aiAdapter != null) { + includedAiAdapterSources.add(aiAdapter); + } + if (aiAdapter != null + && cls.indexOf("com/codename1/ai/vision/") == 0) { + visionSupport = true; + } + if ("com/codename1/ai/inference/InferenceSession".equals(cls)) { + inferenceSupport = true; + } + if (aiAdapter != null + && cls.indexOf("com/codename1/ai/language/") == 0) { + languageSupport = true; + } if (cls.indexOf("com/codename1/ar/") == 0) { // Keeps the ARCore-backed impl sources (deleted for // non-AR apps below) and bumps minSdk to the ARCore @@ -1501,6 +1528,7 @@ public void usesClass(String cls) { @Override public void usesClassMethod(String cls, String method) { + aiAcc.consumeMethod(cls, method); if (cls.indexOf("com/codename1/calendar/LocalCalendarSource") == 0 || (cls.indexOf("com/codename1/calendar/CalendarManager") == 0 && (method.indexOf("getLocalSource") >= 0 @@ -1666,7 +1694,7 @@ public void usesClassMethod(String cls, String method) { // additionalDependencies is written to build.gradle below. StringBuilder aiExtraGradleDependencies = new StringBuilder(); String aiApplicationMetaData = ""; - for (AiDependencyTable.Entry entry : aiAcc.hits()) { + for (PlatformFeatureCatalog.Entry entry : aiAcc.hits()) { for (String perm : entry.androidPermissions()) { String addString = " \n"; xPermissions += permissionAdd(request, perm, addString); @@ -1703,6 +1731,9 @@ public void usesClassMethod(String cls, String method) { aiExtraGradleDependencies.append(" implementation '").append(gav).append("'\n"); } } + if (aiAcc.minimumAndroidSdk() > 0) { + minSDK = maxInt(Integer.toString(aiAcc.minimumAndroidSdk()), minSDK); + } if (aiAcc.anyRequiresBigUpload()) { request.putArgument("cn1.ai.requiresBigUpload", "true"); } @@ -2454,6 +2485,8 @@ public void usesClassMethod(String cls, String method) { arPackage.delete(); } + pruneOptionalAiSources(srcDir); + final String moPubAdUnitId = request.getArg("android.mopubId", null); if (moPubAdUnitId != null && moPubAdUnitId.length() > 0) { integrateMoPub = true; @@ -4605,6 +4638,11 @@ public void usesClassMethod(String cls, String method) { String keepFirebase = "-keep class com.google.android.gms.** { *; }\n\n" + "-keep class com.google.firebase.** { *; }\n\n"; + String keepAi = visionSupport || languageSupport || inferenceSupport + ? "-keep class com.codename1.impl.android.ai.** { *; }\n\n" + : ""; + String keepInferenceRuntime = + androidInferenceProguardRules(inferenceSupport); // workaround broken optimizer in proguard String proguardConfigOverride = "-dontusemixedcaseclassnames\n" + "-dontskipnonpubliclibraryclasses\n" @@ -4615,6 +4653,8 @@ public void usesClassMethod(String cls, String method) { + "\n" + "-dontwarn com.google.android.gms.**\n" + keepFirebase + + keepAi + + keepInferenceRuntime + "-keep class com.codename1.impl.android.AndroidBrowserComponentCallback {\n" + "*;\n" + "}\n\n" @@ -5255,6 +5295,90 @@ public void usesClassMethod(String cls, String method) { return true; } + private void pruneOptionalAiSources(File srcDir) { + File aiDir = new File(srcDir, + "com/codename1/impl/android/ai"); + String[] vision = { + "AndroidTextRecognitionAdapter.java", + "AndroidBarcodeScanningAdapter.java", + "AndroidFaceDetectionAdapter.java", + "AndroidImageLabelingAdapter.java", + "AndroidPoseDetectionAdapter.java", + "AndroidSelfieSegmentationAdapter.java" + }; + String[] language = { + "AndroidLanguageIdAdapter.java", + "AndroidTranslationAdapter.java", + "AndroidSmartReplyAdapter.java" + }; + pruneAiGroup(aiDir, visionSupport, + new String[] {"AndroidVisionImpl.java", + "AndroidVisionAdapter.java"}, vision); + pruneAiGroup(aiDir, languageSupport, + new String[] {"AndroidLanguageImpl.java", + "AndroidLanguageAdapter.java"}, language); + if (!inferenceSupport) { + new File(aiDir, "AndroidInferenceImpl.java").delete(); + } + } + + private void pruneAiGroup(File aiDir, boolean groupIncluded, + String[] sharedSources, String[] adapters) { + if (!groupIncluded) { + for (int i = 0; i < sharedSources.length; i++) { + new File(aiDir, sharedSources[i]).delete(); + } + } + for (int i = 0; i < adapters.length; i++) { + if (!groupIncluded + || !includedAiAdapterSources.contains(adapters[i])) { + new File(aiDir, adapters[i]).delete(); + } + } + } + + static String androidAiAdapterSource(String cls) { + if ("com/codename1/ai/vision/TextRecognizer".equals(cls)) { + return "AndroidTextRecognitionAdapter.java"; + } + if ("com/codename1/ai/vision/BarcodeScanner".equals(cls)) { + return "AndroidBarcodeScanningAdapter.java"; + } + if ("com/codename1/ai/vision/FaceDetector".equals(cls)) { + return "AndroidFaceDetectionAdapter.java"; + } + if ("com/codename1/ai/vision/ImageLabeler".equals(cls)) { + return "AndroidImageLabelingAdapter.java"; + } + if ("com/codename1/ai/vision/PoseDetector".equals(cls)) { + return "AndroidPoseDetectionAdapter.java"; + } + if ("com/codename1/ai/vision/SelfieSegmenter".equals(cls)) { + return "AndroidSelfieSegmentationAdapter.java"; + } + if ("com/codename1/ai/language/LanguageIdentifier".equals(cls)) { + return "AndroidLanguageIdAdapter.java"; + } + if ("com/codename1/ai/language/Translator".equals(cls)) { + return "AndroidTranslationAdapter.java"; + } + if ("com/codename1/ai/language/SmartReply".equals(cls)) { + return "AndroidSmartReplyAdapter.java"; + } + // DocumentScanner is Apple-only. Returning null intentionally prunes + // the Android vision backend for an app that references only it; the + // public API then reports UNSUPPORTED without a native dependency. + return null; + } + + static String androidInferenceProguardRules(boolean inferenceIncluded) { + return inferenceIncluded + ? "-keepclassmembers class org.tensorflow.lite.TensorImpl {\n" + + " void refreshShape();\n" + + "}\n\n" + : ""; + } + static String xmlize(String s) { s = s.replace("&", "&"); s = s.replace("<", "<"); @@ -5499,7 +5623,7 @@ public void extract(InputStream source, File dir, String sdkPath) throws IOExcep while ((entry = zis.getNextEntry()) != null) { debug("Extracting: " + entry); if (entry.isDirectory()) { - File d = new File(dir, entry.getName()); + File d = resolveArchiveEntry(dir, entry.getName()); d.mkdirs(); if (!addedSDKDir && sdkPath != null) { if (is_windows) { @@ -5524,7 +5648,7 @@ public void extract(InputStream source, File dir, String sdkPath) throws IOExcep String sdkPathProperties = "sdk.dir=" + sdkPath + "\n"; // write the files to the disk File destFile; - destFile = new File(dir, entry.getName()); + destFile = resolveArchiveEntry(dir, entry.getName()); destFile.getParentFile().mkdirs(); FileOutputStream fos = new FileOutputStream(destFile); fos.write(sdkPathProperties.getBytes(StandardCharsets.UTF_8)); @@ -5536,7 +5660,7 @@ public void extract(InputStream source, File dir, String sdkPath) throws IOExcep byte[] data = new byte[8192]; // write the files to the disk File destFile; - destFile = new File(dir, entry.getName()); + destFile = resolveArchiveEntry(dir, entry.getName()); destFile.getParentFile().mkdirs(); FileOutputStream fos = new FileOutputStream(destFile); dest = new BufferedOutputStream(fos, data.length); @@ -5637,18 +5761,18 @@ public void extractAAR(InputStream source, File dir, String sdkPath) throws IOEx } - protected File placeXMLFile(ZipEntry entry, File xmlDir, File resDir) { + protected File placeXMLFile(ZipEntry entry, File xmlDir, File resDir) throws IOException { String name = entry.getName(); if (name.endsWith("colors.xml")) { File parent = resDir.getParentFile(); resDir = new File(parent, "res"); File valsDir = new File(resDir, "values"); - return new File(valsDir, entry.getName()); + return resolveArchiveEntry(valsDir, entry.getName()); } else if (name.endsWith("layout.xml")) { File parent = resDir.getParentFile(); resDir = new File(parent, "res"); File layDir = new File(resDir, "layout"); - return new File(layDir, entry.getName()); + return resolveArchiveEntry(layDir, entry.getName()); } else { return super.placeXMLFile(entry, xmlDir, resDir); } @@ -5757,11 +5881,11 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD if (entry.isDirectory()) { if (!entryName.startsWith("raw")) { - File dir = new File(classesDir, entryName); + File dir = resolveArchiveEntry(classesDir, entryName); dir.mkdirs(); - dir = new File(resDir, entryName); + dir = resolveArchiveEntry(resDir, entryName); dir.mkdirs(); - dir = new File(sourceDir, entryName); + dir = resolveArchiveEntry(sourceDir, entryName); dir.mkdirs(); } continue; @@ -5773,24 +5897,26 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD // write the files to the disk File destFile; if (entryName.endsWith(".class")) { - destFile = new File(classesDir, entryName); + destFile = resolveArchiveEntry(classesDir, entryName); } else { if (entryName.endsWith(".java") || entryName.endsWith(".kt") || entryName.endsWith(".swift") || entryName.endsWith(".m") || entryName.endsWith(".h")) { - destFile = new File(sourceDir, entryName); + destFile = resolveArchiveEntry(sourceDir, entryName); } else { if (entryName.endsWith(".jar") || entryName.endsWith(".a") || entryName.endsWith(".dylib") || entryName.endsWith(".andlib") || entryName.endsWith(".aar")) { - destFile = new File(libsDir, entryName); + destFile = resolveArchiveEntry(libsDir, entryName); } else { if (useXMLDir() && entryName.endsWith(".xml")) { destFile = placeXMLFile(entry, xmlDir, resDir); } else if (entryName.startsWith("raw")) { - destFile = new File(xmlDir.getParentFile(), entryName.toLowerCase()); + destFile = resolveArchiveEntry(xmlDir.getParentFile(), + entryName.toLowerCase()); } else if (entryName.contains("notification_sound")) { - destFile = new File(xmlDir.getParentFile(), "raw/" + entryName.toLowerCase()); + destFile = resolveArchiveEntry(xmlDir.getParentFile(), + "raw/" + entryName.toLowerCase()); } else if ("google-services.json".equals(entryName)) { - destFile = new File(libsDir.getParentFile(), entryName); + destFile = resolveArchiveEntry(libsDir.getParentFile(), entryName); } else { - destFile = new File(resDir, entryName); + destFile = resolveArchiveEntry(resDir, entryName); } } } diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java index 8bc5d568271..0eb485669f1 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/Executor.java @@ -1245,15 +1245,21 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD while ((entry = zis.getNextEntry()) != null) { String entryName = entry.getName(); + // Validate the original ZIP name before any archive-specific + // branch can route it into a generated tar and continue. + resolveArchiveEntry(resDir, entryName); + if(!"html.tar".equals(entryName)&& (entryName.startsWith("html") || entryName.startsWith("/html"))) { if(entry.isDirectory()) { continue; } - + entryName = entryName.substring(5); + // The stripped name becomes a member of another archive, + // so validate it independently against that archive's root. + resolveArchiveEntry(resDir, entryName); if(tos == null) { tos = new TarOutputStream(new FileOutputStream(new File(resDir, "html.tar"))); } - entryName = entryName.substring(5); TarEntry tEntry = new TarEntry(new File(entryName), entryName); tEntry.setSize(entry.getSize()); debug("Packaging entry " + entryName + " size: " + entry.getSize()); @@ -1270,10 +1276,11 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD continue; } int podSpecsPrefix = entryName.startsWith("podspecs/") ? "podspecs/".length() : "/podspecs/".length(); + entryName = entryName.substring(podSpecsPrefix); + resolveArchiveEntry(resDir, entryName); if(podspecTos == null) { podspecTos = new TarOutputStream(new FileOutputStream(new File(resDir, "podspecs.tar"))); } - entryName = entryName.substring(podSpecsPrefix); TarEntry tEntry = new TarEntry(new File(entryName), entryName); tEntry.setSize(entry.getSize()); debug("Packaging entry " + entryName + " size: " + entry.getSize()); @@ -1292,10 +1299,11 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD continue; } int libPrefix = entryName.startsWith("javase.lib/") ? "javase.lib/".length() : "/javase.lib/".length(); + entryName = entryName.substring(libPrefix); + resolveArchiveEntry(resDir, entryName); if(libTos == null) { libTos = new TarOutputStream(new FileOutputStream(new File(resDir, "javase.lib.tar"))); } - entryName = entryName.substring(libPrefix); TarEntry tEntry = new TarEntry(new File(entryName), entryName); tEntry.setSize(entry.getSize()); debug("Packaging entry " + entryName + " size: " + entry.getSize()); @@ -1309,11 +1317,11 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD } if (entry.isDirectory()) { - File dir = new File(classesDir, entryName); + File dir = resolveArchiveEntry(classesDir, entryName); dir.mkdirs(); - dir = new File(resDir, entryName); + dir = resolveArchiveEntry(resDir, entryName); dir.mkdirs(); - dir = new File(sourceDir, entryName); + dir = resolveArchiveEntry(sourceDir, entryName); dir.mkdirs(); continue; } @@ -1328,22 +1336,22 @@ public void unzip(InputStream source, File classesDir, File resDir, File sourceD log("!!!!Skipping "+entryName); continue; } else { - destFile = new File(classesDir, entryName); + destFile = resolveArchiveEntry(classesDir, entryName); } } else { if (entryName.endsWith(".java") || entryName.endsWith(".kt") || entryName.endsWith(".swift") || entryName.endsWith(".m") || entryName.endsWith(".h") || entryName.endsWith(".cs")) { - destFile = new File(sourceDir, entryName); + destFile = resolveArchiveEntry(sourceDir, entryName); } else { if (entryName.endsWith(".jar") || entryName.endsWith(".a") || entryName.endsWith(".dylib") || entryName.endsWith(".andlib") || entryName.endsWith(".aar") || entryName.endsWith(dll)) { - destFile = new File(libsDir, entryName); + destFile = resolveArchiveEntry(libsDir, entryName); } else { if (useXMLDir() && entryName.endsWith(".xml")) { destFile = placeXMLFile(entry, xmlDir, resDir); } else { if(entryName.equals("codenameone_settings.properties")) { - destFile = new File(sourceDir.getParentFile(), entryName); + destFile = resolveArchiveEntry(sourceDir.getParentFile(), entryName); } else { - destFile = new File(resDir, entryName); + destFile = resolveArchiveEntry(resDir, entryName); } } } @@ -1390,7 +1398,7 @@ public void extractZip(InputStream source, File destDir, ExtractionFilter filter if (entry.isDirectory()) { currentDir = entryName; - File dir = new File(destDir, entryName); + File dir = resolveArchiveEntry(destDir, entryName); dir.mkdirs(); continue; } @@ -1398,8 +1406,17 @@ public void extractZip(InputStream source, File destDir, ExtractionFilter filter int count; byte[] data = new byte[8192]; - // write the files to the disk + // Validate the untrusted archive name before the trusted + // filter sees it. The filter deliberately owns final routing + // and may select a sibling (for example, project settings), + // so its canonical result is not constrained to destDir. + resolveArchiveEntry(destDir, entryName); File destFile = filter.destFile(currentDir, entryName); + if (destFile == null) { + throw new IOException("Extraction filter returned no destination for " + + entryName); + } + destFile = destFile.getCanonicalFile(); destFile.getParentFile().mkdirs(); FileOutputStream fos = new FileOutputStream(destFile); dest = new BufferedOutputStream(fos, data.length); @@ -1416,7 +1433,7 @@ public void extractZip(InputStream source, File destDir, ExtractionFilter filter } } - protected File placeXMLFile(ZipEntry entry, File xmlDir, File resDir) { + protected File placeXMLFile(ZipEntry entry, File xmlDir, File resDir) throws IOException { boolean putInXMLDir = false; String name = entry.getName(); if (name.contains("/")) { @@ -1431,12 +1448,45 @@ protected File placeXMLFile(ZipEntry entry, File xmlDir, File resDir) { name = name.substring(name.lastIndexOf("/") + 1, name.length()); } if (putInXMLDir) { - return new File(xmlDir, name); + return resolveArchiveEntry(xmlDir, name); } else { - return new File(resDir, entry.getName()); + return resolveArchiveEntry(resDir, entry.getName()); } } + /** + * Resolves an archive entry beneath its intended extraction directory. + * Entries that are absolute or escape through {@code ..} components are + * rejected before any directory or file is created. + * + * @param destinationDirectory extraction root + * @param entryName untrusted name read from the archive + * @return canonical destination contained by {@code destinationDirectory} + * @throws IOException if the entry escapes the extraction root + */ + protected static File resolveArchiveEntry(File destinationDirectory, + String entryName) throws IOException { + if (new File(entryName).isAbsolute()) { + throw new IOException("Archive entry is absolute: " + entryName); + } + return requireArchiveDestination(destinationDirectory, + new File(destinationDirectory, entryName), entryName); + } + + private static File requireArchiveDestination(File destinationDirectory, + File destinationFile, String entryName) throws IOException { + File canonicalDirectory = destinationDirectory.getCanonicalFile(); + File canonicalDestination = destinationFile.getCanonicalFile(); + String directoryPath = canonicalDirectory.getPath(); + String directoryPrefix = directoryPath.endsWith(File.separator) + ? directoryPath : directoryPath + File.separator; + if (!canonicalDestination.getPath().startsWith(directoryPrefix)) { + throw new IOException("Archive entry escapes destination directory: " + + entryName); + } + return canonicalDestination; + } + public int executeProcess(ProcessBuilder pb) throws Exception { return executeProcess(pb, -1); } diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java index c74849d71de..10a7c15606c 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/IPhoneBuilder.java @@ -22,6 +22,7 @@ */ package com.codename1.builders; +import com.codename1.build.shared.PlatformFeatureCatalog; import com.codename1.util.IOSWalletExtensionBuilder; import com.codename1.util.IOSWidgetExtensionBuilder; import org.w3c.dom.Document; @@ -117,6 +118,9 @@ public class IPhoneBuilder extends Executor { private boolean usesBluetoothPeripheral; private boolean usesCn1Camera; private boolean usesCn1Ar; + private boolean usesCn1Vision; + private boolean usesCn1Language; + private boolean usesCn1Inference; // Set when the app references com.codename1.car.* (Apple CarPlay support). Gates the // CN1_USE_CARPLAY native define, CarPlay.framework linkage, the carplay entitlement and the // CarPlay scene in the Info.plist scene manifest. Apps that never touch the API see no change. @@ -334,7 +338,71 @@ private static String append(String str, String separator, String append) { } return str + append; } - + + private static String appendFrameworks(String libraries, + String... frameworks) { + String out = libraries == null ? "" : libraries; + for (String framework : frameworks) { + boolean present = false; + for (String item : out.split(";")) { + if (framework.equalsIgnoreCase(item.trim())) { + present = true; + break; + } + } + if (!present) { + out = out.length() == 0 ? framework : out + ";" + framework; + } + } + return out; + } + + static String appendPodSpecIfAbsent(String pods, String candidate) { + String out = pods == null ? "" : pods; + String candidateName = podName(candidate); + for (String existing : out.split("[,;]")) { + if (candidateName.equalsIgnoreCase(podName(existing))) { + return out; + } + } + return out.length() == 0 ? candidate : out + "," + candidate; + } + + static String deduplicatePodSpecs(String pods) { + String out = ""; + if (pods == null) { + return out; + } + for (String podSpec : pods.split("[,;]")) { + podSpec = podSpec.trim(); + if (podSpec.length() > 0) { + out = appendPodSpecIfAbsent(out, podSpec); + } + } + return out; + } + + private static String podName(String podSpec) { + String value = podSpec == null ? "" : podSpec.trim(); + int separator = value.indexOf(' '); + return separator < 0 ? value : value.substring(0, separator).trim(); + } + + private void applyCatalogPlistEntry(BuildRequest request, + String[] plistEntry) { + String privacyKey = plistEntry[0]; + String requestKey = "ios." + privacyKey; + String value = request.getArg(requestKey, null); + if (value == null) { + value = plistEntry[1]; + request.putArgument(requestKey, value); + } + if (value != null + && !privacyUsageDescriptions.containsKey(privacyKey)) { + privacyUsageDescriptions.put(privacyKey, value); + } + } + private int getDeploymentTargetInt(BuildRequest request) { String target = getDeploymentTarget(request); if (target.indexOf(".") > 0) { @@ -347,6 +415,11 @@ private int getDeploymentTargetInt(BuildRequest request) { @Override public boolean build(File sourceZip, BuildRequest request) throws BuildException { + // Builder instances are normally single-use, but keep scan-derived + // native feature state deterministic if an instance is reused. + usesCn1Vision = false; + usesCn1Language = false; + usesCn1Inference = false; Stopwatch stopwatch = new Stopwatch(); addMinDeploymentTarget(DEFAULT_MIN_DEPLOYMENT_VERSION); if (request.getArg("ios.deployment_target", null) == null) { @@ -791,10 +864,11 @@ public boolean build(File sourceZip, BuildRequest request) throws BuildException } // Accumulator for AI/ML class hits. After the scan we apply - // every matched AiDependencyTable.Entry -- appending pods, + // every matched PlatformFeatureCatalog.Entry -- appending pods, // SPM specs, plist defaults and Android perms -- so the user // doesn't have to declare them by hand. - final AiDependencyTable.Accumulator aiAcc = new AiDependencyTable.Accumulator(); + final PlatformFeatureCatalog.Accumulator aiAcc = new PlatformFeatureCatalog.Accumulator(); + boolean excludeArm64Simulator = false; try { scanClassesForPermissions(classesDir, new Executor.ClassScanner() { @@ -874,6 +948,16 @@ public void usesClass(String cls) { if (!usesCn1Ar && cls.indexOf("com/codename1/ar/") == 0) { usesCn1Ar = true; } + if (!usesCn1Vision && isVisionAnalyzerClass(cls)) { + usesCn1Vision = true; + } + if (!usesCn1Language && isLanguageFeatureClass(cls)) { + usesCn1Language = true; + } + if (!usesCn1Inference + && "com/codename1/ai/inference/InferenceSession".equals(cls)) { + usesCn1Inference = true; + } // Apple CarPlay (com.codename1.car.*). Gated on actual usage so the // CarPlay scene/entitlement/framework are only added for apps that // build an in-car experience. @@ -927,6 +1011,7 @@ public void usesClass(String cls) { @Override public void usesClassMethod(String cls, String method) { + aiAcc.consumeMethod(cls, method); if (cls.indexOf("com/codename1/calendar/LocalCalendarSource") == 0 || (cls.indexOf("com/codename1/calendar/CalendarManager") == 0 && (method.indexOf("getLocalSource") >= 0 @@ -1010,7 +1095,8 @@ public void usesClassMethod(String cls, String method) { // iosPods string and SPM entries through the request build // hints, so the IOSDependencyManager.resolve() call below can // pick them up consistently with manually-declared deps. - if (!aiAcc.hits().isEmpty()) { + Set platformFeatureHits = aiAcc.hits(); + if (!platformFeatureHits.isEmpty()) { // Prefer SPM when the project already uses SPM and the // entry exposes an SPM spec; otherwise pods. A handful // of ML Kit libs are pods-only -- those force pods on @@ -1018,10 +1104,34 @@ public void usesClassMethod(String cls, String method) { // upgrade the effective mode to BOTH below). boolean projectPrefersSpm = dependencyConfig.usesSwiftPackages() && !dependencyConfig.usesCocoaPods(); StringBuilder spmPackages = new StringBuilder(request.getArg("ios.spm.packages", "")); - for (AiDependencyTable.Entry entry : aiAcc.hits()) { + for (PlatformFeatureCatalog.Entry entry : platformFeatureHits) { + // Third-party AI packages may omit Catalyst or arm64 + // simulator slices. Keep framework-only Apple Vision enabled + // for macNative and record the simulator constraint for the + // generated Xcode and Pods projects. + boolean includeApplePackageDependencies = + !macNativeBuilder.isEnabled() + || entry.iosDependenciesSupportMacCatalyst(); + if (includeApplePackageDependencies + && entry.iosMinimumDeploymentTarget() != null) { + addMinDeploymentTarget( + entry.iosMinimumDeploymentTarget()); + } + if (includeApplePackageDependencies + && !entry.iosDependenciesSupportArm64Simulator() + && (!entry.iosPods().isEmpty() + || !entry.iosSpmSpecs().isEmpty())) { + excludeArm64Simulator = true; + log("Catalog-selected iOS dependency \"" + + entry.description() + + "\" has no arm64 simulator slice. The generated " + + "project will use the x86_64 simulator architecture."); + } boolean handledViaSpm = false; - if (projectPrefersSpm && !entry.iosSpmSpecs().isEmpty()) { - for (AiDependencyTable.IosSpm spm : entry.iosSpmSpecs()) { + if (includeApplePackageDependencies + && projectPrefersSpm + && !entry.iosSpmSpecs().isEmpty()) { + for (PlatformFeatureCatalog.IosSpm spm : entry.iosSpmSpecs()) { if (spmPackages.length() > 0) spmPackages.append(';'); spmPackages.append(spm.identity).append('|') .append(spm.url).append('|') @@ -1040,19 +1150,18 @@ public void usesClassMethod(String cls, String method) { } handledViaSpm = true; } - if (!handledViaSpm) { + if (includeApplePackageDependencies && !handledViaSpm) { for (String pod : entry.iosPods()) { - if (iosPods.length() > 0) iosPods += ","; - iosPods += pod; + // User-declared specs are already first in iosPods, so + // they win when the catalog requests the same pod. + iosPods = appendPodSpecIfAbsent(iosPods, pod); } } for (String[] plistEntry : entry.iosPlistEntries()) { - String key = "ios." + plistEntry[0]; - if (request.getArg(key, null) == null) { - request.putArgument(key, plistEntry[1]); - } + applyCatalogPlistEntry(request, plistEntry); } } + iosPods = deduplicatePodSpecs(iosPods); if (spmPackages.length() > 0) { request.putArgument("ios.spm.packages", spmPackages.toString()); } @@ -2401,7 +2510,7 @@ public void usesClassMethod(String cls, String method) { // First-class Bluetooth: weak-link CoreBluetooth and compile in // the CN1Bluetooth natives only when the app references // com.codename1.bluetooth.*. The NSBluetooth* privacy strings - // are defaulted (only-if-unset) by the AiDependencyTable entry + // are defaulted (only-if-unset) by the PlatformFeatureCatalog entry // through the standard plist application above. Background // operation is opt-in through the ios.bluetooth.background hint // ("central", "peripheral" or "central,peripheral"), merged into @@ -2452,7 +2561,7 @@ public void usesClassMethod(String cls, String method) { // INCLUDE_CAMERA_USAGE (the old modal Capture API): the new // AVFoundation natives are only built when the app actually // references com.codename1.camera.*, matching the AVFoundation - // framework injection driven by the same scan via AiDependencyTable. + // framework injection driven by the same scan via PlatformFeatureCatalog. if (usesCn1Camera) { try { replaceInFile(new File(buildinRes, @@ -2468,7 +2577,8 @@ public void usesClassMethod(String cls, String method) { // Augmented reality: uncomment INCLUDE_CN1_AR so the CN1AR // natives (ARKit + ARSCNView) compile in, and link ARKit / // SceneKit explicitly -- neither is default-linked and the - // AiDependencyTable iosFrameworks field is documentation-only. + // Catalog frameworks are linked below. This explicit AR block also + // controls the native compile define and remains for compatibility. // Apps that never reference com.codename1.ar leave the define // commented out so no ARKit symbol is referenced, which keeps // Apple's API-usage scan quiet and tvOS/watchOS slices clean. @@ -2490,6 +2600,53 @@ public void usesClassMethod(String cls, String method) { } } + for (String framework : aiAcc.iosFrameworks()) { + addLibs = appendFrameworks(addLibs, + framework + ".framework"); + } + + if (usesCn1Vision) { + try { + replaceInFile(new File(buildinRes, + "CodenameOne_GLViewController.h"), + "//#define INCLUDE_CN1_VISION", + "#define INCLUDE_CN1_VISION"); + } catch (IOException ex) { + throw new BuildException( + "Failed to enable INCLUDE_CN1_VISION", ex); + } + addLibs = appendFrameworks(addLibs, "Vision.framework", + "CoreImage.framework", "CoreVideo.framework"); + } + + if (usesCn1Language) { + try { + replaceInFile(new File(buildinRes, + "CodenameOne_GLViewController.h"), + "//#define INCLUDE_CN1_LANGUAGE", + "#define INCLUDE_CN1_LANGUAGE"); + } catch (IOException ex) { + throw new BuildException( + "Failed to enable INCLUDE_CN1_LANGUAGE", ex); + } + addLibs = appendFrameworks(addLibs, + "NaturalLanguage.framework"); + } + + if (usesCn1Inference) { + try { + replaceInFile(new File(buildinRes, + "CodenameOne_GLViewController.h"), + "//#define INCLUDE_CN1_INFERENCE", + "#define INCLUDE_CN1_INFERENCE"); + } catch (IOException ex) { + throw new BuildException( + "Failed to enable INCLUDE_CN1_INFERENCE", ex); + } + addLibs = appendFrameworks(addLibs, "CoreML.framework", + "Metal.framework", "Accelerate.framework"); + } + // CarPlay: link CarPlay.framework (+ MediaPlayer for the now-playing template) and // inject the per-category carplay entitlement. The CarPlay entitlements are granted by // Apple per app category, so we only inject the ones the project opts into via the @@ -3042,6 +3199,9 @@ public void usesClassMethod(String cls, String method) { targetStr = "8.0"; } addMinDeploymentTarget(targetStr); + String simulatorArchitectureSettings = excludeArm64Simulator + ? " config.build_settings['EXCLUDED_ARCHS[sdk=iphonesimulator*]'] = 'arm64'\n" + : ""; deploymentTargetStr = "begin\n" + " xcproj.targets.find{|e|e.name=='" + request.getMainClass() + "'}.build_configurations.each{|config| \n" + " config.build_settings['PRODUCT_BUNDLE_IDENTIFIER']='"+request.getPackageName()+"'\n" @@ -3058,6 +3218,7 @@ public void usesClassMethod(String cls, String method) { + " next if target.respond_to?(:product_type) && target.product_type == 'com.apple.product-type.app-extension'\n" + " target.build_configurations.each do |config|\n" + " config.build_settings['IPHONEOS_DEPLOYMENT_TARGET'] = '" + getDeploymentTarget(request) + "'\n" + + simulatorArchitectureSettings + " end\n" + " end\n" + " xcproj.save\n" @@ -3208,6 +3369,20 @@ public void usesClassMethod(String cls, String method) { } + String arcPhaseFixScript = + usesCn1Vision || usesCn1Language || usesCn1Inference + ? " arc_sources = ['CN1Vision.m', 'CN1Language.m', 'CN1Inference.m']\n" + + " main_target.source_build_phase.files.each do |bf|\n" + + " ref = bf.file_ref\n" + + " name = ref && File.basename(ref.path || ref.name || '')\n" + + " next unless arc_sources.include?(name)\n" + + " settings = bf.settings || {}\n" + + " flags = settings['COMPILER_FLAGS'].to_s.split\n" + + " flags << '-fobjc-arc' unless flags.include?('-fobjc-arc')\n" + + " settings['COMPILER_FLAGS'] = flags.join(' ')\n" + + " bf.settings = settings\n" + + " end\n" + : ""; String createSchemesScript = "#!/usr/bin/env ruby\n" + "require 'xcodeproj'\n" + "require 'pathname'\n" + @@ -3318,6 +3493,7 @@ public void usesClassMethod(String cls, String method) { + " end\n" + " raise \"Swift files/resources still present in Copy Bundle Resources: #{names.join(', ')}\"\n" + " end\n" + + arcPhaseFixScript + " end\n" + "rescue => e\n" + " puts \"Error while correcting Swift build phases: #{$!}\"\n" @@ -3353,7 +3529,7 @@ public void usesClassMethod(String cls, String method) { return false; } String podFileContents = "target '" + request.getMainClass() + "' do\n"; - String[] pods = iosPods.split("[,;]"); + String[] pods = deduplicatePodSpecs(iosPods).split("[,;]"); for (String podLib : pods) { podLib = podLib.trim(); if (podLib.isEmpty()) { @@ -3409,6 +3585,14 @@ public void usesClassMethod(String cls, String method) { if (useMetal) { buildSettings += " config.build_settings['CLANG_ENABLE_MODULES'] = \"YES\"\n"; } + if (excludeArm64Simulator) { + // Google ML Kit's binary frameworks contain device + // arm64 and simulator x86_64 slices. Apply the same + // exclusion to every pod target so CocoaPods doesn't + // drop its conflicting per-pod setting while merging + // the aggregate xcconfig. + buildSettings += " config.build_settings['EXCLUDED_ARCHS[sdk=iphonesimulator*]'] = \"arm64\"\n"; + } podFileContents += "\n\npost_install do |installer|\n" + @@ -5482,4 +5666,20 @@ private static String join(String[] strs, String sep) { return out.toString(); } + private static boolean isVisionAnalyzerClass(String cls) { + return "com/codename1/ai/vision/TextRecognizer".equals(cls) + || "com/codename1/ai/vision/BarcodeScanner".equals(cls) + || "com/codename1/ai/vision/FaceDetector".equals(cls) + || "com/codename1/ai/vision/ImageLabeler".equals(cls) + || "com/codename1/ai/vision/PoseDetector".equals(cls) + || "com/codename1/ai/vision/SelfieSegmenter".equals(cls) + || "com/codename1/ai/vision/DocumentScanner".equals(cls); + } + + private static boolean isLanguageFeatureClass(String cls) { + return "com/codename1/ai/language/LanguageIdentifier".equals(cls) + || "com/codename1/ai/language/Translator".equals(cls) + || "com/codename1/ai/language/SmartReply".equals(cls); + } + } diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacNativeBuilder.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacNativeBuilder.java index e0e522494b5..78f906090a9 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacNativeBuilder.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/builders/MacNativeBuilder.java @@ -334,7 +334,7 @@ private void writeEntitlementsFile(BuildRequest request, File appSrcDir, // explicit device entitlements; without them the OS refuses to // open the AVCaptureSession even when the Info.plist usage // descriptions are present. We piggyback on the iOS plist hints - // already populated by AiDependencyTable -- when the build + // already populated by PlatformFeatureCatalog -- when the build // pipeline detected com.codename1.camera.* (or any other API // that triggers NSCameraUsageDescription / NSMicrophoneUsageDescription) // we mirror that into the Mac entitlements. Developers may diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/CN1BuildMojo.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/CN1BuildMojo.java index d4f55d37b00..12700c0df49 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/CN1BuildMojo.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/CN1BuildMojo.java @@ -1466,6 +1466,13 @@ private void doIOSLocalBuild(File tmpProjectDir, Properties props, File distJar) : getGeneratedIOSProjectSourceDirectory(); output.getParentFile().mkdirs(); try { + // This directory is a generated target. Replacing it is + // required when class scanning removes a dependency: + // copyDirectory() alone leaves stale Podfiles, workspaces, + // Pods and optional native sources from the prior build. + if (output.exists()) { + FileUtils.deleteDirectory(output); + } getLog().info("Copying Xcode Project to "+output); FileUtils.copyDirectory(xcodeProject, output); } catch (IOException ex) { diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AiDependencyTableTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AiDependencyTableTest.java deleted file mode 100644 index f3d291de18b..00000000000 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AiDependencyTableTest.java +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Codename One designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Codename One through http://www.codenameone.com/ if you - * need additional information or have any questions. - */ -package com.codename1.builders; - -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class AiDependencyTableTest { - - @Test - void mlkitTextRecognizerMapsToPodAndGradleDep() { - List hits = AiDependencyTable.matchesFor( - "com/codename1/ai/mlkit/text/TextRecognizer"); - assertEquals(1, hits.size(), "expected one entry to fire"); - AiDependencyTable.Entry e = hits.get(0); - assertTrue(e.iosPods().contains("GoogleMLKit/TextRecognition")); - assertTrue(e.androidGradleDeps().get(0).startsWith("com.google.mlkit:text-recognition")); - // Camera plist string is injected because text recognition - // is virtually always used with the camera. - assertNotNull(findPlistDefault(e, "NSCameraUsageDescription")); - } - - @Test - void speechRecognizerInjectsMicAndSpeechPlist() { - List hits = AiDependencyTable.matchesFor( - "com/codename1/media/SpeechRecognizer"); - assertEquals(1, hits.size()); - AiDependencyTable.Entry e = hits.get(0); - assertTrue(e.iosFrameworks().contains("Speech")); - assertNotNull(findPlistDefault(e, "NSMicrophoneUsageDescription")); - assertNotNull(findPlistDefault(e, "NSSpeechRecognitionUsageDescription")); - assertTrue(e.androidPermissions().contains("android.permission.RECORD_AUDIO")); - } - - @Test - void textToSpeechInjectsNoPermissions() { - List hits = AiDependencyTable.matchesFor( - "com/codename1/media/TextToSpeech"); - assertEquals(1, hits.size()); - AiDependencyTable.Entry e = hits.get(0); - assertTrue(e.iosFrameworks().contains("AVFAudio")); - assertTrue(e.androidPermissions().isEmpty(), - "TTS is built-in on every supported OS -- no permission needed"); - assertTrue(e.iosPlistEntries().isEmpty(), - "TTS has no Apple-reviewed restricted entitlement"); - } - - @Test - void llmClientNeedsNothingExtra() { - // The LlmClient entries are intentionally cheap: pure HTTPS - // means no plist string, no extra permission. They still - // register so future diagnostics ("which AI APIs does this - // app use?") can enumerate them. - List hits = AiDependencyTable.matchesFor( - "com/codename1/ai/LlmClient"); - assertEquals(1, hits.size()); - AiDependencyTable.Entry e = hits.get(0); - assertTrue(e.iosPods().isEmpty()); - assertTrue(e.androidGradleDeps().isEmpty()); - assertTrue(e.androidPermissions().isEmpty()); - } - - @Test - void stableDiffusionFlagsBigUpload() { - AiDependencyTable.Accumulator acc = new AiDependencyTable.Accumulator(); - acc.consume("com/codename1/ai/imagegen/StableDiffusion"); - assertTrue(acc.anyRequiresBigUpload(), - "On-device SD ships a 1-2 GB Core ML model -- cloud builds must abort with a friendly message"); - } - - @Test - void mlkitDoesNotFlagBigUpload() { - AiDependencyTable.Accumulator acc = new AiDependencyTable.Accumulator(); - acc.consume("com/codename1/ai/mlkit/text/TextRecognizer"); - acc.consume("com/codename1/ai/mlkit/barcode/BarcodeScanner"); - acc.consume("com/codename1/ai/whisper/WhisperRecognizer"); - assertFalse(acc.anyRequiresBigUpload(), - "ML Kit models stream lazily, Whisper bundles a small static lib -- neither exceeds the 2 GB cap"); - } - - @Test - void unrelatedClassesProduceNoHits() { - // Sanity: we mustn't false-positive on classes outside the - // AI namespace, because the scanner walks every class in - // the user's app. - assertTrue(AiDependencyTable.matchesFor("com/codename1/ui/Form").isEmpty()); - assertTrue(AiDependencyTable.matchesFor("java/lang/Object").isEmpty()); - assertTrue(AiDependencyTable.matchesFor(null).isEmpty()); - } - - @Test - void cameraEntryInjectsAvFoundationAndCameraXGradleDeps() { - // Referencing any class in com.codename1.camera.* must auto-inject - // the iOS frameworks, iOS plist usage descriptions, Android - // permissions, and the four CameraX Gradle dependencies that the - // AndroidCameraImpl reflection layer resolves at runtime. - List hits = AiDependencyTable.matchesFor( - "com/codename1/camera/Camera"); - assertEquals(1, hits.size(), "expected the camera entry to fire"); - AiDependencyTable.Entry e = hits.get(0); - - // iOS side - assertTrue(e.iosFrameworks().contains("AVFoundation")); - assertTrue(e.iosFrameworks().contains("CoreMedia")); - assertTrue(e.iosFrameworks().contains("CoreVideo")); - assertNotNull(findPlistDefault(e, "NSCameraUsageDescription")); - assertNotNull(findPlistDefault(e, "NSMicrophoneUsageDescription")); - assertTrue(e.iosPods().isEmpty(), - "Camera uses AVFoundation framework, not a pod"); - - // Android side - assertTrue(e.androidPermissions().contains("android.permission.CAMERA")); - assertTrue(e.androidPermissions().contains("android.permission.RECORD_AUDIO")); - assertTrue(e.androidFeatures().contains("android.hardware.camera")); - - boolean cameraCore = false, camera2 = false, lifecycle = false, - view = false, video = false; - for (String gav : e.androidGradleDeps()) { - if (gav.startsWith("androidx.camera:camera-core:")) cameraCore = true; - if (gav.startsWith("androidx.camera:camera-camera2:")) camera2 = true; - if (gav.startsWith("androidx.camera:camera-lifecycle:")) lifecycle = true; - if (gav.startsWith("androidx.camera:camera-view:")) view = true; - if (gav.startsWith("androidx.camera:camera-video:")) video = true; - } - assertTrue(cameraCore, "missing androidx.camera:camera-core gradle dep"); - assertTrue(camera2, "missing androidx.camera:camera-camera2 gradle dep"); - assertTrue(lifecycle, "missing androidx.camera:camera-lifecycle gradle dep"); - assertTrue(view, "missing androidx.camera:camera-view gradle dep"); - assertTrue(video, "missing androidx.camera:camera-video gradle dep"); - } - - @Test - void cameraEntryFiresOnAnySubpackageClass() { - // The prefix matcher must hit any class inside com.codename1.camera, - // not just the entry point. - assertEquals(1, AiDependencyTable.matchesFor("com/codename1/camera/CameraView").size()); - assertEquals(1, AiDependencyTable.matchesFor("com/codename1/camera/CameraSession").size()); - assertEquals(1, AiDependencyTable.matchesFor("com/codename1/camera/internal/Foo").size()); - } - - @Test - void tfliteHasBothPodAndSpmSpec() { - // TFLite is published as both a CocoaPod and a Swift Package. - // The table records both so projects can route the dep - // through whichever manager they prefer; the IPhoneBuilder - // applies whichever matches the project's current - // ios.dependencyManager setting. - List hits = AiDependencyTable.matchesFor( - "com/codename1/ai/tflite/Interpreter"); - assertEquals(1, hits.size()); - AiDependencyTable.Entry e = hits.get(0); - assertFalse(e.iosPods().isEmpty(), "expected a CocoaPods spec"); - assertFalse(e.iosSpmSpecs().isEmpty(), "expected an SPM spec"); - } - - @Test - void accumulatorDeduplicates() { - // Same class twice in the same scan shouldn't add the entry - // twice -- otherwise we'd inject duplicate Gradle / pod - // lines on the wire. - AiDependencyTable.Accumulator acc = new AiDependencyTable.Accumulator(); - acc.consume("com/codename1/ai/mlkit/text/TextRecognizer"); - acc.consume("com/codename1/ai/mlkit/text/OptionsBuilder"); - assertEquals(1, acc.hits().size()); - } - - @Test - void arApiInjectsArKitCameraAndArCore() { - List hits = AiDependencyTable.matchesFor( - "com/codename1/ar/AR"); - assertEquals(1, hits.size(), "expected the AR entry to fire"); - AiDependencyTable.Entry e = hits.get(0); - // iOS: ARKit + SceneKit (linked explicitly by IPhoneBuilder) and the - // camera usage string, overridable via ios.NSCameraUsageDescription. - assertTrue(e.iosFrameworks().contains("ARKit")); - assertTrue(e.iosFrameworks().contains("SceneKit")); - assertNotNull(findPlistDefault(e, "NSCameraUsageDescription")); - // Android: the ARCore dependency, the camera permission and the - // optional AR feature/meta-data pair so non-AR devices still install. - assertEquals(1, e.androidGradleDeps().size()); - assertTrue(e.androidGradleDeps().get(0).startsWith("com.google.ar:core")); - assertTrue(e.androidPermissions().contains("android.permission.CAMERA")); - assertTrue(e.androidFeatures().contains("android.hardware.camera.ar")); - assertEquals(1, e.androidMetaDataEntries().size()); - assertEquals("com.google.ar.core", e.androidMetaDataEntries().get(0)[0]); - assertEquals("optional", e.androidMetaDataEntries().get(0)[1]); - } - - @Test - void arEntryMatchesTheWholePackageButNothingElse() { - assertEquals(1, AiDependencyTable.matchesFor("com/codename1/ar/ARSession").size()); - assertEquals(1, AiDependencyTable.matchesFor("com/codename1/ar/ARNode").size()); - // The pure-core VR package must NOT pull the AR native dependencies. - assertTrue(AiDependencyTable.matchesFor("com/codename1/vr/VRView").isEmpty()); - } - - @Test - void nonArEntriesCarryNoMetaData() { - // The meta-data field is new; make sure the existing entries did not - // accidentally gain one. - for (AiDependencyTable.Entry e : AiDependencyTable.entries()) { - if (!e.classPrefix().startsWith("com/codename1/ar/")) { - assertTrue(e.androidMetaDataEntries().isEmpty(), - e.classPrefix() + " should carry no manifest meta-data"); - } - } - } - - @Test - void bluetoothEntryFiresForEverySubPackage() { - String[] classes = { - "com/codename1/bluetooth/Bluetooth", - "com/codename1/bluetooth/le/BlePeripheral", - "com/codename1/bluetooth/le/server/GattServer", - "com/codename1/bluetooth/gatt/GattCharacteristic", - "com/codename1/bluetooth/classic/RfcommConnection" - }; - for (String cls : classes) { - List hits = AiDependencyTable.matchesFor(cls); - assertEquals(1, hits.size(), "expected the bluetooth entry for " + cls); - AiDependencyTable.Entry e = hits.get(0); - assertNotNull(findPlistDefault(e, "NSBluetoothAlwaysUsageDescription")); - assertNotNull(findPlistDefault(e, "NSBluetoothPeripheralUsageDescription")); - assertTrue(e.iosFrameworks().contains("CoreBluetooth")); - // Android permissions deliberately live in - // BluetoothManifestFragments (maxSdkVersion / neverForLocation - // nuances the table cannot express), not in the entry. - assertTrue(e.androidPermissions().isEmpty(), - "bluetooth Android permissions must come from BluetoothManifestFragments"); - } - } - - @Test - void bluetoothEntryDoesNotFireForUnrelatedClasses() { - assertTrue(AiDependencyTable.matchesFor("com/codename1/ui/Form").isEmpty()); - // "bluetoothle" cn1lib package must NOT trigger the core entry - assertTrue(AiDependencyTable.matchesFor( - "com/codename1/bluetoothle/Bluetooth").isEmpty()); - } - - private static String findPlistDefault(AiDependencyTable.Entry e, String key) { - for (String[] entry : e.iosPlistEntries()) { - if (key.equals(entry[0])) { - return entry[1]; - } - } - return null; - } -} diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java new file mode 100644 index 00000000000..92f47cdd870 --- /dev/null +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/AndroidAiSourceSelectionTest.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.builders; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class AndroidAiSourceSelectionTest { + @Test + void selectsOneSourcePerVisionFeature() { + assertEquals("AndroidTextRecognitionAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/TextRecognizer")); + assertEquals("AndroidBarcodeScanningAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/BarcodeScanner")); + assertEquals("AndroidFaceDetectionAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/FaceDetector")); + assertEquals("AndroidImageLabelingAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/ImageLabeler")); + assertEquals("AndroidPoseDetectionAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/PoseDetector")); + assertEquals("AndroidSelfieSegmentationAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/SelfieSegmenter")); + } + + @Test + void selectsOneSourcePerLanguageFeature() { + assertEquals("AndroidLanguageIdAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/language/LanguageIdentifier")); + assertEquals("AndroidTranslationAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/language/Translator")); + assertEquals("AndroidSmartReplyAdapter.java", + AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/language/SmartReply")); + } + + @Test + void ignoresSharedApiAndUnrelatedClasses() { + assertNull(AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/DocumentScanner"), + "DocumentScanner is Apple-only and must not retain an Android backend"); + assertNull(AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ai/vision/VisionPipeline")); + assertNull(AndroidGradleBuilder.androidAiAdapterSource( + "com/codename1/ui/Form")); + } + + @Test + void retainsLiteRtOutputShapeRefreshOnlyForInference() { + String rules = + AndroidGradleBuilder.androidInferenceProguardRules(true); + assertTrue(rules.contains("org.tensorflow.lite.TensorImpl")); + assertTrue(rules.contains("void refreshShape();")); + assertFalse(AndroidGradleBuilder + .androidInferenceProguardRules(false) + .contains("TensorImpl")); + } +} diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java new file mode 100644 index 00000000000..a920e0ed27a --- /dev/null +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/ExecutorArchiveExtractionTest.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.builders; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class ExecutorArchiveExtractionTest { + @TempDir + Path temporaryDirectory; + + @Test + void resolvesNestedEntryInsideDestination() throws Exception { + File destination = temporaryDirectory.toFile(); + + File resolved = Executor.resolveArchiveEntry(destination, + "nested/model.tflite"); + + assertEquals(new File(destination, "nested/model.tflite") + .getCanonicalFile(), resolved); + } + + @Test + void rejectsParentTraversal() { + File destination = temporaryDirectory.resolve("archive").toFile(); + + assertThrows(IOException.class, () -> Executor.resolveArchiveEntry( + destination, "../escaped.txt")); + } + + @Test + void rejectsAbsoluteDestination() { + File destination = temporaryDirectory.resolve("archive").toFile(); + String absoluteEntry = temporaryDirectory.resolve("escaped.txt") + .toAbsolutePath().toString(); + + assertThrows(IOException.class, () -> Executor.resolveArchiveEntry( + destination, absoluteEntry)); + } + + @Test + void rejectsSiblingWithMatchingPrefix() { + File destination = temporaryDirectory.resolve("archive").toFile(); + + assertThrows(IOException.class, () -> Executor.resolveArchiveEntry( + destination, "../archive-escape/payload.bin")); + } + + @Test + void extractionFilterMayRouteSafeEntryToSibling() throws Exception { + File project = temporaryDirectory.resolve("project").toFile(); + File source = new File(project, "src"); + File settings = new File(project, + "codenameone_settings.properties"); + source.mkdirs(); + + new IPhoneBuilder().extractZip( + new ByteArrayInputStream(zipEntry( + "codenameone_settings.properties", + "codename1.mainName=MyApp")), + source, (path, fileName) -> settings); + + assertEquals("codename1.mainName=MyApp", + new String(Files.readAllBytes(settings.toPath()), + StandardCharsets.UTF_8)); + } + + @Test + void rejectsTraversalBeforeArchiveSpecificRouting() throws Exception { + assertSpecialEntryRejected("html/../../payload", "html.tar"); + assertSpecialEntryRejected("javase.lib/../../payload", + "javase.lib.tar"); + } + + @Test + void rejectsTraversalInStrippedTarMemberName() throws Exception { + assertSpecialEntryRejected("html/../payload", "html.tar"); + assertSpecialEntryRejected("podspecs/../payload", "podspecs.tar"); + assertSpecialEntryRejected("javase.lib/../payload", + "javase.lib.tar"); + } + + private static byte[] zipEntry(String name, String contents) + throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + ZipOutputStream zip = new ZipOutputStream(bytes); + zip.putNextEntry(new ZipEntry(name)); + zip.write(contents.getBytes(StandardCharsets.UTF_8)); + zip.closeEntry(); + zip.close(); + return bytes.toByteArray(); + } + + private void assertSpecialEntryRejected(String entryName, + String generatedArchive) throws Exception { + File root = temporaryDirectory.resolve("special-" + + Math.abs(entryName.hashCode())).toFile(); + File classes = new File(root, "classes"); + File resources = new File(root, "resources"); + File sources = new File(root, "sources"); + classes.mkdirs(); + resources.mkdirs(); + sources.mkdirs(); + + new IPhoneBuilder().unzip( + new ByteArrayInputStream(zipEntry(entryName, "payload")), + classes, resources, sources); + + assertFalse(new File(resources, generatedArchive).exists(), + "Unsafe entry must not create " + generatedArchive); + } +} diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java index 3b4f48bae4e..41f5550d158 100644 --- a/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/builders/IPhoneBuilderDependencyConfigTest.java @@ -1,8 +1,33 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ package com.codename1.builders; import org.junit.jupiter.api.Test; +import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.util.List; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -146,6 +171,92 @@ void rejectsUnknownDependencyManagerHint() { assertThrows(BuildException.class, () -> IOSDependencyManager.fromHint("gradle")); } + @Test + void appendsEachRequiredAiFrameworkIndependently() throws Exception { + Method method = IPhoneBuilder.class.getDeclaredMethod( + "appendFrameworks", String.class, String[].class); + method.setAccessible(true); + String value = (String) method.invoke(null, "Vision.framework", + new String[] {"Vision.framework", "CoreImage.framework", + "CoreVideo.framework"}); + assertEquals("Vision.framework;CoreImage.framework;CoreVideo.framework", + value); + + value = (String) method.invoke(null, + "CoreML.framework;Accelerate.framework", + new String[] {"CoreML.framework", "Metal.framework", + "Accelerate.framework"}); + assertEquals("CoreML.framework;Accelerate.framework;Metal.framework", + value); + + value = (String) method.invoke(null, "Vision.framework", + new String[] {"NaturalLanguage.framework"}); + assertEquals("Vision.framework;NaturalLanguage.framework", value); + + value = (String) method.invoke(null, "ThirdPartyVision.framework", + new String[] {"Vision.framework"}); + assertEquals("ThirdPartyVision.framework;Vision.framework", value); + } + + @Test + void catalogPodsDoNotOverrideUserVersionSpecs() { + String pods = IPhoneBuilder.appendPodSpecIfAbsent( + "GoogleMLKit/TextRecognition ~> 7.0,OtherPod", + "GoogleMLKit/TextRecognition"); + assertEquals("GoogleMLKit/TextRecognition ~> 7.0,OtherPod", pods); + + assertEquals("GoogleMLKit/TextRecognition ~> 7.0,OtherPod", + IPhoneBuilder.deduplicatePodSpecs( + pods + ",GoogleMLKit/TextRecognition;OtherPod")); + } + + @Test + void dependencyFloorRaisesExplicitDeploymentTarget() throws Exception { + IPhoneBuilder builder = new IPhoneBuilder(); + Method addTarget = IPhoneBuilder.class.getDeclaredMethod( + "addMinDeploymentTarget", String.class); + addTarget.setAccessible(true); + addTarget.invoke(builder, "15.5"); + + Method getTarget = IPhoneBuilder.class.getDeclaredMethod( + "getDeploymentTarget", BuildRequest.class); + getTarget.setAccessible(true); + String target = (String) getTarget.invoke(builder, + requestWithArgs("ios.deployment_target", "14.0")); + + assertEquals("15.5", target); + } + + @Test + void catalogPrivacyDefaultsReachGeneratedPlistMap() throws Exception { + Method apply = IPhoneBuilder.class.getDeclaredMethod( + "applyCatalogPlistEntry", BuildRequest.class, + String[].class); + apply.setAccessible(true); + Field privacy = IPhoneBuilder.class.getDeclaredField( + "privacyUsageDescriptions"); + privacy.setAccessible(true); + + IPhoneBuilder defaultBuilder = new IPhoneBuilder(); + BuildRequest defaultRequest = new BuildRequest(); + apply.invoke(defaultBuilder, defaultRequest, + new String[] {"NSCameraUsageDescription", "Camera default"}); + assertEquals("Camera default", defaultRequest.getArg( + "ios.NSCameraUsageDescription", null)); + Map defaultMap = (Map) privacy.get(defaultBuilder); + assertEquals("Camera default", + defaultMap.get("NSCameraUsageDescription")); + + IPhoneBuilder explicitBuilder = new IPhoneBuilder(); + BuildRequest explicitRequest = requestWithArgs( + "ios.NSCameraUsageDescription", "Developer value"); + apply.invoke(explicitBuilder, explicitRequest, + new String[] {"NSCameraUsageDescription", "Camera default"}); + Map explicitMap = (Map) privacy.get(explicitBuilder); + assertEquals("Developer value", + explicitMap.get("NSCameraUsageDescription")); + } + private BuildRequest requestWithArgs(String... kvPairs) { BuildRequest out = new BuildRequest(); for (int i = 0; i < kvPairs.length; i += 2) { diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/ChatRequestBuilderTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/ChatRequestBuilderTest.java index 2a17bfade45..cebf1f5a5c8 100644 --- a/maven/core-unittests/src/test/java/com/codename1/ai/ChatRequestBuilderTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/ai/ChatRequestBuilderTest.java @@ -109,4 +109,25 @@ void chatMessageGetTextConcatenatesTextParts() { // ChatView can render a stripped-down preview safely. assertEquals("hello\nworld", m.getText()); } + + @Test + void binaryAiValuesAreDefensivelyCopied() { + byte[] imageBytes = new byte[] {1, 2, 3}; + ImagePart image = new ImagePart(imageBytes, "image/png"); + imageBytes[0] = 9; + assertArrayEquals(new byte[] {1, 2, 3}, image.getData()); + byte[] returnedImage = image.getData(); + returnedImage[1] = 9; + assertArrayEquals(new byte[] {1, 2, 3}, image.getData()); + + float[] coordinates = new float[] {0.25f, 0.75f}; + Embedding embedding = new Embedding(coordinates, 0); + coordinates[0] = 9; + assertArrayEquals(new float[] {0.25f, 0.75f}, + embedding.getVector()); + float[] returnedVector = embedding.getVector(); + returnedVector[1] = 9; + assertArrayEquals(new float[] {0.25f, 0.75f}, + embedding.getVector()); + } } diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java new file mode 100644 index 00000000000..9f512ab83c8 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/ai/inference/InferenceApiTest.java @@ -0,0 +1,586 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.inference; + +import com.codename1.impl.InferenceImpl; +import com.codename1.io.FileSystemStorage; +import com.codename1.junit.UITestBase; +import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; +import org.junit.jupiter.api.Test; + +import java.io.OutputStream; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; + +class InferenceApiTest extends UITestBase { + @Test + void tensorsAreImmutableAndValidateShape() { + float[] source = new float[] {1, 2}; + Tensor tensor = Tensor.floats("input", new int[] {1, 2}, source); + source[0] = 9; + assertArrayEquals(new float[] {1, 2}, (float[]) tensor.getData()); + float[] returned = (float[]) tensor.getData(); + returned[1] = 9; + assertArrayEquals(new float[] {1, 2}, (float[]) tensor.getData()); + assertThrows(IllegalArgumentException.class, () -> + Tensor.floats("bad", new int[] {3}, new float[] {1, 2})); + assertThrows(IllegalArgumentException.class, () -> + Tensor.bytes("overflow", TensorType.UINT8, + new int[] {Integer.MAX_VALUE, 2}, new byte[] {1})); + } + + @Test + void modelSourceOffersExplicitReadOnlyZeroCopyAccess() { + byte[] source = new byte[] {1, 2, 3}; + ModelSource model = ModelSource.bytes(source); + assertNotSame(source, model.getBytesUnsafe()); + assertSame(model.getBytesUnsafe(), model.getBytesUnsafe()); + assertNotSame(model.getBytesUnsafe(), model.getBytes()); + } + + @Test + void modelCacheCompletionIsSingleShot() { + AsyncResource resource = new AsyncResource(); + ModelCache.Completion completion = + new ModelCache.Completion(resource); + completion.complete("first"); + completion.fail(new RuntimeException("late failure")); + completion.complete("second"); + flushSerialCalls(); + assertEquals("first", resource.get()); + } + + @Test + void modelCacheRejectsInsecureAndMalformedRequests() { + assertThrows(IllegalArgumentException.class, + () -> ModelCache.fetch("http://example.com/model.tflite", "model")); + assertThrows(IllegalArgumentException.class, + () -> ModelCache.fetch("https://example.com/model.tflite", "")); + assertThrows(IllegalArgumentException.class, + () -> ModelCache.fetch("https://example.com/model.tflite", "model", "xyz")); + assertThrows(IllegalArgumentException.class, + () -> ModelCache.fetch("https://example.com/model.tflite", "model", + "gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg")); + } + + @Test + void modelCacheRequiresDigestWhenIosHidesRedirects() { + implementation.setPlatformName("ios"); + assertTrue(ModelCache.requiresPinnedModelDigest()); + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> ModelCache.fetch( + "https://example.com/model.tflite", "ios-model")); + assertTrue(error.getMessage().contains("SHA-256")); + } + + @Test + void modelCacheRejectsInsecureRedirects() throws Exception { + FileSystemStorage fs = FileSystemStorage.getInstance(); + String temporary = fs.getAppHomePath() + + "model-cache-redirect-test.download"; + write(temporary, new byte[] {1, 2, 3}); + AsyncResource resource = new AsyncResource(); + AtomicReference error = new AtomicReference(); + resource.except(new SuccessCallback() { + @Override + public void onSucess(Throwable value) { + error.set(value); + } + }); + ModelCache.ModelDownloadRequest request = + new ModelCache.ModelDownloadRequest( + new ModelCache.Completion(resource), + fs, temporary); + + assertFalse(request.onRedirect( + "https://cdn.example.com/model.tflite")); + assertTrue(request.onRedirect( + "http://cdn.example.com/model.tflite")); + flushSerialCalls(); + assertFalse(fs.exists(temporary), + "a downgrade redirect must discard partial model data"); + assertNotNull(error.get(), + "a downgrade redirect must fail the model resource"); + assertTrue(error.get().getCause().getMessage().contains("HTTPS")); + assertTrue(ModelCache.isHttpsUrl( + "HTTPS://cdn.example.com/model.tflite")); + assertFalse(ModelCache.isHttpsUrl( + "http://cdn.example.com/model.tflite")); + } + + @Test + void modelDownloadRequestIdentityIncludesTemporaryPath() { + FileSystemStorage fs = FileSystemStorage.getInstance(); + ModelCache.ModelDownloadRequest first = + new ModelCache.ModelDownloadRequest( + new ModelCache.Completion( + new AsyncResource()), + fs, "model-a.download"); + ModelCache.ModelDownloadRequest same = + new ModelCache.ModelDownloadRequest( + new ModelCache.Completion( + new AsyncResource()), + fs, "model-a.download"); + ModelCache.ModelDownloadRequest differentTarget = + new ModelCache.ModelDownloadRequest( + new ModelCache.Completion( + new AsyncResource()), + fs, "model-b.download"); + first.setUrl("https://example.com/model.tflite"); + same.setUrl("https://example.com/model.tflite"); + differentTarget.setUrl("https://example.com/model.tflite"); + + assertEquals(first, same); + assertEquals(first.hashCode(), same.hashCode()); + assertNotEquals(first, differentTarget); + } + + @Test + void modelCacheCoalescesIdenticalConcurrentFetches() { + String fileName = "model-cache-coalescing-test.tflite"; + ModelCache.FetchRegistration first = ModelCache.registerFetch( + fileName, "https://one.example/model.tflite", null); + ModelCache.FetchRegistration duplicate = ModelCache.registerFetch( + fileName, "https://one.example/model.tflite", null); + assertTrue(first.owner); + assertFalse(duplicate.owner); + assertNotSame(first.resource, duplicate.resource, + "coalesced callers need independent cancellation handles"); + assertTrue(duplicate.resource.cancel(false)); + + ModelCache.FetchRegistration conflict = ModelCache.registerFetch( + fileName, "https://two.example/model.tflite", null); + AtomicReference conflictError = + new AtomicReference(); + conflict.resource.except(new SuccessCallback() { + @Override + public void onSucess(Throwable value) { + conflictError.set(value); + } + }); + flushSerialCalls(); + assertNotNull(conflictError.get(), + "conflicting content for one cache key must fail"); + + first.completion.complete(ModelSource.file("test-model.tflite")); + flushSerialCalls(); + assertEquals("test-model.tflite", first.resource.get().getPath()); + assertTrue(duplicate.resource.isCancelled(), + "one subscriber's cancellation must remain local"); + ModelCache.FetchRegistration afterCompletion = + ModelCache.registerFetch(fileName, + "https://two.example/model.tflite", null); + assertTrue(afterCompletion.owner, + "completion must release the cache-key download slot"); + afterCompletion.completion.complete( + ModelSource.file("test-model-2.tflite")); + flushSerialCalls(); + } + + @Test + void modelCacheNamesDoNotAliasSanitizedKeys() { + assertNotEquals(ModelCache.safeName("model/v1"), + ModelCache.safeName("model?v1")); + assertNotEquals(ModelCache.safeName("model/v1"), + ModelCache.safeName("model_v1")); + assertTrue(ModelCache.safeName("model-v1_2") + .startsWith("model-v1_2-")); + StringBuilder longKey = new StringBuilder(); + for (int i = 0; i < 512; i++) { + longKey.append('a'); + } + assertTrue((ModelCache.safeName(longKey.toString()) + + ".tflite.download").length() <= 255); + } + + @Test + void modelCacheDiscardsPartialAndDigestMismatchFiles() throws Exception { + FileSystemStorage fs = FileSystemStorage.getInstance(); + String temporary = fs.getAppHomePath() + "model-cache-test.download"; + write(temporary, new byte[] {1, 2, 3}); + ModelCache.prepareTemporary(fs, temporary); + assertFalse(fs.exists(temporary), "a previous partial download must not be resumed"); + + write(temporary, new byte[] {4, 5, 6}); + assertThrows(java.io.IOException.class, () -> + ModelCache.verifyDownloaded(fs, temporary, + "0000000000000000000000000000000000000000000000000000000000000000")); + assertFalse(fs.exists(temporary), "a digest mismatch must delete executable data"); + } + + @Test + void modelCacheVerifiesPromotionBeforePublishingPath() throws Exception { + FileSystemStorage fs = FileSystemStorage.getInstance(); + String directory = fs.getAppHomePath(); + String fileName = "model-cache-promoted-test.tflite"; + String target = directory + fileName; + String temporary = target + ".download"; + if (fs.exists(target)) { + fs.delete(target); + } + if (fs.exists(temporary)) { + fs.delete(temporary); + } + + assertThrows(java.io.IOException.class, () -> + ModelCache.promoteDownloaded(fs, temporary, target, + fileName, null)); + assertFalse(fs.exists(target), + "a failed rename must not publish a nonexistent cache path"); + + write(temporary, new byte[] {1, 2, 3}); + assertThrows(java.io.IOException.class, () -> + ModelCache.promoteDownloaded(fs, temporary, target, + fileName, + "00000000000000000000000000000000" + + "00000000000000000000000000000000")); + assertFalse(fs.exists(target), + "a final digest mismatch must remove the promoted path"); + assertFalse(fs.exists(temporary), + "a final digest mismatch must remove the temporary file"); + + write(temporary, new byte[] {1, 2, 3}); + ModelCache.promoteDownloaded(fs, temporary, target, fileName, + "039058c6f2c0cb492c533b0a4d14ef7" + + "7cc0f78abccced5287d84a1a2011cfb81"); + assertTrue(fs.exists(target)); + assertFalse(fs.exists(temporary)); + + fs.delete(target); + } + + private static void write(String path, byte[] value) throws Exception { + OutputStream output = FileSystemStorage.getInstance().openOutputStream(path); + try { + output.write(value); + } finally { + output.close(); + } + } + + @Test + void sessionLifecycleForwardsToBackend() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + implementation.setInferenceImpl(backend); + InferenceSession session = await(InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions().threads(2))); + TensorInfo[] inputs = session.getInputs(); + assertEquals(2, inputs[0].getShape()[1]); + assertNotSame(inputs, session.getInputs()); + inputs[0] = null; + assertNotNull(session.getInputs()[0]); + Tensor[] output = await(session.run(new Tensor[] { + Tensor.floats("input", new int[] {1, 2}, new float[] {1, 2}) + })); + assertArrayEquals(new float[] {3}, (float[]) output[0].getData()); + session.resizeInput("input", new int[] {1, 4}); + assertEquals("input", backend.resizedName); + session.close(); + session.close(); + assertEquals(1, backend.closeCount); + assertThrows(IllegalStateException.class, session::getInputs); + } + + @Test + void sessionSnapshotsOptionsBeforeAsyncOpen() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + implementation.setInferenceImpl(backend); + InferenceOptions options = new InferenceOptions() + .accelerator(InferenceOptions.Accelerator.GPU) + .threads(3) + .allowFallback(false); + + AsyncResource opening = InferenceSession.open( + ModelSource.bytes(new byte[] {1}), options); + options.accelerator(InferenceOptions.Accelerator.CPU) + .threads(8) + .allowFallback(true); + + assertNotSame(options, backend.openOptions); + assertEquals(InferenceOptions.Accelerator.GPU, + backend.openOptions.getAccelerator()); + assertEquals(3, backend.openOptions.getThreads()); + assertFalse(backend.openOptions.isFallbackAllowed()); + await(opening).close(); + } + + @Test + void cancelledOpenClosesLateNativeHandle() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + backend.pendingOpen = new AsyncResource(); + implementation.setInferenceImpl(backend); + + AsyncResource opening = InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions()); + assertTrue(opening.cancel(false)); + backend.pendingOpen.complete(backend.handle); + flushSerialCalls(); + + assertTrue(opening.isCancelled()); + assertEquals(1, backend.closeCount, + "a late native handle must not be orphaned after cancellation"); + assertThrows(AsyncResource.AsyncExecutionException.class, + opening::get); + } + + @Test + void sessionRejectsShapeMismatchBeforeBackendRun() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + implementation.setInferenceImpl(backend); + InferenceSession session = await(InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions())); + + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, () -> session.run( + new Tensor[] {Tensor.floats("input", + new int[] {2, 1}, + new float[] {1, 2})})); + assertTrue(error.getMessage().contains("resizeInput")); + assertNull(backend.receivedInputs, + "shape mismatch must not reach the asynchronous backend"); + session.close(); + } + + @Test + void sessionDefersCloseUntilPendingRunSettles() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + backend.pendingRun = new AsyncResource(); + implementation.setInferenceImpl(backend); + InferenceSession session = await(InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions())); + AsyncResource run = session.run(new Tensor[] { + Tensor.floats("input", new int[] {1, 2}, + new float[] {1, 2}) + }); + assertThrows(IllegalStateException.class, () -> session.run( + new Tensor[] {Tensor.floats("input", new int[] {1, 2}, + new float[] {3, 4})})); + assertThrows(IllegalStateException.class, + () -> session.resizeInput("input", new int[] {1, 4})); + assertThrows(IllegalStateException.class, session::getInputs); + assertThrows(IllegalStateException.class, session::getOutputs); + session.close(); + assertEquals(0, backend.closeCount); + backend.pendingRun.complete(new Tensor[] { + Tensor.floats("output", new int[] {1}, new float[] {3}) + }); + flushSerialCalls(); + assertTrue(run.isDone()); + assertEquals(1, backend.closeCount); + assertThrows(IllegalStateException.class, session::getInputs); + } + + @Test + void cancelledRunStaysSerializedUntilNativeWorkSettles() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + backend.pendingRun = new AsyncResource(); + implementation.setInferenceImpl(backend); + InferenceSession session = await(InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions())); + AsyncResource run = session.run(new Tensor[] { + Tensor.floats("input", new int[] {1, 2}, + new float[] {1, 2}) + }); + + assertTrue(run.cancel(false)); + assertFalse(backend.pendingRun.isCancelled(), + "outward cancellation must not interrupt a native invocation"); + assertThrows(IllegalStateException.class, () -> session.run( + new Tensor[] {Tensor.floats("input", new int[] {1, 2}, + new float[] {3, 4})})); + session.close(); + assertEquals(0, backend.closeCount, + "cancellation must not close an interpreter still in use"); + backend.pendingRun.complete(new Tensor[] { + Tensor.floats("output", new int[] {1}, new float[] {3}) + }); + flushSerialCalls(); + assertTrue(run.isCancelled()); + assertEquals(1, backend.closeCount, + "native settlement must complete deferred close"); + } + + @Test + void settledRunCanStartAnotherRunFromSuccessCallback() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + backend.pendingRun = new AsyncResource(); + implementation.setInferenceImpl(backend); + final InferenceSession session = await(InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions())); + final AtomicReference> second = + new AtomicReference>(); + session.run(new Tensor[] { + Tensor.floats("input", new int[] {1, 2}, + new float[] {1, 2}) + }).ready(new SuccessCallback() { + public void onSucess(Tensor[] ignored) { + backend.pendingRun = null; + second.set(session.run(new Tensor[] { + Tensor.floats("input", new int[] {1, 2}, + new float[] {3, 4}) + })); + } + }); + + backend.pendingRun.complete(new Tensor[] { + Tensor.floats("output", new int[] {1}, new float[] {3}) + }); + flushSerialCalls(); + + assertNotNull(second.get()); + assertArrayEquals(new float[] {3}, + (float[]) await(second.get())[0].getData()); + session.close(); + } + + @Test + void settledRunCanStartAnotherRunFromFailureCallback() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + backend.pendingRun = new AsyncResource(); + implementation.setInferenceImpl(backend); + final InferenceSession session = await(InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions())); + final AtomicReference> second = + new AtomicReference>(); + session.run(new Tensor[] { + Tensor.floats("input", new int[] {1, 2}, + new float[] {1, 2}) + }).except(new SuccessCallback() { + public void onSucess(Throwable ignored) { + backend.pendingRun = null; + second.set(session.run(new Tensor[] { + Tensor.floats("input", new int[] {1, 2}, + new float[] {3, 4}) + })); + } + }); + + backend.pendingRun.error(new RuntimeException("expected")); + flushSerialCalls(); + + assertNotNull(second.get()); + assertArrayEquals(new float[] {3}, + (float[]) await(second.get())[0].getData()); + session.close(); + } + + @Test + void sessionSnapshotsInputArrayBeforeAsyncInference() { + RecordingInferenceImpl backend = new RecordingInferenceImpl(); + backend.pendingRun = new AsyncResource(); + implementation.setInferenceImpl(backend); + InferenceSession session = await(InferenceSession.open( + ModelSource.bytes(new byte[] {1}), new InferenceOptions())); + Tensor original = Tensor.floats("input", new int[] {1, 2}, + new float[] {1, 2}); + Tensor replacement = Tensor.floats("replacement", new int[] {1, 2}, + new float[] {9, 9}); + Tensor[] callerInputs = new Tensor[] {original}; + + AsyncResource run = session.run(callerInputs); + callerInputs[0] = replacement; + + assertNotSame(callerInputs, backend.receivedInputs); + assertSame(original, backend.receivedInputs[0], + "a pending backend must see the run() call-time inputs"); + backend.pendingRun.complete(new Tensor[] { + Tensor.floats("output", new int[] {1}, new float[] {3}) + }); + flushSerialCalls(); + assertTrue(run.isDone()); + session.close(); + } + + private T await(AsyncResource resource) { + final AtomicReference value = new AtomicReference(); + resource.ready(new SuccessCallback() { + public void onSucess(T result) { + value.set(result); + } + }); + flushSerialCalls(); + assertTrue(resource.isDone()); + assertNotNull(value.get()); + return resource.get(); + } + + private static final class RecordingInferenceImpl extends InferenceImpl { + final Object handle = new Object(); + String resizedName; + int closeCount; + AsyncResource pendingRun; + Tensor[] receivedInputs; + InferenceOptions openOptions; + AsyncResource pendingOpen; + + public boolean isSupported() { + return true; + } + + public AsyncResource open(ModelSource source, InferenceOptions options) { + openOptions = options; + if (pendingOpen != null) { + return pendingOpen; + } + AsyncResource result = new AsyncResource(); + result.complete(handle); + return result; + } + + public TensorInfo[] getInputs(Object value) { + assertSame(handle, value); + return new TensorInfo[] { + new TensorInfo("input", TensorType.FLOAT32, new int[] {1, 2}, 0) + }; + } + + public TensorInfo[] getOutputs(Object value) { + return new TensorInfo[] { + new TensorInfo("output", TensorType.FLOAT32, new int[] {1}, 0) + }; + } + + public AsyncResource run(Object value, Tensor[] inputs) { + receivedInputs = inputs; + if (pendingRun != null) { + return pendingRun; + } + AsyncResource result = new AsyncResource(); + result.complete(new Tensor[] { + Tensor.floats("output", new int[] {1}, new float[] {3}) + }); + return result; + } + + public void resizeInput(Object value, String name, int[] shape) { + resizedName = name; + } + + public void close(Object value) { + closeCount++; + } + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java new file mode 100644 index 00000000000..60ea51a3376 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/ai/language/LanguageApiTest.java @@ -0,0 +1,298 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codename1.ai.language; + +import com.codename1.impl.LanguageImpl; +import com.codename1.junit.UITestBase; +import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class LanguageApiTest extends UITestBase { + @Test + void optionsRejectNaNConfidence() { + assertThrows(IllegalArgumentException.class, + () -> new LanguageOptions().minimumConfidence(Float.NaN)); + assertEquals(0f, new LanguageOptions() + .minimumConfidence(Float.NEGATIVE_INFINITY) + .getMinimumConfidence()); + assertEquals(1f, new LanguageOptions() + .minimumConfidence(Float.POSITIVE_INFINITY) + .getMinimumConfidence()); + } + + @Test + void languageOperationsForwardBackendAndOptions() { + RecordingLanguageImpl backend = new RecordingLanguageImpl(); + implementation.setLanguageImpl(backend); + LanguageOptions options = new LanguageOptions() + .backend(LanguageBackends.mlKitLanguageIdentification()) + .minimumConfidence(.4f); + + assertTrue(LanguageIdentifier.isSupported(options)); + assertTrue(Translator.isSupported(options)); + assertTrue(SmartReply.isSupported(options)); + + LanguageCandidate[] candidates = + LanguageIdentifier.identify("bonjour", options).get(); + assertEquals("fr", candidates[0].getLanguageTag()); + assertEquals("language-id", backend.feature); + assertEquals("ml-kit", backend.backend); + assertEquals(.4f, backend.options.getMinimumConfidence()); + + assertEquals("hello", + Translator.translate("bonjour", "fr", "en", options).get()); + assertEquals("translation", backend.feature); + + String[] replies = SmartReply.suggest(new SmartReplyMessage[] { + new SmartReplyMessage("Are you coming?", "remote", false, 1) + }, options).get(); + assertEquals("Yes", replies[0]); + assertEquals("smart-reply", backend.feature); + } + + @Test + void missingBackendReportsUnsupported() { + implementation.setLanguageImpl(null); + assertFalse(LanguageIdentifier.isSupported()); + AsyncResource result = + LanguageIdentifier.identify("hello", null); + assertTrue(result.isDone()); + assertThrows(AsyncResource.AsyncExecutionException.class, result::get); + } + + @Test + void languageOperationsSnapshotMutableArguments() { + RecordingLanguageImpl backend = new RecordingLanguageImpl(); + implementation.setLanguageImpl(backend); + LanguageOptions options = new LanguageOptions() + .backend(LanguageBackends.mlKitLanguageIdentification()) + .minimumConfidence(.35f); + + LanguageIdentifier.identify("bonjour", options); + options.backend(LanguageBackends.auto()).minimumConfidence(.9f); + + assertNotSame(options, backend.options); + assertEquals("ml-kit", backend.backend); + assertEquals(.35f, backend.options.getMinimumConfidence()); + + SmartReplyMessage original = new SmartReplyMessage( + "First", null, false, 1); + assertEquals("remote", original.getParticipantId()); + SmartReplyMessage replacement = new SmartReplyMessage( + "Replacement", "remote", false, 2); + SmartReplyMessage[] conversation = + new SmartReplyMessage[] {original}; + SmartReply.suggest(conversation, null); + conversation[0] = replacement; + assertNotSame(conversation, backend.conversation); + assertEquals(original, backend.conversation[0]); + } + + @Test + void reusableSessionRetainsBackendUntilClosed() { + RecordingLanguageImpl backend = new RecordingLanguageImpl(); + implementation.setLanguageImpl(backend); + + Translator.Session session = Translator.open( + new LanguageOptions().backend( + LanguageBackends.mlKitTranslation())); + assertEquals("hello", + session.translate("bonjour", "fr", "en").get()); + assertEquals("hello", + session.translate("salut", "fr", "en").get()); + assertEquals(2, backend.translationCalls); + assertEquals(0, backend.closeCount); + + session.close(); + session.close(); + assertEquals(1, backend.closeCount); + assertThrows(IllegalStateException.class, + () -> session.translate("bonjour", "fr", "en")); + } + + @Test + void staticConvenienceOperationClosesItsEphemeralBackend() { + RecordingLanguageImpl backend = new RecordingLanguageImpl(); + implementation.setLanguageImpl(backend); + + assertEquals("hello", + Translator.translate("bonjour", "fr", "en", + new LanguageOptions().backend( + LanguageBackends.mlKitTranslation())).get()); + assertEquals(1, backend.closeCount); + } + + @Test + void cancelledOperationSuppressesLateBackendCallbacksAndCloses() + throws Exception { + assertCancelledSettlementDoesNotPublish(false); + assertCancelledSettlementDoesNotPublish(true); + } + + @Test + void throwingCallbackCannotDeferSessionClose() { + assertThrowingCallbackStillCloses(false); + assertThrowingCallbackStillCloses(true); + } + + private void assertThrowingCallbackStillCloses(boolean fail) { + RecordingLanguageImpl backend = new RecordingLanguageImpl(); + backend.deferTranslation = true; + implementation.setLanguageImpl(backend); + Translator.Session session = Translator.open( + new LanguageOptions().backend( + LanguageBackends.mlKitTranslation())); + AsyncResource result = session.translate( + "bonjour", "fr", "en"); + if (fail) { + result.except(new SuccessCallback() { + public void onSucess(Throwable error) { + throw new IllegalStateException("application callback"); + } + }); + } else { + result.ready(new SuccessCallback() { + public void onSucess(String value) { + throw new IllegalStateException("application callback"); + } + }); + } + session.close(); + + assertThrows(IllegalStateException.class, () -> { + if (fail) { + backend.pendingTranslation.error( + new IllegalArgumentException("backend failure")); + } else { + backend.pendingTranslation.complete("hello"); + } + }); + assertEquals(1, backend.closeCount); + } + + private void assertCancelledSettlementDoesNotPublish(boolean fail) { + RecordingLanguageImpl backend = new RecordingLanguageImpl(); + backend.deferTranslation = true; + implementation.setLanguageImpl(backend); + final AtomicInteger successes = new AtomicInteger(); + final AtomicInteger failures = new AtomicInteger(); + + AsyncResource result = Translator.translate( + "bonjour", "fr", "en", + new LanguageOptions().backend( + LanguageBackends.mlKitTranslation())); + result.ready(new SuccessCallback() { + public void onSucess(String value) { + successes.incrementAndGet(); + } + }).except(new SuccessCallback() { + public void onSucess(Throwable error) { + failures.incrementAndGet(); + } + }); + + assertTrue(result.cancel(false)); + if (fail) { + backend.pendingTranslation.error( + new IllegalStateException("late failure")); + } else { + backend.pendingTranslation.complete("late success"); + } + + assertTrue(result.isCancelled()); + assertEquals(0, successes.get()); + assertEquals(0, failures.get()); + assertEquals(1, backend.closeCount); + } + + private static final class RecordingLanguageImpl extends LanguageImpl { + String feature; + String backend; + LanguageOptions options; + SmartReplyMessage[] conversation; + int translationCalls; + int closeCount; + boolean deferTranslation; + AsyncResource pendingTranslation; + + public boolean isSupported(String value, String backendId) { + feature = value; + backend = backendId; + return true; + } + + public AsyncResource identify( + String text, String backendId, LanguageOptions value) { + feature = "language-id"; + backend = backendId; + options = value; + AsyncResource result = + new AsyncResource(); + result.complete(new LanguageCandidate[] { + new LanguageCandidate("fr", .9f) + }); + return result; + } + + public AsyncResource translate( + String text, String sourceLanguage, String targetLanguage, + String backendId, LanguageOptions value) { + feature = "translation"; + translationCalls++; + AsyncResource result = new AsyncResource(); + if (deferTranslation) { + pendingTranslation = result; + } else { + result.complete("hello"); + } + return result; + } + + public AsyncResource suggestReplies( + SmartReplyMessage[] conversation, String backendId, + LanguageOptions value) { + feature = "smart-reply"; + this.conversation = conversation; + AsyncResource result = new AsyncResource(); + result.complete(new String[] {"Yes"}); + return result; + } + + public void close() { + closeCount++; + } + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java new file mode 100644 index 00000000000..e05f6f15a60 --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/ai/vision/VisionApiTest.java @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.ai.vision; + +import com.codename1.camera.CameraFrame; +import com.codename1.camera.FrameFormat; +import com.codename1.impl.VisionImpl; +import com.codename1.junit.UITestBase; +import com.codename1.util.AsyncResource; +import com.codename1.util.SuccessCallback; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; + +class VisionApiTest extends UITestBase { + @Test + void visionImageDefensivelyCopiesInputAndOutput() { + byte[] source = new byte[] {1, 2, 3}; + VisionImage image = VisionImage.encoded(source); + source[0] = 9; + assertEquals(1, image.getEncodedBytes()[0]); + byte[] returned = image.getEncodedBytes(); + returned[1] = 9; + assertEquals(2, image.getEncodedBytes()[1]); + } + + @Test + void encodedInputPreservesExplicitDisplayRotation() { + VisionImage image = VisionImage.encoded( + new byte[] {1, 2, 3}, -90); + assertEquals(270, image.getRotationDegrees()); + assertThrows(IllegalArgumentException.class, + () -> VisionImage.encoded(new byte[] {1}, 45)); + } + + @Test + void pixelInputNormalizesRotation() { + VisionImage image = VisionImage.pixels(new byte[] {1, 2, 3, 4}, + 1, 1, FrameFormat.RGBA8888, -90); + assertEquals(270, image.getRotationDegrees()); + assertThrows(IllegalArgumentException.class, + () -> VisionImage.pixels(new byte[] {1, 2, 3, 4}, + 1, 1, FrameFormat.RGBA8888, 45)); + } + + @Test + void pixelInputRejectsEncodedAndMissingFormats() { + byte[] raw = new byte[] {1, 2, 3, 4}; + assertThrows(IllegalArgumentException.class, + () -> VisionImage.pixels(raw, 1, 1, + FrameFormat.JPEG, 0)); + assertThrows(IllegalArgumentException.class, + () -> VisionImage.pixels(raw, 1, 1, null, 0)); + } + + @Test + void segmentationMaskRejectsOverflowingPixelCount() { + assertThrows(IllegalArgumentException.class, + () -> new SegmentationMask(65536, 65536, new float[0])); + } + + @Test + void optionsRejectNaNConfidence() { + assertThrows(IllegalArgumentException.class, + () -> new VisionOptions().minimumConfidence(Float.NaN)); + assertEquals(0f, new VisionOptions() + .minimumConfidence(Float.NEGATIVE_INFINITY) + .getMinimumConfidence()); + assertEquals(1f, new VisionOptions() + .minimumConfidence(Float.POSITIVE_INFINITY) + .getMinimumConfidence()); + } + + @Test + void cameraFrameCopiesOnlyTheRequestedFormat() { + byte[] jpeg = new byte[] {1, 2}; + byte[] pixels = new byte[] {3, 4, 5, 6}; + VisionImage raw = VisionImage.fromCameraFrame(new CameraFrame( + jpeg, pixels, 1, 1, 0, 7L, FrameFormat.RGBA8888)); + jpeg[0] = 9; + pixels[0] = 9; + assertNull(raw.getEncodedBytes()); + assertArrayEquals(new byte[] {3, 4, 5, 6}, raw.getPixels()); + assertEquals(7L, raw.getTimestampNanos()); + + VisionImage encoded = VisionImage.fromCameraFrame(new CameraFrame( + jpeg, pixels, 1, 1, 0, 8L, FrameFormat.JPEG)); + assertNotNull(encoded.getEncodedBytes()); + assertNull(encoded.getPixels()); + + VisionImage fallback = VisionImage.fromCameraFrame(new CameraFrame( + jpeg, null, 1, 1, 0, 9L, FrameFormat.NV21)); + assertNotNull(fallback.getEncodedBytes()); + assertNull(fallback.getPixels()); + assertEquals(FrameFormat.JPEG, fallback.getFormat()); + } + + @Test + void backendMetadataIsOptionalAndImmutable() { + Map values = new HashMap(); + values.put("nativeType", "qr"); + VisionMetadata metadata = new VisionMetadata("ml-kit", values); + values.put("nativeType", "changed"); + assertEquals("ml-kit", metadata.getBackendId()); + assertEquals("qr", metadata.get("nativeType")); + assertThrows(UnsupportedOperationException.class, + () -> metadata.getValues().put("x", "y")); + Barcode barcode = new Barcode("value", "QR_CODE", null, + VisionRect.EMPTY, null, metadata); + assertSame(metadata, barcode.getMetadata()); + } + + @Test + void analyzerForwardsFeatureBackendAndOptions() { + RecordingVisionImpl backend = new RecordingVisionImpl(); + implementation.setVisionImpl(backend); + VisionOptions options = new VisionOptions() + .backend(VisionBackends.mlKitBarcodeScanning()) + .minimumConfidence(.6f) + .maximumResults(3); + TextRecognizer recognizer = new TextRecognizer(options); + options.backend(VisionBackends.auto()) + .minimumConfidence(.1f) + .maximumResults(1); + TextRecognitionResult result = await(recognizer.process( + VisionImage.encoded(new byte[] {1}))); + assertEquals("hello", result.getText()); + assertEquals(VisionFeature.TEXT_RECOGNITION, backend.feature); + assertEquals("ml-kit", backend.backend); + assertNotSame(options, backend.options); + assertEquals(.6f, backend.options.getMinimumConfidence()); + assertEquals(3, backend.options.getMaximumResults()); + recognizer.close(); + assertEquals(1, backend.closeCount); + } + + @Test + void unsupportedAnalyzerReturnsFailedResource() { + implementation.setVisionImpl(null); + AsyncResource result = + new TextRecognizer().process(VisionImage.encoded(new byte[] {1})); + assertTrue(result.isDone()); + assertThrows(AsyncResource.AsyncExecutionException.class, result::get); + } + + @Test + void analyzerCreationAndCloseAreSerialized() throws Exception { + final RecordingVisionImpl backend = new RecordingVisionImpl(); + implementation.setVisionImpl(backend); + final CountDownLatch creationEntered = new CountDownLatch(1); + final CountDownLatch allowCreation = new CountDownLatch(1); + implementation.setVisionImplCreationHook(new Runnable() { + public void run() { + creationEntered.countDown(); + try { + allowCreation.await(); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new RuntimeException(error); + } + } + }); + final TextRecognizer recognizer = new TextRecognizer(); + final AtomicReference failure = + new AtomicReference(); + Thread processor = new Thread(new Runnable() { + public void run() { + try { + recognizer.process( + VisionImage.encoded(new byte[] {1})); + } catch (Throwable error) { + failure.set(error); + } + } + }, "vision-process"); + Thread closer = new Thread(new Runnable() { + public void run() { + recognizer.close(); + } + }, "vision-close"); + + processor.start(); + assertTrue(creationEntered.await(2, TimeUnit.SECONDS)); + closer.start(); + try { + long deadline = System.currentTimeMillis() + 2000; + while (closer.getState() != Thread.State.BLOCKED + && closer.isAlive() + && System.currentTimeMillis() < deadline) { + Thread.yield(); + } + assertEquals(Thread.State.BLOCKED, closer.getState()); + } finally { + allowCreation.countDown(); + } + processor.join(2000); + closer.join(2000); + + assertFalse(processor.isAlive()); + assertFalse(closer.isAlive()); + assertNull(failure.get()); + assertEquals(1, backend.closeCount); + assertThrows(IllegalStateException.class, + () -> recognizer.process( + VisionImage.encoded(new byte[] {1}))); + } + + private T await(AsyncResource resource) { + final AtomicReference value = new AtomicReference(); + resource.ready(new SuccessCallback() { + public void onSucess(T result) { + value.set(result); + } + }); + flushSerialCalls(); + assertTrue(resource.isDone()); + assertNotNull(value.get()); + return resource.get(); + } + + private static final class RecordingVisionImpl extends VisionImpl { + VisionFeature feature; + String backend; + VisionOptions options; + int closeCount; + + public boolean isSupported(VisionFeature feature, String backendId) { + return true; + } + + @SuppressWarnings("unchecked") + public AsyncResource analyze(VisionFeature value, String backendId, + VisionImage image, VisionOptions opts) { + feature = value; + backend = backendId; + options = opts; + AsyncResource result = new AsyncResource(); + result.complete((T) new TextRecognitionResult("hello", null)); + return result; + } + + public void close() { + closeCount++; + } + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/camera/VisionPipelineTest.java b/maven/core-unittests/src/test/java/com/codename1/camera/VisionPipelineTest.java new file mode 100644 index 00000000000..acccc3fa26a --- /dev/null +++ b/maven/core-unittests/src/test/java/com/codename1/camera/VisionPipelineTest.java @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.camera; + +import com.codename1.ai.vision.VisionAnalyzer; +import com.codename1.ai.vision.VisionImage; +import com.codename1.ai.vision.VisionPipeline; +import com.codename1.ai.vision.VisionPipelineListener; +import com.codename1.junit.UITestBase; +import com.codename1.util.AsyncResource; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +class VisionPipelineTest extends UITestBase { + private RecordingCameraImpl implementationBackend; + private CameraSession session; + private VisionPipeline pipeline; + + @BeforeEach + void openSession() { + implementationBackend = new RecordingCameraImpl(); + implementation.setCameraImpl(implementationBackend); + session = Camera.open(new CameraInfo("back", CameraFacing.BACK, + null, null, true, true), new CameraSessionOptions()); + } + + @AfterEach + void closeSession() { + if (pipeline != null) { + pipeline.close(); + } + if (session != null) { + session.close(); + } + } + + @Test + void listenerFailureDoesNotStrandPendingFrame() { + final RecordingAnalyzer analyzer = new RecordingAnalyzer(); + final int[] notifications = new int[1]; + pipeline = new VisionPipeline(session, analyzer, + new VisionPipelineListener() { + public void result(String value, VisionImage image) { + notifications[0]++; + if (notifications[0] == 1) { + throw new IllegalStateException( + "Application listener failed"); + } + } + + public void error(Throwable error) { + } + }); + + implementationBackend.lastFrameListener.onFrame(frame(1)); + implementationBackend.lastFrameListener.onFrame(frame(2)); + assertEquals(1, analyzer.operations.size()); + + analyzer.operations.get(0).complete("first"); + try { + flushSerialCalls(); + } catch (IllegalStateException expected) { + // The application exception may propagate from the EDT harness, + // but the pending operation must already have started. + } + assertEquals(2, analyzer.operations.size()); + + analyzer.operations.get(1).complete("second"); + flushSerialCalls(); + assertEquals(2, notifications[0]); + assertFalse(pipeline.isBusy()); + } + + @Test + void listenerCloseDiscardsAlreadySelectedPendingFrame() { + final RecordingAnalyzer analyzer = new RecordingAnalyzer(); + pipeline = new VisionPipeline(session, analyzer, + new VisionPipelineListener() { + public void result(String value, VisionImage image) { + pipeline.close(); + } + + public void error(Throwable error) { + } + }); + + implementationBackend.lastFrameListener.onFrame(frame(1)); + implementationBackend.lastFrameListener.onFrame(frame(2)); + assertEquals(1, analyzer.operations.size()); + + analyzer.operations.get(0).complete("first"); + flushSerialCalls(); + + assertEquals(1, analyzer.operations.size(), + "close from the listener must discard the pending frame"); + assertEquals(1, analyzer.closeCount); + assertFalse(pipeline.isBusy()); + } + + @Test + void busyPipelineKeepsOnlyNewestPendingFrame() { + final RecordingAnalyzer analyzer = new RecordingAnalyzer(); + pipeline = new VisionPipeline(session, analyzer, + new VisionPipelineListener() { + public void result(String value, VisionImage image) { + } + + public void error(Throwable error) { + } + }); + + implementationBackend.lastFrameListener.onFrame(frame(1)); + implementationBackend.lastFrameListener.onFrame(frame(2)); + implementationBackend.lastFrameListener.onFrame(frame(3)); + analyzer.operations.get(0).complete("first"); + flushSerialCalls(); + + assertEquals(2, analyzer.operations.size()); + assertEquals(Integer.valueOf(3), analyzer.inputValues.get(1)); + } + + @Test + void synchronousAnalyzerFailureDoesNotStallPipeline() { + final int[] errors = new int[1]; + VisionAnalyzer analyzer = new VisionAnalyzer() { + int calls; + + public boolean isSupported() { + return true; + } + + public AsyncResource process(VisionImage image) { + calls++; + if (calls == 1) { + throw new IllegalStateException("synchronous failure"); + } + AsyncResource result = + new AsyncResource(); + result.complete("recovered"); + return result; + } + + public void close() { + } + }; + pipeline = new VisionPipeline(session, analyzer, + new VisionPipelineListener() { + public void result(String value, VisionImage image) { + } + + public void error(Throwable error) { + errors[0]++; + } + }); + + implementationBackend.lastFrameListener.onFrame(frame(1)); + implementationBackend.lastFrameListener.onFrame(frame(2)); + flushSerialCalls(); + assertEquals(1, errors[0]); + assertFalse(pipeline.isBusy()); + } + + private static CameraFrame frame(int value) { + return new CameraFrame(new byte[] {(byte) value}, null, + 1, 1, 0, value, FrameFormat.JPEG); + } + + private static final class RecordingAnalyzer + implements VisionAnalyzer { + final List> operations = + new ArrayList>(); + final List inputValues = new ArrayList(); + int closeCount; + + public boolean isSupported() { + return true; + } + + public AsyncResource process(VisionImage image) { + inputValues.add((int) image.getEncodedBytesUnsafe()[0]); + AsyncResource operation = new AsyncResource(); + operations.add(operation); + return operation; + } + + public void close() { + closeCount++; + } + } +} diff --git a/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java b/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java index d9d71e2e30d..287dcb61a92 100644 --- a/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java +++ b/maven/core-unittests/src/test/java/com/codename1/testing/TestCodenameOneImplementation.java @@ -936,6 +936,52 @@ public com.codename1.impl.ARImpl createARImpl() { return arImpl; } + private com.codename1.impl.VisionImpl visionImpl; + private Runnable visionImplCreationHook; + private com.codename1.impl.InferenceImpl inferenceImpl; + private com.codename1.impl.LanguageImpl languageImpl; + + public void setVisionImpl(com.codename1.impl.VisionImpl visionImpl) { + this.visionImpl = visionImpl; + } + + /** + * Installs a test hook invoked immediately before the vision backend is + * returned from {@link #createVisionImpl()}. Tests can use this to hold + * backend creation at a deterministic concurrency boundary. + * + * @param hook hook to invoke, or {@code null} to clear it + */ + public void setVisionImplCreationHook(Runnable hook) { + visionImplCreationHook = hook; + } + + @Override + public com.codename1.impl.VisionImpl createVisionImpl() { + if (visionImplCreationHook != null) { + visionImplCreationHook.run(); + } + return visionImpl; + } + + public void setInferenceImpl(com.codename1.impl.InferenceImpl inferenceImpl) { + this.inferenceImpl = inferenceImpl; + } + + @Override + public com.codename1.impl.InferenceImpl createInferenceImpl() { + return inferenceImpl; + } + + public void setLanguageImpl(com.codename1.impl.LanguageImpl languageImpl) { + this.languageImpl = languageImpl; + } + + @Override + public com.codename1.impl.LanguageImpl createLanguageImpl() { + return languageImpl; + } + private com.codename1.sensors.MotionSensorManager motionSensorManager; /** @@ -1307,6 +1353,7 @@ public void reset() { largerTextScale = 1f; cameraImpl = null; arImpl = null; + visionImplCreationHook = null; motionSensorManager = null; platformName = "test"; } diff --git a/maven/platform-feature-catalog/pom.xml b/maven/platform-feature-catalog/pom.xml new file mode 100644 index 00000000000..5a94c125518 --- /dev/null +++ b/maven/platform-feature-catalog/pom.xml @@ -0,0 +1,31 @@ + + + 4.0.0 + + com.codenameone + codenameone + 8.0-SNAPSHOT + + codenameone-platform-feature-catalog + Codename One Platform Feature Catalog + Shared builder registry for optional platform APIs and native dependencies + + 1.8 + 1.8 + UTF-8 + + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + diff --git a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java new file mode 100644 index 00000000000..4da4061e87a --- /dev/null +++ b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java @@ -0,0 +1,887 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.build.shared; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * Central registry that maps class-name prefixes used by the + * {@code com.codename1.ai.*} family (and the speech/TTS sister APIs + * in {@code com.codename1.media}) to the native dependencies and + * permissions each one requires. + * + *

The build server's class scanners ({@code IPhoneBuilder} and + * {@code AndroidGradleBuilder}) call into this table from inside their + * existing {@code Executor.ClassScanner.usesClass(String)} blocks; the + * resulting set of {@link Entry} records is then applied just before + * iOS pods / SPM are resolved and just before the Android Gradle + * dependencies / manifest fragments are written.

+ * + *

Keep this table small and declarative: any class prefix whose + * needs change (different pod version, additional plist entry) should + * be edited here, not in the builder hot loop.

+ */ +public final class PlatformFeatureCatalog { + + private static final String MLKIT_LANGUAGE_ID = + "com.google.mlkit:language-id:17.0.6"; + private static final String MLKIT_TRANSLATE = + "com.google.mlkit:translate:17.0.3"; + private static final String MLKIT_SMART_REPLY = + "com.google.mlkit:smart-reply:17.0.4"; + private static final String MLKIT_SELFIE_SEGMENTATION = + "com.google.mlkit:segmentation-selfie:16.0.0-beta5"; + private static final List ENTRIES; + private static final List CLASS_PREFIXES; + private static final Set METHOD_KEYS; + + static { + List e = new ArrayList(); + + // LLM clients: pure HTTPS. INTERNET is on by default on + // Android so no permission needed; we still register the + // entry so the scanner has a positive hit for diagnostics. + e.add(new Entry("com/codename1/ai/LlmClient") + .description("LLM client (OpenAI / Anthropic / Gemini / Ollama)")); + e.add(new Entry("com/codename1/ai/OpenAiClient").description("OpenAI client")); + e.add(new Entry("com/codename1/ai/AnthropicClient").description("Anthropic client")); + e.add(new Entry("com/codename1/ai/GeminiClient").description("Gemini client")); + + // Core speech recognition: iOS Speech framework + mic & speech plist + // strings; Android record-audio permission. The TTS API has no + // plist requirement (AVSpeech is unrestricted) and no Android + // permission (built-in). + e.add(new Entry("com/codename1/media/SpeechRecognizer") + .iosFrameworks("Speech", "AVFoundation") + .iosPlist("NSSpeechRecognitionUsageDescription", + "Used to transcribe your voice into text.") + .iosPlist("NSMicrophoneUsageDescription", + "Required to capture audio for speech recognition.") + .androidPermissions("android.permission.RECORD_AUDIO") + .description("On-device speech-to-text")); + + e.add(new Entry("com/codename1/media/TextToSpeech") + .iosFrameworks("AVFoundation") + .description("Text-to-speech")); + + // Compatibility mappings for the retired AI cn1libs. The artifacts + // already published with these package names remain usable even + // though new applications should use the built-in vision, language, + // and inference APIs below. + e.add(new Entry("com/codename1/ai/mlkit/text/") + .iosPod("GoogleMLKit/TextRecognition") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle("com.google.mlkit:text-recognition:16.0.0") + .androidMinimumSdk(21) + .iosPlist("NSCameraUsageDescription", + "Used to recognise text from your camera.") + .description("Legacy ML Kit Text Recognition cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/barcode/") + .iosPod("GoogleMLKit/BarcodeScanning") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle("com.google.mlkit:barcode-scanning:17.2.0") + .androidMinimumSdk(21) + .iosPlist("NSCameraUsageDescription", + "Used to scan barcodes with your camera.") + .androidFeatures("android.hardware.camera") + .description("Legacy ML Kit Barcode Scanning cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/face/") + .iosPod("GoogleMLKit/FaceDetection") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle("com.google.mlkit:face-detection:16.1.5") + .androidMinimumSdk(21) + .iosPlist("NSCameraUsageDescription", + "Used to detect faces in images.") + .description("Legacy ML Kit Face Detection cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/labeling/") + .iosPod("GoogleMLKit/ImageLabeling") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle("com.google.mlkit:image-labeling:17.0.7") + .androidMinimumSdk(21) + .description("Legacy ML Kit Image Labeling cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/translate/") + .iosPod("GoogleMLKit/Translate") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle(MLKIT_TRANSLATE) + .androidMinimumSdk(21) + .description("Legacy ML Kit Translation cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/smartreply/") + .iosPod("GoogleMLKit/SmartReply") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle(MLKIT_SMART_REPLY) + .androidMinimumSdk(21) + .description("Legacy ML Kit Smart Reply cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/langid/") + .iosPod("GoogleMLKit/LanguageID") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle(MLKIT_LANGUAGE_ID) + .androidMinimumSdk(21) + .description("Legacy ML Kit Language ID cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/pose/") + .iosPod("GoogleMLKit/PoseDetection") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle("com.google.mlkit:pose-detection:18.0.0-beta3") + .androidMinimumSdk(21) + .description("Legacy ML Kit Pose Detection cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/segmentation/") + .iosPod("GoogleMLKit/SegmentationSelfie") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .androidGradle(MLKIT_SELFIE_SEGMENTATION) + .androidMinimumSdk(21) + .description("Legacy ML Kit Selfie Segmentation cn1lib")); + e.add(new Entry("com/codename1/ai/mlkit/docscan/") + .iosPod("GoogleMLKit/DocumentScanner") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .iosFrameworks("VisionKit") + .androidGradle( + "com.google.android.gms:play-services-mlkit-document-scanner:16.0.0-beta1") + .androidMinimumSdk(21) + .description("Legacy ML Kit Document Scanner cn1lib")); + e.add(new Entry("com/codename1/ai/tflite/") + .iosPod("TensorFlowLiteSwift") + .iosSpm("TensorFlowLiteSwift", + "https://github.com/tensorflow/tensorflow.git", + "from:2.13.0", "TensorFlowLite") + .androidGradle("org.tensorflow:tensorflow-lite:2.13.0") + .androidGradle( + "org.tensorflow:tensorflow-lite-support:0.4.4") + .androidMinimumSdk(21) + .description("Legacy TensorFlow Lite interpreter cn1lib")); + + // Built-in vision APIs. Android uses ML Kit by default. iOS uses + // Apple Vision/VisionKit unless VisionBackends.mlKit() is selected. + // The compound entries require both the feature and selector method, + // so iOS only bundles ML Kit pods for features actually used. + e.add(new Entry("com/codename1/ai/vision/TextRecognizer") + .iosFrameworks("Vision", "CoreImage") + .androidGradle("com.google.mlkit:text-recognition:16.0.0") + .androidMinimumSdk(21) + .description("Text recognition")); + e.add(new Entry("com/codename1/ai/vision/TextRecognizer") + .requiresMethod("com/codename1/ai/vision/VisionBackends", + "mlKitTextRecognition") + .iosPod("GoogleMLKit/TextRecognition") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .description("ML Kit iOS text-recognition backend")); + + e.add(new Entry("com/codename1/ai/vision/BarcodeScanner") + .iosFrameworks("Vision", "CoreImage") + .androidGradle("com.google.mlkit:barcode-scanning:17.2.0") + .androidMinimumSdk(21) + .description("Barcode scanning")); + e.add(new Entry("com/codename1/ai/vision/BarcodeScanner") + .requiresMethod("com/codename1/ai/vision/VisionBackends", + "mlKitBarcodeScanning") + .iosPod("GoogleMLKit/BarcodeScanning") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .description("ML Kit iOS barcode backend")); + + e.add(new Entry("com/codename1/ai/vision/FaceDetector") + .iosFrameworks("Vision", "CoreImage") + .androidGradle("com.google.mlkit:face-detection:16.1.5") + .androidMinimumSdk(21) + .description("Face detection")); + e.add(new Entry("com/codename1/ai/vision/FaceDetector") + .requiresMethod("com/codename1/ai/vision/VisionBackends", + "mlKitFaceDetection") + .iosPod("GoogleMLKit/FaceDetection") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .description("ML Kit iOS face-detection backend")); + + e.add(new Entry("com/codename1/ai/vision/ImageLabeler") + .iosFrameworks("Vision", "CoreML") + .androidGradle("com.google.mlkit:image-labeling:17.0.7") + .androidMinimumSdk(21) + .description("Image labeling")); + e.add(new Entry("com/codename1/ai/vision/ImageLabeler") + .requiresMethod("com/codename1/ai/vision/VisionBackends", + "mlKitImageLabeling") + .iosPod("GoogleMLKit/ImageLabeling") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .description("ML Kit iOS image-labeling backend")); + + e.add(new Entry("com/codename1/ai/vision/PoseDetector") + .iosFrameworks("Vision", "CoreML") + .androidGradle("com.google.mlkit:pose-detection:18.0.0-beta3") + .androidMinimumSdk(21) + .description("Pose detection")); + e.add(new Entry("com/codename1/ai/vision/PoseDetector") + .requiresMethod("com/codename1/ai/vision/VisionBackends", + "mlKitPoseDetection") + .iosPod("GoogleMLKit/PoseDetection") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .description("ML Kit iOS pose-detection backend")); + + e.add(new Entry("com/codename1/ai/vision/SelfieSegmenter") + .iosFrameworks("Vision", "CoreML") + .androidGradle(MLKIT_SELFIE_SEGMENTATION) + .androidMinimumSdk(21) + .description("Selfie segmentation")); + e.add(new Entry("com/codename1/ai/vision/SelfieSegmenter") + .requiresMethod("com/codename1/ai/vision/VisionBackends", + "mlKitSelfieSegmentation") + .iosPod("GoogleMLKit/SegmentationSelfie") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .description("ML Kit iOS selfie-segmentation backend")); + + e.add(new Entry("com/codename1/ai/vision/DocumentScanner") + .iosFrameworks("VisionKit", "Vision", "CoreImage") + .description("Still-image document correction (Apple platforms)")); + + // The common inference API is backed by LiteRT/TensorFlow Lite. + // iOS may select the Core ML delegate without changing + // model formats. + e.add(new Entry("com/codename1/ai/inference/InferenceSession") + .iosPod("TensorFlowLiteObjC/CoreML") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosFrameworks("CoreML", "Metal", "Accelerate") + .androidGradle("com.google.ai.edge.litert:litert:1.0.1") + .androidMinimumSdk(21) + .description("LiteRT inference")); + + e.add(new Entry("com/codename1/ai/language/LanguageIdentifier") + .iosFrameworks("NaturalLanguage") + .androidGradle(MLKIT_LANGUAGE_ID) + .androidMinimumSdk(21) + .description("On-device language identification")); + e.add(new Entry("com/codename1/ai/language/LanguageIdentifier") + .requiresMethod("com/codename1/ai/language/LanguageBackends", + "mlKitLanguageIdentification") + .iosPod("GoogleMLKit/LanguageID") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .description("ML Kit iOS language-identification backend")); + // CN1Language.m contains the dependency-free Apple identifier beside + // the ML Kit adapters, so every target that compiles this source must + // link the small system NaturalLanguage framework. + e.add(new Entry("com/codename1/ai/language/Translator") + .iosFrameworks("NaturalLanguage") + .androidGradle(MLKIT_TRANSLATE) + .androidMinimumSdk(21) + .description("On-device translation")); + e.add(new Entry("com/codename1/ai/language/Translator") + .requiresMethod("com/codename1/ai/language/LanguageBackends", + "mlKitTranslation") + .iosPod("GoogleMLKit/Translate") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .description("ML Kit iOS translation backend")); + e.add(new Entry("com/codename1/ai/language/SmartReply") + .iosFrameworks("NaturalLanguage") + .androidGradle(MLKIT_SMART_REPLY) + .androidMinimumSdk(21) + .description("On-device smart reply")); + e.add(new Entry("com/codename1/ai/language/SmartReply") + .requiresMethod("com/codename1/ai/language/LanguageBackends", + "mlKitSmartReply") + .iosPod("GoogleMLKit/SmartReply") + .iosMinimumDeploymentTarget("15.5") + .iosDependenciesUnsupportedOnMacCatalyst() + .iosDependenciesUnsupportedOnArm64Simulator() + .description("ML Kit iOS Smart Reply backend")); + + e.add(new Entry("com/codename1/ai/whisper/") + .iosFrameworks("Accelerate") + .description("On-device Whisper transcription (libwhisper.a ships with the cn1lib)")); + + // Low-level cross-platform camera API: live preview + frame + // stream + photo + video. iOS uses AVFoundation (framework + // only, no pod); Android uses CameraX (androidx.camera) which + // is added as Gradle deps below. Just referencing classes in + // com.codename1.camera causes the build to inject the right + // permissions and plist strings; developers may still override + // the plist text via the ios.NSCameraUsageDescription build + // hint. + e.add(new Entry("com/codename1/camera/") + .iosFrameworks("AVFoundation", "CoreMedia", "CoreVideo") + .iosPlist("NSCameraUsageDescription", + "Used to capture photos and video.") + .iosPlist("NSMicrophoneUsageDescription", + "Used to capture audio for video recording.") + .androidPermissions("android.permission.CAMERA", + "android.permission.RECORD_AUDIO") + .androidFeatures("android.hardware.camera", + "android.hardware.camera.autofocus") + .androidGradle("androidx.camera:camera-core:1.3.4") + .androidGradle("androidx.camera:camera-camera2:1.3.4") + .androidGradle("androidx.camera:camera-lifecycle:1.3.4") + .androidGradle("androidx.camera:camera-view:1.3.4") + .androidGradle("androidx.camera:camera-video:1.3.4") + .androidMinimumSdk(21) + .description("Cross-platform camera (preview + frames + photo + video)")); + + // First-class Bluetooth (com.codename1.bluetooth.*): CoreBluetooth + // on iOS with the two privacy strings defaulted only-if-unset via + // the standard entry application. Android permissions are + // deliberately NOT listed here -- the Android 12 permission split + // needs maxSdkVersion / usesPermissionFlags attributes this table + // cannot express, so AndroidGradleBuilder injects nuanced manifest + // fragments through BluetoothManifestFragments instead. The + // framework linking + CN1_INCLUDE_BLUETOOTH define flip likewise + // happen in IPhoneBuilder (iosFrameworks is documentation-only). + e.add(new Entry("com/codename1/bluetooth/") + .iosFrameworks("CoreBluetooth") + .iosPlist("NSBluetoothAlwaysUsageDescription", + "Communicates with nearby Bluetooth accessories.") + .iosPlist("NSBluetoothPeripheralUsageDescription", + "Communicates with nearby Bluetooth accessories.") + .description("Cross-platform Bluetooth (BLE central/peripheral, L2CAP, classic RFCOMM)")); + + // On-device Stable Diffusion: bundled Core ML model on iOS, + // ONNX runtime on Android. Flag the >2 GB upload concern so + // the cloud build server can abort early with a helpful + // message. + e.add(new Entry("com/codename1/ai/imagegen/StableDiffusion") + .iosFrameworks("CoreML", "Vision") + .androidGradle("com.microsoft.onnxruntime:onnxruntime-android:1.16.3") + .androidMinimumSdk(21) + .markBigUpload() + .description("On-device Stable Diffusion (local-build only)")); + + // Cross-platform augmented reality (com.codename1.ar): ARKit + + // SceneKit on iOS (linked explicitly by IPhoneBuilder, gated by + // the INCLUDE_CN1_AR define since neither is default-linked), + // Google Play Services for AR (ARCore) on Android. The camera + // permission and plist string ride along because AR always + // drives the camera. The com.google.ar.core meta-data marks + // ARCore optional so the app still installs on non-AR devices; + // the android.ar.required=true build hint flips it to required. + e.add(new Entry("com/codename1/ar/") + .iosFrameworks("ARKit", "SceneKit") + .iosPlist("NSCameraUsageDescription", + "Used to display augmented reality content.") + .androidGradle("com.google.ar:core:1.44.0") + .androidPermissions("android.permission.CAMERA") + .androidFeatures("android.hardware.camera.ar") + .androidMetaData("com.google.ar.core", "optional") + .description("Cross-platform augmented reality (world/image/face tracking)")); + + ENTRIES = Collections.unmodifiableList(e); + Set classPrefixes = new LinkedHashSet(); + Set methodKeys = new LinkedHashSet(); + for (Entry entry : e) { + classPrefixes.add(entry.classPrefix); + if (entry.hasMethodRequirement()) { + methodKeys.add(entry.methodOwner + "#" + entry.methodName); + } + } + CLASS_PREFIXES = Collections.unmodifiableList( + new ArrayList(classPrefixes)); + METHOD_KEYS = Collections.unmodifiableSet(methodKeys); + } + + private PlatformFeatureCatalog() { + } + + /** + * Returns every registered platform feature in declaration order. + * + * @return immutable catalog entry list used by builders and tooling + */ + public static List entries() { + return ENTRIES; + } + + /** + * Returns every entry whose {@link Entry#classPrefix} matches the + * given internal-form class name (slashes, not dots). When the + * prefix ends with a slash, package-prefix matching is used; + * otherwise an exact class match is required. + * + * @param internalClassName JVM internal-form class name + * @return immutable matching entries that have no method requirement + */ + public static List matchesFor(String internalClassName) { + if (internalClassName == null) { + return Collections.emptyList(); + } + List out = new ArrayList(); + for (Entry e : ENTRIES) { + if (e.matchesClass(internalClassName) && !e.hasMethodRequirement()) { + out.add(e); + } + } + return out; + } + + /** + * Builder/scanner output: the de-duplicated union of entries whose class + * and optional method requirements were observed. + */ + public static final class Accumulator { + private final Set classes = new LinkedHashSet(); + private final Set methods = new LinkedHashSet(); + private Set cachedHits = Collections.emptySet(); + private boolean dirty = true; + + /** + * Records a class only when it can match at least one catalog entry. + * The scanner calls this for the full application and framework + * graph, so filtering here keeps memory proportional to catalog usage + * rather than application size. + * + * @param internalClassName JVM internal-form class name + */ + public void consume(String internalClassName) { + if (isCatalogClass(internalClassName) + && classes.add(internalClassName)) { + dirty = true; + } + } + + /** + * Records a method reference only when an entry requires that exact + * owner and method name. + * + * @param internalClassName JVM internal-form owner name + * @param methodName referenced method name + */ + public void consumeMethod(String internalClassName, String methodName) { + if (internalClassName != null && methodName != null) { + String key = internalClassName + "#" + methodName; + if (METHOD_KEYS.contains(key) && methods.add(key)) { + dirty = true; + } + } + } + + /** + * Returns the immutable matched-entry set. The result is recomputed + * only after a relevant class or method is newly observed. + * + * @return matched catalog entries in declaration order + */ + public Set hits() { + if (!dirty) { + return cachedHits; + } + Set hits = new LinkedHashSet(); + for (Entry entry : ENTRIES) { + if (entry.requirementsMet(classes, methods)) { + hits.add(entry); + } + } + cachedHits = Collections.unmodifiableSet(hits); + dirty = false; + return cachedHits; + } + + /** + * Tests whether any observed feature requires the builder's larger + * upload allowance. + * + * @return {@code true} when at least one matched entry is marked as a + * large upload + */ + public boolean anyRequiresBigUpload() { + for (Entry e : hits()) { + if (e.requiresBigUpload) { + return true; + } + } + return false; + } + + /** + * Returns the highest Android API level required by the matched + * catalog entries. + * + *

Builders should combine this value with the application's + * {@code android.min_sdk_version} and retain the larger value before + * writing either the manifest or Gradle configuration. This prevents + * Android's manifest merger from rejecting a dependency whose own + * minimum SDK is newer than Codename One's default.

+ * + * @return the required Android API level, or {@code 0} when none of + * the matched entries imposes an additional minimum + */ + public int minimumAndroidSdk() { + int minimum = 0; + for (Entry entry : hits()) { + minimum = Math.max(minimum, entry.androidMinimumSdk()); + } + return minimum; + } + + /** + * Returns the de-duplicated Apple system frameworks required by the + * matched entries. + * + *

Names are returned without the {@code .framework} suffix, exactly + * as they appear in the catalog. Builders can append the suffix while + * merging these values into an application's existing library list. + * The returned set is immutable.

+ * + * @return the framework names required by the observed API usage + */ + public Set iosFrameworks() { + Set frameworks = new LinkedHashSet(); + for (Entry entry : hits()) { + frameworks.addAll(entry.iosFrameworks()); + } + return Collections.unmodifiableSet(frameworks); + } + } + + private static boolean isCatalogClass(String internalClassName) { + if (internalClassName == null) { + return false; + } + for (String prefix : CLASS_PREFIXES) { + if (prefix.endsWith("/")) { + if (internalClassName.startsWith(prefix)) { + return true; + } + } else if (internalClassName.equals(prefix)) { + return true; + } + } + return false; + } + + /** + * A single registry record. Mutable while the table is being + * built (the fluent setters); semantically immutable once exposed + * via {@link #entries()}. + */ + public static final class Entry { + private final String classPrefix; + private String methodOwner; + private String methodName; + private final List iosPods = new ArrayList(); + private final List iosSpm = new ArrayList(); + private final List iosFrameworks = new ArrayList(); + private final List iosPlist = new ArrayList(); + private final List androidGradle = new ArrayList(); + private final List androidPermissions = new ArrayList(); + private final List androidFeatures = new ArrayList(); + private final List androidMetaData = new ArrayList(); + private int androidMinimumSdk; + private boolean iosDependenciesSupportMacCatalyst = true; + private boolean iosDependenciesSupportArm64Simulator = true; + private String iosMinimumDeploymentTarget; + private boolean requiresBigUpload; + private String description = ""; + + Entry(String classPrefix) { + this.classPrefix = classPrefix; + } + + boolean matchesClass(String internalClassName) { + if (classPrefix.endsWith("/")) { + return internalClassName.startsWith(classPrefix); + } + return internalClassName.equals(classPrefix); + } + + boolean hasMethodRequirement() { + return methodOwner != null; + } + + boolean requirementsMet(Set classes, Set methods) { + boolean classSeen = false; + for (String cls : classes) { + if (matchesClass(cls)) { + classSeen = true; + break; + } + } + if (!classSeen) { + return false; + } + if (!hasMethodRequirement()) { + return true; + } + for (String method : methods) { + if (method.equals(methodOwner + "#" + methodName)) { + return true; + } + } + return false; + } + + Entry requiresMethod(String owner, String methodName) { + this.methodOwner = owner; + this.methodName = methodName; + return this; + } + + Entry iosPod(String pod) { + iosPods.add(pod); + return this; + } + + Entry iosMinimumDeploymentTarget(String target) { + iosMinimumDeploymentTarget = target; + return this; + } + + Entry iosSpm(String identity, String url, String requirement, String... products) { + iosSpm.add(new IosSpm(identity, url, requirement, + Arrays.asList(products))); + return this; + } + + Entry iosDependenciesUnsupportedOnMacCatalyst() { + iosDependenciesSupportMacCatalyst = false; + return this; + } + + Entry iosDependenciesUnsupportedOnArm64Simulator() { + iosDependenciesSupportArm64Simulator = false; + return this; + } + + Entry iosFrameworks(String... fws) { + for (String f : fws) { + iosFrameworks.add(f); + } + return this; + } + + Entry iosPlist(String key, String defaultValue) { + iosPlist.add(new String[]{key, defaultValue}); + return this; + } + + Entry androidGradle(String gav) { + androidGradle.add(gav); + return this; + } + + Entry androidMinimumSdk(int apiLevel) { + androidMinimumSdk = apiLevel; + return this; + } + + Entry androidPermissions(String... perms) { + for (String p : perms) { + androidPermissions.add(p); + } + return this; + } + + Entry androidFeatures(String... feats) { + for (String f : feats) { + androidFeatures.add(f); + } + return this; + } + + Entry androidMetaData(String name, String value) { + androidMetaData.add(new String[]{name, value}); + return this; + } + + Entry markBigUpload() { + this.requiresBigUpload = true; + return this; + } + + Entry description(String d) { + this.description = d; + return this; + } + + /** @return exact class or package prefix that activates this entry */ + public String classPrefix() { + return classPrefix; + } + + /** @return immutable CocoaPods dependency specifications */ + public List iosPods() { + return Collections.unmodifiableList(iosPods); + } + + /** @return immutable Swift Package Manager dependency descriptors */ + public List iosSpmSpecs() { + return Collections.unmodifiableList(iosSpm); + } + + /** + * Whether this entry's CocoaPod/SPM payload has a Mac Catalyst slice. + * System frameworks are not affected by this flag. + * + * @return {@code true} when catalog dependencies support Catalyst + */ + public boolean iosDependenciesSupportMacCatalyst() { + return iosDependenciesSupportMacCatalyst; + } + + /** + * Whether this entry's CocoaPod/SPM payload has an arm64 iOS + * Simulator slice. Dependencies that return {@code false} still + * support the x86_64 simulator. + * + * @return {@code true} when catalog dependencies support arm64 + * simulator builds + */ + public boolean iosDependenciesSupportArm64Simulator() { + return iosDependenciesSupportArm64Simulator; + } + + /** + * Returns the minimum iOS deployment target required by this entry's + * CocoaPod or Swift package payload. + * + *

The iOS builder combines this value with the application's + * requested deployment target and all other dependency floors, then + * uses the highest version for the generated app target and Podfile. + * A {@code null} value means that the entry does not impose an + * additional deployment floor.

+ * + * @return the required iOS version, such as {@code "15.5"}, or + * {@code null} when the dependency has no catalog-specific minimum + */ + public String iosMinimumDeploymentTarget() { + return iosMinimumDeploymentTarget; + } + + /** @return immutable Apple system-framework names without suffixes */ + public List iosFrameworks() { + return Collections.unmodifiableList(iosFrameworks); + } + + /** Each entry is {key, defaultValue}. The builder injects the + * value only if the app hasn't already declared one for the + * same key in its build hints. + * + * @return immutable list of two-element property-list entries + */ + public List iosPlistEntries() { + return Collections.unmodifiableList(iosPlist); + } + + /** @return immutable Android Gradle dependency coordinates */ + public List androidGradleDeps() { + return Collections.unmodifiableList(androidGradle); + } + + /** + * Returns the minimum Android API level required by this entry's + * native dependencies. + * + *

The Android builder raises the application's requested minimum + * to at least this value before generating the manifest and Gradle + * configuration. A value of {@code 0} means that the entry does not + * impose an additional floor.

+ * + * @return the required Android API level, or {@code 0} when no + * catalog-specific minimum is required + */ + public int androidMinimumSdk() { + return androidMinimumSdk; + } + + /** @return immutable Android manifest permission names */ + public List androidPermissions() { + return Collections.unmodifiableList(androidPermissions); + } + + /** @return immutable Android manifest feature names */ + public List androidFeatures() { + return Collections.unmodifiableList(androidFeatures); + } + + /** Each entry is {name, value}: an application-level manifest + * <meta-data> element the Android builder injects unless the + * app already declares the same name. + * + * @return immutable list of two-element manifest metadata entries + */ + public List androidMetaDataEntries() { + return Collections.unmodifiableList(androidMetaData); + } + + /** @return whether this entry requires the larger upload allowance */ + public boolean requiresBigUpload() { + return requiresBigUpload; + } + + /** @return user-readable feature or dependency description */ + public String description() { + return description; + } + } + + /** + * Immutable Swift Package Manager dependency descriptor consumed by the + * iOS builder. + */ + public static final class IosSpm { + /** Stable package identity used for de-duplication. */ + public final String identity; + /** Package repository URL. */ + public final String url; + /** Builder-formatted version or branch requirement. */ + public final String requirement; + /** Immutable package product names linked into the app. */ + public final List products; + + IosSpm(String identity, String url, String requirement, List products) { + this.identity = identity; + this.url = url; + this.requirement = requirement; + this.products = Collections.unmodifiableList(new ArrayList(products)); + } + } +} diff --git a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java new file mode 100644 index 00000000000..4110478d811 --- /dev/null +++ b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java @@ -0,0 +1,493 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.build.shared; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class PlatformFeatureCatalogTest { + + @Test + void builtInTextRecognizerMapsToAppleVisionAndAndroidMlKit() { + List hits = PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/vision/TextRecognizer"); + assertEquals(1, hits.size(), "expected one entry to fire"); + PlatformFeatureCatalog.Entry e = hits.get(0); + assertTrue(e.iosFrameworks().contains("Vision")); + assertTrue(e.iosPods().isEmpty()); + assertTrue(e.androidGradleDeps().get(0).startsWith("com.google.mlkit:text-recognition")); + assertEquals(21, e.androidMinimumSdk()); + assertTrue(e.iosPlistEntries().isEmpty(), + "Still-image analysis must not imply camera permission"); + } + + @Test + void retiredCn1libsRetainPublishedDependencyMappings() { + List mlKit = + PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/mlkit/text/TextRecognizer"); + assertEquals(1, mlKit.size()); + assertTrue(mlKit.get(0).iosPods().contains( + "GoogleMLKit/TextRecognition")); + assertTrue(mlKit.get(0).androidGradleDeps().get(0).startsWith( + "com.google.mlkit:text-recognition:")); + + List tflite = + PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/tflite/Interpreter"); + assertEquals(1, tflite.size()); + assertFalse(tflite.get(0).iosPods().isEmpty()); + assertFalse(tflite.get(0).iosSpmSpecs().isEmpty()); + assertEquals(2, tflite.get(0).androidGradleDeps().size()); + } + + @Test + void explicitMlKitBackendAddsOnlyUsedFeaturePod() { + PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); + acc.consume("com/codename1/ai/vision/TextRecognizer"); + acc.consumeMethod("com/codename1/ai/vision/VisionBackends", + "mlKitTextRecognition"); + boolean foundTextPod = false; + for (PlatformFeatureCatalog.Entry e : acc.hits()) { + foundTextPod |= e.iosPods().contains("GoogleMLKit/TextRecognition"); + assertFalse(e.iosPods().contains("GoogleMLKit/FaceDetection"), + "Unused vision features must not be bundled"); + if (!e.iosPods().isEmpty()) { + assertFalse(e.iosDependenciesSupportArm64Simulator(), + "Google ML Kit binaries require the x86_64 iOS simulator"); + } + } + assertTrue(foundTextPod); + } + + @Test + void currentIosMlKitPodsRequireIos155() { + PlatformFeatureCatalog.Accumulator acc = + new PlatformFeatureCatalog.Accumulator(); + acc.consume("com/codename1/ai/vision/TextRecognizer"); + acc.consumeMethod("com/codename1/ai/vision/VisionBackends", + "mlKitTextRecognition"); + acc.consume("com/codename1/ai/language/Translator"); + acc.consume("com/codename1/ai/language/SmartReply"); + + int podEntries = 0; + for (PlatformFeatureCatalog.Entry entry : acc.hits()) { + if (!entry.iosPods().isEmpty()) { + podEntries++; + assertEquals("15.5", + entry.iosMinimumDeploymentTarget(), + entry.description()); + } + } + assertEquals(1, podEntries, + "Language class references alone must preserve the arm64 simulator"); + + acc.consumeMethod("com/codename1/ai/language/LanguageBackends", + "mlKitTranslation"); + acc.consumeMethod("com/codename1/ai/language/LanguageBackends", + "mlKitSmartReply"); + podEntries = 0; + for (PlatformFeatureCatalog.Entry entry : acc.hits()) { + if (!entry.iosPods().isEmpty()) { + podEntries++; + assertEquals("15.5", entry.iosMinimumDeploymentTarget(), + entry.description()); + } + } + assertEquals(3, podEntries); + } + + @Test + void speechRecognizerInjectsMicAndSpeechPlist() { + List hits = PlatformFeatureCatalog.matchesFor( + "com/codename1/media/SpeechRecognizer"); + assertEquals(1, hits.size()); + PlatformFeatureCatalog.Entry e = hits.get(0); + assertTrue(e.iosFrameworks().contains("Speech")); + assertNotNull(findPlistDefault(e, "NSMicrophoneUsageDescription")); + assertNotNull(findPlistDefault(e, "NSSpeechRecognitionUsageDescription")); + assertTrue(e.androidPermissions().contains("android.permission.RECORD_AUDIO")); + } + + @Test + void textToSpeechInjectsNoPermissions() { + List hits = PlatformFeatureCatalog.matchesFor( + "com/codename1/media/TextToSpeech"); + assertEquals(1, hits.size()); + PlatformFeatureCatalog.Entry e = hits.get(0); + assertTrue(e.iosFrameworks().contains("AVFoundation")); + assertTrue(e.androidPermissions().isEmpty(), + "TTS is built-in on every supported OS -- no permission needed"); + assertTrue(e.iosPlistEntries().isEmpty(), + "TTS has no Apple-reviewed restricted entitlement"); + } + + @Test + void llmClientNeedsNothingExtra() { + // The LlmClient entries are intentionally cheap: pure HTTPS + // means no plist string, no extra permission. They still + // register so future diagnostics ("which AI APIs does this + // app use?") can enumerate them. + List hits = PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/LlmClient"); + assertEquals(1, hits.size()); + PlatformFeatureCatalog.Entry e = hits.get(0); + assertTrue(e.iosPods().isEmpty()); + assertTrue(e.androidGradleDeps().isEmpty()); + assertTrue(e.androidPermissions().isEmpty()); + } + + @Test + void stableDiffusionFlagsBigUpload() { + PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); + acc.consume("com/codename1/ai/imagegen/StableDiffusion"); + assertTrue(acc.anyRequiresBigUpload(), + "On-device SD ships a 1-2 GB Core ML model -- cloud builds must abort with a friendly message"); + assertEquals(21, acc.minimumAndroidSdk(), + "ONNX Runtime Android requires API 21"); + } + + @Test + void builtInVisionDoesNotFlagBigUpload() { + PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); + acc.consume("com/codename1/ai/vision/TextRecognizer"); + acc.consume("com/codename1/ai/vision/BarcodeScanner"); + acc.consume("com/codename1/ai/whisper/WhisperRecognizer"); + assertFalse(acc.anyRequiresBigUpload(), + "ML Kit models stream lazily, Whisper bundles a small static lib -- neither exceeds the 2 GB cap"); + } + + @Test + void unrelatedClassesProduceNoHits() { + // Sanity: we mustn't false-positive on classes outside the + // AI namespace, because the scanner walks every class in + // the user's app. + assertTrue(PlatformFeatureCatalog.matchesFor("com/codename1/ui/Form").isEmpty()); + assertTrue(PlatformFeatureCatalog.matchesFor("java/lang/Object").isEmpty()); + assertTrue(PlatformFeatureCatalog.matchesFor(null).isEmpty()); + } + + @Test + void cameraEntryInjectsAvFoundationAndCameraXGradleDeps() { + // Referencing any class in com.codename1.camera.* must auto-inject + // the iOS frameworks, iOS plist usage descriptions, Android + // permissions, and the four CameraX Gradle dependencies that the + // AndroidCameraImpl reflection layer resolves at runtime. + List hits = PlatformFeatureCatalog.matchesFor( + "com/codename1/camera/Camera"); + assertEquals(1, hits.size(), "expected the camera entry to fire"); + PlatformFeatureCatalog.Entry e = hits.get(0); + + // iOS side + assertTrue(e.iosFrameworks().contains("AVFoundation")); + assertTrue(e.iosFrameworks().contains("CoreMedia")); + assertTrue(e.iosFrameworks().contains("CoreVideo")); + assertNotNull(findPlistDefault(e, "NSCameraUsageDescription")); + assertNotNull(findPlistDefault(e, "NSMicrophoneUsageDescription")); + assertTrue(e.iosPods().isEmpty(), + "Camera uses AVFoundation framework, not a pod"); + + // Android side + assertTrue(e.androidPermissions().contains("android.permission.CAMERA")); + assertTrue(e.androidPermissions().contains("android.permission.RECORD_AUDIO")); + assertTrue(e.androidFeatures().contains("android.hardware.camera")); + + boolean cameraCore = false, camera2 = false, lifecycle = false, + view = false, video = false; + for (String gav : e.androidGradleDeps()) { + if (gav.startsWith("androidx.camera:camera-core:")) cameraCore = true; + if (gav.startsWith("androidx.camera:camera-camera2:")) camera2 = true; + if (gav.startsWith("androidx.camera:camera-lifecycle:")) lifecycle = true; + if (gav.startsWith("androidx.camera:camera-view:")) view = true; + if (gav.startsWith("androidx.camera:camera-video:")) video = true; + } + assertTrue(cameraCore, "missing androidx.camera:camera-core gradle dep"); + assertTrue(camera2, "missing androidx.camera:camera-camera2 gradle dep"); + assertTrue(lifecycle, "missing androidx.camera:camera-lifecycle gradle dep"); + assertTrue(view, "missing androidx.camera:camera-view gradle dep"); + assertTrue(video, "missing androidx.camera:camera-video gradle dep"); + } + + @Test + void cameraEntryFiresOnAnySubpackageClass() { + // The prefix matcher must hit any class inside com.codename1.camera, + // not just the entry point. + assertEquals(1, PlatformFeatureCatalog.matchesFor("com/codename1/camera/CameraView").size()); + assertEquals(1, PlatformFeatureCatalog.matchesFor("com/codename1/camera/CameraSession").size()); + assertEquals(1, PlatformFeatureCatalog.matchesFor("com/codename1/camera/internal/Foo").size()); + } + + @Test + void inferenceUsesCoreMlEnabledObjectiveCPod() { + // The Objective-C native bridge uses the Core ML-enabled TFLite + // subspec. The Swift package exposes a different surface and must + // not be selected for this implementation. + List hits = PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/inference/InferenceSession"); + assertEquals(1, hits.size()); + PlatformFeatureCatalog.Entry e = hits.get(0); + assertTrue(e.iosPods().contains("TensorFlowLiteObjC/CoreML")); + assertTrue(e.iosSpmSpecs().isEmpty()); + assertFalse(e.iosDependenciesSupportMacCatalyst(), + "The official TensorFlow Lite Objective-C XCFramework has no Catalyst slice"); + assertTrue(e.iosDependenciesSupportArm64Simulator(), + "TensorFlow Lite's XCFramework includes an arm64 simulator slice"); + assertEquals(21, e.androidMinimumSdk(), + "LiteRT requires Android API 21"); + } + + @Test + void accumulatorCombinesAndroidFloorAndAppleFrameworks() { + PlatformFeatureCatalog.Accumulator acc = + new PlatformFeatureCatalog.Accumulator(); + acc.consume("com/codename1/ai/vision/DocumentScanner"); + acc.consume("com/codename1/ai/vision/ImageLabeler"); + + assertEquals(21, acc.minimumAndroidSdk()); + assertTrue(acc.iosFrameworks().contains("VisionKit")); + assertTrue(acc.iosFrameworks().contains("CoreML")); + assertTrue(acc.iosFrameworks().contains("Vision")); + } + + @Test + void everyMlKitAndLiteRtDependencyDeclaresAndroidFloor() { + int checked = 0; + for (PlatformFeatureCatalog.Entry entry + : PlatformFeatureCatalog.entries()) { + for (String dependency : entry.androidGradleDeps()) { + if (dependency.startsWith("com.google.mlkit:") + || dependency.startsWith( + "com.google.ai.edge.litert:") + || dependency.startsWith( + "com.google.android.gms:play-services-mlkit-") + || dependency.startsWith( + "org.tensorflow:tensorflow-lite")) { + checked++; + assertEquals(21, entry.androidMinimumSdk(), dependency); + } + } + } + assertEquals(22, checked, + "If an AI dependency is intentionally added or removed, " + + "update this lock count after verifying its Android floor"); + } + + @Test + void thirdPartyAppleAiPackagesAreExcludedFromCatalyst() { + PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); + acc.consume("com/codename1/ai/vision/TextRecognizer"); + acc.consumeMethod("com/codename1/ai/vision/VisionBackends", + "mlKitTextRecognition"); + acc.consume("com/codename1/ai/language/LanguageIdentifier"); + acc.consume("com/codename1/ai/inference/InferenceSession"); + + boolean foundSystemVision = false; + for (PlatformFeatureCatalog.Entry e : acc.hits()) { + if (e.iosPods().isEmpty()) { + foundSystemVision |= e.iosFrameworks().contains("Vision"); + assertTrue(e.iosDependenciesSupportMacCatalyst(), + "Apple system-framework entries should remain enabled"); + } else { + assertFalse(e.iosDependenciesSupportMacCatalyst(), + e.description() + " must not inject an iOS-only package into Catalyst"); + } + } + assertTrue(foundSystemVision); + } + + @Test + void selectorMethodsAreMatchedExactlyAndLanguageIdDefaultsToApple() { + PlatformFeatureCatalog.Accumulator acc = + new PlatformFeatureCatalog.Accumulator(); + acc.consume("com/codename1/ai/vision/TextRecognizer"); + acc.consumeMethod("com/codename1/ai/vision/VisionBackends", + "mlKitTextRecognitionFuture"); + for (PlatformFeatureCatalog.Entry entry : acc.hits()) { + assertTrue(entry.iosPods().isEmpty(), + "A method-name prefix must not select an optional pod"); + } + + List language = + PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/language/LanguageIdentifier"); + assertEquals(1, language.size()); + assertTrue(language.get(0).iosPods().isEmpty()); + assertTrue(language.get(0).iosFrameworks().contains("NaturalLanguage")); + assertTrue(language.get(0).iosDependenciesSupportArm64Simulator()); + } + + @Test + void androidAdaptersReceiveOnlyTheirFeatureDependency() { + List vision = PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/vision/TextRecognizer"); + assertEquals(1, vision.get(0).androidGradleDeps().size()); + assertTrue(vision.get(0).androidGradleDeps().get(0) + .startsWith("com.google.mlkit:text-recognition:")); + + List language = PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/language/LanguageIdentifier"); + assertEquals(1, language.get(0).androidGradleDeps().size()); + assertTrue(language.get(0).androidGradleDeps().get(0) + .startsWith("com.google.mlkit:language-id:")); + } + + @Test + void documentCorrectionDoesNotInjectUnusedAndroidScanner() { + List hits = PlatformFeatureCatalog.matchesFor( + "com/codename1/ai/vision/DocumentScanner"); + assertEquals(1, hits.size()); + assertTrue(hits.get(0).androidGradleDeps().isEmpty(), + "The interactive Google scanner cannot implement a still-image analyzer"); + assertTrue(hits.get(0).iosFrameworks().contains("VisionKit")); + } + + @Test + void accumulatorFiltersUnrelatedSymbolsAndMemoizesHits() { + PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); + for (int i = 0; i < 10000; i++) { + acc.consume("example/unrelated/Class" + i); + acc.consumeMethod("example/unrelated/Class" + i, "method" + i); + } + Set empty = acc.hits(); + assertTrue(empty.isEmpty()); + assertSame(empty, acc.hits(), + "Unchanged scans should reuse the memoized result"); + + acc.consume("com/codename1/ai/vision/TextRecognizer"); + acc.consume("com/codename1/ai/vision/TextRecognizer"); + assertEquals(1, acc.hits().size()); + } + + @Test + void catalogUsesOneVersionPerAndroidArtifact() { + Map versions = new HashMap(); + for (PlatformFeatureCatalog.Entry entry + : PlatformFeatureCatalog.entries()) { + for (String dependency : entry.androidGradleDeps()) { + String[] parts = dependency.split(":"); + if (parts.length < 3) { + continue; + } + String artifact = parts[0] + ":" + parts[1]; + String previous = versions.put(artifact, parts[2]); + if (previous != null) { + assertEquals(previous, parts[2], + "Version drift for " + artifact); + } + } + } + } + + @Test + void arApiInjectsArKitCameraAndArCore() { + List hits = PlatformFeatureCatalog.matchesFor( + "com/codename1/ar/AR"); + assertEquals(1, hits.size(), "expected the AR entry to fire"); + PlatformFeatureCatalog.Entry e = hits.get(0); + // iOS: ARKit + SceneKit (linked explicitly by IPhoneBuilder) and the + // camera usage string, overridable via ios.NSCameraUsageDescription. + assertTrue(e.iosFrameworks().contains("ARKit")); + assertTrue(e.iosFrameworks().contains("SceneKit")); + assertNotNull(findPlistDefault(e, "NSCameraUsageDescription")); + // Android: the ARCore dependency, the camera permission and the + // optional AR feature/meta-data pair so non-AR devices still install. + assertEquals(1, e.androidGradleDeps().size()); + assertTrue(e.androidGradleDeps().get(0).startsWith("com.google.ar:core")); + assertTrue(e.androidPermissions().contains("android.permission.CAMERA")); + assertTrue(e.androidFeatures().contains("android.hardware.camera.ar")); + assertEquals(1, e.androidMetaDataEntries().size()); + assertEquals("com.google.ar.core", e.androidMetaDataEntries().get(0)[0]); + assertEquals("optional", e.androidMetaDataEntries().get(0)[1]); + } + + @Test + void arEntryMatchesTheWholePackageButNothingElse() { + assertEquals(1, PlatformFeatureCatalog.matchesFor("com/codename1/ar/ARSession").size()); + assertEquals(1, PlatformFeatureCatalog.matchesFor("com/codename1/ar/ARNode").size()); + // The pure-core VR package must NOT pull the AR native dependencies. + assertTrue(PlatformFeatureCatalog.matchesFor("com/codename1/vr/VRView").isEmpty()); + } + + @Test + void nonArEntriesCarryNoMetaData() { + // The meta-data field is new; make sure the existing entries did not + // accidentally gain one. + for (PlatformFeatureCatalog.Entry e : PlatformFeatureCatalog.entries()) { + if (!e.classPrefix().startsWith("com/codename1/ar/")) { + assertTrue(e.androidMetaDataEntries().isEmpty(), + e.classPrefix() + " should carry no manifest meta-data"); + } + } + } + + @Test + void bluetoothEntryFiresForEverySubPackage() { + String[] classes = { + "com/codename1/bluetooth/Bluetooth", + "com/codename1/bluetooth/le/BlePeripheral", + "com/codename1/bluetooth/le/server/GattServer", + "com/codename1/bluetooth/gatt/GattCharacteristic", + "com/codename1/bluetooth/classic/RfcommConnection" + }; + for (String cls : classes) { + List hits = PlatformFeatureCatalog.matchesFor(cls); + assertEquals(1, hits.size(), "expected the bluetooth entry for " + cls); + PlatformFeatureCatalog.Entry e = hits.get(0); + assertNotNull(findPlistDefault(e, "NSBluetoothAlwaysUsageDescription")); + assertNotNull(findPlistDefault(e, "NSBluetoothPeripheralUsageDescription")); + assertTrue(e.iosFrameworks().contains("CoreBluetooth")); + // Android permissions deliberately live in + // BluetoothManifestFragments (maxSdkVersion / neverForLocation + // nuances the table cannot express), not in the entry. + assertTrue(e.androidPermissions().isEmpty(), + "bluetooth Android permissions must come from BluetoothManifestFragments"); + } + } + + @Test + void bluetoothEntryDoesNotFireForUnrelatedClasses() { + assertTrue(PlatformFeatureCatalog.matchesFor("com/codename1/ui/Form").isEmpty()); + // "bluetoothle" cn1lib package must NOT trigger the core entry + assertTrue(PlatformFeatureCatalog.matchesFor( + "com/codename1/bluetoothle/Bluetooth").isEmpty()); + } + + private static String findPlistDefault(PlatformFeatureCatalog.Entry e, String key) { + for (String[] entry : e.iosPlistEntries()) { + if (key.equals(entry[0])) { + return entry[1]; + } + } + return null; + } +} diff --git a/maven/pom.xml b/maven/pom.xml index 2a683e8df92..86a4dfa3640 100644 --- a/maven/pom.xml +++ b/maven/pom.xml @@ -61,6 +61,7 @@ cn1-binaries + platform-feature-catalog java-runtime core factory @@ -80,17 +81,6 @@ cn1app-archetype cn1lib-archetype cn1-debug-proxy - cn1-ai-mlkit-text - cn1-ai-mlkit-barcode - cn1-ai-mlkit-face - cn1-ai-mlkit-labeling - cn1-ai-mlkit-translate - cn1-ai-mlkit-smartreply - cn1-ai-mlkit-langid - cn1-ai-mlkit-pose - cn1-ai-mlkit-segmentation - cn1-ai-mlkit-docscan - cn1-ai-tflite cn1-ai-whisper cn1-ai-stablediffusion cn1-admob diff --git a/scripts/gen-ai-cn1libs.py b/scripts/gen-ai-cn1libs.py index b10fb73f5f9..646ba1a5792 100644 --- a/scripts/gen-ai-cn1libs.py +++ b/scripts/gen-ai-cn1libs.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 """ -Regenerates every AI cn1lib (cn1-ai-*) under maven/ as a proper multi-module -Maven project: root + common + ios + android + javase + lib. Each module's -sources are emitted by this script -- there are NO empty stubs. +Regenerates the deliberately external AI cn1libs listed in ``LIBS``. -Run with `python3 scripts/gen-ai-cn1libs.py` from any working directory. -Existing cn1-ai-* directories are wiped and rewritten. +Small platform AI features (vision, language services, and LiteRT/Core ML +inference) are built into Codename One and selected by the platform builders. +Only features with exceptionally large native runtimes or model payloads, +currently Whisper and Stable Diffusion, remain cn1libs. + +Run with ``python3 scripts/gen-ai-cn1libs.py`` from any working directory. +Only the cn1lib directories listed in ``LIBS`` are wiped and rewritten. """ from __future__ import annotations @@ -784,1557 +787,6 @@ def test_java(pkg: str, facade: str, mock_methods: str, test_methods: str) -> st # -------------------------------------------------------------------------- -def lib_mlkit_text() -> Lib: - facade_methods = textwrap.dedent("""\ - /// Runs OCR on the supplied image bytes (JPEG or PNG). Completes with - /// the recognised text. Empty image -> empty string. No text -> empty - /// string. Hard errors fire `AsyncResource.error(...)`. - public static AsyncResource recognize(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - if (imageBytes == null || imageBytes.length == 0) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(""); } - }); - return out; - } - final NativeTextRecognizer bridge = NativeLookup.create(NativeTextRecognizer.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException( - "TextRecognizer.recognize is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String result = bridge.recognize(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(result == null ? "" : result); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("TextRecognizer.recognize failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } - """) - ni_methods = "String recognize(byte[] imageBytes);\n" - - ios_h_decls = "-(NSString*)recognize:(NSData*)param;\n" - ios_impl = textwrap.dedent("""\ - -(NSString*)recognize:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return @""; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKTextRecognizerOptions *opts = [[MLKTextRecognizerOptions alloc] init]; - MLKTextRecognizer *recognizer = [MLKTextRecognizer textRecognizerWithOptions:opts]; - __block NSString *result = @""; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [recognizer processImage:vision completion:^(MLKText * _Nullable text, NSError * _Nullable err) { - if (text && !err) { - result = text.text ?: @""; - } else if (err) { - result = @""; - } - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; - } - """) - # ML Kit ships per-framework umbrella headers (named after the - # framework itself, e.g. MLKitTextRecognition.h). Importing the - # umbrella pulls every public class -- the per-class header paths - # we previously used aren't actually exported. - # MLKText (the result type) lives in MLKitTextRecognitionCommon, - # which the recognizer's main framework re-exports but clang's - # modules system still requires the explicit import. - ios_imports = textwrap.dedent("""\ - #import - #import - #import - """) - - android_impl = textwrap.dedent("""\ - public String recognize(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return ""; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.text.TextRecognizer rec = - com.google.mlkit.vision.text.TextRecognition.getClient( - com.google.mlkit.vision.text.latin.TextRecognizerOptions.DEFAULT_OPTIONS); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference(""); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - rec.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.vision.text.Text>() { - public void onSuccess(com.google.mlkit.vision.text.Text t) { - out.set(t.getText() == null ? "" : t.getText()); - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - """) - android_imports = "" - - javase_impl = textwrap.dedent("""\ - public String recognize(byte[] imageBytes) { - if (imageBytes == null || imageBytes.length == 0) return ""; - return "[mlkit-text simulator stub] " + imageBytes.length + " bytes"; - } - """) - - test_mock_methods = textwrap.dedent("""\ - String response = "hello"; - public String recognize(byte[] imageBytes) { - if (imageBytes == null) throw new NullPointerException(); - return response; - } - """) - test_methods = textwrap.dedent("""\ - @Test - void bridge_returns_canned_string() { - MockBridge b = new MockBridge(); - assertEquals("hello", b.recognize(new byte[]{1, 2, 3})); - } - - @Test - void bridge_reports_supported() { - MockBridge b = new MockBridge(); - assertTrue(b.isSupported()); - } - - @Test - void bridge_rejects_null_input() { - MockBridge b = new MockBridge(); - assertThrows(NullPointerException.class, () -> b.recognize(null)); - } - """) - - return Lib( - artifact="cn1-ai-mlkit-text", - pkg="mlkit.text", - facade="TextRecognizer", - short_desc="ML Kit Text Recognition (OCR)", - long_desc=( - "Extracts text strings from images entirely on-device via Google's ML Kit.\n" - "Bridges to `GoogleMLKit/TextRecognition` on iOS and\n" - "`com.google.mlkit:text-recognition` on Android." - ), - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls=ios_h_decls, - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports=android_imports, - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("recognize__byte_1ARRAY", 1),), - simulator_hints=( - ("ios.NSCameraUsageDescription", - "This app uses the camera to recognise text."), - ), - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/TextRecognition", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:text-recognition:16.0.0'", - }, - ) - - -# --- mlkit-barcode ------------------------------------------------------- - - -def lib_mlkit_barcode() -> Lib: - facade_methods = textwrap.dedent("""\ - public static AsyncResource scan(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - if (imageBytes == null || imageBytes.length == 0) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(new String[0]); } - }); - return out; - } - final NativeBarcodeScanner bridge = NativeLookup.create(NativeBarcodeScanner.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("BarcodeScanner.scan is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String[] r = bridge.scan(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new String[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("BarcodeScanner.scan failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } - """) - ni_methods = "String[] scan(byte[] imageBytes);\n" - - ios_impl = textwrap.dedent("""\ - -(NSData*)scan:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [self packStrings:@[]]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKBarcodeScannerOptions *opts = [[MLKBarcodeScannerOptions alloc] init]; - MLKBarcodeScanner *scanner = [MLKBarcodeScanner barcodeScannerWithOptions:opts]; - __block NSArray *values = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [scanner processImage:vision completion:^(NSArray * _Nullable barcodes, - NSError * _Nullable error) { - NSMutableArray *m = [NSMutableArray array]; - for (MLKBarcode *b in barcodes ?: @[]) { - if (b.rawValue) [m addObject:b.rawValue]; - } - values = m; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return [self packStrings:values]; - } - - -(NSData*)packStrings:(NSArray *)strings { - // Encode as length-prefixed UTF-8 (network byte order int + bytes). - NSMutableData *out = [NSMutableData data]; - uint32_t count = htonl((uint32_t)strings.count); - [out appendBytes:&count length:sizeof(count)]; - for (NSString *s in strings) { - NSData *u = [s dataUsingEncoding:NSUTF8StringEncoding]; - uint32_t len = htonl((uint32_t)u.length); - [out appendBytes:&len length:sizeof(len)]; - [out appendData:u]; - } - return out; - } - """) - ios_imports = textwrap.dedent("""\ - #import - #import - #import - """) - android_impl = textwrap.dedent("""\ - public String[] scan(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new String[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.barcode.BarcodeScannerOptions o = - new com.google.mlkit.vision.barcode.BarcodeScannerOptions.Builder().build(); - com.google.mlkit.vision.barcode.BarcodeScanner scanner = - com.google.mlkit.vision.barcode.BarcodeScanning.getClient(o); - final java.util.List out = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - scanner.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - java.util.List>() { - public void onSuccess(java.util.List rs) { - for (com.google.mlkit.vision.barcode.common.Barcode b : rs) { - String v = b.getRawValue(); - if (v != null) out.add(v); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.toArray(new String[0]); - } - """) - javase_impl = textwrap.dedent("""\ - public String[] scan(byte[] imageBytes) { - if (imageBytes == null || imageBytes.length == 0) return new String[0]; - // Deterministic stub for simulator runs. - return new String[]{"SIMULATOR_BARCODE_" + imageBytes.length}; - } - """) - test_mock_methods = textwrap.dedent("""\ - public String[] scan(byte[] imageBytes) { - return new String[]{"x", "y"}; - } - """) - test_methods = textwrap.dedent("""\ - @Test - void mock_bridge_returns_two_codes() { - MockBridge b = new MockBridge(); - String[] r = b.scan(new byte[]{1, 2, 3}); - assertEquals(2, r.length); - assertEquals("x", r[0]); - } - - @Test - void bridge_reports_supported() { - assertTrue(new MockBridge().isSupported()); - } - """) - return Lib( - artifact="cn1-ai-mlkit-barcode", - pkg="mlkit.barcode", - facade="BarcodeScanner", - short_desc="ML Kit Barcode Scanning", - long_desc=( - "Decodes barcodes (QR, EAN, UPC, Data Matrix, PDF417, etc.) from images.\n" - "Bridges to `MLKitBarcodeScanning` on iOS and\n" - "`com.google.mlkit:barcode-scanning` on Android." - ), - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSData*)scan:(NSData*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("scan__byte_1ARRAY", 1),), - simulator_hints=( - ("ios.NSCameraUsageDescription", - "This app uses the camera to scan barcodes."), - ), - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/BarcodeScanning", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:barcode-scanning:17.2.0'", - "codename1.arg.android.xpermissions": "", - }, - ) - - -# --- mlkit-face ---------------------------------------------------------- - - -def lib_mlkit_face() -> Lib: - facade_methods = textwrap.dedent("""\ - /// Returns an array of face bounding-box quadruples - /// (x, y, width, height) packed as `int[4 * n]`. - public static AsyncResource detect(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativeFaceDetector bridge = NativeLookup.create(NativeFaceDetector.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("FaceDetector.detect is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final int[] r = bridge.detect(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new int[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("FaceDetector.detect failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } - """) - ni_methods = "int[] detect(byte[] imageBytes);\n" - - ios_impl = textwrap.dedent("""\ - -(NSData*)detect:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [NSData data]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKFaceDetectorOptions *opts = [[MLKFaceDetectorOptions alloc] init]; - MLKFaceDetector *det = [MLKFaceDetector faceDetectorWithOptions:opts]; - __block NSArray *faces = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [det processImage:vision completion:^(NSArray * _Nullable f, NSError * _Nullable e) { - faces = f ?: @[]; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - NSMutableData *out = [NSMutableData data]; - for (MLKFace *face in faces) { - CGRect r = face.frame; - int32_t v[4] = { htonl((int32_t)r.origin.x), htonl((int32_t)r.origin.y), - htonl((int32_t)r.size.width), htonl((int32_t)r.size.height) }; - [out appendBytes:v length:sizeof(v)]; - } - return out; - } - """) - ios_imports = textwrap.dedent("""\ - #import - #import - #import - """) - android_impl = textwrap.dedent("""\ - public int[] detect(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new int[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.face.FaceDetector det = - com.google.mlkit.vision.face.FaceDetection.getClient( - new com.google.mlkit.vision.face.FaceDetectorOptions.Builder().build()); - final java.util.List rs = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - det.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - java.util.List>() { - public void onSuccess(java.util.List faces) { - for (com.google.mlkit.vision.face.Face f : faces) { - android.graphics.Rect r = f.getBoundingBox(); - rs.add(new int[]{r.left, r.top, r.width(), r.height()}); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - int[] flat = new int[rs.size() * 4]; - int i = 0; - for (int[] r : rs) { System.arraycopy(r, 0, flat, i, 4); i += 4; } - return flat; - } - """) - javase_impl = textwrap.dedent("""\ - public int[] detect(byte[] imageBytes) { - // Deterministic 1-face stub for simulator runs. - if (imageBytes == null || imageBytes.length == 0) return new int[0]; - return new int[]{10, 20, 100, 120}; - } - """) - test_mock_methods = textwrap.dedent("""\ - public int[] detect(byte[] imageBytes) { - return new int[]{1, 2, 3, 4, 5, 6, 7, 8}; - } - """) - test_methods = textwrap.dedent("""\ - @Test - void mock_bridge_returns_two_faces() { - MockBridge b = new MockBridge(); - int[] r = b.detect(new byte[]{1}); - assertEquals(8, r.length); - } - """) - return Lib( - artifact="cn1-ai-mlkit-face", - pkg="mlkit.face", - facade="FaceDetector", - short_desc="ML Kit Face Detection", - long_desc=( - "Detects faces in images and returns bounding boxes.\n" - "Bridges to `MLKitFaceDetection` on iOS and\n" - "`com.google.mlkit:face-detection` on Android." - ), - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSData*)detect:(NSData*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("detect__byte_1ARRAY", 1),), - simulator_hints=( - ("ios.NSCameraUsageDescription", - "This app uses the camera to detect faces."), - ), - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/FaceDetection", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:face-detection:16.1.5'", - }, - ) - - -# Below: shorter helpers for the remaining libs. Each follows the same -# pattern; the native bodies use the appropriate ML Kit / TFLite / whisper / -# CoreML API. - - -def _string_facade_methods(facade: str, ni: str, method: str, arg_kind: str = "bytes"): - """Build the AsyncResource facade body for a String-returning bridge.""" - arg_type = "byte[] imageBytes" if arg_kind == "bytes" else "String input" - arg_pass = "imageBytes" if arg_kind == "bytes" else "input" - return textwrap.dedent(f"""\ - public static AsyncResource {method}(final {arg_type}) {{ - final AsyncResource out = new AsyncResource(); - final {ni} bridge = NativeLookup.create({ni}.class); - if (bridge == null || !bridge.isSupported()) {{ - out.error(new LlmException("{facade}.{method} is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - }} - Display.getInstance().scheduleBackgroundTask(new Runnable() {{ - @Override public void run() {{ - try {{ - final String r = bridge.{method}({arg_pass}); - Display.getInstance().callSerially(new Runnable() {{ - @Override public void run() {{ out.complete(r == null ? "" : r); }} - }}); - }} catch (final Throwable t) {{ - Display.getInstance().callSerially(new Runnable() {{ - @Override public void run() {{ - out.error(new LlmException("{facade}.{method} failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - }} - }}); - }} - }} - }}); - return out; - }} - """) - - -def _strings_facade_methods(facade: str, ni: str, method: str, arg_kind: str = "bytes"): - arg_type = "byte[] imageBytes" if arg_kind == "bytes" else "String input" - arg_pass = "imageBytes" if arg_kind == "bytes" else "input" - return textwrap.dedent(f"""\ - public static AsyncResource {method}(final {arg_type}) {{ - final AsyncResource out = new AsyncResource(); - final {ni} bridge = NativeLookup.create({ni}.class); - if (bridge == null || !bridge.isSupported()) {{ - out.error(new LlmException("{facade}.{method} is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - }} - Display.getInstance().scheduleBackgroundTask(new Runnable() {{ - @Override public void run() {{ - try {{ - final String[] r = bridge.{method}({arg_pass}); - Display.getInstance().callSerially(new Runnable() {{ - @Override public void run() {{ out.complete(r == null ? new String[0] : r); }} - }}); - }} catch (final Throwable t) {{ - Display.getInstance().callSerially(new Runnable() {{ - @Override public void run() {{ - out.error(new LlmException("{facade}.{method} failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - }} - }}); - }} - }} - }}); - return out; - }} - """) - - -def lib_mlkit_labeling() -> Lib: - facade_methods = _strings_facade_methods("ImageLabeler", "NativeImageLabeler", "label") - ios_impl = textwrap.dedent("""\ - -(NSData*)label:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [self packStrings:@[]]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKImageLabelerOptions *opts = [[MLKImageLabelerOptions alloc] init]; - MLKImageLabeler *labeler = [MLKImageLabeler imageLabelerWithOptions:opts]; - __block NSArray *labels = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [labeler processImage:vision completion:^(NSArray * _Nullable r, NSError * _Nullable e) { - labels = r ?: @[]; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - NSMutableArray *m = [NSMutableArray array]; - for (MLKImageLabel *l in labels) { if (l.text) [m addObject:l.text]; } - return [self packStrings:m]; - } - - -(NSData*)packStrings:(NSArray *)strings { - NSMutableData *out = [NSMutableData data]; - uint32_t count = htonl((uint32_t)strings.count); - [out appendBytes:&count length:sizeof(count)]; - for (NSString *s in strings) { - NSData *u = [s dataUsingEncoding:NSUTF8StringEncoding]; - uint32_t len = htonl((uint32_t)u.length); - [out appendBytes:&len length:sizeof(len)]; - [out appendData:u]; - } - return out; - } - """) - ios_imports = textwrap.dedent("""\ - #import - #import - #import - #import - """) - android_impl = textwrap.dedent("""\ - public String[] label(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new String[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.label.ImageLabeler labeler = - com.google.mlkit.vision.label.ImageLabeling.getClient( - com.google.mlkit.vision.label.defaults.ImageLabelerOptions.DEFAULT_OPTIONS); - final java.util.List out = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - labeler.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - java.util.List>() { - public void onSuccess(java.util.List rs) { - for (com.google.mlkit.vision.label.ImageLabel l : rs) out.add(l.getText()); - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.toArray(new String[0]); - } - """) - javase_impl = textwrap.dedent("""\ - public String[] label(byte[] imageBytes) { - if (imageBytes == null || imageBytes.length == 0) return new String[0]; - return new String[]{"object", "stub", "simulator"}; - } - """) - test_mock_methods = textwrap.dedent("""\ - public String[] label(byte[] imageBytes) { return new String[]{"a", "b"}; } - """) - test_methods = textwrap.dedent("""\ - @Test - void mock_bridge_returns_labels() { - MockBridge b = new MockBridge(); - assertArrayEquals(new String[]{"a", "b"}, b.label(new byte[]{1})); - } - """) - return Lib( - artifact="cn1-ai-mlkit-labeling", - pkg="mlkit.labeling", - facade="ImageLabeler", - short_desc="ML Kit Image Labeling", - long_desc=( - "Returns descriptive labels for the contents of an image.\n" - "Bridges to `MLKitImageLabeling` on iOS and\n" - "`com.google.mlkit:image-labeling` on Android." - ), - facade_methods=facade_methods, - ni_methods="String[] label(byte[] imageBytes);\n", - ios_h_decls="-(NSData*)label:(NSData*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("label__byte_1ARRAY", 1),), - win_decls="public string[] label(byte[] param) { return null; }", - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/ImageLabeling", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:image-labeling:17.0.7'", - }, - ) - - -def lib_mlkit_translate() -> Lib: - facade_methods = textwrap.dedent("""\ - public static AsyncResource translate(final String text, - final String sourceLang, - final String targetLang) { - final AsyncResource out = new AsyncResource(); - final NativeTranslator bridge = NativeLookup.create(NativeTranslator.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("Translator.translate is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final String r = bridge.translate(text, sourceLang, targetLang); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? "" : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("Translator.translate failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } - """) - ni_methods = "String translate(String text, String sourceLang, String targetLang);\n" - ios_impl = textwrap.dedent("""\ - -(NSString*)translate:(NSString*)param param1:(NSString*)param1 param2:(NSString*)param2 { - MLKTranslatorOptions *opts = [[MLKTranslatorOptions alloc] - initWithSourceLanguage:param1 - targetLanguage:param2]; - MLKTranslator *t = [MLKTranslator translatorWithOptions:opts]; - MLKModelDownloadConditions *cond = [[MLKModelDownloadConditions alloc] - initWithAllowsCellularAccess:YES - allowsBackgroundDownloading:YES]; - __block NSString *result = @""; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [t downloadModelIfNeededWithConditions:cond completion:^(NSError * _Nullable err) { - if (err) { dispatch_semaphore_signal(sem); return; } - [t translateText:param completion:^(NSString * _Nullable r, NSError * _Nullable e) { - if (r && !e) result = r; - dispatch_semaphore_signal(sem); - }]; - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; - } - """) - ios_imports = textwrap.dedent("""\ - #import - #import - """) - android_impl = textwrap.dedent("""\ - public String translate(String text, String sourceLang, String targetLang) { - com.google.mlkit.nl.translate.TranslatorOptions opts = - new com.google.mlkit.nl.translate.TranslatorOptions.Builder() - .setSourceLanguage(sourceLang) - .setTargetLanguage(targetLang) - .build(); - com.google.mlkit.nl.translate.Translator t = - com.google.mlkit.nl.translate.Translation.getClient(opts); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference(""); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - t.downloadModelIfNeeded() - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener() { - public void onSuccess(Void v) { - t.translate(text) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener() { - public void onSuccess(String r) { out.set(r); latch.countDown(); } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - """) - javase_impl = textwrap.dedent("""\ - public String translate(String text, String sourceLang, String targetLang) { - if (text == null) return ""; - return "[" + sourceLang + "->" + targetLang + "] " + text; - } - """) - test_mock_methods = textwrap.dedent("""\ - public String translate(String text, String sourceLang, String targetLang) { - return text + "@" + sourceLang + "->" + targetLang; - } - """) - test_methods = textwrap.dedent("""\ - @Test - void mock_translate_round_trip() { - MockBridge b = new MockBridge(); - assertEquals("hi@en->fr", b.translate("hi", "en", "fr")); - } - """) - return Lib( - artifact="cn1-ai-mlkit-translate", - pkg="mlkit.translate", - facade="Translator", - short_desc="ML Kit on-device Translation", - long_desc="Translates short text between language pairs entirely on-device.", - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSString*)translate:(NSString*)param param1:(NSString*)param1 param2:(NSString*)param2;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("translate__java_lang_String_java_lang_String_java_lang_String", 3),), - win_decls="public string translate(string param, string param1, string param2) { return null; }", - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/Translate", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:translate:17.0.3'", - }, - ) - - -def lib_mlkit_smartreply() -> Lib: - facade_methods = _strings_facade_methods("SmartReply", "NativeSmartReply", "suggest", arg_kind="text") - ni_methods = "String[] suggest(String conversationJson);\n" - ios_impl = textwrap.dedent("""\ - -(NSData*)suggest:(NSString*)param { - // param is a JSON array of {role,message,timestamp,userId}. - NSError *err = nil; - NSArray *items = [NSJSONSerialization JSONObjectWithData: - [param dataUsingEncoding:NSUTF8StringEncoding] - options:0 error:&err]; - NSMutableArray *messages = [NSMutableArray array]; - if ([items isKindOfClass:[NSArray class]]) { - for (NSDictionary *d in items) { - if (![d isKindOfClass:[NSDictionary class]]) continue; - NSString *role = d[@"role"] ?: @"user"; - BOOL isLocalUser = [role isEqualToString:@"user"]; - NSString *text = d[@"message"] ?: @""; - NSNumber *ts = d[@"timestamp"] ?: @0; - MLKTextMessage *m = [[MLKTextMessage alloc] - initWithText:text timestamp:[ts doubleValue] - userID:(d[@"userId"] ?: @"u") - isLocalUser:isLocalUser]; - [messages addObject:m]; - } - } - __block NSArray *out = @[]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [[MLKSmartReply smartReply] suggestRepliesForMessages:messages - completion:^(MLKSmartReplySuggestionResult * _Nullable result, NSError * _Nullable e) { - NSMutableArray *m = [NSMutableArray array]; - for (MLKSmartReplySuggestion *s in result.suggestions ?: @[]) { - if (s.text) [m addObject:s.text]; - } - out = m; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return [self packStrings:out]; - } - - -(NSData*)packStrings:(NSArray *)strings { - NSMutableData *out = [NSMutableData data]; - uint32_t count = htonl((uint32_t)strings.count); - [out appendBytes:&count length:sizeof(count)]; - for (NSString *s in strings) { - NSData *u = [s dataUsingEncoding:NSUTF8StringEncoding]; - uint32_t len = htonl((uint32_t)u.length); - [out appendBytes:&len length:sizeof(len)]; - [out appendData:u]; - } - return out; - } - """) - ios_imports = textwrap.dedent("""\ - #import - #import - """) - android_impl = textwrap.dedent("""\ - public String[] suggest(String conversationJson) { - java.util.List msgs = - new java.util.ArrayList(); - try { - org.json.JSONArray a = new org.json.JSONArray(conversationJson); - for (int i = 0; i < a.length(); i++) { - org.json.JSONObject o = a.getJSONObject(i); - String role = o.optString("role", "user"); - long ts = o.optLong("timestamp", 0); - String text = o.optString("message", ""); - String userId = o.optString("userId", "u"); - if ("user".equals(role)) { - msgs.add(com.google.mlkit.nl.smartreply.TextMessage.createForLocalUser(text, ts)); - } else { - msgs.add(com.google.mlkit.nl.smartreply.TextMessage.createForRemoteUser(text, ts, userId)); - } - } - } catch (org.json.JSONException jex) { - return new String[0]; - } - final java.util.List out = new java.util.ArrayList(); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - com.google.mlkit.nl.smartreply.SmartReplyGenerator gen = - com.google.mlkit.nl.smartreply.SmartReply.getClient(); - gen.suggestReplies(msgs) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.nl.smartreply.SmartReplySuggestionResult>() { - public void onSuccess(com.google.mlkit.nl.smartreply.SmartReplySuggestionResult r) { - for (com.google.mlkit.nl.smartreply.SmartReplySuggestion s : r.getSuggestions()) { - out.add(s.getText()); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.toArray(new String[0]); - } - """) - javase_impl = textwrap.dedent("""\ - public String[] suggest(String conversationJson) { - return new String[]{"Sounds good", "Thanks!", "Got it"}; - } - """) - test_mock_methods = "public String[] suggest(String c) { return new String[]{\"ok\"}; }\n" - test_methods = textwrap.dedent("""\ - @Test - void mock_returns_single_suggestion() { - MockBridge b = new MockBridge(); - assertEquals(1, b.suggest("[]").length); - } - """) - return Lib( - artifact="cn1-ai-mlkit-smartreply", - pkg="mlkit.smartreply", - facade="SmartReply", - short_desc="ML Kit Smart Reply", - long_desc="Generates short reply suggestions for chat conversations on-device.", - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSData*)suggest:(NSString*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("suggest__java_lang_String", 1),), - win_decls="public string[] suggest(string param) { return null; }", - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/SmartReply", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:smart-reply:17.0.4'", - }, - ) - - -def lib_mlkit_langid() -> Lib: - facade_methods = _string_facade_methods("LanguageIdentifier", "NativeLanguageIdentifier", "identify", - arg_kind="text") - ni_methods = "String identify(String input);\n" - ios_impl = textwrap.dedent("""\ - -(NSString*)identify:(NSString*)param { - MLKLanguageIdentification *id = [MLKLanguageIdentification languageIdentification]; - __block NSString *result = @"und"; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [id identifyLanguageForText:param completion:^(NSString * _Nullable lang, NSError * _Nullable e) { - if (lang) result = lang; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; - } - """) - ios_imports = "#import \n" - android_impl = textwrap.dedent("""\ - public String identify(String input) { - com.google.mlkit.nl.languageid.LanguageIdentifier id = - com.google.mlkit.nl.languageid.LanguageIdentification.getClient(); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference("und"); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - id.identifyLanguage(input) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener() { - public void onSuccess(String s) { if (s != null) out.set(s); latch.countDown(); } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - """) - javase_impl = textwrap.dedent("""\ - public String identify(String input) { - // Crude language ID stub for simulator. - if (input == null || input.isEmpty()) return "und"; - return "en"; - } - """) - test_mock_methods = "public String identify(String input) { return \"en\"; }\n" - test_methods = textwrap.dedent("""\ - @Test - void mock_identifies_english() { - MockBridge b = new MockBridge(); - assertEquals("en", b.identify("hello")); - } - """) - return Lib( - artifact="cn1-ai-mlkit-langid", - pkg="mlkit.langid", - facade="LanguageIdentifier", - short_desc="ML Kit Language Identification", - long_desc="Identifies the language of a given text string.", - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSString*)identify:(NSString*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("identify__java_lang_String", 1),), - win_decls="public string identify(string param) { return null; }", - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/LanguageID", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:language-id:17.0.6'", - }, - ) - - -def lib_mlkit_pose() -> Lib: - facade_methods = textwrap.dedent("""\ - /// Returns 33 landmark triples (x,y,confidence) per detected pose - /// packed as `float[3 * 33]`. - public static AsyncResource detect(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativePoseDetector bridge = NativeLookup.create(NativePoseDetector.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("PoseDetector.detect is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final float[] r = bridge.detect(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new float[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("PoseDetector.detect failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } - """) - ni_methods = "float[] detect(byte[] imageBytes);\n" - ios_impl = textwrap.dedent("""\ - -(NSData*)detect:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [NSData data]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKPoseDetectorOptions *opts = [[MLKPoseDetectorOptions alloc] init]; - MLKPoseDetector *det = [MLKPoseDetector poseDetectorWithOptions:opts]; - __block MLKPose *pose = nil; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [det processImage:vision completion:^(NSArray * _Nullable r, NSError * _Nullable e) { - if (r.count > 0) pose = r[0]; - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - float buf[99] = {0}; - if (pose) { - for (NSInteger i = 0; i < 33 && i < pose.landmarks.count; i++) { - MLKPoseLandmark *lm = pose.landmarks[i]; - buf[i * 3] = (float)lm.position.x; - buf[i * 3 + 1] = (float)lm.position.y; - buf[i * 3 + 2] = (float)lm.inFrameLikelihood; - } - } - // Pack as big-endian float bytes (matches JAVA_ARRAY_FLOAT on iOS port). - return [NSData dataWithBytes:buf length:sizeof(buf)]; - } - """) - ios_imports = textwrap.dedent("""\ - #import - #import - #import - """) - android_impl = textwrap.dedent("""\ - public float[] detect(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new float[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.pose.PoseDetector det = - com.google.mlkit.vision.pose.PoseDetection.getClient( - new com.google.mlkit.vision.pose.defaults.PoseDetectorOptions.Builder().build()); - final float[] out = new float[33 * 3]; - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - det.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.vision.pose.Pose>() { - public void onSuccess(com.google.mlkit.vision.pose.Pose p) { - java.util.List lms = p.getAllPoseLandmarks(); - for (int i = 0; i < 33 && i < lms.size(); i++) { - com.google.mlkit.vision.pose.PoseLandmark lm = lms.get(i); - out[i * 3] = lm.getPosition().x; - out[i * 3 + 1] = lm.getPosition().y; - out[i * 3 + 2] = lm.getInFrameLikelihood(); - } - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out; - } - """) - javase_impl = textwrap.dedent("""\ - public float[] detect(byte[] imageBytes) { - float[] out = new float[99]; - for (int i = 0; i < 33; i++) { out[i * 3] = i; out[i * 3 + 2] = 0.5f; } - return out; - } - """) - test_mock_methods = "public float[] detect(byte[] imageBytes) { return new float[99]; }\n" - test_methods = textwrap.dedent("""\ - @Test - void mock_returns_33_landmarks() { - MockBridge b = new MockBridge(); - assertEquals(99, b.detect(new byte[]{1}).length); - } - """) - return Lib( - artifact="cn1-ai-mlkit-pose", - pkg="mlkit.pose", - facade="PoseDetector", - short_desc="ML Kit Pose Detection", - long_desc="Returns skeletal landmarks for human bodies detected in an image.", - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSData*)detect:(NSData*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("detect__byte_1ARRAY", 1),), - win_decls="public float[] detect(byte[] param) { return null; }", - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/PoseDetection", - "codename1.arg.android.gradleDep": "implementation 'com.google.mlkit:pose-detection:18.0.0-beta3'", - }, - ) - - -def lib_mlkit_segmentation() -> Lib: - facade_methods = textwrap.dedent("""\ - /// Returns a per-pixel mask separating foreground (person) from - /// background as `byte[width * height]` (0=background, 255=foreground). - public static AsyncResource segment(final byte[] imageBytes) { - final AsyncResource out = new AsyncResource(); - final NativeSelfieSegmenter bridge = NativeLookup.create(NativeSelfieSegmenter.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("SelfieSegmenter.segment is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final byte[] r = bridge.segment(imageBytes); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new byte[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("SelfieSegmenter.segment failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } - """) - ni_methods = "byte[] segment(byte[] imageBytes);\n" - ios_impl = textwrap.dedent("""\ - -(NSData*)segment:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return [NSData data]; - MLKVisionImage *vision = [[MLKVisionImage alloc] initWithImage:image]; - MLKSelfieSegmenterOptions *opts = [[MLKSelfieSegmenterOptions alloc] init]; - opts.segmenterMode = MLKSegmenterModeSingleImage; - MLKSegmenter *seg = [MLKSegmenter segmenterWithOptions:opts]; - __block NSData *result = [NSData data]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - [seg processImage:vision completion:^(MLKSegmentationMask * _Nullable mask, NSError * _Nullable e) { - if (mask) { - // MLKSegmentationMask exposes only `buffer` (a - // CVPixelBuffer); dimensions come from CVPixelBufferGet*. - CVPixelBufferRef buf = mask.buffer; - size_t w = CVPixelBufferGetWidth(buf); - size_t h = CVPixelBufferGetHeight(buf); - CVPixelBufferLockBaseAddress(buf, kCVPixelBufferLock_ReadOnly); - void *base = CVPixelBufferGetBaseAddress(buf); - NSMutableData *m = [NSMutableData dataWithLength:w * h]; - uint8_t *out = m.mutableBytes; - float *src = (float *)base; - for (size_t i = 0; i < w * h; i++) { - float v = src[i]; - out[i] = (uint8_t)(v * 255.0f); - } - CVPixelBufferUnlockBaseAddress(buf, kCVPixelBufferLock_ReadOnly); - result = m; - } - dispatch_semaphore_signal(sem); - }]; - dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); - return result; - } - """) - ios_imports = textwrap.dedent("""\ - #import - #import - #import - #import - """) - android_impl = textwrap.dedent("""\ - public byte[] segment(byte[] imageBytes) { - android.graphics.Bitmap bm = android.graphics.BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.length); - if (bm == null) return new byte[0]; - com.google.mlkit.vision.common.InputImage img = - com.google.mlkit.vision.common.InputImage.fromBitmap(bm, 0); - com.google.mlkit.vision.segmentation.Segmenter seg = - com.google.mlkit.vision.segmentation.Segmentation.getClient( - new com.google.mlkit.vision.segmentation.SelfieSegmenterOptions.Builder() - .setDetectorMode( - com.google.mlkit.vision.segmentation.SelfieSegmenterOptions.SINGLE_IMAGE_MODE) - .build()); - final java.util.concurrent.atomic.AtomicReference out = - new java.util.concurrent.atomic.AtomicReference(new byte[0]); - final java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(1); - seg.process(img) - .addOnSuccessListener(new com.google.android.gms.tasks.OnSuccessListener< - com.google.mlkit.vision.segmentation.SegmentationMask>() { - public void onSuccess(com.google.mlkit.vision.segmentation.SegmentationMask mask) { - int w = mask.getWidth(), h = mask.getHeight(); - java.nio.ByteBuffer buf = mask.getBuffer(); - buf.rewind(); - byte[] outb = new byte[w * h]; - for (int i = 0; i < w * h; i++) { - float v = buf.getFloat(); - outb[i] = (byte)(int)(v * 255); - } - out.set(outb); - latch.countDown(); - } - }) - .addOnFailureListener(new com.google.android.gms.tasks.OnFailureListener() { - public void onFailure(Exception e) { latch.countDown(); } - }); - try { latch.await(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } - return out.get(); - } - """) - javase_impl = textwrap.dedent("""\ - public byte[] segment(byte[] imageBytes) { - // 8x8 checkerboard stub. - byte[] out = new byte[64]; - for (int i = 0; i < 64; i++) out[i] = (byte)(((i / 8) + (i % 8)) % 2 == 0 ? 255 : 0); - return out; - } - """) - test_mock_methods = "public byte[] segment(byte[] imageBytes) { return new byte[16]; }\n" - test_methods = textwrap.dedent("""\ - @Test - void mock_returns_mask_bytes() { - MockBridge b = new MockBridge(); - assertEquals(16, b.segment(new byte[]{1}).length); - } - """) - return Lib( - artifact="cn1-ai-mlkit-segmentation", - pkg="mlkit.segmentation", - facade="SelfieSegmenter", - short_desc="ML Kit Selfie Segmentation", - long_desc="Returns a per-pixel mask separating a person in the foreground from the background.", - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSData*)segment:(NSData*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("segment__byte_1ARRAY", 1),), - win_decls="public byte[] segment(byte[] param) { return null; }", - build_hints={ - "codename1.arg.ios.pods": "GoogleMLKit/SegmentationSelfie", - "codename1.arg.android.gradleDep": - "implementation 'com.google.mlkit:segmentation-selfie:16.0.0-beta5'", - }, - ) - - -def lib_mlkit_docscan() -> Lib: - facade_methods = _string_facade_methods("DocumentScanner", "NativeDocumentScanner", "scanToFile", - arg_kind="bytes") - ni_methods = "String scanToFile(byte[] imageBytes);\n" - ios_impl = textwrap.dedent("""\ - // VisionKit-based fallback: Apple's VNDocumentCameraViewController is - // interactive; this bridge accepts a pre-captured image and returns its - // cropped JPEG path. On iOS 13+ VisionKit handles the live UI flow; the - // sample app drives that flow and feeds the bytes into the cn1lib. - -(NSString*)scanToFile:(NSData*)param { - UIImage *image = [UIImage imageWithData:param]; - if (!image) return @""; - CIImage *ci = [CIImage imageWithCGImage:image.CGImage]; - CIContext *ctx = [CIContext context]; - CIDetector *det = [CIDetector detectorOfType:CIDetectorTypeRectangle context:ctx - options:@{CIDetectorAccuracy: CIDetectorAccuracyHigh}]; - NSArray *features = [det featuresInImage:ci]; - UIImage *cropped = image; - if (features.count > 0) { - CIRectangleFeature *rf = (CIRectangleFeature *)features.firstObject; - CIImage *flat = [ci imageByApplyingFilter:@"CIPerspectiveCorrection" withInputParameters:@{ - @"inputTopLeft": [CIVector vectorWithCGPoint:rf.topLeft], - @"inputTopRight": [CIVector vectorWithCGPoint:rf.topRight], - @"inputBottomLeft": [CIVector vectorWithCGPoint:rf.bottomLeft], - @"inputBottomRight": [CIVector vectorWithCGPoint:rf.bottomRight] - }]; - CGImageRef cg = [ctx createCGImage:flat fromRect:flat.extent]; - cropped = [UIImage imageWithCGImage:cg]; - CGImageRelease(cg); - } - NSString *path = [NSString stringWithFormat:@"%@/docscan-%@.jpg", - NSTemporaryDirectory(), [[NSUUID UUID] UUIDString]]; - [UIImageJPEGRepresentation(cropped, 0.92) writeToFile:path atomically:YES]; - return path; - } - """) - ios_imports = textwrap.dedent("""\ - #import - """) - android_impl = textwrap.dedent("""\ - public String scanToFile(byte[] imageBytes) { - try { - java.io.File f = java.io.File.createTempFile("docscan-", ".jpg"); - java.io.FileOutputStream fos = new java.io.FileOutputStream(f); - fos.write(imageBytes); - fos.close(); - return f.getAbsolutePath(); - } catch (java.io.IOException ioe) { - return ""; - } - } - """) - javase_impl = textwrap.dedent("""\ - public String scanToFile(byte[] imageBytes) { - try { - java.io.File f = java.io.File.createTempFile("docscan-stub-", ".jpg"); - java.nio.file.Files.write(f.toPath(), imageBytes); - return f.getAbsolutePath(); - } catch (java.io.IOException ioe) { - return ""; - } - } - """) - test_mock_methods = "public String scanToFile(byte[] imageBytes) { return \"/tmp/x.jpg\"; }\n" - test_methods = textwrap.dedent("""\ - @Test - void mock_returns_path() { - MockBridge b = new MockBridge(); - assertEquals("/tmp/x.jpg", b.scanToFile(new byte[]{1})); - } - """) - return Lib( - artifact="cn1-ai-mlkit-docscan", - pkg="mlkit.docscan", - facade="DocumentScanner", - short_desc="ML Kit / VisionKit Document Scanner", - long_desc=( - "Captures and crops document photos. On iOS uses Apple's VisionKit + Core Image " - "rectangle detection (no extra pod). On Android uses the Google Play services " - "document-scanner module." - ), - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSString*)scanToFile:(NSData*)param;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("scanToFile__byte_1ARRAY", 1),), - win_decls="public string scanToFile(byte[] param) { return null; }", - build_hints={ - "codename1.arg.ios.add_frameworks": "VisionKit", - "codename1.arg.android.gradleDep": - "implementation 'com.google.android.gms:play-services-mlkit-document-scanner:16.0.0-beta1'", - }, - ) - - -def lib_tflite() -> Lib: - facade_methods = textwrap.dedent("""\ - /// Loads a TensorFlow Lite model from the supplied bytes and runs - /// inference against a float32 input tensor. Returns the output as - /// `float[]`. The model file is held in a native handle keyed by - /// the SHA-1 of the input bytes; repeated calls reuse the loaded - /// model. - public static AsyncResource run(final byte[] modelBytes, - final float[] input, - final int outputLength) { - final AsyncResource out = new AsyncResource(); - final NativeInterpreter bridge = NativeLookup.create(NativeInterpreter.class); - if (bridge == null || !bridge.isSupported()) { - out.error(new LlmException("Interpreter.run is not supported on this platform.", - -1, null, null, null, LlmException.ErrorType.UNKNOWN)); - return out; - } - Display.getInstance().scheduleBackgroundTask(new Runnable() { - @Override public void run() { - try { - final float[] r = bridge.run(modelBytes, input, outputLength); - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { out.complete(r == null ? new float[0] : r); } - }); - } catch (final Throwable t) { - Display.getInstance().callSerially(new Runnable() { - @Override public void run() { - out.error(new LlmException("Interpreter.run failed: " + t.getMessage(), - -1, null, null, t, LlmException.ErrorType.UNKNOWN)); - } - }); - } - } - }); - return out; - } - """) - ni_methods = "float[] run(byte[] modelBytes, float[] input, int outputLength);\n" - ios_impl = textwrap.dedent("""\ - -(NSData*)run:(NSData*)param param1:(NSData*)param1 param2:(int)param2 { - NSError *err = nil; - NSString *modelPath = [NSString stringWithFormat:@"%@/tflite-%@.tflite", - NSTemporaryDirectory(), [[NSUUID UUID] UUIDString]]; - [param writeToFile:modelPath atomically:YES]; - TFLInterpreter *interp = [[TFLInterpreter alloc] initWithModelPath:modelPath error:&err]; - if (err) return [NSData data]; - [interp allocateTensorsWithError:&err]; - if (err) return [NSData data]; - TFLTensor *in0 = [interp inputTensorAtIndex:0 error:&err]; - [in0 copyData:param1 error:&err]; - [interp invokeWithError:&err]; - TFLTensor *out0 = [interp outputTensorAtIndex:0 error:&err]; - NSData *outBytes = [out0 dataWithError:&err]; - return outBytes ?: [NSData data]; - } - """) - # TensorFlowLiteObjC pod installs the `TFLTensorFlowLite` framework - # with umbrella header `TFLTensorFlowLite.h`. The framework dir - # name (used as the angle-bracket prefix) is the module name - # `TFLTensorFlowLite`, not the pod name `TensorFlowLiteObjC`. - ios_imports = "#import \n" - android_impl = textwrap.dedent("""\ - public float[] run(byte[] modelBytes, float[] input, int outputLength) { - java.nio.ByteBuffer bb = java.nio.ByteBuffer.allocateDirect(modelBytes.length); - bb.order(java.nio.ByteOrder.nativeOrder()); - bb.put(modelBytes); - bb.rewind(); - org.tensorflow.lite.Interpreter interp = new org.tensorflow.lite.Interpreter(bb); - float[][] out = new float[1][outputLength]; - interp.run(new float[][]{input}, out); - interp.close(); - return out[0]; - } - """) - javase_impl = textwrap.dedent("""\ - public float[] run(byte[] modelBytes, float[] input, int outputLength) { - // Identity stub: returns first outputLength entries of input - // (or zero-padded if input shorter). Lets simulator test plumbing. - float[] out = new float[outputLength]; - int n = Math.min(input.length, outputLength); - System.arraycopy(input, 0, out, 0, n); - return out; - } - """) - test_mock_methods = textwrap.dedent("""\ - public float[] run(byte[] modelBytes, float[] input, int outputLength) { - float[] r = new float[outputLength]; - for (int i = 0; i < r.length; i++) r[i] = i; - return r; - } - """) - test_methods = textwrap.dedent("""\ - @Test - void mock_returns_increasing_vector() { - MockBridge b = new MockBridge(); - float[] r = b.run(new byte[0], new float[]{1f, 2f}, 4); - assertEquals(4, r.length); - assertEquals(3.0f, r[3], 1e-6); - } - """) - return Lib( - artifact="cn1-ai-tflite", - pkg="tflite", - facade="Interpreter", - short_desc="TensorFlow Lite on-device inference", - long_desc=( - "Loads a `.tflite` model and runs inference against `float[]` inputs.\n" - "Bridges to `TensorFlowLiteObjC` on iOS and `org.tensorflow:tensorflow-lite`\n" - "on Android." - ), - facade_methods=facade_methods, - ni_methods=ni_methods, - ios_h_decls="-(NSData*)run:(NSData*)param param1:(NSData*)param1 param2:(int)param2;\n", - ios_impl=ios_impl, - ios_imports=ios_imports, - android_imports="", - android_impl=android_impl, - javase_impl=javase_impl, - test_methods=test_methods, - test_mock_methods=test_mock_methods, - js_method_keys=(("run__byte_1ARRAY_float_1ARRAY_int", 3),), - win_decls="public float[] run(byte[] param, float[] param1, int param2) { return null; }", - build_hints={ - "codename1.arg.ios.pods": "TensorFlowLiteObjC", - "codename1.arg.android.gradleDep": "implementation 'org.tensorflow:tensorflow-lite:2.14.0'", - }, - ) - - def lib_whisper() -> Lib: facade_methods = textwrap.dedent("""\ /// Transcribes audio using a whisper.cpp model. `modelPath` is the @@ -2606,17 +1058,6 @@ def lib_stablediffusion() -> Lib: LIBS: list[Lib] = [ - lib_mlkit_text(), - lib_mlkit_barcode(), - lib_mlkit_face(), - lib_mlkit_labeling(), - lib_mlkit_translate(), - lib_mlkit_smartreply(), - lib_mlkit_langid(), - lib_mlkit_pose(), - lib_mlkit_segmentation(), - lib_mlkit_docscan(), - lib_tflite(), lib_whisper(), lib_stablediffusion(), ] diff --git a/scripts/hellocodenameone/README.adoc b/scripts/hellocodenameone/README.adoc index caaea701d8a..c50f15c6bdd 100644 --- a/scripts/hellocodenameone/README.adoc +++ b/scripts/hellocodenameone/README.adoc @@ -124,6 +124,15 @@ When adding a conformance test: . Run the validator before pushing. The `Validate port status contract` workflow enforces the same rules in CI. +The on-device AI rows are permanent assertion tests rather than screenshots. +They validate immutable image, camera-frame, tensor, model-source, and options +contracts on every translated runtime; query every native analyzer or service; +and verify deterministic failed-resource behavior when a port has no backend. +They do not open camera hardware, download language models, or bundle a test +inference model. Registering each concrete entry point in the shared application +also exercises the builders' granular native source and dependency selection on +every source-build job. + CN1SS normalizes each run to `pass`, `fail`, `skip`, or `not-run` per test. A public feature checkbox is checked only when every mapped test passes. Skips, incomplete runs, stale reports, and failures remain distinct states; none is diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java index 30dd761a081..3b31c5bb407 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.java @@ -121,6 +121,13 @@ private static int testTimeoutMs(BaseTest testClass) { && testClass instanceof ToastBarTopPositionScreenshotTest) { return 45000; } + if (!"HTML5".equals(Display.getInstance().getPlatformName()) + && testClass instanceof VisionOnDeviceApiTest) { + // The first Apple Vision request may compile its detector model on + // a cold simulator. Keep the EDT free and allow that one-time + // native initialization a bounded minute. + return TEST_TIMEOUT_MS_NATIVE * 2; + } return "HTML5".equals(Display.getInstance().getPlatformName()) ? TEST_TIMEOUT_MS_HTML5 : TEST_TIMEOUT_MS_NATIVE; @@ -366,6 +373,16 @@ private static int testTimeoutMs(BaseTest testClass) { // prompts). Self-skips on iOS / Android / JS where the open // call would surface an OS dialog. new CameraApiTest(), + // Built-in on-device AI coverage. These are assertion-only tests: + // portable value/lifecycle contracts run everywhere, while + // capability queries exercise each port's native bridge without + // requiring a camera, a downloaded language model, or a bundled + // inference model. Referencing the individual entry points also + // keeps the builders' granular dependency selection under the + // permanent cross-platform source-build suite. + new VisionOnDeviceApiTest(), + new LanguageOnDeviceApiTest(), + new InferenceOnDeviceApiTest(), // Exercises com.codename1.ar end-to-end: the unsupported // contract on the CI platforms (none has an AR runtime) and a // full session round trip when a backend is present. diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/CommonWorkloadBenchmarkTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/CommonWorkloadBenchmarkTest.java index 1fa93512e43..fe249ea193a 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/CommonWorkloadBenchmarkTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/CommonWorkloadBenchmarkTest.java @@ -24,8 +24,18 @@ package com.codenameone.examples.hellocodenameone.tests; import com.bench.CommonWorkloads; +import com.codename1.ui.CN; +import com.codename1.ui.Display; -/** Runs the canonical ParparVM common workloads inside every port app. */ +/** + * Runs the canonical ParparVM common workloads inside every port app. + * + *

The iOS simulator omits the allocation-heavy workloads because ParparVM's + * simulator mark-sweep collector can require several gigabytes for a single + * invocation. Device builds still exercise the complete workload set. Skipped + * workloads are emitted explicitly so that the port-status report distinguishes + * them from missing benchmark output.

+ */ public class CommonWorkloadBenchmarkTest extends BaseTest { private static final int WARMUP = 3; private static final int MEASUREMENTS = 5; @@ -54,15 +64,21 @@ public boolean runTest() { run("arrayRandom", new Workload() { public long run() { return CommonWorkloads.arrayRandom(); } }); - run("objectAllocation", new Workload() { - public long run() { return CommonWorkloads.objectAllocation(); } - }); - run("hashMapChurn", new Workload() { - public long run() { return CommonWorkloads.hashMapChurn(); } - }); - run("stringBuilding", new Workload() { - public long run() { return CommonWorkloads.stringBuilding(); } - }); + if (isIosSimulator()) { + skip("objectAllocation", "ios-simulator-gc-footprint"); + skip("hashMapChurn", "ios-simulator-gc-footprint"); + skip("stringBuilding", "ios-simulator-gc-footprint"); + } else { + run("objectAllocation", new Workload() { + public long run() { return CommonWorkloads.objectAllocation(); } + }); + run("hashMapChurn", new Workload() { + public long run() { return CommonWorkloads.hashMapChurn(); } + }); + run("stringBuilding", new Workload() { + public long run() { return CommonWorkloads.stringBuilding(); } + }); + } run("recursion", new Workload() { public long run() { return CommonWorkloads.recursion(); } }); @@ -96,6 +112,14 @@ private static void run(String id, Workload workload) { emit("benchmark id=" + id + " duration_ns=" + minimumNanos + " checksum=" + checksum); } + private static boolean isIosSimulator() { + return CN.isSimulator() && "ios".equals(Display.getInstance().getPlatformName()); + } + + private static void skip(String id, String reason) { + emit("skipped id=" + id + " reason=" + reason); + } + private static void emit(String value) { System.out.println("CN1SS:PERF:" + value); } diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/InferenceOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/InferenceOnDeviceApiTest.java new file mode 100644 index 00000000000..ae11d70a0a7 --- /dev/null +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/InferenceOnDeviceApiTest.java @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codenameone.examples.hellocodenameone.tests; + +import com.codename1.ai.inference.InferenceOptions; +import com.codename1.ai.inference.InferenceSession; +import com.codename1.ai.inference.ModelSource; +import com.codename1.ai.inference.Tensor; +import com.codename1.ai.inference.TensorInfo; +import com.codename1.ai.inference.TensorType; +import com.codename1.io.Log; +import com.codename1.util.AsyncResource; + +/** + * Cross-port, non-visual contract coverage for LiteRT model inference. + * + *

The portable tensor and model-source contracts run everywhere. The test + * also queries the native runtime so the builders select it where supported; + * an actual model is deliberately not bundled into the conformance app.

+ */ +public class InferenceOnDeviceApiTest extends BaseTest { + @Override + public boolean shouldTakeScreenshot() { + return false; + } + + @Override + public boolean runTest() { + try { + checkTensorContracts(); + checkModelAndOptions(); + checkCapability(); + done(); + return true; + } catch (Throwable t) { + fail("On-device inference API test failed: " + t); + return false; + } + } + + private void checkTensorContracts() { + int[] shape = new int[] {1, 2}; + float[] values = new float[] {1f, 2f}; + Tensor tensor = Tensor.floats("input", shape, values); + shape[1] = 9; + values[0] = 9f; + checkEqual(2, tensor.getShape()[1], + "Tensor must copy the input shape"); + checkEqual(1f, ((float[]) tensor.getData())[0], + "Tensor must copy the input data"); + int[] outputShape = tensor.getShape(); + float[] outputData = (float[]) tensor.getData(); + outputShape[1] = 8; + outputData[1] = 8f; + checkEqual(2, tensor.getShape()[1], + "Tensor must copy the output shape"); + checkEqual(2f, ((float[]) tensor.getData())[1], + "Tensor must copy the output data"); + check(tensor.getType() == TensorType.FLOAT32, + "Tensor FLOAT32 type"); + + TensorInfo info = new TensorInfo( + "output", TensorType.INT32, new int[] {1, 3}, 2); + int[] infoShape = info.getShape(); + infoShape[1] = 7; + checkEqual(3, info.getShape()[1], + "TensorInfo must copy the output shape"); + checkEqual(2, info.getIndex(), "TensorInfo index"); + + try { + Tensor.floats("bad", new int[] {3}, new float[] {1f, 2f}); + throw new IllegalStateException( + "Tensor accepted data that did not match its shape"); + } catch (IllegalArgumentException expected) { + // Shape/data validation is part of the public contract. + } + } + + private void checkModelAndOptions() { + byte[] bytes = new byte[] {1, 2, 3}; + ModelSource source = ModelSource.bytes(bytes); + bytes[0] = 9; + checkEqual(1, source.getBytes()[0], + "ModelSource must copy input bytes"); + byte[] returned = source.getBytes(); + returned[1] = 9; + checkEqual(2, source.getBytes()[1], + "ModelSource must copy output bytes"); + checkEqual(ModelSource.BYTES, source.getKind(), + "ModelSource byte kind"); + check("model.tflite".equals( + ModelSource.resource("model.tflite").getPath()), + "ModelSource resource path"); + + InferenceOptions options = new InferenceOptions() + .accelerator(InferenceOptions.Accelerator.CORE_ML) + .threads(-1) + .allowFallback(false); + check(options.getAccelerator() + == InferenceOptions.Accelerator.CORE_ML, + "inference accelerator"); + checkEqual(0, options.getThreads(), "thread lower clamp"); + check(!options.isFallbackAllowed(), "inference fallback option"); + } + + private void checkCapability() { + boolean supported = InferenceSession.isSupported(); + Log.p("InferenceOnDeviceApiTest: supported=" + supported); + if (!supported) { + AsyncResource result = InferenceSession.open( + ModelSource.bytes(new byte[] {1}), null); + check(result.isDone(), + "unsupported inference must complete immediately"); + try { + result.get(); + throw new IllegalStateException( + "unsupported inference unexpectedly opened a session"); + } catch (AsyncResource.AsyncExecutionException expected) { + // The documented unsupported-resource contract. + } + } + } + + private void check(boolean value, String label) { + if (!value) { + throw new IllegalStateException(label); + } + } + + private void checkEqual(int expected, int actual, String label) { + if (expected != actual) { + throw new IllegalStateException(label + ": expected " + + expected + " got " + actual); + } + } + + private void checkEqual(float expected, float actual, String label) { + if (Math.abs(expected - actual) > 0.0001f) { + throw new IllegalStateException(label + ": expected " + + expected + " got " + actual); + } + } +} diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java new file mode 100644 index 00000000000..71512751c02 --- /dev/null +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/LanguageOnDeviceApiTest.java @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + */ +package com.codenameone.examples.hellocodenameone.tests; + +import com.codename1.ai.language.LanguageBackends; +import com.codename1.ai.language.LanguageCandidate; +import com.codename1.ai.language.LanguageIdentifier; +import com.codename1.ai.language.LanguageOptions; +import com.codename1.ai.language.SmartReply; +import com.codename1.ai.language.SmartReplyMessage; +import com.codename1.ai.language.Translator; +import com.codename1.io.Log; +import com.codename1.util.AsyncResource; + +/** + * Cross-port, non-visual contract coverage for on-device language services. + * + *

Capability checks are safe on every port and make the three independent + * services visible to the dependency scanner. Unsupported ports additionally + * exercise their immediate failed-resource fallback. Supported ports do not + * download mutable language models during the unattended suite.

+ */ +public class LanguageOnDeviceApiTest extends BaseTest { + @Override + public boolean shouldTakeScreenshot() { + return false; + } + + @Override + public boolean runTest() { + try { + checkValuesAndOptions(); + checkCapabilities(); + checkReusableSessionLifecycle(); + done(); + return true; + } catch (Throwable t) { + fail("On-device language API test failed: " + t); + return false; + } + } + + private void checkValuesAndOptions() { + LanguageOptions options = new LanguageOptions() + .minimumConfidence(2f); + checkEqual(1f, options.getMinimumConfidence(), + "confidence upper clamp"); + check("auto".equals(options.getBackend().getId()), + "default language backend"); + options.backend(null).minimumConfidence(-1f); + check(options.getBackend() == LanguageBackends.auto(), + "null language backend must restore auto"); + checkEqual(0f, options.getMinimumConfidence(), + "confidence lower clamp"); + + LanguageCandidate candidate = new LanguageCandidate("fr", .75f); + check("fr".equals(candidate.getLanguageTag()), + "language candidate tag"); + checkEqual(.75f, candidate.getConfidence(), + "language candidate confidence"); + + SmartReplyMessage message = + new SmartReplyMessage(null, "remote", false, 42L); + check("".equals(message.getText()), + "null smart-reply text must normalize to empty"); + check("remote".equals(message.getParticipantId()), + "smart-reply participant"); + check(!message.isLocalUser(), "smart-reply remote participant"); + checkEqual(42L, message.getTimestampMillis(), + "smart-reply timestamp"); + } + + private void checkCapabilities() { + boolean languageId = LanguageIdentifier.isSupported(); + boolean translation = Translator.isSupported(); + boolean smartReply = SmartReply.isSupported(); + Log.p("LanguageOnDeviceApiTest: languageId=" + languageId + + " translation=" + translation + + " smartReply=" + smartReply); + + if (!languageId) { + assertImmediateFailure(LanguageIdentifier.identify("hello", null), + "language identification"); + } + if (!translation) { + assertImmediateFailure(Translator.translate( + "bonjour", "fr", "en", null), "translation"); + } + if (!smartReply) { + assertImmediateFailure(SmartReply.suggest( + new SmartReplyMessage[] { + new SmartReplyMessage("Hello", "remote", false, 1) + }, null), "smart reply"); + } + } + + private void checkReusableSessionLifecycle() { + LanguageIdentifier.Session identifier = + LanguageIdentifier.open(new LanguageOptions()); + Translator.Session translator = + Translator.open(new LanguageOptions()); + SmartReply.Session smartReply = + SmartReply.open(new LanguageOptions()); + identifier.close(); + translator.close(); + smartReply.close(); + // Closing is intentionally idempotent so owners can use one cleanup path. + identifier.close(); + translator.close(); + smartReply.close(); + + assertClosedSessionThrows(new ClosedSessionOperation() { + public void run() { + identifier.identify("hello"); + } + }, "closed language-identification session"); + assertClosedSessionThrows(new ClosedSessionOperation() { + public void run() { + translator.translate("bonjour", "fr", "en"); + } + }, "closed translation session"); + assertClosedSessionThrows(new ClosedSessionOperation() { + public void run() { + smartReply.suggest(new SmartReplyMessage[] { + new SmartReplyMessage("Hello", "remote", false, 1) + }); + } + }, "closed smart-reply session"); + } + + private void assertClosedSessionThrows(ClosedSessionOperation operation, + String label) { + try { + operation.run(); + } catch (IllegalStateException expected) { + // The documented closed-session contract. + return; + } + throw new IllegalStateException(label + + " unexpectedly accepted a new request"); + } + + private void assertImmediateFailure(AsyncResource resource, + String label) { + check(resource.isDone(), label + + " unsupported fallback must complete immediately"); + try { + resource.get(); + throw new IllegalStateException(label + + " unsupported fallback unexpectedly succeeded"); + } catch (AsyncResource.AsyncExecutionException expected) { + // The documented unsupported-resource contract. + } + } + + private void check(boolean value, String label) { + if (!value) { + throw new IllegalStateException(label); + } + } + + private void checkEqual(long expected, long actual, String label) { + if (expected != actual) { + throw new IllegalStateException(label + ": expected " + + expected + " got " + actual); + } + } + + private void checkEqual(float expected, float actual, String label) { + if (Math.abs(expected - actual) > 0.0001f) { + throw new IllegalStateException(label + ": expected " + + expected + " got " + actual); + } + } + + private interface ClosedSessionOperation { + void run(); + } +} diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/SimdApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/SimdApiTest.java index 36c07f55a6e..db06c7d5b21 100644 --- a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/SimdApiTest.java +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/SimdApiTest.java @@ -1,3 +1,26 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + package com.codenameone.examples.hellocodenameone.tests; import com.codename1.ui.CN; @@ -69,7 +92,7 @@ public boolean runTest() { return false; } - if (CN.isSimulator()) { + if (CN.isSimulator() && "javase".equals(CN.getPlatformName())) { try { simd.add(new int[4], new int[4], new int[4], 0, 4); fail("Expected simulator registry guard to reject non-alloc arrays"); diff --git a/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java new file mode 100644 index 00000000000..c0936cf509a --- /dev/null +++ b/scripts/hellocodenameone/common/src/main/java/com/codenameone/examples/hellocodenameone/tests/VisionOnDeviceApiTest.java @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.examples.hellocodenameone.tests; + +import com.codename1.ai.vision.BarcodeScanner; +import com.codename1.ai.vision.Barcode; +import com.codename1.ai.vision.DocumentScanner; +import com.codename1.ai.vision.FaceDetector; +import com.codename1.ai.vision.ImageLabeler; +import com.codename1.ai.vision.PoseDetector; +import com.codename1.ai.vision.SelfieSegmenter; +import com.codename1.ai.vision.TextRecognizer; +import com.codename1.ai.vision.VisionAnalyzer; +import com.codename1.ai.vision.VisionBackends; +import com.codename1.ai.vision.VisionImage; +import com.codename1.ai.vision.VisionOptions; +import com.codename1.camera.CameraFrame; +import com.codename1.camera.FrameFormat; +import com.codename1.io.Log; +import com.codename1.util.AsyncResource; +import com.codename1.util.Base64; +import com.codename1.util.SuccessCallback; + +/** + * Cross-port, non-visual contract coverage for the built-in vision API. + * + *

In addition to portable ownership/lifecycle checks, supported mobile + * ports decode a bundled deterministic QR image. That assertion crosses the + * Java/native image boundary, native detector, JSON result mapping, format + * normalization, and geometry mapping without camera hardware.

+ */ +public class VisionOnDeviceApiTest extends BaseTest { + @Override + public boolean shouldTakeScreenshot() { + return false; + } + + @Override + public boolean runTest() { + try { + checkImageOwnership(); + checkOptions(); + checkAnalyzerCapabilitiesAndLifecycle(); + return checkNativeQrDecode(); + } catch (Throwable t) { + fail("On-device vision API test failed: " + t); + return false; + } + } + + private boolean checkNativeQrDecode() throws Exception { + final BarcodeScanner scanner = new BarcodeScanner(); + if (!scanner.isSupported()) { + Log.p("VisionOnDeviceApiTest: native QR assertion skipped; " + + "barcode backend unsupported"); + scanner.close(); + done(); + return true; + } + String png = "iVBORw0KGgoAAAANSUhEUgAAAMAAAADAAQAAAAB6p1GqAAAA5UlEQVR4Xu2UQQ7DIAwEnVOewU8L/JRncIJ6TaqkJT17pbBSImBy2HhtpP+R/B58tMCkBSY9AyRRBX1vFauNA2R9akh72Wo4tgygqtHQRKJiCVzA7LKBjAUT6IhWT/V9bAnAGIMM9j0GrmBIcTx3/kDjbHZlmNdy/Q9H0NBkJVZEm3kAxlJkR9u1a7SuAEUEQx31CxIA4VSxWecBuCnKS9B5mQPYAGi3aQVxa0QOAH82ALGijqddX2BGm5lOdCD0PrqNCCTrNgRMArpFW5Gu3GTuA8YYZKtjul4ZruBOC0xaYNJzwRtNmtsMgNUTWwAAAABJRU5ErkJggg=="; + AsyncResource operation = scanner.process( + VisionImage.encoded(Base64.decode(png.getBytes("UTF-8")))); + operation.ready(new SuccessCallback() { + public void onSucess(Barcode[] values) { + try { + check(values.length > 0, + "native QR detector returned no result"); + check("CN1-VISION-QR".equals(values[0].getValue()), + "native QR payload mismatch"); + check("QR_CODE".equals(values[0].getFormat()), + "native QR format was not normalized"); + check(values[0].getCorners().length == 4, + "native QR detector did not return four corners"); + done(); + } catch (Throwable t) { + fail("On-device vision API test failed: " + t); + } finally { + scanner.close(); + } + } + }); + operation.except(new SuccessCallback() { + public void onSucess(Throwable error) { + try { + fail("On-device vision API test failed: " + error); + } finally { + scanner.close(); + } + } + }); + // Native completion is asynchronous. Returning lets the EDT service + // Apple Vision/ML Kit callbacks while the suite runner polls isDone(). + return true; + } + + private void checkImageOwnership() { + byte[] encodedSource = new byte[] {1, 2, 3}; + VisionImage encoded = VisionImage.encoded(encodedSource); + encodedSource[0] = 9; + checkEqual(1, encoded.getEncodedBytes()[0], + "VisionImage must copy encoded input"); + byte[] encodedOutput = encoded.getEncodedBytes(); + encodedOutput[1] = 9; + checkEqual(2, encoded.getEncodedBytes()[1], + "VisionImage must copy encoded output"); + VisionImage rotatedEncoded = + VisionImage.encoded(new byte[] {7}, -90); + checkEqual(270, rotatedEncoded.getRotationDegrees(), + "encoded image rotation normalization"); + + byte[] jpeg = new byte[] {4, 5}; + byte[] pixels = new byte[] {10, 20, 30, 40}; + CameraFrame frame = new CameraFrame(jpeg, pixels, 1, 1, -90, + 123456789L, FrameFormat.RGBA8888); + VisionImage cameraImage = VisionImage.fromCameraFrame(frame); + jpeg[0] = 99; + pixels[0] = 99; + check(cameraImage.getEncodedBytes() == null, + "raw VisionImage must not retain camera JPEG bytes"); + checkEqual(10, cameraImage.getPixels()[0], + "VisionImage must own camera pixel bytes"); + checkEqual(1, cameraImage.getWidth(), "camera image width"); + checkEqual(1, cameraImage.getHeight(), "camera image height"); + checkEqual(270, cameraImage.getRotationDegrees(), + "camera image rotation normalization"); + checkEqual(123456789L, cameraImage.getTimestampNanos(), + "camera image timestamp"); + check(cameraImage.getFormat() == FrameFormat.RGBA8888, + "camera image format"); + + byte[] fallbackJpeg = new byte[] {50, 60}; + VisionImage fallbackImage = VisionImage.fromCameraFrame( + new CameraFrame(fallbackJpeg, null, 1, 1, 0, 1L, + FrameFormat.NV21)); + fallbackJpeg[0] = 99; + checkEqual(50, fallbackImage.getEncodedBytes()[0], + "JPEG-only port fallback must be copied"); + check(fallbackImage.getPixels() == null, + "JPEG-only port fallback must not expose raw pixels"); + check(fallbackImage.getFormat() == FrameFormat.JPEG, + "JPEG-only port fallback must report JPEG"); + } + + private void checkOptions() { + VisionOptions options = new VisionOptions() + .backend(VisionBackends.appleVision()) + .minimumConfidence(2f) + .maximumResults(-1); + check("apple-vision".equals(options.getBackend().getId()), + "explicit Apple Vision backend id"); + checkEqual(1f, options.getMinimumConfidence(), + "confidence upper clamp"); + checkEqual(0, options.getMaximumResults(), + "maximum result lower clamp"); + options.backend(null).minimumConfidence(-1f); + check("auto".equals(options.getBackend().getId()), + "null backend must restore auto"); + checkEqual(0f, options.getMinimumConfidence(), + "confidence lower clamp"); + } + + private void checkAnalyzerCapabilitiesAndLifecycle() { + VisionAnalyzer[] analyzers = new VisionAnalyzer[] { + new TextRecognizer(), + new BarcodeScanner(), + new FaceDetector(), + new ImageLabeler(), + new PoseDetector(), + new SelfieSegmenter(), + new DocumentScanner() + }; + String[] names = new String[] { + "text", "barcode", "face", "label", "pose", "segmentation", + "document" + }; + VisionImage input = VisionImage.encoded(new byte[] {1}); + for (int i = 0; i < analyzers.length; i++) { + VisionAnalyzer analyzer = analyzers[i]; + boolean supported = analyzer.isSupported(); + Log.p("VisionOnDeviceApiTest: " + names[i] + + " supported=" + supported); + analyzer.close(); + check(!analyzer.isSupported(), + names[i] + " analyzer must report unsupported after close"); + boolean rejected = false; + try { + analyzer.process(input); + } catch (IllegalStateException expected) { + rejected = true; + } + check(rejected, names[i] + " analyzer accepted input after close"); + } + } + + private void check(boolean value, String label) { + if (!value) { + throw new IllegalStateException(label); + } + } + + private void checkEqual(int expected, int actual, String label) { + if (expected != actual) { + throw new IllegalStateException(label + ": expected " + + expected + " got " + actual); + } + } + + private void checkEqual(long expected, long actual, String label) { + if (expected != actual) { + throw new IllegalStateException(label + ": expected " + + expected + " got " + actual); + } + } + + private void checkEqual(float expected, float actual, String label) { + if (Math.abs(expected - actual) > 0.0001f) { + throw new IllegalStateException(label + ": expected " + + expected + " got " + actual); + } + } +} diff --git a/scripts/hellocodenameone/conformance/port_status.py b/scripts/hellocodenameone/conformance/port_status.py index 20f34f9f748..e4e11980c26 100755 --- a/scripts/hellocodenameone/conformance/port_status.py +++ b/scripts/hellocodenameone/conformance/port_status.py @@ -33,6 +33,9 @@ PERF_BENCH_RE = re.compile( r"CN1SS:PERF:benchmark id=([A-Za-z0-9-]+) duration_ns=(\d+) checksum=(-?\d+)" ) +PERF_SKIP_RE = re.compile( + r"CN1SS:PERF:skipped id=([A-Za-z0-9-]+) reason=([A-Za-z0-9-]+)" +) PERF_COMPLETE_RE = re.compile( r"CN1SS:PERF:complete benchmark_version=(\d+) checksum=(-?\d+)" ) @@ -211,6 +214,19 @@ def validate(manifest: dict) -> dict: problems.append(f"Stored report {report_path} has an invalid result for {test}") elif result.get("status") == "skip": skipped_tests.add(test) + actual_summary = Counter( + result.get("status") + for result in report_tests.values() + if isinstance(result, dict) + ) + expected_summary = { + key: actual_summary.get(key, 0) + for key in ("pass", "fail", "skip", "not-run") + } + if report.get("summary") != expected_summary: + problems.append( + f"Stored report {report_path} summary does not match its test results" + ) manual_feature_count = 0 try: @@ -393,6 +409,7 @@ def parse_logs(paths: list[Path], states: dict[str, dict]) -> bool: def parse_performance(paths: list[Path], expected_benchmarks: list[str], binary_size: int | None) -> dict: benchmarks: dict[str, dict] = {} + skipped: dict[str, str] = {} benchmark_version: int | None = None suite_checksum: int | None = None for path in paths: @@ -408,11 +425,24 @@ def parse_performance(paths: list[Path], expected_benchmarks: list[str], binary_ "duration_ns": int(match.group(2)), "checksum": match.group(3), } + skipped.pop(benchmark_id, None) + match = PERF_SKIP_RE.search(line) + if match: + benchmark_id = match.group(1) + if benchmark_id not in expected_benchmarks: + raise ContractError( + f"Unknown skipped performance benchmark {benchmark_id} in {path}" + ) + if benchmark_id not in benchmarks: + skipped[benchmark_id] = match.group(2) match = PERF_COMPLETE_RE.search(line) if match: benchmark_version = int(match.group(1)) suite_checksum = int(match.group(2)) - missing = [item for item in expected_benchmarks if item not in benchmarks] + missing = [ + item for item in expected_benchmarks + if item not in benchmarks and item not in skipped + ] status = "complete" if ( not missing and benchmark_version is not None and suite_checksum is not None ) else "partial" @@ -421,6 +451,7 @@ def parse_performance(paths: list[Path], expected_benchmarks: list[str], binary_ "benchmark_version": benchmark_version, "method": "minimum of five measured runs after three in-process warm-ups", "benchmarks": {item: benchmarks[item] for item in expected_benchmarks if item in benchmarks}, + "skipped": {item: skipped[item] for item in expected_benchmarks if item in skipped}, "missing": missing, "suite_checksum": suite_checksum, } diff --git a/scripts/hellocodenameone/conformance/test_port_status.py b/scripts/hellocodenameone/conformance/test_port_status.py index 9b788e906c5..b5817f12521 100755 --- a/scripts/hellocodenameone/conformance/test_port_status.py +++ b/scripts/hellocodenameone/conformance/test_port_status.py @@ -16,9 +16,9 @@ def setUpClass(cls): def test_contract_covers_registered_tests_and_goldens(self): counts = port_status.validate(self.manifest) - self.assertEqual(167, counts["tests"]) + self.assertEqual(170, counts["tests"]) self.assertEqual(1, counts["performance_tests"]) - self.assertGreaterEqual(counts["features"], 51) + self.assertGreaterEqual(counts["features"], 54) self.assertEqual(11, counts["ports"]) self.assertEqual(20, counts["manual_features"]) self.assertEqual(8, counts["deployment_platforms"]) @@ -27,6 +27,9 @@ def test_contract_covers_registered_tests_and_goldens(self): features = {feature["id"]: feature["tests"] for feature in self.manifest["features"]} self.assertEqual(["ARApiTest", "MotionSensorDeviceTest"], features["ar-motion-sensors"]) self.assertEqual(["CameraApiTest"], features["camera-access"]) + self.assertEqual(["VisionOnDeviceApiTest"], features["on-device-vision"]) + self.assertEqual(["LanguageOnDeviceApiTest"], features["on-device-language"]) + self.assertEqual(["InferenceOnDeviceApiTest"], features["on-device-inference"]) self.assertEqual(["CalendarApiTest"], features["calendar-integration"]) self.assertEqual(["VideoIODecodedFramesScreenshotTest"], features["video-decoding"]) self.assertEqual(["VideoIORoundTripTest"], features["video-round-trip"]) @@ -111,6 +114,43 @@ def test_error_lines_allow_messages_or_no_message(self): self.assertEqual("fail", report["tests"]["StringApiTest"]["status"]) self.assertEqual(["suite-error"], report["tests"]["StringApiTest"]["reasons"]) + def test_performance_skips_are_complete_and_preserve_reasons(self): + expected = self.manifest["performance_benchmarks"] + skipped = {"objectAllocation", "hashMapChurn", "stringBuilding"} + log_text = "\n".join( + [ + *[ + ( + f"CN1SS:PERF:skipped id={benchmark} " + "reason=ios-simulator-gc-footprint" + if benchmark in skipped + else ( + f"CN1SS:PERF:benchmark id={benchmark} " + "duration_ns=12000000 checksum=42" + ) + ) + for benchmark in expected + ], + "CN1SS:PERF:complete benchmark_version=1 checksum=42", + ] + ) + with tempfile.TemporaryDirectory() as tmp: + log_path = Path(tmp) / "suite.log" + log_path.write_text(log_text, encoding="utf-8") + performance = port_status.parse_performance([log_path], expected, None) + + self.assertEqual("complete", performance["status"]) + self.assertEqual([], performance["missing"]) + self.assertEqual( + { + benchmark: "ios-simulator-gc-footprint" + for benchmark in expected + if benchmark in skipped + }, + performance["skipped"], + ) + self.assertNotIn("objectAllocation", performance["benchmarks"]) + def test_strict_report_errors_reject_failures_missing_tests_and_incomplete_suite(self): report = { "suite_finished": False, @@ -183,6 +223,34 @@ def test_cli_strict_gate_writes_report_before_returning_failure(self): self.assertFalse(report["suite_finished"]) self.assertGreater(report["summary"]["not-run"], 0) + def test_validate_rejects_inconsistent_stored_report_summary(self): + original_directory = self.manifest["report_directory"] + with tempfile.TemporaryDirectory(dir=port_status.REPO_ROOT) as tmp: + report_root = Path(tmp) + for port in self.manifest["ports"]: + source = port_status.REPO_ROOT / original_directory / ( + port["id"] + ".json" + ) + (report_root / source.name).write_text( + source.read_text(encoding="utf-8"), encoding="utf-8" + ) + android_path = report_root / "android.json" + android = port_status.read_json(android_path) + android["summary"]["not-run"] += 1 + android_path.write_text( + json.dumps(android, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + manifest = dict(self.manifest) + manifest["report_directory"] = str( + report_root.relative_to(port_status.REPO_ROOT) + ) + with self.assertRaisesRegex( + port_status.ContractError, + "summary does not match its test results", + ): + port_status.validate(manifest) + if __name__ == "__main__": unittest.main() diff --git a/scripts/initializr/common/src/main/resources/skill/SKILL.md b/scripts/initializr/common/src/main/resources/skill/SKILL.md index 1beefb158d7..e3d88dbd20b 100644 --- a/scripts/initializr/common/src/main/resources/skill/SKILL.md +++ b/scripts/initializr/common/src/main/resources/skill/SKILL.md @@ -37,7 +37,7 @@ This skill teaches you how to write code for a Codename One (CN1) cross-platform - `references/mobile-adaptability.md` — Density-independent units (mm), `convertToPixels`, `LayeredLayout` for responsive design, `Display.isTablet()`, font scaling. - `references/native-interfaces.md` — Authoring native interfaces for iOS/Android/JavaScript/Desktop with `cn1:generate-native-interfaces` and platform callbacks. - `references/cn1libs.md` — Creating, packaging, and consuming Codename One libraries (Maven and legacy `.cn1lib`). -- `references/ai-and-speech.md` — LLM client (`com.codename1.ai`), `ChatView`, `SpeechRecognizer`, `TextToSpeech`, non-prompting `SecureStorage` overloads, the ML Kit cn1libs, and the simulator's offline Ollama redirect. Read this when the user asks for chat, voice, embeddings, image generation, barcode/document/face detection, or wants to store an LLM API key. +- `references/ai-and-speech.md` — LLM client (`com.codename1.ai`), built-in vision/LiteRT/language APIs, `ChatView`, `SpeechRecognizer`, `TextToSpeech`, non-prompting `SecureStorage` overloads, and the simulator's offline Ollama redirect. Read this when the user asks for chat, voice, embeddings, image generation, barcode/document/face detection, or wants to store an LLM API key. - `references/printing.md` — Cross-platform printing (`com.codename1.printing`): `Printer.printPDF` / `printImage` / `print`, the `PrintResult` outcome, and per-platform caveats (iOS AirPrint, Android, desktop, native Windows, web). Read this when the user wants to print a document, report, image, or the current screen. - `references/games.md` — Game development (`com.codename1.gaming`): the `GameView` update loop, `Sprite` / `AnimatedSprite` / `SpriteSheet`, pollable `GameInput`, `TouchControls`, `SoundPool`, and 2D `com.codename1.gaming.physics` (Box2D). Read this for arcade/casual/scroller/board games or any real-time animated canvas. - `references/3d-graphics.md` — Portable GPU 3D (`com.codename1.gpu`): the `RenderView` + `Renderer` loop, declarative `Material` / `VertexFormat` (engine-generated shaders — no GLSL), `Primitives`, `GltfLoader` for glTF models, `Camera` / `Light` / `Matrix4`, and platform backends. Read this for product viewers, 3D scenes, or custom GPU rendering. diff --git a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md index 330d62b3b55..66c58eab62b 100644 --- a/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md +++ b/scripts/initializr/common/src/main/resources/skill/references/ai-and-speech.md @@ -1,6 +1,6 @@ # AI, Chat UI, and Speech Reference -Codename One ships a portable LLM client, a streaming chat component, speech recognition, text-to-speech, and ML Kit cn1lib bridges. All of it sits in the cross-platform `common/` module — the same call site runs on iOS, Android, JavaSE, and (where the backend supports it) JavaScript. +Codename One ships a portable LLM client, a streaming chat component, speech recognition, text-to-speech, and built-in on-device vision, language, and LiteRT inference APIs. **Read this reference when** the user asks to integrate an LLM, build a chat UI, voice input, voice output, image generation, embeddings, on-device barcode/face/document scanning, or wants to store an API key. @@ -13,11 +13,11 @@ Codename One ships a portable LLM client, a streaming chat component, speech rec | Speech-to-text | `com.codename1.media.SpeechRecognizer` | core (built-in) | | Text-to-speech | `com.codename1.media.TextToSpeech` | core (built-in) | | Silent secret storage (LLM API keys, etc.) | Single-arg overloads on `com.codename1.security.SecureStorage` | core (built-in) | -| Barcode scanning | `com.codename1.ai.mlkit.barcode.BarcodeScanner` | cn1lib `cn1-ai-mlkit-barcode` | -| Document scanning | `com.codename1.ai.mlkit.docscan.DocumentScanner` | cn1lib `cn1-ai-mlkit-docscan` | -| Face detection | `com.codename1.ai.mlkit.face.FaceDetector` | cn1lib `cn1-ai-mlkit-face` | +| Vision analysis | `com.codename1.ai.vision.*` | core (built-in) | +| LiteRT inference | `com.codename1.ai.inference.*` | core (built-in) | +| Language services | `com.codename1.ai.language.*` | core (built-in) | -The build-time scanner in the Codename One Maven plugin (`AiDependencyTable`) picks up references to any `com.codename1.ai.*` or `com.codename1.media.{Speech,Tts}*` class and automatically wires Pods (iOS), Swift Packages (iOS SPM), Gradle dependencies (Android), `Info.plist` usage strings, and Android permissions. You don't edit `codenameone_settings.properties` build hints for these classes. +The build-time scanner uses `PlatformFeatureCatalog` to add the required Apple frameworks, CocoaPods, Android dependencies, and permissions. Applications don't add AI cn1libs or build hints. ## LlmClient — chat, embeddings, image generation @@ -307,61 +307,68 @@ LlmClient client = LlmClient.openai(key); Base class returns `null` / `false` on platforms without an implementation, so you can wire this in without a platform check. -## ML Kit cn1libs - -Three cn1libs, each backed by ML Kit on iOS and Android. Drop the `pom` dependency in `common/pom.xml` and the build-time scanner takes care of Pods, Gradle deps, permissions, and usage strings. - -```xml - - com.codenameone - cn1-ai-mlkit-barcode-lib - ${cn1.version} - pom - -``` - -### Barcode scanner - -```java -import com.codename1.ai.mlkit.barcode.BarcodeScanner; - -Capture.capturePhoto(evt -> { - String path = (String) evt.getSource(); - byte[] bytes = readAllBytes(path); - BarcodeScanner.scan(bytes).ready(values -> { - for (String v : values) Log.p("Detected: " + v); - }); -}); -``` - -Decodes QR, EAN, Code 128, and the other ML Kit-supported formats. - -### Document scanner - -```java -import com.codename1.ai.mlkit.docscan.DocumentScanner; - -DocumentScanner.scanToFile(jpegBytes).ready(filePath -> { - // filePath points to the cropped, corrected document image -}); -``` - -iOS uses `VisionKit`. Android uses the Google Play Services document scanner — on devices without Play Services, the call resolves through `AsyncResource.except(...)`. - -### Face detector +## Built-in on-device ML ```java -import com.codename1.ai.mlkit.face.FaceDetector; - -FaceDetector.detect(jpegBytes).ready(rects -> { - for (int i = 0; i < rects.length; i += 4) { - int x = rects[i], y = rects[i + 1], w = rects[i + 2], h = rects[i + 3]; - // draw rectangle, crop, etc. - } -}); +new TextRecognizer() + .process(VisionImage.encoded(jpegBytes)) + .ready(result -> Log.p(result.getText())); + +InferenceSession.open( + ModelSource.resource("/model.tflite"), + new InferenceOptions()) + .ready(session -> session.run(new Tensor[] {input})); + +LanguageOptions translationOptions = new LanguageOptions() + .backend(LanguageBackends.mlKitTranslation()); +Translator.translate("Bonjour", "fr", "en", translationOptions) + .ready(result -> Log.p(result)); ``` -Returns a packed `int[]` — four ints per face. +`VisionImage` accepts JPEG, PNG, NV21, and RGBA8888. Use +`VisionPipeline` to connect a reusable analyzer to camera frames; it +copies callback-owned data and drops stale pending frames under load. +Use `ModelCache.fetch(httpsUrl, cacheKey, sha256)` for models too large +to bundle. The initial model URL must remain HTTPS. Observable redirects +are rejected if they downgrade to HTTP; iOS requires the digest because +its network stack follows redirects below the portable callback. +Identical concurrent fetches coalesce instead of sharing a temporary +file unsafely. iOS and Mac native default to Apple Vision; iOS also +supports explicit ML Kit selection. Android uses ML Kit. + +The Apple barcode backend uses Vision request revision 1 in the iOS +simulator because newer simulator runtimes can return no observations +for valid QR images. Devices use the latest OS revision. Validate newer +Apple symbologies on a device, or select the ML Kit barcode backend. + +Dependency selection is per entry point. Referencing `TextRecognizer` +retains only the Android text adapter/artifact; it does not pull +barcode, face, labeling, pose, or segmentation. `LanguageIdentifier`, +`Translator`, and `SmartReply` likewise select independent artifacts. +On iOS, an ML Kit vision pod is selected only when both its analyzer +and its matching feature-specific selector are referenced. For example, +`VisionBackends.mlKitBarcodeScanning()` selects only the barcode pod. +LiteRT is selected only by +`InferenceSession`. NPU acceleration is best effort when fallback is +enabled. Android and iOS reject `Accelerator.NPU` together with +`allowFallback(false)`: NNAPI cannot prove full-graph delegation, while +the Core ML delegate may also schedule work on CPU or GPU. + +Language identification defaults to Apple Natural Language on iOS and ML +Kit on Android. iOS translation and Smart Reply require their +`mlKitTranslation()` and `mlKitSmartReply()` selectors; referencing the API +class alone does not disable the arm64 simulator. The static methods own a +one-shot backend. For repeated operations, reuse and close the feature session +returned by `LanguageIdentifier.open()`, `Translator.open()`, or +`SmartReply.open()`. Call `isSupported()` before exposing a feature. Vision has native backends +on Android, iOS, and Mac native. Language services and LiteRT inference +have native backends on Android and iOS. JavaSE, JavaScript, native +Windows/Linux, watchOS, and tvOS return unsupported; Mac native returns +unsupported for language and LiteRT. An opt-in runtime can add another +backend, and the built-in fallback never uploads input to a cloud service. + +Do not add the retired `cn1-ai-mlkit-*` or `cn1-ai-tflite` dependencies. +There are no compatibility aliases for `com.codename1.ai.mlkit.*`. ## Common patterns @@ -419,5 +426,5 @@ view.setOnVoice(e -> SpeechRecognizer.recognize( - **Don't store an API key in source.** Use `SecureStorage.get("openai.key")` (single-arg overload) or pull it from a server-side proxy. Hard-coded keys leak through reverse-engineered binaries. - **Don't call `chatStream` from a tight UI loop.** A streaming call holds an HTTP connection until the response completes; one per user turn is correct, one per keystroke is a bug. - **Don't mutate `ChatView` on a non-EDT thread without going through the documented mutators.** `addMessage`, `appendToLastMessage`, and `setTypingIndicatorVisible` are thread-safe; arbitrary `view.add(...)` calls are not. -- **Don't assume the document scanner works on every Android device.** It requires Google Play Services. Wrap the call and fall back to `Capture.capturePhoto(...)` if the scanner returns an error. +- **Don't assume `DocumentScanner` is available on Android.** The built-in API corrects an existing image, while Google's Android scanner is an interactive camera flow. Check `isSupported()` and use your capture flow when it is false. - **Don't ship a project that bundles `cn1-ai-stablediffusion` to the cloud build server without checking.** The cn1lib carries multi-GB native blobs; the build will reject the upload with a `cn1.ai.requiresBigUpload` hint. Build locally for those projects. diff --git a/scripts/ios/screenshots/FloatingActionButtonTheme_light.tolerance b/scripts/ios/screenshots/FloatingActionButtonTheme_light.tolerance new file mode 100644 index 00000000000..ea902126126 --- /dev/null +++ b/scripts/ios/screenshots/FloatingActionButtonTheme_light.tolerance @@ -0,0 +1,7 @@ +# The floating action button shadow has run-to-run anti-aliasing jitter on the +# iOS GL simulator. Three independent runs differed in at most 4 channel values +# across 0.279% of the frame while all geometry and the other 167 screenshots +# matched. Keep the pre-tightening gate for this screen only; a real layout, +# color, or missing-button regression still exceeds these narrow bounds. +maxChannelDelta=4 +maxMismatchPercent=0.30 diff --git a/scripts/run-ios-native-tests.sh b/scripts/run-ios-native-tests.sh index a194c170dd8..f1238f94fe7 100755 --- a/scripts/run-ios-native-tests.sh +++ b/scripts/run-ios-native-tests.sh @@ -209,6 +209,18 @@ fi if [ -n "${CN1_TEST_OPT_LEVEL:-}" ]; then DERIVED_ARGS+=("GCC_OPTIMIZATION_LEVEL=$CN1_TEST_OPT_LEVEL") fi +SIMULATOR_ARCH_ARGS=() +PODFILE_LOCK="$PROJECT_DIR/Podfile.lock" +if [ -f "$PODFILE_LOCK" ] && grep -q "GoogleMLKit/" "$PODFILE_LOCK"; then + # Match the UI build and the generated project: Google ML Kit currently + # supplies an x86_64 simulator slice, not an arm64 simulator slice. + SIMULATOR_ARCH_ARGS=( + "ARCHS=x86_64" + "ONLY_ACTIVE_ARCH=YES" + "EXCLUDED_ARCHS=arm64 armv7 armv7s" + ) + ri_log "Google ML Kit detected; selecting its supported x86_64 simulator slice" +fi ri_log "Running xcodebuild test (scheme=$TEST_SCHEME, destination=$DESTINATION)" set +e xcodebuild \ @@ -216,6 +228,7 @@ xcodebuild \ -scheme "$TEST_SCHEME" \ -destination "$DESTINATION" \ ${DERIVED_ARGS[@]+"${DERIVED_ARGS[@]}"} \ + ${SIMULATOR_ARCH_ARGS[@]+"${SIMULATOR_ARCH_ARGS[@]}"} \ test | tee "$TEST_LOG" RC=${PIPESTATUS[0]} set -e diff --git a/scripts/run-ios-ui-tests.sh b/scripts/run-ios-ui-tests.sh index 4dec3acb7f2..350224c7890 100755 --- a/scripts/run-ios-ui-tests.sh +++ b/scripts/run-ios-ui-tests.sh @@ -625,6 +625,18 @@ case "$HOST_ARCH" in arm64|x86_64) BUILD_ARCH="$HOST_ARCH" ;; *) BUILD_ARCH="arm64" ;; esac +SIMULATOR_EXCLUDED_ARCHS="armv7 armv7s" +FORCE_SIMULATOR_ARCH="false" +PODFILE_LOCK="$(dirname "$WORKSPACE_PATH")/Podfile.lock" +if [ -f "$PODFILE_LOCK" ] && grep -q "GoogleMLKit/" "$PODFILE_LOCK"; then + # Google ML Kit's iOS binaries contain a device arm64 slice and an + # x86_64 simulator slice, but no arm64 simulator slice. Xcode otherwise + # selects arm64 on Apple Silicon and the linker rejects the device object. + BUILD_ARCH="x86_64" + SIMULATOR_EXCLUDED_ARCHS="arm64 armv7 armv7s" + FORCE_SIMULATOR_ARCH="true" + ri_log "Google ML Kit detected; selecting its supported x86_64 simulator slice" +fi # A shared/stable derived-data dir can be supplied via CN1_IOS_DERIVED_DATA so the compiled # app + core are reused by a later step in the same job (e.g. the native test build) and across @@ -652,12 +664,12 @@ XCODE_BUILD_CMD=( -destination-timeout 120 -derivedDataPath "$DERIVED_DATA_DIR" ) -if [ "$USE_GENERIC_BUILD_DESTINATION" = "true" ]; then - ri_log "Forcing simulator ARCHS=$BUILD_ARCH for generic build destination" +if [ "$USE_GENERIC_BUILD_DESTINATION" = "true" ] || [ "$FORCE_SIMULATOR_ARCH" = "true" ]; then + ri_log "Forcing simulator ARCHS=$BUILD_ARCH" XCODE_BUILD_CMD+=( "ARCHS=$BUILD_ARCH" "ONLY_ACTIVE_ARCH=YES" - "EXCLUDED_ARCHS=armv7 armv7s" + "EXCLUDED_ARCHS=$SIMULATOR_EXCLUDED_ARCHS" ) fi # Optimize the translated C (Xcode's Debug config defaults to -O0). With -O0 the diff --git a/scripts/settings/common/src/main/java/com/codename1/settings/extensions/MavenCentralSearch.java b/scripts/settings/common/src/main/java/com/codename1/settings/extensions/MavenCentralSearch.java index 03b9ce67866..c7b8663b85d 100644 --- a/scripts/settings/common/src/main/java/com/codename1/settings/extensions/MavenCentralSearch.java +++ b/scripts/settings/common/src/main/java/com/codename1/settings/extensions/MavenCentralSearch.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ package com.codename1.settings.extensions; import com.codename1.io.ConnectionRequest; @@ -20,14 +42,6 @@ public static List curated() { ArrayList out = new ArrayList(); out.add(maven("Google Maps", "Native Google Maps integration for Codename One apps.", "com.codenameone", "googlemaps-lib", "1.0.1", "pom")); - out.add(maven("ML Kit Barcode", "Decode QR, EAN, Code 128, and other ML Kit barcode formats.", - "com.codenameone", "cn1-ai-mlkit-barcode-lib", "LATEST", "pom")); - out.add(maven("ML Kit Document Scanner", "Capture and crop document photos with native VisionKit and Google Play Services support.", - "com.codenameone", "cn1-ai-mlkit-docscan-lib", "LATEST", "pom")); - out.add(maven("ML Kit Face Detection", "Detect faces and bounding rectangles using ML Kit-backed native providers.", - "com.codenameone", "cn1-ai-mlkit-face-lib", "LATEST", "pom")); - out.add(maven("TensorFlow Lite", "TensorFlow Lite inference bridge packaged as a Codename One cn1lib.", - "com.codenameone", "cn1-ai-tflite-lib", "LATEST", "pom")); out.add(maven("Whisper", "Speech-to-text support through the Codename One AI Whisper cn1lib.", "com.codenameone", "cn1-ai-whisper-lib", "LATEST", "pom")); out.add(legacy("Bouncy Castle SDK", "Legacy cn1lib catalog entry for Bouncy Castle cryptography support."));