Skip to content

feat(typing): adaptive key geometry - learns where you tap (opt-in) #107

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
SHAWNERZZ wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

feat(typing): adaptive key geometry - learns where you tap (opt-in) #107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 46 additions & 7 deletions app/src/main/java/com/android/inputmethod/keyboard/ProximityInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

package com.android.inputmethod.keyboard;

import android.content.Context;
import android.graphics.Rect;
import helium314.keyboard.latin.utils.Log;

Expand All @@ -14,6 +15,10 @@
import helium314.keyboard.keyboard.Key;
import helium314.keyboard.keyboard.internal.TouchPositionCorrection;
import helium314.keyboard.latin.common.Constants;
import helium314.keyboard.latin.database.TouchModelDao;
import helium314.keyboard.latin.database.TouchModelManager;
import helium314.keyboard.latin.settings.Settings;
import helium314.keyboard.latin.settings.SettingsValues;
import helium314.keyboard.latin.utils.JniUtils;

import java.util.ArrayList;
Expand Down Expand Up @@ -48,12 +53,17 @@ public class ProximityInfo {
private final List<Key> mSortedKeys;
@NonNull
private final List<Key>[] mGridNeighbors;
// Adaptive typing: the keyboard element this proximity info is for (e.g. alphabet vs symbols).
// Keys the learned touch model by (key, this element, orientation) so layouts stay separate.
private final int mLayoutElementId;

@SuppressWarnings("unchecked")
public ProximityInfo(final int gridWidth, final int gridHeight, final int minWidth, final int height,
final int mostCommonKeyWidth, final int mostCommonKeyHeight,
@NonNull final List<Key> sortedKeys,
@NonNull final TouchPositionCorrection touchPositionCorrection) {
@NonNull final TouchPositionCorrection touchPositionCorrection,
final int layoutElementId) {
mLayoutElementId = layoutElementId;
mGridWidth = gridWidth;
mGridHeight = gridHeight;
mGridSize = mGridWidth * mGridHeight;
Expand Down Expand Up @@ -165,14 +175,34 @@ private long createNativeProximityInfo(@NonNull final TouchPositionCorrection to
infoIndex++;
}

if (touchPositionCorrection.isValid()) {
// Adaptive typing (opt-in): when enabled, shift each key's sweet-spot center by the
// learned per-user landing offset (capped) so the native recognizer matches swipes — and
// tap-correction candidates — against keys where the user's hand actually goes. This
// path runs even when the layout ships no static touch-position-correction data, so we
// still generate sweet spots in that case. The shift is computed in Java and crosses the
// existing JNI; no native change. See docs/ADAPTIVE_TYPING.md.
final boolean tpcValid = touchPositionCorrection.isValid();
final SettingsValues sv = Settings.getValues();
final boolean adaptiveOn = sv != null && sv.mAdaptiveKeyGeometry;
final int adaptiveStrength = sv != null ? sv.mAdaptiveKeyGeometryStrength : 0;
TouchModelDao adaptiveDao = null;
int adaptiveOrientation = 0;
if (adaptiveOn && adaptiveStrength > 0) {
final Context ctx = Settings.getCurrentContext();
if (ctx != null) {
adaptiveDao = TouchModelDao.getInstance(ctx);
adaptiveOrientation = ctx.getResources().getConfiguration().orientation;
}
}
final boolean applyAdaptive = adaptiveDao != null;
if (tpcValid || applyAdaptive) {
if (DEBUG) {
Log.d(TAG, "touchPositionCorrection: ON");
Log.d(TAG, "sweet spots: ON (tpc=" + tpcValid + " adaptive=" + applyAdaptive + ")");
}
sweetSpotCenterXs = new float[keyCount];
sweetSpotCenterYs = new float[keyCount];
sweetSpotRadii = new float[keyCount];
final int rows = touchPositionCorrection.getRows();
final int rows = tpcValid ? touchPositionCorrection.getRows() : 0;
final float defaultRadius = DEFAULT_TOUCH_POSITION_CORRECTION_RADIUS
* (float)Math.hypot(mMostCommonKeyWidth, mMostCommonKeyHeight);
for (int infoIndex = 0, keyIndex = 0; keyIndex < sortedKeys.size(); keyIndex++) {
Expand All @@ -186,7 +216,7 @@ private long createNativeProximityInfo(@NonNull final TouchPositionCorrection to
sweetSpotCenterYs[infoIndex] = hitBox.exactCenterY();
sweetSpotRadii[infoIndex] = defaultRadius;
final int row = hitBox.top / mMostCommonKeyHeight;
if (row < rows) {
if (tpcValid && row < rows) {
final int hitBoxWidth = hitBox.width();
final int hitBoxHeight = hitBox.height();
final float hitBoxDiagonal = (float)Math.hypot(hitBoxWidth, hitBoxHeight);
Expand All @@ -197,19 +227,28 @@ private long createNativeProximityInfo(@NonNull final TouchPositionCorrection to
sweetSpotRadii[infoIndex] =
touchPositionCorrection.getRadius(row) * hitBoxDiagonal;
}
if (applyAdaptive) {
final TouchModelDao.Stat stat = adaptiveDao.get(key.getCode(),
Integer.toString(mLayoutElementId), adaptiveOrientation);
final float[] off = TouchModelManager.adjustedOffset(
stat, key.getWidth(), key.getHeight(), adaptiveStrength,
System.currentTimeMillis(), sv.adaptiveForgetHalfLifeMs());
sweetSpotCenterXs[infoIndex] += off[0];
sweetSpotCenterYs[infoIndex] += off[1];
}
if (DEBUG) {
Log.d(TAG, String.format(Locale.US,
" [%2d] row=%d x/y/r=%7.2f/%7.2f/%5.2f %s code=%s", infoIndex, row,
sweetSpotCenterXs[infoIndex], sweetSpotCenterYs[infoIndex],
sweetSpotRadii[infoIndex], (row < rows ? "correct" : "default"),
sweetSpotRadii[infoIndex], (tpcValid && row < rows ? "correct" : "default"),
Constants.printableCode(key.getCode())));
}
infoIndex++;
}
} else {
sweetSpotCenterXs = sweetSpotCenterYs = sweetSpotRadii = null;
if (DEBUG) {
Log.d(TAG, "touchPositionCorrection: OFF");
Log.d(TAG, "sweet spots: OFF");
}
}

Expand Down
136 changes: 136 additions & 0 deletions app/src/main/java/helium314/keyboard/keyboard/AdaptiveKeyContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
/*
* SPDX-License-Identifier: GPL-3.0-only
*/
package helium314.keyboard.keyboard;

import helium314.keyboard.latin.SuggestedWords;

import java.util.HashMap;
import java.util.Map;

/**
* Holds a transient "likely next key" prior derived from the current suggestion strip, used to
* gently enlarge the touch target of likely next keys (adaptive typing, see
* docs/ADAPTIVE_TYPING.md).
*
* <p>It is rebuilt between keystrokes — whenever the suggestions change — on the UI thread, and
* read per tap in {@link KeyDetector}. Reads are lock-free via {@code volatile} parallel arrays,
* which are tiny (at most a handful of distinct next-characters), so the tap hot path stays fast
* even for very fast typists. Because suggestions are computed asynchronously, the prior may lag
* the latest keystroke by one tap under very fast typing; that is harmless, as the prior is only
* a soft, capped bias.
*
* <p>Suggestions are weighted EQUALLY (averaged) rather than by score, so the result is not skewed
* toward the single top suggestion.
*/
public final class AdaptiveKeyContext {
/** How many suggestions to average over. */
private static final int TOP_N = 5;

private static volatile int[] sCodes;
private static volatile float[] sWeights;

/**
* Optional observer notified whenever the prior changes, on the same (UI) thread that mutates
* it. Used only by the debug overlay (see AdaptiveTargetsDrawingPreview) to repaint the live
* keyboard as the prior shifts between keystrokes; null in normal operation. Volatile so the
* keyboard view can register/clear it from its own lifecycle without extra locking.
*/
private static volatile Runnable sChangeListener;

private AdaptiveKeyContext() {}

/** Register (or clear, with {@code null}) the debug repaint observer. */
public static void setChangeListener(final Runnable listener) {
sChangeListener = listener;
}

private static void fireChanged() {
final Runnable l = sChangeListener;
if (l != null) l.run();
}

/**
* Rebuild the prior from the top suggestions.
*
* @param words the current suggestion strip contents.
* @param position index of the NEXT character within each suggestion — the current
* composing-word length while a word is being built, or 0 for a new word
* (using next-word predictions, whose first letter is the likely next key).
*/
public static void update(final SuggestedWords words, final int position) {
if (words == null || words.isEmpty() || position < 0) {
clear();
return;
}
final int n = Math.min(TOP_N, words.size());
final HashMap<Integer, Integer> tally = new HashMap<>();
int considered = 0;
for (int i = 0; i < n; i++) {
final String w = words.getWord(i);
if (w == null || position >= w.length()) continue; // typed word itself / too short
final int cp = Character.toLowerCase(w.charAt(position));
if (!Character.isLetter(cp)) continue;
tally.merge(cp, 1, Integer::sum);
considered++;
}
if (considered == 0) {
clear();
return;
}
final int[] codes = new int[tally.size()];
final float[] weights = new float[tally.size()];
int j = 0;
for (final Map.Entry<Integer, Integer> e : tally.entrySet()) {
codes[j] = e.getKey();
weights[j] = (float) e.getValue() / considered; // 0..1, equal-weight average
j++;
}
sCodes = codes;
sWeights = weights;
fireChanged();
}

public static void clear() {
// Already clear: do NOT fire. This is the default path — with the context prior off,
// InputLogic.setSuggestedWords calls clear() on every suggestion update, and firing here
// would invalidate the overlay view on every keystroke for users who never enabled the
// feature. Only fire on an actual transition from non-empty to empty.
if (sCodes == null && sWeights == null) return;
sCodes = null;
sWeights = null;
fireChanged();
}

/** Prior weight in [0, 1] for the given key code (0 if none / no prior). Case-insensitive:
* the prior stores lowercase next-characters, but a shifted keyboard reports uppercase key
* codes, so we fold to lowercase to match (otherwise the bias/overlay miss capital letters). */
public static float weight(final int code) {
final int[] c = sCodes;
final float[] w = sWeights;
if (c == null || w == null) return 0f;
final int lower = Character.toLowerCase(code);
for (int i = 0; i < c.length && i < w.length; i++) {
if (c[i] == lower) return w[i];
}
return 0f;
}

public static boolean hasPrior() {
return sCodes != null;
}

/** Human-readable snapshot of the current prior, e.g. {@code [e=0.60,o=0.40]}, for debug logs. */
public static String debugString() {
final int[] c = sCodes;
final float[] w = sWeights;
if (c == null || w == null) return "(none)";
final StringBuilder sb = new StringBuilder("[");
for (int i = 0; i < c.length && i < w.length; i++) {
if (i > 0) sb.append(',');
sb.append((char) c[i]).append('=')
.append(String.format(java.util.Locale.US, "%.2f", w[i]));
}
return sb.append(']').toString();
}
}
121 changes: 121 additions & 0 deletions app/src/main/java/helium314/keyboard/keyboard/KeyDetector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,26 @@

package helium314.keyboard.keyboard;

import android.content.Context;
import android.graphics.Rect;

import helium314.keyboard.latin.database.TouchModelDao;
import helium314.keyboard.latin.database.TouchModelManager;
import helium314.keyboard.latin.settings.Settings;
import helium314.keyboard.latin.settings.SettingsValues;
import helium314.keyboard.latin.utils.Log;

/**
* This class handles key detection.
*/
public class KeyDetector {
// Adaptive typing: the context prior may enlarge a likely key's effective target by at most
// this fraction of the key (deliberately a bit less than the learned-geometry cap, so the
// prior nudges rather than dominates). See docs/ADAPTIVE_TYPING.md.
private static final float PRIOR_MAX_FRACTION = 0.18f;
// A neighbor is only allowed to win a tap if the touch is within this fraction of its hitbox.
private static final float CONSIDER_MARGIN_FRACTION = 0.40f;

private final int mKeyHysteresisDistanceSquared;
private final int mKeyHysteresisDistanceForSlidingModifierSquared;

Expand Down Expand Up @@ -101,6 +117,111 @@ public Key detectHitKey(final int x, final int y) {
primaryKey = key;
}
}
// Adaptive typing (opt-in): for a plain tap (not a gesture/swipe), let the learned
// per-key landing offset and the current next-key prior gently bias which key wins —
// bounded so only genuinely ambiguous, near-boundary taps can flip.
if (primaryKey != null && !PointerTracker.isInGestureOrKeySwipe()) {
final Key biased = applyAdaptiveBias(touchX, touchY, primaryKey);
if (biased != null) return biased;
}
return primaryKey;
}

/** Returns a key that should win this tap instead of {@code geo} due to learned/prior bias,
* or {@code null} to keep the plain geometric result. */
private Key applyAdaptiveBias(final int touchX, final int touchY, final Key geo) {
final SettingsValues sv = Settings.getValues();
if (sv == null || sv.mAdaptiveKeyGeometryStrength <= 0) return null;
// The two halves are independently toggleable: learned per-key offset, and the
// context prior. Either alone is enough to bias a tap; both share the strength slider.
final boolean learn = sv.mAdaptiveKeyGeometry;
final boolean usePrior = sv.mAdaptiveContextPrior;
if (!learn && !usePrior) return null;
final Context ctx = Settings.getCurrentContext();
if (ctx == null || mKeyboard == null) return null;
final TouchModelDao dao = learn ? TouchModelDao.getInstance(ctx) : null;
final boolean hasPrior = usePrior && AdaptiveKeyContext.hasPrior();
if (dao == null && !hasPrior) return null; // nothing to bias with
final String layout = Integer.toString(mKeyboard.mId.mElementId);
final int orientation = ctx.getResources().getConfiguration().orientation;
final int strength = sv.mAdaptiveKeyGeometryStrength;
final long now = System.currentTimeMillis();
final long halfLifeMs = sv.adaptiveForgetHalfLifeMs();

// Guard the flip: bias must never override a press the user made clearly inside a key. A
// neighbor may win ONLY if the touch also lands inside ITS hitbox, or it is geometrically
// no farther (true, unshifted center) than the pressed key. Combined with the per-neighbor
// checks below, this caps the learned-offset + prior combination so an off-center-but-
// unambiguous tap can't be stolen — only a genuinely near-boundary tap can flip.
final float geoGeomSq = geomCenterDistSq(geo, touchX, touchY);
final TouchModelDao.Stat geoStat = (dao == null) ? null : dao.get(geo.getCode(), layout, orientation);

Key best = geo;
float bestScore = adjustedDistance(geo, touchX, touchY, geoStat, strength, usePrior, now, halfLifeMs);
for (final Key k : mKeyboard.getNearestKeys(touchX, touchY)) {
if (k == geo) continue;
final int code = k.getCode();
if (code <= 0 || !Character.isLetter(code)) continue;
// Only a genuinely-favored neighbor (a confident learned offset and/or a next-key
// prior) may steal a near-boundary tap; otherwise leave the geometric result alone.
final float prior = usePrior ? AdaptiveKeyContext.weight(code) : 0f;
final TouchModelDao.Stat st = (dao == null) ? null : dao.get(code, layout, orientation);
final boolean hasLearned = st != null && st.getCount() >= TouchModelDao.MIN_CONFIDENT_SAMPLES;
if (prior <= 0f && !hasLearned) continue;
// Bound: the touch must be within a margin of the neighbor's hitbox...
final float margin = CONSIDER_MARGIN_FRACTION * k.getWidth();
if (k.squaredDistanceToEdge(touchX, touchY) > margin * margin) continue;
// ...and (the flip guard) the neighbor must contain the touch, or be no farther than geo.
if (!k.isOnKey(touchX, touchY) && geomCenterDistSq(k, touchX, touchY) > geoGeomSq) continue;
final float s = adjustedDistance(k, touchX, touchY, st, strength, usePrior, now, halfLifeMs);
if (s < bestScore) {
bestScore = s;
best = k;
}
}
if (sv.mAdaptiveDebugOverlay && hasPrior) {
Log.d("AdaptivePrior", "tap geo='" + (char) geo.getCode()
+ "' priorOnGeo=" + AdaptiveKeyContext.weight(geo.getCode())
+ " chosen='" + (char) best.getCode() + "'"
+ (best == geo ? "" : " (FLIPPED by bias)")
+ " prior=" + AdaptiveKeyContext.debugString());
}
return best == geo ? null : best;
}

/** Distance from the touch to the key's effective center (center shifted by the learned
* landing offset {@code st}, when present) minus the capped next-key prior boost (when
* enabled). Smaller wins. Takes the pre-fetched {@code st} so a tap does ONE DAO lookup per
* key rather than two. */
private float adjustedDistance(final Key k, final int touchX, final int touchY,
final TouchModelDao.Stat st, final int strength, final boolean usePrior,
final long now, final long halfLifeMs) {
final Rect hb = k.getHitBox();
float cx = hb.exactCenterX();
float cy = hb.exactCenterY();
if (st != null) {
// Cap the shift against the HIT BOX — the same basis the offset was recorded against
// and the center we shift from — so the 0.25-key safety bound is geometrically
// consistent (key.getWidth() excludes the gap and would misstate the bound). The
// now/halfLifeMs apply the recency "forget window" so stale learning fades out.
final float[] off = TouchModelManager.adjustedOffset(st, hb.width(), hb.height(), strength, now, halfLifeMs);
cx += off[0];
cy += off[1];
}
final float dx = touchX - cx;
final float dy = touchY - cy;
final float dist = (float) Math.sqrt(dx * dx + dy * dy);
final float boost = usePrior
? AdaptiveKeyContext.weight(k.getCode()) * PRIOR_MAX_FRACTION * k.getWidth() * (strength / 100f)
: 0f;
return dist - boost;
}

/** Squared distance from the touch to the key's true (unshifted) hit-box center. */
private static float geomCenterDistSq(final Key k, final int touchX, final int touchY) {
final Rect hb = k.getHitBox();
final float dx = touchX - hb.exactCenterX();
final float dy = touchY - hb.exactCenterY();
return dx * dx + dy * dy;
}
}
Loading
Loading