From 5769bc8e63106054f8276456ec38e92d74ec0b48 Mon Sep 17 00:00:00 2001 From: Max Starikov Date: Tue, 4 Aug 2026 12:01:06 +0200 Subject: [PATCH 1/6] add android keyboard --- .../VoiceKeyboardInstrumentedTest.kt | 46 ++++ app/src/main/AndroidManifest.xml | 13 + .../me/maxistar/voiceinbox/MainActivity.kt | 23 ++ .../voiceinbox/VoiceKeyboardAudioRecorder.kt | 109 ++++++++ .../voiceinbox/VoiceKeyboardController.kt | 69 +++++ .../VoiceKeyboardInputMethodService.kt | 260 ++++++++++++++++++ .../maxistar/voiceinbox/VoiceKeyboardSetup.kt | 6 + .../res/layout/input_view_voice_keyboard.xml | 80 ++++++ app/src/main/res/values/strings.xml | 22 ++ .../main/res/xml/voice_keyboard_method.xml | 7 + .../voiceinbox/VoiceKeyboardControllerTest.kt | 73 +++++ 11 files changed, 708 insertions(+) create mode 100644 app/src/androidTest/java/me/maxistar/voiceinbox/VoiceKeyboardInstrumentedTest.kt create mode 100644 app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardAudioRecorder.kt create mode 100644 app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardController.kt create mode 100644 app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardInputMethodService.kt create mode 100644 app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardSetup.kt create mode 100644 app/src/main/res/layout/input_view_voice_keyboard.xml create mode 100644 app/src/main/res/xml/voice_keyboard_method.xml create mode 100644 app/src/test/java/me/maxistar/voiceinbox/VoiceKeyboardControllerTest.kt diff --git a/app/src/androidTest/java/me/maxistar/voiceinbox/VoiceKeyboardInstrumentedTest.kt b/app/src/androidTest/java/me/maxistar/voiceinbox/VoiceKeyboardInstrumentedTest.kt new file mode 100644 index 0000000..dd47b73 --- /dev/null +++ b/app/src/androidTest/java/me/maxistar/voiceinbox/VoiceKeyboardInstrumentedTest.kt @@ -0,0 +1,46 @@ +package me.maxistar.voiceinbox + +import android.content.ComponentName +import android.content.pm.PackageManager +import android.widget.EditText +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class VoiceKeyboardInstrumentedTest { + @Test + fun manifestDeclaresBoundInputMethodService() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + val service = context.packageManager.getServiceInfo( + ComponentName(context, VoiceKeyboardInputMethodService::class.java), + PackageManager.GET_META_DATA, + ) + + assertEquals("android.permission.BIND_INPUT_METHOD", service.permission) + assertTrue(service.metaData?.containsKey("android.view.im") == true) + assertEquals( + PackageManager.PERMISSION_GRANTED, + context.packageManager.checkPermission(android.Manifest.permission.RECORD_AUDIO, context.packageName), + ) + } + + @Test + fun inputConnectionCommitsAndDeletesText() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + InstrumentationRegistry.getInstrumentation().runOnMainSync { + val editor = EditText(context) + editor.setText("start") + editor.setSelection(editor.length()) + val connection = editor.onCreateInputConnection(android.view.inputmethod.EditorInfo()) + + connection.commitText(" text", 1) + connection.deleteSurroundingText(1, 0) + + assertEquals("start tex", editor.text.toString()) + } + } +} diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index f5060d2..149a5d7 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -6,6 +6,7 @@ + + + + + + + diff --git a/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt b/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt index c23c5af..59defbb 100644 --- a/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt +++ b/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt @@ -2,7 +2,9 @@ package me.maxistar.voiceinbox import me.maxistar.voiceinbox.core.* +import android.Manifest import android.content.Intent +import android.content.pm.PackageManager import android.media.MediaPlayer import android.net.Uri import android.os.Bundle @@ -160,6 +162,16 @@ class MainActivity : AppCompatActivity(), StartupProcessingDialogFragment.Listen if (uri != null) acceptModelFolder(uri) } + private val microphonePermissionRequest = registerForActivityResult( + ActivityResultContracts.RequestPermission(), + ) { granted -> + Toast.makeText( + this, + if (granted) R.string.voice_keyboard_permission_granted else R.string.voice_keyboard_permission_denied, + Toast.LENGTH_LONG, + ).show() + } + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) enableEdgeToEdge() @@ -237,6 +249,7 @@ class MainActivity : AppCompatActivity(), StartupProcessingDialogFragment.Listen intent.removeExtra(EXTRA_OPEN_MODEL_FOLDER_PICKER) modelFolderPicker.launch(null) } + requestMicrophonePermissionIfNeeded(intent) } override fun onStart() { @@ -251,7 +264,9 @@ class MainActivity : AppCompatActivity(), StartupProcessingDialogFragment.Listen override fun onNewIntent(intent: Intent) { super.onNewIntent(intent) + setIntent(intent) handleShareIntent(intent) + requestMicrophonePermissionIfNeeded(intent) } override fun onSaveInstanceState(outState: Bundle) { @@ -307,6 +322,14 @@ class MainActivity : AppCompatActivity(), StartupProcessingDialogFragment.Listen super.onDestroy() } + private fun requestMicrophonePermissionIfNeeded(intent: Intent) { + if (!intent.getBooleanExtra(VoiceKeyboardSetup.EXTRA_REQUEST_MICROPHONE_PERMISSION, false)) return + intent.removeExtra(VoiceKeyboardSetup.EXTRA_REQUEST_MICROPHONE_PERMISSION) + if (checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { + microphonePermissionRequest.launch(Manifest.permission.RECORD_AUDIO) + } + } + private fun bindViews() { importAudio = findViewById(R.id.importAudio) newTab = findViewById(R.id.newTab) diff --git a/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardAudioRecorder.kt b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardAudioRecorder.kt new file mode 100644 index 0000000..197d42f --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardAudioRecorder.kt @@ -0,0 +1,109 @@ +package me.maxistar.voiceinbox + +import android.media.AudioFormat +import android.media.AudioRecord +import android.media.MediaRecorder +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.Future + +class VoiceKeyboardAudioRecorder( + private val sampleRate: Int = SAMPLE_RATE, + private val recorderFactory: (Int) -> AudioRecord = { bufferSize -> + AudioRecord.Builder() + .setAudioSource(MediaRecorder.AudioSource.VOICE_RECOGNITION) + .setAudioFormat( + AudioFormat.Builder() + .setSampleRate(SAMPLE_RATE) + .setChannelMask(AudioFormat.CHANNEL_IN_MONO) + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .build(), + ) + .setBufferSizeInBytes(bufferSize) + .build() + }, + private val readerExecutor: ExecutorService = Executors.newSingleThreadExecutor(), +) { + private val lock = Any() + private val chunks = mutableListOf() + private var recorder: AudioRecord? = null + private var reader: Future<*>? = null + private var reading = false + + fun start(): Result = runCatching { + synchronized(lock) { + check(!reading) { "Microphone recording is already active" } + val minimum = AudioRecord.getMinBufferSize( + sampleRate, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + ) + check(minimum > 0) { "This device does not support 16 kHz mono recording" } + val created = recorderFactory(minimum * 2) + try { + check(created.state == AudioRecord.STATE_INITIALIZED) { "Microphone recorder could not initialize" } + chunks.clear() + recorder = created + reading = true + created.startRecording() + reader = readerExecutor.submit { readSamples(created) } + } catch (error: Throwable) { + recorder = null + reading = false + created.release() + throw error + } + } + } + + fun stop(): Result = runCatching { + val activeRecorder = synchronized(lock) { + reading = false + recorder + } ?: return@runCatching FloatArray(0) + runCatching { activeRecorder.stop() } + val readerResult = runCatching { reader?.get() } + val result = synchronized(lock) { + recorder = null + reader = null + activeRecorder.release() + val sampleCount = chunks.sumOf(FloatArray::size) + FloatArray(sampleCount).also { result -> + var offset = 0 + chunks.forEach { chunk -> + chunk.copyInto(result, destinationOffset = offset) + offset += chunk.size + } + chunks.clear() + } + } + readerResult.getOrThrow() + result + } + + fun cancel() { + stop() + } + + fun close() { + cancel() + readerExecutor.shutdownNow() + } + + private fun readSamples(activeRecorder: AudioRecord) { + val buffer = ShortArray(READ_BUFFER_SAMPLES) + while (synchronized(lock) { reading }) { + val read = activeRecorder.read(buffer, 0, buffer.size, AudioRecord.READ_BLOCKING) + if (read <= 0) continue + val normalized = FloatArray(read) { index -> buffer[index] / 32768f } + synchronized(lock) { + if (reading) chunks += normalized + } + } + } + + private companion object { + const val SAMPLE_RATE = 16_000 + const val READ_BUFFER_SAMPLES = 2_048 + } +} diff --git a/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardController.kt b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardController.kt new file mode 100644 index 0000000..5909f18 --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardController.kt @@ -0,0 +1,69 @@ +package me.maxistar.voiceinbox + +enum class VoiceKeyboardPhase { + IDLE, + PREPARING, + RECORDING, + TRANSCRIBING, + RESULT_PENDING, + ERROR, +} + +/** Keeps IME request state deterministic and independent from Android view callbacks. */ +class VoiceKeyboardController { + var phase: VoiceKeyboardPhase = VoiceKeyboardPhase.IDLE + private set + + var pendingText: String? = null + private set + + fun beginPreparation(): Boolean { + if (phase !in setOf(VoiceKeyboardPhase.IDLE, VoiceKeyboardPhase.ERROR)) return false + phase = VoiceKeyboardPhase.PREPARING + return true + } + + fun recordingStarted(): Boolean { + if (phase != VoiceKeyboardPhase.PREPARING) return false + phase = VoiceKeyboardPhase.RECORDING + return true + } + + fun recordingStopped(): Boolean { + if (phase != VoiceKeyboardPhase.RECORDING) return false + phase = VoiceKeyboardPhase.TRANSCRIBING + return true + } + + fun completeTranscription(text: String?): String? { + if (phase != VoiceKeyboardPhase.TRANSCRIBING) return null + phase = VoiceKeyboardPhase.IDLE + return text?.trim()?.takeIf(String::isNotEmpty) + } + + fun deferResult(text: String) { + pendingText = text + phase = VoiceKeyboardPhase.RESULT_PENDING + } + + fun pendingResult(): String? = pendingText + + fun markPendingCommitted() { + pendingText = null + phase = VoiceKeyboardPhase.IDLE + } + + fun dismissPendingResult() { + pendingText = null + if (phase == VoiceKeyboardPhase.RESULT_PENDING) phase = VoiceKeyboardPhase.IDLE + } + + fun fail() { + phase = VoiceKeyboardPhase.ERROR + } + + fun cancel() { + pendingText = null + phase = VoiceKeyboardPhase.IDLE + } +} diff --git a/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardInputMethodService.kt b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardInputMethodService.kt new file mode 100644 index 0000000..0dfd8ca --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardInputMethodService.kt @@ -0,0 +1,260 @@ +package me.maxistar.voiceinbox + +import android.Manifest +import android.content.Intent +import android.content.pm.PackageManager +import android.inputmethodservice.InputMethodService +import android.os.Build +import android.os.Handler +import android.os.Looper +import android.view.KeyEvent +import android.view.LayoutInflater +import android.view.View +import android.view.inputmethod.EditorInfo +import android.view.inputmethod.InputConnection +import android.view.inputmethod.InputMethodManager +import android.widget.Button +import android.widget.TextView +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +class VoiceKeyboardInputMethodService : InputMethodService() { + private val controller = VoiceKeyboardController() + private val mainHandler = Handler(Looper.getMainLooper()) + private val workExecutor: ExecutorService = Executors.newSingleThreadExecutor() + private lateinit var recorder: VoiceKeyboardAudioRecorder + private var inputActive = false + private var requestGeneration = 0L + + private var statusView: TextView? = null + private var recordButton: Button? = null + private var dismissButton: Button? = null + private var setupButton: Button? = null + private var backspaceButton: Button? = null + private var spaceButton: Button? = null + private var enterButton: Button? = null + private var nextKeyboardButton: Button? = null + + override fun onCreate() { + super.onCreate() + recorder = VoiceKeyboardAudioRecorder() + } + + override fun onCreateInputView(): View { + val view = LayoutInflater.from(this).inflate(R.layout.input_view_voice_keyboard, null) + statusView = view.findViewById(R.id.voiceKeyboardStatus) + recordButton = view.findViewById(R.id.voiceKeyboardRecord) + dismissButton = view.findViewById(R.id.voiceKeyboardDismiss) + setupButton = view.findViewById(R.id.voiceKeyboardSetup) + backspaceButton = view.findViewById(R.id.voiceKeyboardBackspace) + spaceButton = view.findViewById(R.id.voiceKeyboardSpace) + enterButton = view.findViewById(R.id.voiceKeyboardEnter) + nextKeyboardButton = view.findViewById(R.id.voiceKeyboardNextKeyboard) + + recordButton?.setOnClickListener { handleRecordButton() } + dismissButton?.setOnClickListener { + controller.dismissPendingResult() + render(R.string.voice_keyboard_ready) + } + setupButton?.setOnClickListener { openVoiceInboxSetup() } + backspaceButton?.setOnClickListener { currentInputConnection?.deleteSurroundingText(1, 0) } + spaceButton?.setOnClickListener { currentInputConnection?.commitText(" ", 1) } + enterButton?.setOnClickListener(::performEnterAction) + nextKeyboardButton?.setOnClickListener { switchKeyboard() } + render(R.string.voice_keyboard_ready) + return view + } + + override fun onStartInput(attribute: EditorInfo, restarting: Boolean) { + super.onStartInput(attribute, restarting) + inputActive = true + } + + override fun onStartInputView(info: EditorInfo, restarting: Boolean) { + super.onStartInputView(info, restarting) + inputActive = true + flushPendingResult() + } + + override fun onFinishInput() { + inputActive = false + super.onFinishInput() + } + + override fun onDestroy() { + requestGeneration += 1 + recorder.close() + controller.cancel() + workExecutor.shutdownNow() + super.onDestroy() + } + + private fun handleRecordButton() { + when (controller.phase) { + VoiceKeyboardPhase.RECORDING -> stopRecording() + VoiceKeyboardPhase.PREPARING, + VoiceKeyboardPhase.TRANSCRIBING, + -> cancelCurrentRequest() + VoiceKeyboardPhase.RESULT_PENDING -> { + controller.dismissPendingResult() + render(R.string.voice_keyboard_ready) + } + VoiceKeyboardPhase.IDLE, + VoiceKeyboardPhase.ERROR, + -> prepareAndStartRecording() + } + } + + private fun prepareAndStartRecording() { + if (checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { + controller.fail() + render(R.string.voice_keyboard_microphone_permission_missing, showSetup = true) + return + } + if (!controller.beginPreparation()) return + val generation = ++requestGeneration + render(R.string.voice_keyboard_preparing) + workExecutor.execute { + val preparation = runCatching { + val repository = SpeechModelRepository.forActive(noBackupFilesDir.resolve("models")) + SpeechModelPreparation.prepare(repository, NativeTranscriptionBridge::initialize).getOrThrow() + } + mainHandler.post { + if (generation != requestGeneration || controller.phase != VoiceKeyboardPhase.PREPARING) return@post + preparation.onSuccess { + recorder.start().onSuccess { + if (controller.recordingStarted()) { + render(R.string.voice_keyboard_listening) + mainHandler.postDelayed({ stopForDurationLimit(generation) }, MAX_PHRASE_DURATION_MS) + } + }.onFailure { + controller.fail() + render(R.string.voice_keyboard_microphone_unavailable, showSetup = false) + } + }.onFailure { + controller.fail() + render(R.string.voice_keyboard_model_unavailable, showSetup = true) + } + } + } + } + + private fun stopForDurationLimit(generation: Long) { + if (generation == requestGeneration && controller.phase == VoiceKeyboardPhase.RECORDING) { + stopRecording() + } + } + + private fun stopRecording() { + if (!controller.recordingStopped()) return + mainHandler.removeCallbacksAndMessages(null) + val generation = requestGeneration + render(R.string.voice_keyboard_transcribing) + workExecutor.execute { + val result = recorder.stop().mapCatching { samples -> + if (samples.isEmpty()) null else NativeTranscriptionBridge.transcribeChunk(samples)?.text + } + mainHandler.post { + if (generation != requestGeneration || controller.phase != VoiceKeyboardPhase.TRANSCRIBING) return@post + result.onSuccess { text -> + val completed = controller.completeTranscription(text) + when { + completed == null -> render(R.string.voice_keyboard_no_speech) + commitToCurrentEditor(completed) -> render(R.string.voice_keyboard_ready) + else -> { + controller.deferResult(completed) + render(R.string.voice_keyboard_result_pending) + } + } + }.onFailure { + controller.fail() + render(R.string.voice_keyboard_transcription_failed) + } + } + } + } + + private fun cancelCurrentRequest() { + requestGeneration += 1 + mainHandler.removeCallbacksAndMessages(null) + if (controller.phase == VoiceKeyboardPhase.RECORDING) { + workExecutor.execute { recorder.cancel() } + } + controller.cancel() + render(R.string.voice_keyboard_ready) + } + + private fun flushPendingResult() { + val pending = controller.pendingResult() ?: return + if (commitToCurrentEditor(pending)) { + controller.markPendingCommitted() + render(R.string.voice_keyboard_ready) + } + } + + private fun commitToCurrentEditor(text: String): Boolean { + if (!inputActive || text.isBlank()) return false + return currentInputConnection?.commitText(text, 1) == true + } + + private fun performEnterAction(@Suppress("UNUSED_PARAMETER") view: View) { + val connection = currentInputConnection ?: return + val info = currentInputEditorInfo + val action = info?.imeOptions?.and(EditorInfo.IME_MASK_ACTION) ?: EditorInfo.IME_ACTION_NONE + val supportsAction = action in setOf( + EditorInfo.IME_ACTION_GO, + EditorInfo.IME_ACTION_SEARCH, + EditorInfo.IME_ACTION_SEND, + EditorInfo.IME_ACTION_NEXT, + ) && info?.imeOptions?.and(EditorInfo.IME_FLAG_NO_ENTER_ACTION) == 0 + if (supportsAction) { + connection.performEditorAction(action) + } else { + connection.sendKeyEvent(KeyEvent(KeyEvent.ACTION_DOWN, KeyEvent.KEYCODE_ENTER)) + connection.sendKeyEvent(KeyEvent(KeyEvent.ACTION_UP, KeyEvent.KEYCODE_ENTER)) + } + } + + private fun switchKeyboard() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + switchToNextInputMethod(false) + } else { + getSystemService(InputMethodManager::class.java).showInputMethodPicker() + } + } + + private fun openVoiceInboxSetup() { + startActivity( + Intent(this, MainActivity::class.java) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_SINGLE_TOP) + .putExtra(VoiceKeyboardSetup.EXTRA_REQUEST_MICROPHONE_PERMISSION, true), + ) + } + + private fun render(status: Int, showSetup: Boolean = false) { + statusView?.setText(status) + val phase = controller.phase + recordButton?.apply { + isEnabled = true + text = when (phase) { + VoiceKeyboardPhase.RECORDING -> context.getString(R.string.voice_keyboard_stop) + VoiceKeyboardPhase.PREPARING, + VoiceKeyboardPhase.TRANSCRIBING, + -> context.getString(R.string.voice_keyboard_cancel) + VoiceKeyboardPhase.RESULT_PENDING -> context.getString(R.string.voice_keyboard_dismiss) + else -> context.getString(R.string.voice_keyboard_record) + } + } + dismissButton?.visibility = if (phase == VoiceKeyboardPhase.RESULT_PENDING) View.VISIBLE else View.GONE + setupButton?.visibility = if (showSetup) View.VISIBLE else View.GONE + val editorReady = inputActive && phase !in setOf(VoiceKeyboardPhase.PREPARING, VoiceKeyboardPhase.TRANSCRIBING) + backspaceButton?.isEnabled = editorReady + spaceButton?.isEnabled = editorReady + enterButton?.isEnabled = editorReady + nextKeyboardButton?.isEnabled = phase != VoiceKeyboardPhase.RECORDING + } + + private companion object { + const val MAX_PHRASE_DURATION_MS = 30_000L + } +} diff --git a/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardSetup.kt b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardSetup.kt new file mode 100644 index 0000000..39bc0b1 --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/VoiceKeyboardSetup.kt @@ -0,0 +1,6 @@ +package me.maxistar.voiceinbox + +object VoiceKeyboardSetup { + const val EXTRA_REQUEST_MICROPHONE_PERMISSION = + "me.maxistar.voiceinbox.extra.REQUEST_MICROPHONE_PERMISSION" +} diff --git a/app/src/main/res/layout/input_view_voice_keyboard.xml b/app/src/main/res/layout/input_view_voice_keyboard.xml new file mode 100644 index 0000000..7e8e068 --- /dev/null +++ b/app/src/main/res/layout/input_view_voice_keyboard.xml @@ -0,0 +1,80 @@ + + + + + +