Skip to content

Commit cca6d0b

Browse files
committed
feat(chat): send, store and open photo messages
The transport and storage half of chat photos, ported from code-ios-app#959. Nothing in the app calls it yet; the UI follows in a stacked PR. - Encoding: photos are downscaled and encoded as JPEG down a quality ladder of 0.9, 0.8, 0.7, 0.6 until they fit the blob policy, with EXIF orientation baked in and a BlurHash for the placeholder. The chat_media.json vectors are the iOS file, byte for byte. - Encrypted chats: the blob is sealed with the shared-core ChatCipher, uploaded with the `chat` field, and sent as EncryptedContent. A photo that can't be opened loads as ChatPhotoUnavailable. - Sending: one message per photo, each retried on its own. Finalization polls GetBlobs every 2s, at most 30 times, counted only while the app is in the foreground. - Storage: pending uploads live in Room (schema 43) with the JPEG in filesDir, so a send resumes after the app is killed. - Loading: a Coil fetcher keyed `chat-media-<blobId>`, registered in FlipcashApp. - Fixes found on the way: captions were never sent, because `caption` inside the builder's `apply` resolved to the builder's own field; and a sent message disappeared when the stream delivered its stored copy before the send returned, because clearing the pending row deleted by its client id.
1 parent d7a8a07 commit cca6d0b

69 files changed

Lines changed: 7192 additions & 56 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/flipcash/app/build.gradle.kts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,7 @@ dependencies {
232232
implementation(project(":apps:flipcash:shared:contacts"))
233233
implementation(project(":apps:flipcash:shared:common-ui"))
234234
implementation(project(":apps:flipcash:shared:notifications"))
235+
implementation(project(":apps:flipcash:shared:chat"))
235236
implementation(project(":apps:flipcash:shared:onramp:coinbase"))
236237
implementation(project(":apps:flipcash:shared:onramp:deeplinks"))
237238
implementation(libs.phantom.connect) {

‎apps/flipcash/app/src/main/kotlin/com/flipcash/app/FlipcashApp.kt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ import coil3.request.crossfade
1414
import com.flipcash.app.auth.AuthManager
1515
import okio.Path.Companion.toOkioPath
1616
import com.flipcash.app.core.android.ActivityProvider
17+
import com.flipcash.shared.chat.media.ChatPhotoFetcher
18+
import com.flipcash.shared.chat.media.ChatPhotoKeyer
1719
import com.flipcash.app.currency.PreferredCurrencyController
1820
import com.flipcash.app.bills.share.TipCodePreviewCache
1921
import com.getcode.opencode.repositories.EventRepository
@@ -51,6 +53,9 @@ class FlipcashApp : Application(), Configuration.Provider, SingletonImageLoader.
5153
@Inject
5254
lateinit var preferredCurrencyController: Lazy<PreferredCurrencyController>
5355

56+
@Inject
57+
lateinit var chatPhotoFetcher: Lazy<ChatPhotoFetcher.Factory>
58+
5459
@Inject
5560
lateinit var workerFactory: Lazy<HiltWorkerFactory>
5661

@@ -109,6 +114,8 @@ class FlipcashApp : Application(), Configuration.Provider, SingletonImageLoader.
109114
// would flash the BlurHash again). The default strategy respects HTTP cache headers and
110115
// would revalidate/re-fetch the ephemeral, expiring download URLs; blobs are static.
111116
.components {
117+
add(ChatPhotoKeyer())
118+
add(chatPhotoFetcher.get())
112119
add(OkHttpNetworkFetcherFactory(cacheStrategy = { ImmutableBlobCacheStrategy }))
113120
}
114121
.build()

‎apps/flipcash/shared/blob/build.gradle.kts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,7 @@ dependencies {
1919
testImplementation(kotlin("test"))
2020
testImplementation(libs.bundles.unit.testing)
2121
testImplementation(libs.robolectric)
22+
// The BlurHash round-trip test decodes with the real decoder; common-ui is compose-versioned.
23+
testImplementation(platform(libs.compose.bom))
24+
testImplementation(project(":apps:flipcash:shared:common-ui"))
2225
}

‎apps/flipcash/shared/blob/src/main/kotlin/com/flipcash/app/blob/BlobStorageCoordinator.kt‎

Lines changed: 77 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,15 @@ import androidx.datastore.preferences.preferencesDataStoreFile
1111
import com.flipcash.libs.coroutines.DispatcherProvider
1212
import com.flipcash.services.controllers.BlobStorageController
1313
import com.flipcash.services.models.InitiateExternalUploadError
14+
import com.flipcash.services.models.blob.EncryptedConstraints
1415
import com.flipcash.services.models.blob.MimeTypeConstraints
1516
import com.flipcash.services.models.blob.UploadPolicy
1617
import com.flipcash.services.models.chat.BlobId
18+
import com.flipcash.services.models.chat.ChatId
1719
import dagger.hilt.android.qualifiers.ApplicationContext
1820
import kotlinx.coroutines.CoroutineScope
1921
import kotlinx.coroutines.SupervisorJob
22+
import kotlinx.coroutines.delay
2023
import kotlinx.coroutines.flow.Flow
2124
import kotlinx.coroutines.flow.first
2225
import kotlinx.coroutines.flow.map
@@ -75,12 +78,75 @@ class BlobStorageCoordinator @Inject constructor(
7578
* poll are all handled inside the controller. A policy-driven rejection invalidates the cached
7679
* policy (the server echoes a newer policy version on such denials).
7780
*/
78-
suspend fun upload(bytes: ByteArray, mimeType: String): Result<BlobId> {
79-
val result = blobStorageController.upload(bytes, mimeType)
81+
suspend fun upload(
82+
bytes: ByteArray,
83+
mimeType: String,
84+
onProgress: ((sentBytes: Long, totalBytes: Long) -> Unit)? = null,
85+
): Result<BlobId> {
86+
val result = blobStorageController.upload(bytes, mimeType, onProgress)
8087
result.exceptionOrNull()?.let { refreshIfPolicyChanged(it) }
8188
return result
8289
}
8390

91+
/**
92+
* Stores an encoded chat photo and returns its READY [BlobId], retrying what a retry can fix.
93+
*
94+
* [jpeg] is the encoder's output. With [sealing] it is encrypted for that chat and uploaded as
95+
* opaque bytes; without, it goes up as plain `image/jpeg` for the server to moderate. Each
96+
* attempt reserves afresh, so a retry gets a new blob id and re-seals under it.
97+
*
98+
* Up to [ChatMediaRetry.BACKOFFS].size retries, 1 s, 2 s, then 4 s apart, for failures in
99+
* transit or while finalizing; a refusal that repeating can't change (see
100+
* [ChatMediaRetry.isRetryable]) is returned at once. [onProgress] restarts from zero on a retry.
101+
*/
102+
suspend fun storeChatMedia(
103+
jpeg: ByteArray,
104+
sealing: ChatMediaSealing? = null,
105+
onProgress: ((sentBytes: Long, totalBytes: Long) -> Unit)? = null,
106+
): Result<BlobId> = withStoreRetries { _ ->
107+
if (sealing != null) {
108+
blobStorageController.uploadSealed(jpeg, sealing.chatId, sealing.seal, onProgress)
109+
} else {
110+
blobStorageController.uploadChatMedia(jpeg, ChatMediaEncoder.UPLOAD_MIME_TYPE, onProgress)
111+
}
112+
}
113+
114+
/**
115+
* [storeChatMedia] up to the point the bytes are in storage, without waiting for finalization.
116+
* Retries only what happens before that, so a returned id means "stored": a caller that then
117+
* fails to see it finalize re-polls it with [awaitChatMediaReady] instead of uploading again.
118+
* [onAttempt] runs before each attempt, first included, so progress can restart with a retry.
119+
*/
120+
suspend fun storeChatMediaUnfinalized(
121+
jpeg: ByteArray,
122+
sealing: ChatMediaSealing? = null,
123+
onAttempt: (() -> Unit)? = null,
124+
onProgress: ((sentBytes: Long, totalBytes: Long) -> Unit)? = null,
125+
): Result<BlobId> = withStoreRetries { _ ->
126+
onAttempt?.invoke()
127+
if (sealing != null) {
128+
blobStorageController.storeSealed(jpeg, sealing.chatId, sealing.seal, onProgress)
129+
} else {
130+
blobStorageController.storeChatMedia(jpeg, ChatMediaEncoder.UPLOAD_MIME_TYPE, onProgress)
131+
}
132+
}
133+
134+
/** Waits for a blob [storeChatMediaUnfinalized] stored to finalize. */
135+
suspend fun awaitChatMediaReady(blobId: BlobId): Result<BlobId> =
136+
blobStorageController.awaitChatMediaReady(blobId)
137+
138+
private suspend fun withStoreRetries(attemptStore: suspend (attempt: Int) -> Result<BlobId>): Result<BlobId> {
139+
var attempt = 0
140+
while (true) {
141+
val result = attemptStore(attempt)
142+
val failure = result.exceptionOrNull() ?: return result
143+
144+
refreshIfPolicyChanged(failure)
145+
if (!ChatMediaRetry.isRetryable(failure) || attempt >= ChatMediaRetry.BACKOFFS.size) return result
146+
delay(ChatMediaRetry.BACKOFFS[attempt++])
147+
}
148+
}
149+
84150
suspend fun reset() {
85151
dataStore.edit { it.remove(KEY_UPLOAD_POLICY) }
86152
}
@@ -128,26 +194,32 @@ class BlobStorageCoordinator @Inject constructor(
128194
private fun now(): Long = System.currentTimeMillis()
129195

130196
companion object {
131-
private val KEY_UPLOAD_POLICY = stringPreferencesKey("cached_upload_policy")
197+
// v2: entries cached before the policy carried `encrypted` would read back as "encrypted
198+
// uploads not allowed" for the rest of their ttl; a new key makes them refetch instead.
199+
private val KEY_UPLOAD_POLICY = stringPreferencesKey("cached_upload_policy_v2")
132200
}
133201
}
134202

135203
/**
136204
* On-disk form. [UploadPolicy.ttl] is a [kotlin.time.Duration] (not kotlinx-serializable) so it is
137205
* stored as milliseconds; [fetchedAtMillis] stamps when it was cached so freshness can be checked
138-
* against the ttl; [MimeTypeConstraints] is already `@Serializable` and stored as-is.
206+
* against the ttl; [MimeTypeConstraints] and [EncryptedConstraints] are already `@Serializable` and stored as-is.
139207
*/
140208
@kotlinx.serialization.Serializable
141209
private data class CachedUploadPolicy(
142210
val version: String,
143211
val ttlMillis: Long,
144212
val fetchedAtMillis: Long,
145213
val mimeTypeConstraints: List<MimeTypeConstraints>,
214+
// Absent in entries cached before encrypted uploads existed, which read back as "not allowed"
215+
// until the next refresh.
216+
val encrypted: EncryptedConstraints? = null,
146217
) {
147218
fun toDomain(): UploadPolicy = UploadPolicy(
148219
version = version,
149220
ttl = ttlMillis.milliseconds,
150221
mimeTypeConstraints = mimeTypeConstraints,
222+
encrypted = encrypted,
151223
)
152224

153225
companion object {
@@ -156,6 +228,7 @@ private data class CachedUploadPolicy(
156228
ttlMillis = policy.ttl.inWholeMilliseconds,
157229
fetchedAtMillis = fetchedAtMillis,
158230
mimeTypeConstraints = policy.mimeTypeConstraints,
231+
encrypted = policy.encrypted,
159232
)
160233
}
161234
}
Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
package com.flipcash.app.blob
2+
3+
import kotlin.math.PI
4+
import kotlin.math.abs
5+
import kotlin.math.cos
6+
import kotlin.math.floor
7+
import kotlin.math.max
8+
import kotlin.math.min
9+
import kotlin.math.pow
10+
import kotlin.math.sign
11+
12+
/**
13+
* Encoder for [BlurHash](https://blurha.sh) strings, the counterpart of the decoder in
14+
* `shared/common-ui`. A sealed photo can't be previewed from the server, so its hash is computed
15+
* here from the plaintext and travels inside the encrypted message.
16+
*
17+
* Works on an ARGB [IntArray] so the maths runs on the JVM; [fromThumbnail] is the only part that
18+
* needs a [android.graphics.Bitmap].
19+
*/
20+
object BlurHashEncoder {
21+
22+
private const val THUMBNAIL_EDGE = 64
23+
24+
/**
25+
* Hash of [pixels] (row-major ARGB, alpha ignored) using [componentsX] × [componentsY] DCT
26+
* components. Returns "" when the input is unusable; a missing preview must never fail a send.
27+
*/
28+
fun encode(pixels: IntArray, width: Int, height: Int, componentsX: Int, componentsY: Int): String {
29+
if (width <= 0 || height <= 0 || pixels.size < width * height) return ""
30+
if (componentsX !in 1..9 || componentsY !in 1..9) return ""
31+
32+
val factors = Array(componentsX * componentsY) { index ->
33+
factor(pixels, width, height, index % componentsX, index / componentsX)
34+
}
35+
val dc = factors[0]
36+
val ac = factors.drop(1)
37+
38+
val hash = StringBuilder()
39+
hash.append(encode83((componentsX - 1) + (componentsY - 1) * 9, 1))
40+
41+
val maxValue: Float
42+
if (ac.isNotEmpty()) {
43+
val actualMax = ac.maxOf { channels -> channels.maxOf { abs(it) } }
44+
val quantisedMax = floor(actualMax * 166f - 0.5f).toInt().coerceIn(0, 82)
45+
maxValue = (quantisedMax + 1) / 166f
46+
hash.append(encode83(quantisedMax, 1))
47+
} else {
48+
maxValue = 1f
49+
hash.append(encode83(0, 1))
50+
}
51+
52+
hash.append(encode83(encodeDc(dc), 4))
53+
for (factor in ac) hash.append(encode83(encodeAc(factor, maxValue), 2))
54+
return hash.toString()
55+
}
56+
57+
/**
58+
* Hash of an image of [width] × [height] from a [THUMBNAIL_EDGE]-px thumbnail of [source],
59+
* 4×3 components for landscape (and square) and 3×4 for portrait. "" on any failure.
60+
*/
61+
fun fromThumbnail(source: android.graphics.Bitmap, width: Int, height: Int): String =
62+
runCatching {
63+
val scale = min(1f, THUMBNAIL_EDGE.toFloat() / max(source.width, source.height))
64+
val tw = max((source.width * scale).toInt(), 1)
65+
val th = max((source.height * scale).toInt(), 1)
66+
val thumb = android.graphics.Bitmap.createScaledBitmap(source, tw, th, true)
67+
val pixels = IntArray(tw * th)
68+
thumb.getPixels(pixels, 0, tw, 0, 0, tw, th)
69+
if (thumb !== source) thumb.recycle()
70+
val landscape = width >= height
71+
encode(pixels, tw, th, if (landscape) 4 else 3, if (landscape) 3 else 4)
72+
}.getOrDefault("")
73+
74+
private fun factor(pixels: IntArray, width: Int, height: Int, i: Int, j: Int): FloatArray {
75+
var r = 0f
76+
var g = 0f
77+
var b = 0f
78+
val normalisation = if (i == 0 && j == 0) 1f else 2f
79+
for (y in 0 until height) {
80+
val basisY = cos(PI * j * y / height).toFloat()
81+
for (x in 0 until width) {
82+
val basis = normalisation * cos(PI * i * x / width).toFloat() * basisY
83+
val pixel = pixels[y * width + x]
84+
r += basis * srgbToLinear(pixel shr 16 and 255)
85+
g += basis * srgbToLinear(pixel shr 8 and 255)
86+
b += basis * srgbToLinear(pixel and 255)
87+
}
88+
}
89+
val scale = 1f / (width * height)
90+
return floatArrayOf(r * scale, g * scale, b * scale)
91+
}
92+
93+
private fun encodeDc(value: FloatArray): Int =
94+
(linearToSrgb(value[0]) shl 16) or (linearToSrgb(value[1]) shl 8) or linearToSrgb(value[2])
95+
96+
private fun encodeAc(value: FloatArray, maxValue: Float): Int {
97+
fun quantise(v: Float): Int =
98+
floor(signPow(v / maxValue, 0.5f) * 9f + 9.5f).toInt().coerceIn(0, 18)
99+
return quantise(value[0]) * 19 * 19 + quantise(value[1]) * 19 + quantise(value[2])
100+
}
101+
102+
private fun signPow(value: Float, exp: Float): Float = abs(value).pow(exp) * sign(value)
103+
104+
private fun srgbToLinear(value: Int): Float {
105+
val v = value / 255f
106+
return if (v <= 0.04045f) v / 12.92f else ((v + 0.055f) / 1.055f).pow(2.4f)
107+
}
108+
109+
private fun linearToSrgb(value: Float): Int {
110+
val v = value.coerceIn(0f, 1f)
111+
val srgb = if (v <= 0.0031308f) v * 12.92f else 1.055f * v.pow(1f / 2.4f) - 0.055f
112+
return (srgb * 255f + 0.5f).toInt()
113+
}
114+
115+
private fun encode83(value: Int, length: Int): String {
116+
val out = CharArray(length)
117+
var remaining = value
118+
for (i in length - 1 downTo 0) {
119+
out[i] = CHARS[remaining % 83]
120+
remaining /= 83
121+
}
122+
return String(out)
123+
}
124+
125+
private const val CHARS =
126+
"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#\$%*+,-.:;=?@[]^_{|}~"
127+
}
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package com.flipcash.app.blob
2+
3+
/** Picks which upload-policy entry governs a MIME type. */
4+
object ChatMediaConstraints {
5+
6+
/**
7+
* Index of the first entry in [patterns], in policy order, that matches [mimeType]: the
8+
* catch-all pattern matches anything, a type wildcard matches on the type alone, and anything
9+
* else is an exact `type/subtype`.
10+
* Null when [mimeType] has no `/` or nothing matches.
11+
*
12+
* Policy order is authoritative — a later, more specific entry never overrides an earlier
13+
* catch-all. Must agree with iOS (`test-vectors/chat_media.json`, `constraintSelection`).
14+
*/
15+
fun firstMatchIndex(patterns: List<String>, mimeType: String): Int? {
16+
val slash = mimeType.indexOf('/')
17+
if (slash < 0) return null
18+
val type = mimeType.substring(0, slash)
19+
20+
val index = patterns.indexOfFirst { pattern ->
21+
when {
22+
pattern == "*/*" -> true
23+
pattern.endsWith("/*") -> pattern.dropLast(2).equals(type, ignoreCase = true)
24+
else -> pattern.equals(mimeType, ignoreCase = true)
25+
}
26+
}
27+
return index.takeIf { it >= 0 }
28+
}
29+
}
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package com.flipcash.app.blob
2+
3+
import kotlin.math.max
4+
import kotlin.math.min
5+
import kotlin.math.sqrt
6+
7+
data class PixelSize(val width: Int, val height: Int)
8+
9+
/**
10+
* The pixel size a photo is scaled to before encoding. A bound of 0 means unbounded, and an image
11+
* is never scaled up.
12+
*/
13+
object ChatMediaDownscale {
14+
15+
/**
16+
* [width] and [height] are display-space — after the EXIF rotation has been applied.
17+
*
18+
* The scale is computed in doubles and truncated, which can land a pixel or two over
19+
* [maxPixels], so the longer side is then walked down until the area fits. Must agree with iOS
20+
* (`test-vectors/chat_media.json`, `downscale`) to the pixel.
21+
*/
22+
fun target(width: Int, height: Int, maxWidth: Int, maxHeight: Int, maxPixels: Long): PixelSize {
23+
if (width <= 0 || height <= 0) return PixelSize(max(width, 1), max(height, 1))
24+
25+
var scale = 1.0
26+
if (maxWidth > 0) scale = min(scale, maxWidth.toDouble() / width)
27+
if (maxHeight > 0) scale = min(scale, maxHeight.toDouble() / height)
28+
if (maxPixels > 0) scale = min(scale, sqrt(maxPixels.toDouble() / (width.toDouble() * height)))
29+
30+
var w = max((width * scale).toInt(), 1)
31+
var h = max((height * scale).toInt(), 1)
32+
33+
if (maxPixels > 0) {
34+
while (w.toLong() * h > maxPixels && (w > 1 || h > 1)) {
35+
if (w >= h) w-- else h--
36+
}
37+
}
38+
return PixelSize(w, h)
39+
}
40+
}

0 commit comments

Comments
 (0)