From 4fbb5465eae7c801dd95dae98c84564ae4c21a5b Mon Sep 17 00:00:00 2001 From: Joshua Blum Date: Wed, 6 May 2026 13:22:39 -0400 Subject: [PATCH 1/5] classify audio attachments --- go/chat/attachments/preprocess.go | 1 + go/chat/attachments/preview.go | 10 ++++++++++ go/chat/attachments/preview_android.go | 10 +++++++++- go/chat/attachments/preview_darwin.go | 12 ++++++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/go/chat/attachments/preprocess.go b/go/chat/attachments/preprocess.go index b9307129df3b..68f21c19b707 100644 --- a/go/chat/attachments/preprocess.go +++ b/go/chat/attachments/preprocess.go @@ -278,6 +278,7 @@ func PreprocessAsset(ctx context.Context, g *globals.Context, log utils.DebugLab p.PreviewDim = &Dimension{Width: previewRes.PreviewWidth, Height: previewRes.PreviewHeight} } p.BaseDurationMs = previewRes.BaseDurationMs + p.BaseIsAudio = previewRes.BaseIsAudio p.PreviewDurationMs = previewRes.PreviewDurationMs } diff --git a/go/chat/attachments/preview.go b/go/chat/attachments/preview.go index f6e927999337..b2fb9d7a219b 100644 --- a/go/chat/attachments/preview.go +++ b/go/chat/attachments/preview.go @@ -13,6 +13,7 @@ import ( "image/jpeg" "image/png" "io" + "path/filepath" "strings" "github.com/keybase/client/go/chat/types" @@ -31,12 +32,21 @@ const ( previewImageHeight = 640 ) +func isAudioExtension(basename string) bool { + switch strings.ToLower(filepath.Ext(basename)) { + case ".m4a", ".mp3", ".aac", ".ogg", ".flac", ".wav": + return true + } + return false +} + type PreviewRes struct { Source []byte ContentType string BaseWidth int BaseHeight int BaseDurationMs int + BaseIsAudio bool PreviewWidth int PreviewHeight int PreviewDurationMs int diff --git a/go/chat/attachments/preview_android.go b/go/chat/attachments/preview_android.go index 483fb8241469..f24f26e6e0eb 100644 --- a/go/chat/attachments/preview_android.go +++ b/go/chat/attachments/preview_android.go @@ -23,7 +23,15 @@ func previewVideo(ctx context.Context, log utils.DebugLabeler, src io.Reader, log.Debug(ctx, "previewVideo: size: %d duration: %d", len(dat), duration) if len(dat) == 0 { log.Debug(ctx, "failed to generate preview from native, using blank image") - return previewVideoBlank(ctx, log, src, basename) + blank, blankErr := previewVideoBlank(ctx, log, src, basename) + if blankErr != nil { + return res, blankErr + } + if duration > 1 { + blank.BaseDurationMs = duration + blank.BaseIsAudio = isAudioExtension(basename) + } + return blank, nil } imagePreview, err := previewImage(ctx, log, bytes.NewReader(dat), basename, "image/jpeg") if err != nil { diff --git a/go/chat/attachments/preview_darwin.go b/go/chat/attachments/preview_darwin.go index 7fbe050d2a27..24196cf233ba 100644 --- a/go/chat/attachments/preview_darwin.go +++ b/go/chat/attachments/preview_darwin.go @@ -137,6 +137,18 @@ func previewVideo(ctx context.Context, log utils.DebugLabeler, src io.Reader, } log.Debug(ctx, "previewVideo: length: %d duration: %ds", result.imageLength, duration) if result.imageLength == 0 { + // Audio-only files (e.g. M4A) have no video track, so no thumbnail, but AVFoundation + // can still read their duration. Return a blank preview carrying the real duration so + // the snippet and UI show the correct length instead of "00:00". + if duration > 1 { + blank, blankErr := previewVideoBlank(ctx, log, src, basename) + if blankErr != nil { + return res, blankErr + } + blank.BaseDurationMs = duration + blank.BaseIsAudio = isAudioExtension(basename) + return blank, nil + } return res, errors.New("no data returned from native") } localDat := make([]byte, result.imageLength) From 98330ad2fdd09de5a6a887946941406f04d2bbf5 Mon Sep 17 00:00:00 2001 From: Joshua Blum Date: Wed, 6 May 2026 14:48:04 -0400 Subject: [PATCH 2/5] compute amps --- go/bind/keybase.go | 18 +++ go/chat/attachments/preprocess.go | 1 + go/chat/attachments/preview.go | 10 +- go/chat/attachments/preview_android.go | 14 +- go/chat/attachments/preview_darwin.go | 150 ++++++++++++++++-- go/chat/attachments/preview_native.go | 55 +++++++ go/chat/attachments/preview_native_test.go | 22 +++ go/chat/types/interfaces.go | 1 + go/chat/types/types.go | 4 + .../io/keybase/ossifrage/util/VideoHelper.kt | 110 +++++++++++++ 10 files changed, 356 insertions(+), 29 deletions(-) create mode 100644 go/chat/attachments/preview_native.go create mode 100644 go/chat/attachments/preview_native_test.go diff --git a/go/bind/keybase.go b/go/bind/keybase.go index 029b5ab3fd9c..3a7b6d1489c3 100644 --- a/go/bind/keybase.go +++ b/go/bind/keybase.go @@ -5,9 +5,11 @@ package keybase import ( "context" + "encoding/binary" "encoding/json" "errors" "fmt" + "math" "net" "os" "path/filepath" @@ -102,6 +104,9 @@ type PushNotifier interface { type NativeVideoHelper interface { Thumbnail(filename string) []byte Duration(filename string) int + // AudioAmps returns IEEE 754 float32 samples encoded as little-endian bytes, + // representing RMS amplitude values in [0,1] for waveform visualization. + AudioAmps(filename string) []byte } // ShareIntentDonator is implemented by the native iOS layer to donate INSendMessageIntent @@ -189,6 +194,19 @@ func (v videoHelper) ThumbnailAndDuration(ctx context.Context, filename string) return v.nvh.Thumbnail(filename), v.nvh.Duration(filename), nil } +func (v videoHelper) AudioAmps(ctx context.Context, filename string) ([]float64, error) { + data := v.nvh.AudioAmps(filename) + if len(data) == 0 || len(data)%4 != 0 { + return nil, nil + } + amps := make([]float64, len(data)/4) + for i := range amps { + bits := binary.LittleEndian.Uint32(data[i*4:]) + amps[i] = float64(math.Float32frombits(bits)) + } + return amps, nil +} + type ExternalDNSNSFetcher interface { GetServers() []byte } diff --git a/go/chat/attachments/preprocess.go b/go/chat/attachments/preprocess.go index 68f21c19b707..c49b82b689e9 100644 --- a/go/chat/attachments/preprocess.go +++ b/go/chat/attachments/preprocess.go @@ -279,6 +279,7 @@ func PreprocessAsset(ctx context.Context, g *globals.Context, log utils.DebugLab } p.BaseDurationMs = previewRes.BaseDurationMs p.BaseIsAudio = previewRes.BaseIsAudio + p.PreviewAudioAmps = previewRes.AudioAmps p.PreviewDurationMs = previewRes.PreviewDurationMs } diff --git a/go/chat/attachments/preview.go b/go/chat/attachments/preview.go index b2fb9d7a219b..d40b6a8f5e45 100644 --- a/go/chat/attachments/preview.go +++ b/go/chat/attachments/preview.go @@ -13,7 +13,6 @@ import ( "image/jpeg" "image/png" "io" - "path/filepath" "strings" "github.com/keybase/client/go/chat/types" @@ -32,14 +31,6 @@ const ( previewImageHeight = 640 ) -func isAudioExtension(basename string) bool { - switch strings.ToLower(filepath.Ext(basename)) { - case ".m4a", ".mp3", ".aac", ".ogg", ".flac", ".wav": - return true - } - return false -} - type PreviewRes struct { Source []byte ContentType string @@ -47,6 +38,7 @@ type PreviewRes struct { BaseHeight int BaseDurationMs int BaseIsAudio bool + AudioAmps []float64 PreviewWidth int PreviewHeight int PreviewDurationMs int diff --git a/go/chat/attachments/preview_android.go b/go/chat/attachments/preview_android.go index f24f26e6e0eb..32c6c35f24ca 100644 --- a/go/chat/attachments/preview_android.go +++ b/go/chat/attachments/preview_android.go @@ -21,17 +21,13 @@ func previewVideo(ctx context.Context, log utils.DebugLabeler, src io.Reader, return res, err } log.Debug(ctx, "previewVideo: size: %d duration: %d", len(dat), duration) + if len(dat) == 0 && duration > 1 && isAudioExtension(basename) { + amps, _ := nvh.AudioAmps(ctx, basename) + return previewAudio(duration, amps) + } if len(dat) == 0 { log.Debug(ctx, "failed to generate preview from native, using blank image") - blank, blankErr := previewVideoBlank(ctx, log, src, basename) - if blankErr != nil { - return res, blankErr - } - if duration > 1 { - blank.BaseDurationMs = duration - blank.BaseIsAudio = isAudioExtension(basename) - } - return blank, nil + return previewVideoBlank(ctx, log, src, basename) } imagePreview, err := previewImage(ctx, log, bytes.NewReader(dat), basename, "image/jpeg") if err != nil { diff --git a/go/chat/attachments/preview_darwin.go b/go/chat/attachments/preview_darwin.go index 24196cf233ba..ac6efa7b4e94 100644 --- a/go/chat/attachments/preview_darwin.go +++ b/go/chat/attachments/preview_darwin.go @@ -12,6 +12,7 @@ package attachments #include #include #include +#include #include #if TARGET_OS_IPHONE #include @@ -31,6 +32,11 @@ typedef struct { int imageLength; } ImageConversionResult; +typedef struct { + float* data; + int length; +} AudioAmpsResult; + VideoPreviewResult MakeVideoThumbnail(const char* inFilename) { VideoPreviewResult result = {NULL, 0, 0}; NSString* filename = [NSString stringWithUTF8String:inFilename]; @@ -71,6 +77,118 @@ VideoPreviewResult MakeVideoThumbnail(const char* inFilename) { return result; } +// GetAudioAmplitudes reads PCM samples from an audio file via AVAssetReader and +// returns numSamples RMS amplitude values in [0,1]. The caller must free result.data. +AudioAmpsResult GetAudioAmplitudes(const char* inFilename, int numSamples) { + AudioAmpsResult result = {NULL, 0}; + if (numSamples <= 0) return result; + + NSString* filename = [NSString stringWithUTF8String:inFilename]; + NSURL* url = [NSURL fileURLWithPath:filename]; + AVURLAsset* asset = [AVURLAsset URLAssetWithURL:url options:nil]; + + NSArray* audioTracks = [asset tracksWithMediaType:AVMediaTypeAudio]; + if (audioTracks.count == 0) return result; + AVAssetTrack* audioTrack = audioTracks[0]; + + Float64 durationSeconds = CMTimeGetSeconds(asset.duration); + if (!(durationSeconds > 0)) return result; + + Float64 sampleRate = 0; + for (id formatDescription in audioTrack.formatDescriptions) { + CMAudioFormatDescriptionRef desc = (__bridge CMAudioFormatDescriptionRef)formatDescription; + const AudioStreamBasicDescription* asbd = + CMAudioFormatDescriptionGetStreamBasicDescription(desc); + if (asbd && asbd->mSampleRate > 0) { + sampleRate = asbd->mSampleRate; + break; + } + } + if (sampleRate <= 0) { + sampleRate = 44100; + } + long long totalSamples = llround(durationSeconds * sampleRate); + if (totalSamples < numSamples) { + totalSamples = numSamples; + } + + NSError* error = nil; + AVAssetReader* reader = [AVAssetReader assetReaderWithAsset:asset error:&error]; + if (!reader || error) return result; + + NSDictionary* outputSettings = @{ + AVFormatIDKey: @(kAudioFormatLinearPCM), + AVLinearPCMBitDepthKey: @32, + AVLinearPCMIsFloatKey: @YES, + AVLinearPCMIsNonInterleaved: @NO, + AVNumberOfChannelsKey: @1, + }; + + AVAssetReaderTrackOutput* output = [AVAssetReaderTrackOutput + assetReaderTrackOutputWithTrack:audioTrack + outputSettings:outputSettings]; + output.alwaysCopiesSampleData = NO; + + if (![reader canAddOutput:output]) return result; + [reader addOutput:output]; + if (![reader startReading]) return result; + + float* sumSq = (float*)calloc(numSamples, sizeof(float)); + unsigned int* counts = (unsigned int*)calloc(numSamples, sizeof(unsigned int)); + if (!sumSq || !counts) { + free(sumSq); + free(counts); + return result; + } + + long long sampleIndex = 0; + while (reader.status == AVAssetReaderStatusReading) { + CMSampleBufferRef sampleBuffer = [output copyNextSampleBuffer]; + if (!sampleBuffer) break; + CMBlockBufferRef blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer); + if (blockBuffer) { + size_t length = CMBlockBufferGetDataLength(blockBuffer); + if (length >= sizeof(float)) { + float* chunk = (float*)malloc(length); + if (chunk && CMBlockBufferCopyDataBytes(blockBuffer, 0, length, chunk) == kCMBlockBufferNoErr) { + size_t floatCount = length / sizeof(float); + for (size_t i = 0; i < floatCount; i++) { + int bucket = (int)(((sampleIndex + (long long)i) * numSamples) / totalSamples); + if (bucket >= numSamples) { + bucket = numSamples - 1; + } + float s = chunk[i]; + sumSq[bucket] += s * s; + counts[bucket]++; + } + sampleIndex += (long long)floatCount; + } + free(chunk); + } + } + CFRelease(sampleBuffer); + } + + float* amps = (float*)calloc(numSamples, sizeof(float)); + if (!amps) { + free(sumSq); + free(counts); + return result; + } + + for (int i = 0; i < numSamples; i++) { + if (counts[i] > 0) { + amps[i] = sqrtf(sumSq[i] / (float)counts[i]); + } + } + free(sumSq); + free(counts); + + result.data = amps; + result.length = numSamples; + return result; +} + #if TARGET_OS_IPHONE ImageConversionResult HEICToJPEG(const char* inFilename) { ImageConversionResult result = {NULL, 0}; @@ -121,6 +239,22 @@ import ( "github.com/keybase/client/go/chat/utils" ) +func getAudioAmps(basename string) []float64 { + cbasename := C.CString(basename) + defer C.free(unsafe.Pointer(cbasename)) + result := C.GetAudioAmplitudes(cbasename, C.int(audioAmpsCount)) + if result.length == 0 || result.data == nil { + return nil + } + defer C.free(unsafe.Pointer(result.data)) + amps := make([]float64, int(result.length)) + cData := (*[1 << 20]C.float)(unsafe.Pointer(result.data))[:int(result.length):int(result.length)] + for i, v := range cData { + amps[i] = float64(v) + } + return amps +} + func previewVideo(ctx context.Context, log utils.DebugLabeler, src io.Reader, basename string, nvh types.NativeVideoHelper, ) (res *PreviewRes, err error) { @@ -137,17 +271,11 @@ func previewVideo(ctx context.Context, log utils.DebugLabeler, src io.Reader, } log.Debug(ctx, "previewVideo: length: %d duration: %ds", result.imageLength, duration) if result.imageLength == 0 { - // Audio-only files (e.g. M4A) have no video track, so no thumbnail, but AVFoundation - // can still read their duration. Return a blank preview carrying the real duration so - // the snippet and UI show the correct length instead of "00:00". - if duration > 1 { - blank, blankErr := previewVideoBlank(ctx, log, src, basename) - if blankErr != nil { - return res, blankErr - } - blank.BaseDurationMs = duration - blank.BaseIsAudio = isAudioExtension(basename) - return blank, nil + // Audio-only files (e.g. M4A) have no video track so no thumbnail, but AVFoundation + // can still read their duration. Extract amplitude data for the waveform visualization. + if duration > 1 && isAudioExtension(basename) { + amps := getAudioAmps(basename) + return previewAudio(duration, amps) } return res, errors.New("no data returned from native") } diff --git a/go/chat/attachments/preview_native.go b/go/chat/attachments/preview_native.go new file mode 100644 index 000000000000..474bb9a79f23 --- /dev/null +++ b/go/chat/attachments/preview_native.go @@ -0,0 +1,55 @@ +//go:build darwin || android +// +build darwin android + +package attachments + +import ( + "math" + "path/filepath" + "strings" +) + +const audioAmpsCount = 60 + +func isAudioExtension(basename string) bool { + switch strings.ToLower(filepath.Ext(basename)) { + case ".m4a", ".mp3", ".aac", ".ogg", ".flac", ".wav": + return true + } + return false +} + +func normalizeAudioAmps(amps []float64) []float64 { + if len(amps) == 0 { + return make([]float64, audioAmpsCount) + } + return amps +} + +// previewAudio generates a waveform preview image and packages amplitude data +// for an audio-only file. amps are linear RMS values in [0,1]. +func previewAudio(duration int, amps []float64) (*PreviewRes, error) { + amps = normalizeAudioAmps(amps) + // audioVisualizer expects dB values; convert from linear RMS. + dbAmps := make([]float64, len(amps)) + for i, a := range amps { + if a <= 0 { + dbAmps[i] = -80 + } else { + dbAmps[i] = 20 * math.Log10(a) + } + } + v := newAudioVisualizer(dbAmps) + dat, width := v.visualize() + return &PreviewRes{ + Source: dat, + ContentType: "image/png", + BaseWidth: width, + BaseHeight: v.height, + BaseDurationMs: duration, + BaseIsAudio: true, + AudioAmps: amps, + PreviewWidth: width, + PreviewHeight: v.height, + }, nil +} diff --git a/go/chat/attachments/preview_native_test.go b/go/chat/attachments/preview_native_test.go new file mode 100644 index 000000000000..72f7fbb21f24 --- /dev/null +++ b/go/chat/attachments/preview_native_test.go @@ -0,0 +1,22 @@ +//go:build darwin || android +// +build darwin android + +package attachments + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPreviewAudioEmptyAmps(t *testing.T) { + res, err := previewAudio(1234, nil) + require.NoError(t, err) + require.NotNil(t, res) + require.True(t, res.BaseIsAudio) + require.Equal(t, 1234, res.BaseDurationMs) + require.Len(t, res.AudioAmps, audioAmpsCount) + require.NotEmpty(t, res.Source) + require.Positive(t, res.PreviewWidth) + require.Positive(t, res.PreviewHeight) +} diff --git a/go/chat/types/interfaces.go b/go/chat/types/interfaces.go index 88495819ca49..43708f0b88dc 100644 --- a/go/chat/types/interfaces.go +++ b/go/chat/types/interfaces.go @@ -444,6 +444,7 @@ type AttachmentUploader interface { type NativeVideoHelper interface { ThumbnailAndDuration(ctx context.Context, filename string) ([]byte, int, error) + AudioAmps(ctx context.Context, filename string) ([]float64, error) } // ShareConversation holds data for donating a conversation to the iOS share sheet. diff --git a/go/chat/types/types.go b/go/chat/types/types.go index 7e2614335428..10d07040250c 100644 --- a/go/chat/types/types.go +++ b/go/chat/types/types.go @@ -563,6 +563,10 @@ func (d DummyNativeVideoHelper) ThumbnailAndDuration(ctx context.Context, filena return nil, 0, nil } +func (d DummyNativeVideoHelper) AudioAmps(ctx context.Context, filename string) ([]float64, error) { + return nil, nil +} + type UnfurlerTaskStatus int const ( diff --git a/shared/android/app/src/main/java/io/keybase/ossifrage/util/VideoHelper.kt b/shared/android/app/src/main/java/io/keybase/ossifrage/util/VideoHelper.kt index 9877c31fdc15..32803015aa56 100644 --- a/shared/android/app/src/main/java/io/keybase/ossifrage/util/VideoHelper.kt +++ b/shared/android/app/src/main/java/io/keybase/ossifrage/util/VideoHelper.kt @@ -1,9 +1,15 @@ package io.keybase.ossifrage.util import android.graphics.Bitmap +import android.media.MediaCodec +import android.media.MediaExtractor +import android.media.MediaFormat import android.media.MediaMetadataRetriever import keybase.NativeVideoHelper import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder +import kotlin.math.sqrt class VideoHelper : NativeVideoHelper { override fun thumbnail(filename: String): ByteArray { @@ -35,4 +41,108 @@ class VideoHelper : NativeVideoHelper { 0 } } + + override fun audioAmps(filename: String): ByteArray { + val numSamples = 60 + val extractor = MediaExtractor() + var codec: MediaCodec? = null + try { + extractor.setDataSource(filename) + val audioTrackIndex = (0 until extractor.trackCount).firstOrNull { i -> + extractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME)?.startsWith("audio/") == true + } ?: return ByteArray(0) + + extractor.selectTrack(audioTrackIndex) + val format = extractor.getTrackFormat(audioTrackIndex) + val mime = format.getString(MediaFormat.KEY_MIME) ?: return ByteArray(0) + + codec = MediaCodec.createDecoderByType(mime) + codec.configure(format, null, null, 0) + codec.start() + + val sampleRate = if (format.containsKey(MediaFormat.KEY_SAMPLE_RATE)) { + format.getInteger(MediaFormat.KEY_SAMPLE_RATE) + } else { + 44_100 + } + val channelCount = if (format.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) { + format.getInteger(MediaFormat.KEY_CHANNEL_COUNT) + } else { + 1 + } + val durationUs = if (format.containsKey(MediaFormat.KEY_DURATION)) { + format.getLong(MediaFormat.KEY_DURATION) + } else { + 0L + } + val totalSamplesEstimate = maxOf( + numSamples.toLong(), + durationUs * sampleRate.toLong() * channelCount.toLong() / 1_000_000L, + ) + val sumSq = DoubleArray(numSamples) + val counts = IntArray(numSamples) + var sampleIndex = 0L + val info = MediaCodec.BufferInfo() + var inputDone = false + var outputDone = false + + while (!outputDone) { + if (!inputDone) { + val inputIndex = codec.dequeueInputBuffer(10_000L) + if (inputIndex >= 0) { + val inputBuffer = codec.getInputBuffer(inputIndex)!! + val sampleSize = extractor.readSampleData(inputBuffer, 0) + if (sampleSize < 0) { + codec.queueInputBuffer(inputIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM) + inputDone = true + } else { + codec.queueInputBuffer(inputIndex, 0, sampleSize, extractor.sampleTime, 0) + extractor.advance() + } + } + } + val outputIndex = codec.dequeueOutputBuffer(info, 10_000L) + if (outputIndex >= 0) { + val outputBuffer = codec.getOutputBuffer(outputIndex) + if (outputBuffer != null && info.size > 0) { + // PCM output is 16-bit signed by default on Android. + val pcmBuffer = outputBuffer.duplicate() + pcmBuffer.position(info.offset) + pcmBuffer.limit(info.offset + info.size) + val shortBuf = pcmBuffer.slice().order(ByteOrder.nativeOrder()).asShortBuffer() + while (shortBuf.hasRemaining()) { + val sample = shortBuf.get().toDouble() / Short.MAX_VALUE.toDouble() + val bucket = minOf( + numSamples - 1, + ((sampleIndex * numSamples) / totalSamplesEstimate).toInt(), + ) + sumSq[bucket] += sample * sample + counts[bucket]++ + sampleIndex++ + } + } + codec.releaseOutputBuffer(outputIndex, false) + if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) outputDone = true + } + } + + if (sampleIndex == 0L) return ByteArray(0) + + val result = ByteBuffer.allocate(numSamples * 4).order(ByteOrder.LITTLE_ENDIAN) + for (i in 0 until numSamples) { + if (counts[i] > 0) { + result.putFloat(sqrt(sumSq[i] / counts[i]).toFloat()) + } else { + result.putFloat(0f) + } + } + return result.array() + } catch (e: Exception) { + return ByteArray(0) + } finally { + runCatching { codec?.stop() } + codec?.release() + extractor.release() + } + } } From 0261fdeb63866c661d8600e09f08b3b5d1da6574 Mon Sep 17 00:00:00 2001 From: Joshua Blum Date: Mon, 11 May 2026 14:00:55 -0400 Subject: [PATCH 3/5] better audio attachment support --- go/chat/attachments/preview_android.go | 5 +- go/chat/attachments/preview_native.go | 2 +- .../attachment-fullscreen/hooks.tsx | 10 +-- .../attachment-fullscreen/index.desktop.tsx | 6 +- .../attachment-fullscreen/index.native.tsx | 8 +- .../messages/attachment/audio.tsx | 72 ++++++++------- .../conversation/messages/attachment/file.tsx | 5 +- .../messages/attachment/shared.tsx | 90 ++++++++++++------- 8 files changed, 116 insertions(+), 82 deletions(-) diff --git a/go/chat/attachments/preview_android.go b/go/chat/attachments/preview_android.go index 32c6c35f24ca..b64fad100271 100644 --- a/go/chat/attachments/preview_android.go +++ b/go/chat/attachments/preview_android.go @@ -22,7 +22,10 @@ func previewVideo(ctx context.Context, log utils.DebugLabeler, src io.Reader, } log.Debug(ctx, "previewVideo: size: %d duration: %d", len(dat), duration) if len(dat) == 0 && duration > 1 && isAudioExtension(basename) { - amps, _ := nvh.AudioAmps(ctx, basename) + amps, ampErr := nvh.AudioAmps(ctx, basename) + if ampErr != nil { + log.Debug(ctx, "previewVideo: AudioAmps failed: %v", ampErr) + } return previewAudio(duration, amps) } if len(dat) == 0 { diff --git a/go/chat/attachments/preview_native.go b/go/chat/attachments/preview_native.go index 474bb9a79f23..1e46056be29c 100644 --- a/go/chat/attachments/preview_native.go +++ b/go/chat/attachments/preview_native.go @@ -13,7 +13,7 @@ const audioAmpsCount = 60 func isAudioExtension(basename string) bool { switch strings.ToLower(filepath.Ext(basename)) { - case ".m4a", ".mp3", ".aac", ".ogg", ".flac", ".wav": + case ".m4a", ".mp3", ".aac", ".ogg", ".flac", ".wav", ".opus", ".aiff", ".caf": return true } return false diff --git a/shared/chat/conversation/attachment-fullscreen/hooks.tsx b/shared/chat/conversation/attachment-fullscreen/hooks.tsx index 584d3cdeacc9..7bb897748eea 100644 --- a/shared/chat/conversation/attachment-fullscreen/hooks.tsx +++ b/shared/chat/conversation/attachment-fullscreen/hooks.tsx @@ -51,7 +51,7 @@ export const useData = (initialOrdinal: T.Chat.Ordinal) => { maxHeight ) - const isVideo = message.fileType.startsWith('video') + const isPlayableMedia = message.fileType.startsWith('video') || message.fileType.startsWith('audio') const showPreview = !fileType.includes('png') const onAllMedia = () => showInfoPanel(true, 'attachments') const onClose = () => navigateUp() @@ -75,7 +75,7 @@ export const useData = (initialOrdinal: T.Chat.Ordinal) => { return { fullHeight, fullWidth, - isVideo, + isPlayableMedia, message, onAllMedia, onClose, @@ -101,12 +101,12 @@ const seenPaths = new Set() export const usePreviewFallback = ( path: string, previewPath: string, - isVideo: boolean, + isPlayableMedia: boolean, showPreview: boolean, preload: (path: string, onLoad: () => void, onError: () => void) => void ) => { const [imgSrc, setImgSrc] = React.useState('') - const canUseFallback = path && previewPath && !isVideo && showPreview + const canUseFallback = path && previewPath && !isPlayableMedia && showPreview React.useEffect(() => { const onLoad = () => { @@ -128,7 +128,7 @@ export const usePreviewFallback = ( return () => { clearTimeout(id) } - }, [path, previewPath, isVideo, preload]) + }, [path, previewPath, isPlayableMedia, preload]) if (seenPaths.has(path)) { return path diff --git a/shared/chat/conversation/attachment-fullscreen/index.desktop.tsx b/shared/chat/conversation/attachment-fullscreen/index.desktop.tsx index 0c122da1e608..054d320b60a5 100644 --- a/shared/chat/conversation/attachment-fullscreen/index.desktop.tsx +++ b/shared/chat/conversation/attachment-fullscreen/index.desktop.tsx @@ -34,7 +34,7 @@ const Fullscreen = React.memo(function Fullscreen(p: Props) { const data = useData(p.ordinal) const {message, ordinal, path, title, progress, previewPath} = data const {progressLabel, onNextAttachment, onPreviousAttachment, onClose} = data - const {onDownloadAttachment, onShowInFinder, isVideo} = data + const {onDownloadAttachment, onShowInFinder, isPlayableMedia} = data const {fullWidth, fullHeight} = data const [isZoomed, setIsZoomed] = React.useState(false) @@ -49,7 +49,7 @@ const Fullscreen = React.memo(function Fullscreen(p: Props) { img.onerror = onError }, []) - const imgSrc = usePreviewFallback(path, previewPath, isVideo, data.showPreview, preload) + const imgSrc = usePreviewFallback(path, previewPath, isPlayableMedia, data.showPreview, preload) const forceDims = React.useMemo(() => { return fullHeight && fullWidth ? {height: fullHeight, width: fullWidth} : undefined @@ -108,7 +108,7 @@ const Fullscreen = React.memo(function Fullscreen(p: Props) { style={Kb.Styles.globalStyles.flexGrow} key={path} > - {isVideo ? ( + {isPlayableMedia ? (