From 17cd2e385e585c58a6b3dba2ffd133574517e79c Mon Sep 17 00:00:00 2001 From: timo <44401485+Timo972@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:01:06 +0200 Subject: [PATCH] fix: rotate ReplayKit frames upright on landscape ReplayKit delivers the screen as a portrait buffer and carries the device orientation only in RPVideoSampleOrientationKey. Since e0d899d1 the pipeline swaps the encoder footprint on rotation, but the pixel transfer only letterboxed the portrait buffer into it, so landscape streamed a narrow portrait strip with the UI sideways. Rotate the frame before encoding. iOS has no rotating pixel transfer and a VTPixelRotationSession cannot scale, so a rotated frame is scaled into an encoder-sized buffer in the un-rotated orientation and then rotated into the encoder buffer. Upright frames keep the existing path, and the direct-source shortcut is skipped while a rotation is needed. Rotation needs iOS 16; iOS 15 keeps the previous behaviour. The quarter-turn mapping was measured on an iPhone 15 Pro (iOS 26.6.1): it reads upright opposite to the CGImagePropertyOrientation docs. Refs DRO-3359 Co-Authored-By: Claude Opus 5.5 --- .../FBExtSessionPipeline.m | 155 +++++++++++++++++- .../Utilities/FBBroadcastProtocol.h | 15 ++ .../Utilities/FBBroadcastProtocol.m | 16 ++ .../UnitTests/FBVideoStreamSessionTests.m | 22 +++ 4 files changed, 205 insertions(+), 3 deletions(-) diff --git a/WebDriverAgentBroadcast/FBExtSessionPipeline.m b/WebDriverAgentBroadcast/FBExtSessionPipeline.m index ac2a7fd31..5f600b537 100644 --- a/WebDriverAgentBroadcast/FBExtSessionPipeline.m +++ b/WebDriverAgentBroadcast/FBExtSessionPipeline.m @@ -49,6 +49,14 @@ @interface FBExtSessionPipeline () { @property (atomic) BOOL active; @property (atomic) uint8_t currentOrientation; @property (nonatomic) BOOL directSourceEncodingDisabled; +/** VTPixelRotationSessionRef (iOS 16+), created on the first rotated frame. */ +@property (nonatomic, nullable) CFTypeRef rotationSession; +/** The rotation currently set on rotationSession. */ +@property (nonatomic) FBBroadcastRotation sessionRotation; +/** Encoder-sized buffers in the un-rotated orientation, the rotation session's input. */ +@property (nonatomic, nullable) CVPixelBufferPoolRef unrotatedPool; +@property (nonatomic) NSUInteger unrotatedWidth; +@property (nonatomic) NSUInteger unrotatedHeight; @property (nonatomic, nullable, copy) NSData *lastSentParameterSets; @property (nonatomic) NSUInteger failedReconfigureWidth; @property (nonatomic) NSUInteger failedReconfigureHeight; @@ -427,7 +435,15 @@ - (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer } } + // ReplayKit buffers stay portrait-shaped; the orientation attachment says how to turn them. + // Without the turn, a landscape frame is letterboxed sideways into the swapped footprint. + FBBroadcastRotation rotation = FBBroadcastRotationNone; + if (@available(iOS 16.0, *)) { + rotation = FBBroadcastFrameRotation(orientation); + } + if (!self.directSourceEncodingDisabled + && rotation == FBBroadcastRotationNone && CVPixelBufferGetWidth(sourceBuffer) == self.width && CVPixelBufferGetHeight(sourceBuffer) == self.height) { NSError *error; @@ -454,9 +470,10 @@ - (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer return; } - OSStatus transferStatus = VTPixelTransferSessionTransferImage(self.transferSession, sourceBuffer, scaledBuffer); - if (transferStatus != noErr) { - FBExtLogError("Session %u: pixel transfer failed (status %d)", self.sessionId, (int)transferStatus); + BOOL transferred = rotation == FBBroadcastRotationNone + ? [self transferSourceBuffer:sourceBuffer intoBuffer:scaledBuffer] + : [self transferSourceBuffer:sourceBuffer intoBuffer:scaledBuffer rotation:rotation]; + if (!transferred) { CVPixelBufferRelease(scaledBuffer); return; } @@ -471,6 +488,127 @@ - (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer CVPixelBufferRelease(scaledBuffer); } +/** Scales sourceBuffer into destination (letterboxed). */ +- (BOOL)transferSourceBuffer:(CVPixelBufferRef)sourceBuffer intoBuffer:(CVPixelBufferRef)destination +{ + OSStatus transferStatus = VTPixelTransferSessionTransferImage(self.transferSession, sourceBuffer, destination); + if (transferStatus != noErr) { + FBExtLogError("Session %u: pixel transfer failed (status %d)", self.sessionId, (int)transferStatus); + return NO; + } + return YES; +} + +/** + Scales sourceBuffer into an un-rotated buffer of destination's size (axes swapped for a quarter + turn), then rotates that into destination. iOS has no rotating pixel transfer, and a rotation + session cannot scale, so the turn runs on the small encoder-sized buffer. + */ +- (BOOL)transferSourceBuffer:(CVPixelBufferRef)sourceBuffer + intoBuffer:(CVPixelBufferRef)destination + rotation:(FBBroadcastRotation)rotation API_AVAILABLE(ios(16.0)) +{ + BOOL quarterTurn = rotation == FBBroadcastRotationClockwise90 + || rotation == FBBroadcastRotationCounterclockwise90; + NSUInteger width = quarterTurn ? self.height : self.width; + NSUInteger height = quarterTurn ? self.width : self.height; + if (![self prepareRotationForWidth:width height:height rotation:rotation]) { + return NO; + } + + CVPixelBufferRef unrotatedBuffer = NULL; + NSDictionary *auxAttributes = @{(id)kCVPixelBufferPoolAllocationThresholdKey: @(POOL_ALLOCATION_THRESHOLD)}; + CVReturn poolStatus = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault, + self.unrotatedPool, + (__bridge CFDictionaryRef)auxAttributes, + &unrotatedBuffer); + if (poolStatus != kCVReturnSuccess || NULL == unrotatedBuffer) { + self.droppedPoolCount += 1; + return NO; + } + BOOL rotated = NO; + if ([self transferSourceBuffer:sourceBuffer intoBuffer:unrotatedBuffer]) { + OSStatus rotateStatus = VTPixelRotationSessionRotateImage((VTPixelRotationSessionRef)self.rotationSession, + unrotatedBuffer, + destination); + if (rotateStatus == noErr) { + rotated = YES; + } else { + FBExtLogError("Session %u: pixel rotation failed (status %d)", self.sessionId, (int)rotateStatus); + } + } + CVPixelBufferRelease(unrotatedBuffer); + return rotated; +} + +/** + Makes the rotation session and the un-rotated buffer pool match the requested turn and size, + recreating only what changed. + */ +- (BOOL)prepareRotationForWidth:(NSUInteger)width + height:(NSUInteger)height + rotation:(FBBroadcastRotation)rotation API_AVAILABLE(ios(16.0)) +{ + if (NULL == self.rotationSession) { + VTPixelRotationSessionRef session = NULL; + OSStatus status = VTPixelRotationSessionCreate(kCFAllocatorDefault, &session); + if (status != noErr || NULL == session) { + FBExtLogError("Session %u: cannot create a pixel rotation session (status %d)", self.sessionId, (int)status); + return NO; + } + self.rotationSession = session; + self.sessionRotation = FBBroadcastRotationNone; + } + if (rotation != self.sessionRotation) { + CFStringRef value = kVTRotation_0; + switch (rotation) { + case FBBroadcastRotationClockwise90: + value = kVTRotation_CW90; + break; + case FBBroadcastRotation180: + value = kVTRotation_180; + break; + case FBBroadcastRotationCounterclockwise90: + value = kVTRotation_CCW90; + break; + case FBBroadcastRotationNone: + break; + } + OSStatus status = VTSessionSetProperty(self.rotationSession, kVTPixelRotationPropertyKey_Rotation, value); + if (status != noErr) { + FBExtLogError("Session %u: cannot set the pixel rotation (status %d)", self.sessionId, (int)status); + return NO; + } + self.sessionRotation = rotation; + } + if (NULL == self.unrotatedPool || width != self.unrotatedWidth || height != self.unrotatedHeight) { + if (NULL != self.unrotatedPool) { + CVPixelBufferPoolRelease(self.unrotatedPool); + self.unrotatedPool = NULL; + } + NSDictionary *pixelBufferAttributes = @{ + (id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange), + (id)kCVPixelBufferWidthKey: @(width), + (id)kCVPixelBufferHeightKey: @(height), + (id)kCVPixelBufferIOSurfacePropertiesKey: @{}, + }; + NSDictionary *poolAttributes = @{(id)kCVPixelBufferPoolMinimumBufferCountKey: @(POOL_ALLOCATION_THRESHOLD)}; + CVPixelBufferPoolRef pool = NULL; + CVReturn poolStatus = CVPixelBufferPoolCreate(kCFAllocatorDefault, + (__bridge CFDictionaryRef)poolAttributes, + (__bridge CFDictionaryRef)pixelBufferAttributes, + &pool); + if (poolStatus != kCVReturnSuccess || NULL == pool) { + FBExtLogError("Session %u: cannot create the rotation buffer pool (status %d)", self.sessionId, (int)poolStatus); + return NO; + } + self.unrotatedPool = pool; + self.unrotatedWidth = width; + self.unrotatedHeight = height; + } + return YES; +} + - (BOOL)reconfigureForWidth:(NSUInteger)width height:(NSUInteger)height { VTPixelTransferSessionRef transferSession = NULL; @@ -595,6 +733,17 @@ - (void)releaseScalerResources CVPixelBufferPoolRelease(self.bufferPool); self.bufferPool = NULL; } + if (NULL != self.unrotatedPool) { + CVPixelBufferPoolRelease(self.unrotatedPool); + self.unrotatedPool = NULL; + } + if (NULL != self.rotationSession) { + if (@available(iOS 16.0, *)) { + VTPixelRotationSessionInvalidate((VTPixelRotationSessionRef)self.rotationSession); + } + CFRelease(self.rotationSession); + self.rotationSession = NULL; + } } - (void)dealloc diff --git a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h index 4ca17e7e9..131343bfd 100644 --- a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h +++ b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h @@ -135,6 +135,21 @@ FBBroadcastDimensions FBBroadcastTargetDimensions(NSUInteger configuredWidth, NSUInteger sourceBufferHeight, uint8_t orientation); +/** The turn a ReplayKit buffer needs before it reads upright. */ +typedef NS_ENUM(NSUInteger, FBBroadcastRotation) { + FBBroadcastRotationNone = 0, + FBBroadcastRotationClockwise90, + FBBroadcastRotation180, + FBBroadcastRotationCounterclockwise90, +}; + +/** + Maps a frame's RPVideoSampleOrientationKey (a CGImagePropertyOrientation) to + the rotation that makes its portrait-shaped buffer upright. Unknown and + mirrored values, which ReplayKit does not emit, map to no rotation. + */ +FBBroadcastRotation FBBroadcastFrameRotation(uint8_t orientation); + /** Milliseconds to wait after a failed encoder reconfiguration attempt. */ uint64_t FBBroadcastReconfigureRetryDelayMs(NSUInteger failureCount); diff --git a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m index b6932f611..a23fa9bbb 100644 --- a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m +++ b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m @@ -60,6 +60,22 @@ FBBroadcastDimensions FBBroadcastTargetDimensions(NSUInteger configuredWidth, : (FBBroadcastDimensions){shortSide, longSide}; } +FBBroadcastRotation FBBroadcastFrameRotation(uint8_t orientation) +{ + switch (orientation) { + case 3: // kCGImagePropertyOrientationDown + return FBBroadcastRotation180; + // Measured on an iPhone 15 Pro (iOS 26.6.1) through VTPixelRotationSession: the quarter + // turns read upright opposite to the CGImagePropertyOrientation documentation. + case 6: // kCGImagePropertyOrientationRight + return FBBroadcastRotationCounterclockwise90; + case 8: // kCGImagePropertyOrientationLeft + return FBBroadcastRotationClockwise90; + default: + return FBBroadcastRotationNone; + } +} + uint64_t FBBroadcastReconfigureRetryDelayMs(NSUInteger failureCount) { if (failureCount == 0) { diff --git a/WebDriverAgentTests/UnitTests/FBVideoStreamSessionTests.m b/WebDriverAgentTests/UnitTests/FBVideoStreamSessionTests.m index 924bac5b9..babe735bd 100644 --- a/WebDriverAgentTests/UnitTests/FBVideoStreamSessionTests.m +++ b/WebDriverAgentTests/UnitTests/FBVideoStreamSessionTests.m @@ -116,6 +116,28 @@ - (void)testBroadcastTargetSizeFollowsEffectiveFrameOrientation XCTAssertEqual(portrait.height, (NSUInteger)1278); } +- (void)testBroadcastFrameRotationUprightsReplayKitBuffers +{ + // RPVideoSampleOrientationKey is a CGImagePropertyOrientation; the expected + // quarter turns were measured on hardware (see FBBroadcastFrameRotation). + XCTAssertEqual(FBBroadcastFrameRotation(1), FBBroadcastRotationNone); + XCTAssertEqual(FBBroadcastFrameRotation(3), FBBroadcastRotation180); + XCTAssertEqual(FBBroadcastFrameRotation(6), FBBroadcastRotationCounterclockwise90); + XCTAssertEqual(FBBroadcastFrameRotation(8), FBBroadcastRotationClockwise90); +} + +- (void)testBroadcastFrameRotationLeavesUnknownAndMirroredOrientationsAlone +{ + // ReplayKit does not emit mirrored orientations; an unknown or mirrored + // value keeps the buffer as delivered rather than guessing a flip. + XCTAssertEqual(FBBroadcastFrameRotation(0), FBBroadcastRotationNone); + XCTAssertEqual(FBBroadcastFrameRotation(2), FBBroadcastRotationNone); + XCTAssertEqual(FBBroadcastFrameRotation(4), FBBroadcastRotationNone); + XCTAssertEqual(FBBroadcastFrameRotation(5), FBBroadcastRotationNone); + XCTAssertEqual(FBBroadcastFrameRotation(7), FBBroadcastRotationNone); + XCTAssertEqual(FBBroadcastFrameRotation(9), FBBroadcastRotationNone); +} + - (void)testBroadcastTargetSizeFallsBackToBufferAspectForUnknownOrientation { FBBroadcastDimensions landscape = FBBroadcastTargetDimensions(588, 1278, 2532, 1170, 0);