diff --git a/WebDriverAgentBroadcast/FBExtSessionPipeline.m b/WebDriverAgentBroadcast/FBExtSessionPipeline.m index ac2a7fd31..5f600b537 100644 --- a/WebDriverAgentBroadcast/FBExtSessionPipeline.m +++ b/WebDriverAgentBroadcast/FBExtSessionPipeline.m @@ -49,6 +49,14 @@ @interface FBExtSessionPipeline () { @property (atomic) BOOL active; @property (atomic) uint8_t currentOrientation; @property (nonatomic) BOOL directSourceEncodingDisabled; +/** VTPixelRotationSessionRef (iOS 16+), created on the first rotated frame. */ +@property (nonatomic, nullable) CFTypeRef rotationSession; +/** The rotation currently set on rotationSession. */ +@property (nonatomic) FBBroadcastRotation sessionRotation; +/** Encoder-sized buffers in the un-rotated orientation, the rotation session's input. */ +@property (nonatomic, nullable) CVPixelBufferPoolRef unrotatedPool; +@property (nonatomic) NSUInteger unrotatedWidth; +@property (nonatomic) NSUInteger unrotatedHeight; @property (nonatomic, nullable, copy) NSData *lastSentParameterSets; @property (nonatomic) NSUInteger failedReconfigureWidth; @property (nonatomic) NSUInteger failedReconfigureHeight; @@ -427,7 +435,15 @@ - (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer } } + // ReplayKit buffers stay portrait-shaped; the orientation attachment says how to turn them. + // Without the turn, a landscape frame is letterboxed sideways into the swapped footprint. + FBBroadcastRotation rotation = FBBroadcastRotationNone; + if (@available(iOS 16.0, *)) { + rotation = FBBroadcastFrameRotation(orientation); + } + if (!self.directSourceEncodingDisabled + && rotation == FBBroadcastRotationNone && CVPixelBufferGetWidth(sourceBuffer) == self.width && CVPixelBufferGetHeight(sourceBuffer) == self.height) { NSError *error; @@ -454,9 +470,10 @@ - (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer return; } - OSStatus transferStatus = VTPixelTransferSessionTransferImage(self.transferSession, sourceBuffer, scaledBuffer); - if (transferStatus != noErr) { - FBExtLogError("Session %u: pixel transfer failed (status %d)", self.sessionId, (int)transferStatus); + BOOL transferred = rotation == FBBroadcastRotationNone + ? [self transferSourceBuffer:sourceBuffer intoBuffer:scaledBuffer] + : [self transferSourceBuffer:sourceBuffer intoBuffer:scaledBuffer rotation:rotation]; + if (!transferred) { CVPixelBufferRelease(scaledBuffer); return; } @@ -471,6 +488,127 @@ - (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer CVPixelBufferRelease(scaledBuffer); } +/** Scales sourceBuffer into destination (letterboxed). */ +- (BOOL)transferSourceBuffer:(CVPixelBufferRef)sourceBuffer intoBuffer:(CVPixelBufferRef)destination +{ + OSStatus transferStatus = VTPixelTransferSessionTransferImage(self.transferSession, sourceBuffer, destination); + if (transferStatus != noErr) { + FBExtLogError("Session %u: pixel transfer failed (status %d)", self.sessionId, (int)transferStatus); + return NO; + } + return YES; +} + +/** + Scales sourceBuffer into an un-rotated buffer of destination's size (axes swapped for a quarter + turn), then rotates that into destination. iOS has no rotating pixel transfer, and a rotation + session cannot scale, so the turn runs on the small encoder-sized buffer. + */ +- (BOOL)transferSourceBuffer:(CVPixelBufferRef)sourceBuffer + intoBuffer:(CVPixelBufferRef)destination + rotation:(FBBroadcastRotation)rotation API_AVAILABLE(ios(16.0)) +{ + BOOL quarterTurn = rotation == FBBroadcastRotationClockwise90 + || rotation == FBBroadcastRotationCounterclockwise90; + NSUInteger width = quarterTurn ? self.height : self.width; + NSUInteger height = quarterTurn ? self.width : self.height; + if (![self prepareRotationForWidth:width height:height rotation:rotation]) { + return NO; + } + + CVPixelBufferRef unrotatedBuffer = NULL; + NSDictionary *auxAttributes = @{(id)kCVPixelBufferPoolAllocationThresholdKey: @(POOL_ALLOCATION_THRESHOLD)}; + CVReturn poolStatus = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault, + self.unrotatedPool, + (__bridge CFDictionaryRef)auxAttributes, + &unrotatedBuffer); + if (poolStatus != kCVReturnSuccess || NULL == unrotatedBuffer) { + self.droppedPoolCount += 1; + return NO; + } + BOOL rotated = NO; + if ([self transferSourceBuffer:sourceBuffer intoBuffer:unrotatedBuffer]) { + OSStatus rotateStatus = VTPixelRotationSessionRotateImage((VTPixelRotationSessionRef)self.rotationSession, + unrotatedBuffer, + destination); + if (rotateStatus == noErr) { + rotated = YES; + } else { + FBExtLogError("Session %u: pixel rotation failed (status %d)", self.sessionId, (int)rotateStatus); + } + } + CVPixelBufferRelease(unrotatedBuffer); + return rotated; +} + +/** + Makes the rotation session and the un-rotated buffer pool match the requested turn and size, + recreating only what changed. + */ +- (BOOL)prepareRotationForWidth:(NSUInteger)width + height:(NSUInteger)height + rotation:(FBBroadcastRotation)rotation API_AVAILABLE(ios(16.0)) +{ + if (NULL == self.rotationSession) { + VTPixelRotationSessionRef session = NULL; + OSStatus status = VTPixelRotationSessionCreate(kCFAllocatorDefault, &session); + if (status != noErr || NULL == session) { + FBExtLogError("Session %u: cannot create a pixel rotation session (status %d)", self.sessionId, (int)status); + return NO; + } + self.rotationSession = session; + self.sessionRotation = FBBroadcastRotationNone; + } + if (rotation != self.sessionRotation) { + CFStringRef value = kVTRotation_0; + switch (rotation) { + case FBBroadcastRotationClockwise90: + value = kVTRotation_CW90; + break; + case FBBroadcastRotation180: + value = kVTRotation_180; + break; + case FBBroadcastRotationCounterclockwise90: + value = kVTRotation_CCW90; + break; + case FBBroadcastRotationNone: + break; + } + OSStatus status = VTSessionSetProperty(self.rotationSession, kVTPixelRotationPropertyKey_Rotation, value); + if (status != noErr) { + FBExtLogError("Session %u: cannot set the pixel rotation (status %d)", self.sessionId, (int)status); + return NO; + } + self.sessionRotation = rotation; + } + if (NULL == self.unrotatedPool || width != self.unrotatedWidth || height != self.unrotatedHeight) { + if (NULL != self.unrotatedPool) { + CVPixelBufferPoolRelease(self.unrotatedPool); + self.unrotatedPool = NULL; + } + NSDictionary *pixelBufferAttributes = @{ + (id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange), + (id)kCVPixelBufferWidthKey: @(width), + (id)kCVPixelBufferHeightKey: @(height), + (id)kCVPixelBufferIOSurfacePropertiesKey: @{}, + }; + NSDictionary *poolAttributes = @{(id)kCVPixelBufferPoolMinimumBufferCountKey: @(POOL_ALLOCATION_THRESHOLD)}; + CVPixelBufferPoolRef pool = NULL; + CVReturn poolStatus = CVPixelBufferPoolCreate(kCFAllocatorDefault, + (__bridge CFDictionaryRef)poolAttributes, + (__bridge CFDictionaryRef)pixelBufferAttributes, + &pool); + if (poolStatus != kCVReturnSuccess || NULL == pool) { + FBExtLogError("Session %u: cannot create the rotation buffer pool (status %d)", self.sessionId, (int)poolStatus); + return NO; + } + self.unrotatedPool = pool; + self.unrotatedWidth = width; + self.unrotatedHeight = height; + } + return YES; +} + - (BOOL)reconfigureForWidth:(NSUInteger)width height:(NSUInteger)height { VTPixelTransferSessionRef transferSession = NULL; @@ -595,6 +733,17 @@ - (void)releaseScalerResources CVPixelBufferPoolRelease(self.bufferPool); self.bufferPool = NULL; } + if (NULL != self.unrotatedPool) { + CVPixelBufferPoolRelease(self.unrotatedPool); + self.unrotatedPool = NULL; + } + if (NULL != self.rotationSession) { + if (@available(iOS 16.0, *)) { + VTPixelRotationSessionInvalidate((VTPixelRotationSessionRef)self.rotationSession); + } + CFRelease(self.rotationSession); + self.rotationSession = NULL; + } } - (void)dealloc diff --git a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h index 4ca17e7e9..131343bfd 100644 --- a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h +++ b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.h @@ -135,6 +135,21 @@ FBBroadcastDimensions FBBroadcastTargetDimensions(NSUInteger configuredWidth, NSUInteger sourceBufferHeight, uint8_t orientation); +/** The turn a ReplayKit buffer needs before it reads upright. */ +typedef NS_ENUM(NSUInteger, FBBroadcastRotation) { + FBBroadcastRotationNone = 0, + FBBroadcastRotationClockwise90, + FBBroadcastRotation180, + FBBroadcastRotationCounterclockwise90, +}; + +/** + Maps a frame's RPVideoSampleOrientationKey (a CGImagePropertyOrientation) to + the rotation that makes its portrait-shaped buffer upright. Unknown and + mirrored values, which ReplayKit does not emit, map to no rotation. + */ +FBBroadcastRotation FBBroadcastFrameRotation(uint8_t orientation); + /** Milliseconds to wait after a failed encoder reconfiguration attempt. */ uint64_t FBBroadcastReconfigureRetryDelayMs(NSUInteger failureCount); diff --git a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m index b6932f611..a23fa9bbb 100644 --- a/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m +++ b/WebDriverAgentLib/Utilities/FBBroadcastProtocol.m @@ -60,6 +60,22 @@ FBBroadcastDimensions FBBroadcastTargetDimensions(NSUInteger configuredWidth, : (FBBroadcastDimensions){shortSide, longSide}; } +FBBroadcastRotation FBBroadcastFrameRotation(uint8_t orientation) +{ + switch (orientation) { + case 3: // kCGImagePropertyOrientationDown + return FBBroadcastRotation180; + // Measured on an iPhone 15 Pro (iOS 26.6.1) through VTPixelRotationSession: the quarter + // turns read upright opposite to the CGImagePropertyOrientation documentation. + case 6: // kCGImagePropertyOrientationRight + return FBBroadcastRotationCounterclockwise90; + case 8: // kCGImagePropertyOrientationLeft + return FBBroadcastRotationClockwise90; + default: + return FBBroadcastRotationNone; + } +} + uint64_t FBBroadcastReconfigureRetryDelayMs(NSUInteger failureCount) { if (failureCount == 0) { diff --git a/WebDriverAgentTests/UnitTests/FBVideoStreamSessionTests.m b/WebDriverAgentTests/UnitTests/FBVideoStreamSessionTests.m index 924bac5b9..babe735bd 100644 --- a/WebDriverAgentTests/UnitTests/FBVideoStreamSessionTests.m +++ b/WebDriverAgentTests/UnitTests/FBVideoStreamSessionTests.m @@ -116,6 +116,28 @@ - (void)testBroadcastTargetSizeFollowsEffectiveFrameOrientation XCTAssertEqual(portrait.height, (NSUInteger)1278); } +- (void)testBroadcastFrameRotationUprightsReplayKitBuffers +{ + // RPVideoSampleOrientationKey is a CGImagePropertyOrientation; the expected + // quarter turns were measured on hardware (see FBBroadcastFrameRotation). + XCTAssertEqual(FBBroadcastFrameRotation(1), FBBroadcastRotationNone); + XCTAssertEqual(FBBroadcastFrameRotation(3), FBBroadcastRotation180); + XCTAssertEqual(FBBroadcastFrameRotation(6), FBBroadcastRotationCounterclockwise90); + XCTAssertEqual(FBBroadcastFrameRotation(8), FBBroadcastRotationClockwise90); +} + +- (void)testBroadcastFrameRotationLeavesUnknownAndMirroredOrientationsAlone +{ + // ReplayKit does not emit mirrored orientations; an unknown or mirrored + // value keeps the buffer as delivered rather than guessing a flip. + XCTAssertEqual(FBBroadcastFrameRotation(0), FBBroadcastRotationNone); + XCTAssertEqual(FBBroadcastFrameRotation(2), FBBroadcastRotationNone); + XCTAssertEqual(FBBroadcastFrameRotation(4), FBBroadcastRotationNone); + XCTAssertEqual(FBBroadcastFrameRotation(5), FBBroadcastRotationNone); + XCTAssertEqual(FBBroadcastFrameRotation(7), FBBroadcastRotationNone); + XCTAssertEqual(FBBroadcastFrameRotation(9), FBBroadcastRotationNone); +} + - (void)testBroadcastTargetSizeFallsBackToBufferAspectForUnknownOrientation { FBBroadcastDimensions landscape = FBBroadcastTargetDimensions(588, 1278, 2532, 1170, 0);