fix: rotate ReplayKit frames upright on landscape

WebDriverAgentBroadcast/FBExtSessionPipeline.m

@@ -49,6 +49,14 @@ @interface FBExtSessionPipeline () <FBVideoEncoderDelegate> {
 @property (atomic) BOOL active;
 @property (atomic) uint8_t currentOrientation;
 @property (nonatomic) BOOL directSourceEncodingDisabled;
+/** VTPixelRotationSessionRef (iOS 16+), created on the first rotated frame. */
+@property (nonatomic, nullable) CFTypeRef rotationSession;
+/** The rotation currently set on rotationSession. */
+@property (nonatomic) FBBroadcastRotation sessionRotation;
+/** Encoder-sized buffers in the un-rotated orientation, the rotation session's input. */
+@property (nonatomic, nullable) CVPixelBufferPoolRef unrotatedPool;
+@property (nonatomic) NSUInteger unrotatedWidth;
+@property (nonatomic) NSUInteger unrotatedHeight;
 @property (nonatomic, nullable, copy) NSData *lastSentParameterSets;
 @property (nonatomic) NSUInteger failedReconfigureWidth;
 @property (nonatomic) NSUInteger failedReconfigureHeight;
@@ -427,7 +435,15 @@ - (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer
     }
   }
 
+  // ReplayKit buffers stay portrait-shaped; the orientation attachment says how to turn them.
+  // Without the turn, a landscape frame is letterboxed sideways into the swapped footprint.
+  FBBroadcastRotation rotation = FBBroadcastRotationNone;
+  if (@available(iOS 16.0, *)) {
+    rotation = FBBroadcastFrameRotation(orientation);
+  }
+
   if (!self.directSourceEncodingDisabled
+      && rotation == FBBroadcastRotationNone
       && CVPixelBufferGetWidth(sourceBuffer) == self.width
       && CVPixelBufferGetHeight(sourceBuffer) == self.height) {
     NSError *error;
@@ -454,9 +470,10 @@ - (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer
     return;
   }
 
-  OSStatus transferStatus = VTPixelTransferSessionTransferImage(self.transferSession, sourceBuffer, scaledBuffer);
-  if (transferStatus != noErr) {
-    FBExtLogError("Session %u: pixel transfer failed (status %d)", self.sessionId, (int)transferStatus);
+  BOOL transferred = rotation == FBBroadcastRotationNone
+    ? [self transferSourceBuffer:sourceBuffer intoBuffer:scaledBuffer]
+    : [self transferSourceBuffer:sourceBuffer intoBuffer:scaledBuffer rotation:rotation];
+  if (!transferred) {
     CVPixelBufferRelease(scaledBuffer);
     return;
   }
@@ -471,6 +488,127 @@ - (void)processRetainedSampleBuffer:(CMSampleBufferRef)sampleBuffer
   CVPixelBufferRelease(scaledBuffer);
 }
 
+/** Scales sourceBuffer into destination (letterboxed). */
+- (BOOL)transferSourceBuffer:(CVPixelBufferRef)sourceBuffer intoBuffer:(CVPixelBufferRef)destination
+{
+  OSStatus transferStatus = VTPixelTransferSessionTransferImage(self.transferSession, sourceBuffer, destination);
+  if (transferStatus != noErr) {
+    FBExtLogError("Session %u: pixel transfer failed (status %d)", self.sessionId, (int)transferStatus);
+    return NO;
+  }
+  return YES;
+}
+
+/**
+ Scales sourceBuffer into an un-rotated buffer of destination's size (axes swapped for a quarter
+ turn), then rotates that into destination. iOS has no rotating pixel transfer, and a rotation
+ session cannot scale, so the turn runs on the small encoder-sized buffer.
+ */
+- (BOOL)transferSourceBuffer:(CVPixelBufferRef)sourceBuffer
+                  intoBuffer:(CVPixelBufferRef)destination
+                    rotation:(FBBroadcastRotation)rotation API_AVAILABLE(ios(16.0))
+{
+  BOOL quarterTurn = rotation == FBBroadcastRotationClockwise90
+    || rotation == FBBroadcastRotationCounterclockwise90;
+  NSUInteger width = quarterTurn ? self.height : self.width;
+  NSUInteger height = quarterTurn ? self.width : self.height;
+  if (![self prepareRotationForWidth:width height:height rotation:rotation]) {
+    return NO;
+  }
+
+  CVPixelBufferRef unrotatedBuffer = NULL;
+  NSDictionary *auxAttributes = @{(id)kCVPixelBufferPoolAllocationThresholdKey: @(POOL_ALLOCATION_THRESHOLD)};
+  CVReturn poolStatus = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault,
+                                                                            self.unrotatedPool,
+                                                                            (__bridge CFDictionaryRef)auxAttributes,
+                                                                            &unrotatedBuffer);
+  if (poolStatus != kCVReturnSuccess || NULL == unrotatedBuffer) {
+    self.droppedPoolCount += 1;
+    return NO;
+  }
+  BOOL rotated = NO;
+  if ([self transferSourceBuffer:sourceBuffer intoBuffer:unrotatedBuffer]) {
+    OSStatus rotateStatus = VTPixelRotationSessionRotateImage((VTPixelRotationSessionRef)self.rotationSession,
+                                                              unrotatedBuffer,
+                                                              destination);
+    if (rotateStatus == noErr) {
+      rotated = YES;
+    } else {
+      FBExtLogError("Session %u: pixel rotation failed (status %d)", self.sessionId, (int)rotateStatus);
+    }
+  }
+  CVPixelBufferRelease(unrotatedBuffer);
+  return rotated;
+}
+
+/**
+ Makes the rotation session and the un-rotated buffer pool match the requested turn and size,
+ recreating only what changed.
+ */
+- (BOOL)prepareRotationForWidth:(NSUInteger)width
+                         height:(NSUInteger)height
+                       rotation:(FBBroadcastRotation)rotation API_AVAILABLE(ios(16.0))
+{
+  if (NULL == self.rotationSession) {
+    VTPixelRotationSessionRef session = NULL;
+    OSStatus status = VTPixelRotationSessionCreate(kCFAllocatorDefault, &session);
+    if (status != noErr || NULL == session) {
+      FBExtLogError("Session %u: cannot create a pixel rotation session (status %d)", self.sessionId, (int)status);
+      return NO;
+    }
+    self.rotationSession = session;
+    self.sessionRotation = FBBroadcastRotationNone;
+  }
+  if (rotation != self.sessionRotation) {
+    CFStringRef value = kVTRotation_0;
+    switch (rotation) {
+      case FBBroadcastRotationClockwise90:
+        value = kVTRotation_CW90;
+        break;
+      case FBBroadcastRotation180:
+        value = kVTRotation_180;
+        break;
+      case FBBroadcastRotationCounterclockwise90:
+        value = kVTRotation_CCW90;
+        break;
+      case FBBroadcastRotationNone:
+        break;
+    }
+    OSStatus status = VTSessionSetProperty(self.rotationSession, kVTPixelRotationPropertyKey_Rotation, value);
+    if (status != noErr) {
+      FBExtLogError("Session %u: cannot set the pixel rotation (status %d)", self.sessionId, (int)status);
+      return NO;
+    }
+    self.sessionRotation = rotation;
+  }
+  if (NULL == self.unrotatedPool || width != self.unrotatedWidth || height != self.unrotatedHeight) {
+    if (NULL != self.unrotatedPool) {
+      CVPixelBufferPoolRelease(self.unrotatedPool);
+      self.unrotatedPool = NULL;
+    }
+    NSDictionary *pixelBufferAttributes = @{
+      (id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange),
+      (id)kCVPixelBufferWidthKey: @(width),
+      (id)kCVPixelBufferHeightKey: @(height),
+      (id)kCVPixelBufferIOSurfacePropertiesKey: @{},
+    };
+    NSDictionary *poolAttributes = @{(id)kCVPixelBufferPoolMinimumBufferCountKey: @(POOL_ALLOCATION_THRESHOLD)};
+    CVPixelBufferPoolRef pool = NULL;
+    CVReturn poolStatus = CVPixelBufferPoolCreate(kCFAllocatorDefault,
+                                                  (__bridge CFDictionaryRef)poolAttributes,
+                                                  (__bridge CFDictionaryRef)pixelBufferAttributes,
+                                                  &pool);
+    if (poolStatus != kCVReturnSuccess || NULL == pool) {
+      FBExtLogError("Session %u: cannot create the rotation buffer pool (status %d)", self.sessionId, (int)poolStatus);
+      return NO;
+    }
+    self.unrotatedPool = pool;
+    self.unrotatedWidth = width;
+    self.unrotatedHeight = height;
+  }
+  return YES;
+}
+
 - (BOOL)reconfigureForWidth:(NSUInteger)width height:(NSUInteger)height
 {
   VTPixelTransferSessionRef transferSession = NULL;
@@ -595,6 +733,17 @@ - (void)releaseScalerResources
     CVPixelBufferPoolRelease(self.bufferPool);
     self.bufferPool = NULL;
   }
+  if (NULL != self.unrotatedPool) {
+    CVPixelBufferPoolRelease(self.unrotatedPool);
+    self.unrotatedPool = NULL;
+  }
+  if (NULL != self.rotationSession) {
+    if (@available(iOS 16.0, *)) {
+      VTPixelRotationSessionInvalidate((VTPixelRotationSessionRef)self.rotationSession);
+    }
+    CFRelease(self.rotationSession);
+    self.rotationSession = NULL;
+  }
 }
 
 - (void)dealloc

WebDriverAgentLib/Utilities/FBBroadcastProtocol.h

@@ -135,6 +135,21 @@ FBBroadcastDimensions FBBroadcastTargetDimensions(NSUInteger configuredWidth,
                                                    NSUInteger sourceBufferHeight,
                                                    uint8_t orientation);
 
+/** The turn a ReplayKit buffer needs before it reads upright. */
+typedef NS_ENUM(NSUInteger, FBBroadcastRotation) {
+  FBBroadcastRotationNone = 0,
+  FBBroadcastRotationClockwise90,
+  FBBroadcastRotation180,
+  FBBroadcastRotationCounterclockwise90,
+};
+
+/**
+ Maps a frame's RPVideoSampleOrientationKey (a CGImagePropertyOrientation) to
+ the rotation that makes its portrait-shaped buffer upright. Unknown and
+ mirrored values, which ReplayKit does not emit, map to no rotation.
+ */
+FBBroadcastRotation FBBroadcastFrameRotation(uint8_t orientation);
+
 /** Milliseconds to wait after a failed encoder reconfiguration attempt. */
 uint64_t FBBroadcastReconfigureRetryDelayMs(NSUInteger failureCount);
 

WebDriverAgentLib/Utilities/FBBroadcastProtocol.m

@@ -60,6 +60,22 @@ FBBroadcastDimensions FBBroadcastTargetDimensions(NSUInteger configuredWidth,
     : (FBBroadcastDimensions){shortSide, longSide};
 }
 
+FBBroadcastRotation FBBroadcastFrameRotation(uint8_t orientation)
+{
+  switch (orientation) {
+    case 3: // kCGImagePropertyOrientationDown
+      return FBBroadcastRotation180;
+    // Measured on an iPhone 15 Pro (iOS 26.6.1) through VTPixelRotationSession: the quarter
+    // turns read upright opposite to the CGImagePropertyOrientation documentation.
+    case 6: // kCGImagePropertyOrientationRight
+      return FBBroadcastRotationCounterclockwise90;
+    case 8: // kCGImagePropertyOrientationLeft
+      return FBBroadcastRotationClockwise90;
+    default:
+      return FBBroadcastRotationNone;
+  }
+}
+
 uint64_t FBBroadcastReconfigureRetryDelayMs(NSUInteger failureCount)
 {
   if (failureCount == 0) {

WebDriverAgentTests/UnitTests/FBVideoStreamSessionTests.m

@@ -116,6 +116,28 @@ - (void)testBroadcastTargetSizeFollowsEffectiveFrameOrientation
   XCTAssertEqual(portrait.height, (NSUInteger)1278);
 }
 
+- (void)testBroadcastFrameRotationUprightsReplayKitBuffers
+{
+  // RPVideoSampleOrientationKey is a CGImagePropertyOrientation; the expected
+  // quarter turns were measured on hardware (see FBBroadcastFrameRotation).
+  XCTAssertEqual(FBBroadcastFrameRotation(1), FBBroadcastRotationNone);
+  XCTAssertEqual(FBBroadcastFrameRotation(3), FBBroadcastRotation180);
+  XCTAssertEqual(FBBroadcastFrameRotation(6), FBBroadcastRotationCounterclockwise90);
+  XCTAssertEqual(FBBroadcastFrameRotation(8), FBBroadcastRotationClockwise90);
+}
+
+- (void)testBroadcastFrameRotationLeavesUnknownAndMirroredOrientationsAlone
+{
+  // ReplayKit does not emit mirrored orientations; an unknown or mirrored
+  // value keeps the buffer as delivered rather than guessing a flip.
+  XCTAssertEqual(FBBroadcastFrameRotation(0), FBBroadcastRotationNone);
+  XCTAssertEqual(FBBroadcastFrameRotation(2), FBBroadcastRotationNone);
+  XCTAssertEqual(FBBroadcastFrameRotation(4), FBBroadcastRotationNone);
+  XCTAssertEqual(FBBroadcastFrameRotation(5), FBBroadcastRotationNone);
+  XCTAssertEqual(FBBroadcastFrameRotation(7), FBBroadcastRotationNone);
+  XCTAssertEqual(FBBroadcastFrameRotation(9), FBBroadcastRotationNone);
+}
+
 - (void)testBroadcastTargetSizeFallsBackToBufferAspectForUnknownOrientation
 {
   FBBroadcastDimensions landscape = FBBroadcastTargetDimensions(588, 1278, 2532, 1170, 0);