fix(material/menu): do not pull focus out of an overlay opened by a menu item

src/material/menu/menu-trigger-base.ts

@@ -278,6 +278,9 @@ export abstract class MatMenuTriggerBase implements OnDestroy {
       return;
     }
 
+    // Check where focus is before detaching the overlay, because detaching can move it.
+    const focusMovedToOtherOverlay = reason === 'click' && this._hasFocusMovedToOtherOverlay();
+
     this._closingActionsSubscription.unsubscribe();
     this._pendingRemoval?.unsubscribe();
 
@@ -308,9 +311,11 @@ export abstract class MatMenuTriggerBase implements OnDestroy {
     // Always restore focus if the user is navigating using the keyboard or the menu was opened
     // programmatically. We don't restore for non-root triggers, because it can prevent focus
     // from making it back to the root trigger when closing a long chain of menus by clicking
-    // on the backdrop.
+    // on the backdrop. We also don't restore if clicking on an item moved focus into a different
+    // overlay (e.g. a dialog opened by the item), because it'd pull focus out of that overlay.
     if (
       this.restoreFocus &&
+      !focusMovedToOtherOverlay &&
       (reason === 'keydown' || !this._openedBy || !this._triggersSubmenu())
     ) {
       this.focus(this._openedBy);
@@ -320,6 +325,22 @@ export abstract class MatMenuTriggerBase implements OnDestroy {
     this._setIsMenuOpen(false);
   }
 
+  /**
+   * Checks whether focus is inside an overlay other than a menu or the overlay containing the
+   * trigger, e.g. because a menu item opened a dialog that focused its content.
+   */
+  private _hasFocusMovedToOtherOverlay(): boolean {
+    const trigger = this._element.nativeElement;
+    const focusedElement = trigger.ownerDocument?.activeElement;
+
+    if (!focusedElement || focusedElement.closest('.mat-mdc-menu-panel')) {
+      return false;
+    }
+
+    const pane = focusedElement.closest('.cdk-overlay-pane');
+    return !!pane && !pane.contains(trigger);
+  }
+
   // set state rather than toggle to support triggers sharing a menu
   private _setIsMenuOpen(isOpen: boolean): void {
     if (isOpen !== this._menuOpen) {

src/material/menu/menu.spec.ts

@@ -251,6 +251,74 @@ describe('MatMenu', () => {
     expect(document.activeElement).toBe(triggerEl);
   });
 
+  it('should not restore focus to the trigger if clicking an item moved focus into another overlay', () => {
+    const fixture = TestBed.createComponent(SimpleMenu);
+    fixture.detectChanges();
+    const triggerEl = fixture.componentInstance.triggerEl.nativeElement;
+
+    triggerEl.click();
+    fixture.detectChanges();
+
+    // Simulates an overlay, like a dialog, that the item opens and moves focus into.
+    const otherPane = document.createElement('div');
+    const otherPaneButton = document.createElement('button');
+    otherPane.classList.add('cdk-overlay-pane');
+    otherPane.appendChild(otherPaneButton);
+    overlayContainerElement.appendChild(otherPane);
+
+    const item = overlayContainerElement.querySelector('.mat-mdc-menu-item') as HTMLElement;
+    item.addEventListener('click', () => otherPaneButton.focus());
+    item.click();
+    fixture.detectChanges();
+
+    expect(document.activeElement).toBe(otherPaneButton);
+    otherPane.remove();
+  });
+
+  it('should restore focus to the trigger if clicking an item moved focus outside of an overlay', () => {
+    const fixture = TestBed.createComponent(SimpleMenu);
+    fixture.detectChanges();
+    const triggerEl = fixture.componentInstance.triggerEl.nativeElement;
+    const button = document.createElement('button');
+    document.body.appendChild(button);
+
+    triggerEl.click();
+    fixture.detectChanges();
+
+    const item = overlayContainerElement.querySelector('.mat-mdc-menu-item') as HTMLElement;
+    item.addEventListener('click', () => button.focus());
+    item.click();
+    fixture.detectChanges();
+
+    expect(document.activeElement).toBe(triggerEl);
+    button.remove();
+  });
+
+  it('should restore focus to a trigger inside an overlay if clicking an item moved focus within that overlay', () => {
+    const fixture = TestBed.createComponent(SimpleMenu);
+    fixture.detectChanges();
+    const triggerEl = fixture.componentInstance.triggerEl.nativeElement;
+
+    // Simulates a menu inside an overlay, like a dialog.
+    const pane = document.createElement('div');
+    const paneButton = document.createElement('button');
+    pane.classList.add('cdk-overlay-pane');
+    pane.appendChild(fixture.nativeElement);
+    pane.appendChild(paneButton);
+    overlayContainerElement.appendChild(pane);
+
+    triggerEl.click();
+    fixture.detectChanges();
+
+    const item = overlayContainerElement.querySelector('.mat-mdc-menu-item') as HTMLElement;
+    item.addEventListener('click', () => paneButton.focus());
+    item.click();
+    fixture.detectChanges();
+
+    expect(document.activeElement).toBe(triggerEl);
+    pane.remove();
+  });
+
   it('should move focus to another item if the active item is destroyed', () => {
     const fixture = TestBed.createComponent(MenuWithRepeatedItems);
     fixture.detectChanges();
@@ -2194,6 +2262,23 @@ describe('MatMenu', () => {
         .toBe(0);
     });
 
+    it('should restore focus to the root trigger when an item in a sub-menu is clicked', () => {
+      const rootTrigger = instance.rootTriggerEl.nativeElement;
+      rootTrigger.click();
+      fixture.detectChanges();
+
+      instance.levelOneTrigger.openMenu();
+      fixture.detectChanges();
+
+      const menus = overlay.querySelectorAll('.mat-mdc-menu-panel');
+      const item = menus[1].querySelector('.mat-mdc-menu-item') as HTMLElement;
+      item.focus();
+      item.click();
+      fixture.detectChanges();
+
+      expect(document.activeElement).toBe(rootTrigger);
+    });
+
     it('should close all of the menus when the user tabs away', async () => {
       instance.rootTriggerEl.nativeElement.click();
       fixture.detectChanges();