iommu: apple-dart: Install IOMMU_RESV_TRANSLATED mappings

[insert rant about locked TTBR]

The iommus for the display processors on Apple silicon machines have
locked TTBR registers. This makes it a little akward to install the boot
loader's translated reserved mappings on iommu domain init.
Handle this via a shadow L1 pgtable apple_dart_alloc_pgtable().
apple-dart will after that map the reserved regions and let
io-pgtable-dart transfer the shadow L1 table to the locked HW L1 table.
This has to be done this way since HW L1 table still holds the
bootloader allocated L2 tables. Switching the L2 PTE has to happen
atomically since the display processor is the whole time active with
scan out of the boot frame buffer.

TODO: find a less hacky way for upstream

Signed-off-by: Janne Grunau <j@jannau.net>

Author: jannau

drivers/iommu/Kconfig

@@ -311,6 +311,7 @@ config APPLE_DART
 	depends on !GENERIC_ATOMIC64	# for IOMMU_IO_PGTABLE_DART
 	select IOMMU_API
 	select IOMMU_IO_PGTABLE_DART
+	select OF_IOMMU
 	default ARCH_APPLE
 	help
 	  Support for Apple DART (Device Address Resolution Table) IOMMUs

drivers/iommu/apple-dart.c

@@ -33,6 +33,7 @@
 #include <linux/types.h>
 
 #include "dma-iommu.h"
+#include "io-pgtable-dart.h"
 
 #define DART_MAX_STREAMS 256
 #define DART_MAX_TTBR 4
@@ -557,17 +558,55 @@ apple_dart_setup_translation(struct apple_dart_domain *domain,
 	struct io_pgtable_cfg *pgtbl_cfg =
 		&io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg;
 
-	for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i)
-		apple_dart_hw_set_ttbr(stream_map, i,
-				       pgtbl_cfg->apple_dart_cfg.ttbr[i]);
-	for (; i < stream_map->dart->hw->ttbr_count; ++i)
-		apple_dart_hw_clear_ttbr(stream_map, i);
+	/* Locked DARTs are set up by the bootloader. */
+	if (!stream_map->dart->locked) {
+		for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i)
+			apple_dart_hw_set_ttbr(stream_map, i,
+					pgtbl_cfg->apple_dart_cfg.ttbr[i]);
+		for (; i < stream_map->dart->hw->ttbr_count; ++i)
+			apple_dart_hw_clear_ttbr(stream_map, i);
 
-	apple_dart_hw_enable_translation(stream_map);
+		apple_dart_hw_enable_translation(stream_map);
+	}
 	stream_map->dart->hw->invalidate_tlb(stream_map);
 }
 
+static int apple_dart_setup_resv_locked(struct iommu_domain *domain,
+					struct device *dev, size_t pgsize)
+{
+	struct iommu_resv_region *region;
+	LIST_HEAD(resv_regions);
+	int ret = 0;
+
+	of_iommu_get_resv_regions(dev, &resv_regions);
+	list_for_each_entry(region, &resv_regions, list) {
+		size_t mapped = 0;
+
+		/* Only map translated reserved regions */
+		if (region->type != IOMMU_RESV_TRANSLATED)
+			continue;
+
+		while (mapped < region->length) {
+			phys_addr_t paddr = region->start + mapped;
+			unsigned long iova = region->dva + mapped;
+			size_t length = region->length - mapped;
+			size_t pgcount = length / pgsize;
+
+			ret = apple_dart_map_pages(domain, iova,
+			      paddr, pgsize, pgcount,
+			      region->prot, GFP_KERNEL, &mapped);
+
+			if (ret)
+				goto end_put;
+		}
+	}
+end_put:
+	iommu_put_resv_regions(dev, &resv_regions);
+	return ret;
+}
+
 static int apple_dart_finalize_domain(struct iommu_domain *domain,
+				      struct device *dev,
 				      struct apple_dart_master_cfg *cfg)
 {
 	struct apple_dart_domain *dart_domain = to_dart_domain(domain);
@@ -633,6 +672,11 @@ static int apple_dart_finalize_domain(struct iommu_domain *domain,
 
 	dart_domain->finalized = true;
 
+	if (dart->locked) {
+		/* TODO: error handling */
+		ret = apple_dart_setup_resv_locked(domain, dev, dart->pgsize);
+		io_pgtable_dart_setup_locked(dart_domain->pgtbl_ops);
+	}
 done:
 	mutex_unlock(&dart_domain->init_lock);
 	return ret;
@@ -689,7 +733,7 @@ static int apple_dart_attach_dev(struct iommu_domain *domain,
 	if (dart0->locked && domain->type != IOMMU_DOMAIN_DMA)
 		return -EINVAL;
 
-	ret = apple_dart_finalize_domain(domain, cfg);
+	ret = apple_dart_finalize_domain(domain, dev, cfg);
 	if (ret)
 		return ret;
 

drivers/iommu/io-pgtable-dart.c

@@ -16,6 +16,7 @@
 #include <linux/atomic.h>
 #include <linux/bitfield.h>
 #include <linux/bitops.h>
+#include "linux/export.h"
 #include <linux/io.h>
 #include <linux/io-pgtable.h>
 #include <linux/kernel.h>
@@ -25,6 +26,8 @@
 
 #include <asm/barrier.h>
 
+#include "io-pgtable-dart.h"
+
 #define DART1_MAX_ADDR_BITS	36
 
 #define DART_MAX_TABLES		4
@@ -363,6 +366,32 @@ static phys_addr_t dart_iova_to_phys(struct io_pgtable_ops *ops,
 	return 0;
 }
 
+int io_pgtable_dart_setup_locked(struct io_pgtable_ops *ops)
+{
+	void *l1tbl;
+	struct dart_io_pgtable *data = io_pgtable_ops_to_data(ops);
+	struct io_pgtable_cfg *cfg = &data->iop.cfg;
+	size_t size;
+
+	if (!(cfg->quirks & IO_PGTABLE_QUIRK_APPLE_LOCKED))
+		return 0;
+
+	size = cfg->pgsize_bitmap;
+	l1tbl = devm_memremap(cfg->iommu_dev, cfg->apple_dart_cfg.ttbr[0], size,
+			      MEMREMAP_WB);
+	if (!l1tbl)
+		return -ENOMEM;
+
+	for (int entry = 0; entry < DART_PTES_PER_TABLE(data); entry++)
+		((dart_iopte *)l1tbl)[entry] = ((dart_iopte *)data->pgd[0])[entry];
+
+	free_pages((unsigned long)data->pgd[0], get_order(DART_GRANULE(data)));
+	data->pgd[0] = l1tbl;
+
+	return 0;
+}
+EXPORT_SYMBOL(io_pgtable_dart_setup_locked);
+
 static struct dart_io_pgtable *
 dart_alloc_pgtable(struct io_pgtable_cfg *cfg)
 {
@@ -418,22 +447,19 @@ apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
 
 	cfg->apple_dart_cfg.n_ttbrs = 1 << data->tbl_bits;
 
-	/* Locked DARTs can not modify the TTBR registers. The known locked
-	 * DARTs (dcp, dcpext0) use just a single TTBR so we do not have to
-	 * worry whether the pages are consecutive.
+	/* Locked DARTs can not modify the TTBR registers. Allocate first a shadow
+	 * page table so locked DARTs (disp0, dcp, dcpext*) can map their reserved
+	 * memory regions. They will be later in io_pgtable_dart_setup_locked()
+	 * copied to the locked L1 table.
 	 */
 	if (cfg->quirks & IO_PGTABLE_QUIRK_APPLE_LOCKED) {
-		size_t size = cfg->pgsize_bitmap;
 		if (cfg->apple_dart_cfg.n_ttbrs > 1)
 			goto out_free_data;
 
-		data->pgd[0] = devm_memremap(cfg->iommu_dev,
-					     cfg->apple_dart_cfg.ttbr[0],
-					     size, MEMREMAP_WB);
+		data->pgd[0] = __dart_alloc_pages(DART_GRANULE(data), GFP_KERNEL);
 		if (!data->pgd[0])
 			goto out_free_data;
-		/* start with an empty table */
-		memset(data->pgd[0], 0, size);
+
 		return &data->iop;
 	}
 

drivers/iommu/io-pgtable-dart.h

@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Apple DART page table allocator.
+ *
+ * Copyright (C) 2022 The Asahi Linux Contributors
+ */
+
+/* This will go away on the next iteration of locked DART handling */
+
+#ifndef IO_PGTABLE_DART_H_
+#define IO_PGTABLE_DART_H_
+
+int io_pgtable_dart_setup_locked(struct io_pgtable_ops *ops);
+
+#endif /* IO_PGTABLE_DART_H_ */