diff --git a/drivers/accel/ane/ane_fw_platform.c b/drivers/accel/ane/ane_fw_platform.c index bd2b3f254f2de..1f0d50a1f0d86 100644 --- a/drivers/accel/ane/ane_fw_platform.c +++ b/drivers/accel/ane/ane_fw_platform.c @@ -40,7 +40,7 @@ const struct ane_fw_platform ane_h16_platform = { .dapf_dart = 0, .dapf_offset = 0x10000, .dapf_size = SZ_16K, - /* The DT compatible is t6041; the physical M4 Pro is T6040. */ + /* The DT compatible is t6041 on the M4 Max and the M4 Pro (T6040) alike. */ .soc = 0x6040, .chip_family = 16, .ane_type = 0x110, diff --git a/drivers/accel/ane/ane_fw_transport.h b/drivers/accel/ane/ane_fw_transport.h index 53c44178f528b..7ae897bc90c80 100644 --- a/drivers/accel/ane/ane_fw_transport.h +++ b/drivers/accel/ane/ane_fw_transport.h @@ -8,6 +8,7 @@ #include "ane_fw_protocol.h" #include "ane_h16_execute.h" +#define ANE_FW_TERMINAL 0 #define ANE_FW_IO 1 #define ANE_FW_SHAREDMALLOC 5 #define ANE_FW_IO_T2H 6 @@ -128,6 +129,8 @@ struct ane_fw_transport { size_t pool_used; size_t shared_bytes; u32 shared_count; + u64 terminal_messages; + u64 terminal_bytes; int error; /* ane_fw_transport.h line that raised error, for diagnosis. */ unsigned int error_line; @@ -809,6 +812,39 @@ static inline int ane_fw_transport_irq(struct ane_fw_transport *t, u32 *pending) return 0; } +/* + * The firmware's text log on a unidirectional type 2 ring: a slot is ready when + * bit 0 of its header is clear, and the consumer returns it by storing 1, with + * no doorbell. The text is discarded. An unserved ring fills and the firmware's + * printf blocks behind it. + */ +static inline int ane_fw_transport_terminal(struct ane_fw_transport *t) +{ + const struct ane_fw_channel *c = &t->channels[ANE_FW_TERMINAL]; + unsigned int i; + + if (c->type != 2) + return 0; + for (i = 0; i < c->slots; i++) { + u8 *slot = ane_fw_transport_slot(t, ANE_FW_TERMINAL); + u64 header = t->ops->load64(t->ctx, slot), bytes; + + t->ops->read_barrier(t->ctx); + if (header & 1) + break; + bytes = t->ops->load64(t->ctx, slot + 8); + if (bytes && !ane_fw_transport_report(t, header & ~3ULL, bytes)) + return ane_fw_transport_fail(t, -EPROTO); + t->terminal_messages++; + t->terminal_bytes += bytes; + t->ops->write_barrier(t->ctx); + t->ops->store64(t->ctx, slot, 1); + t->ops->write_barrier(t->ctx); + ane_fw_transport_advance(t, ANE_FW_TERMINAL); + } + return 0; +} + /* The original absolute command deadline survives every provider/IRQ service. */ /* * After a program is torn down the firmware posts FREE for its process pool, @@ -835,6 +871,9 @@ static inline int ane_fw_transport_settle(struct ane_fw_transport *t, u64 quiet_ } else if (ret != -EAGAIN) { return ret; } + ret = ane_fw_transport_terminal(t); + if (ret) + return ret; if (now - last >= quiet_ns || now - start >= limit_ns) return served; t->polls++; @@ -858,6 +897,9 @@ static inline int ane_fw_transport_wait(struct ane_fw_transport *t, unsigned int ret = ane_fw_transport_shared(t); if (ret && ret != -EAGAIN) return ret; + ret = ane_fw_transport_terminal(t); + if (ret) + return ret; /* Recheck before sleeping, including time spent in alloc(). */ now = t->ops->now_ns(t->ctx); if (now >= t->pending[ch].deadline_ns) diff --git a/drivers/accel/ane/ane_fw_transport_test.c b/drivers/accel/ane/ane_fw_transport_test.c index f1a69baa86630..4f4b87cf98974 100644 --- a/drivers/accel/ane/ane_fw_transport_test.c +++ b/drivers/accel/ane/ane_fw_transport_test.c @@ -1477,7 +1477,69 @@ static void ane_transport_poll_sleep_test(struct kunit *test) KUNIT_EXPECT_EQ(test, ane_fw_poll_sleep_us(U64_MAX, &max_us), 1000U); } +static void ane_transport_terminal_ack_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + u8 *slot, *next; + u32 last_reg; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + slot = ane_test_slot(f, ANE_FW_TERMINAL); + next = slot + ANE_FW_SLOT_SIZE; + put_unaligned_le64(TEST_POOL_DVA | 2, slot); + put_unaligned_le64(1, slot + 8); + put_unaligned_le64(4, slot + 16); + put_unaligned_le64(3, next); + last_reg = f->last_reg; + KUNIT_EXPECT_EQ(test, ane_fw_transport_terminal(&f->t), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot), 1ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot + 8), 1ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot + 16), 4ULL); + KUNIT_EXPECT_EQ(test, f->t.head[ANE_FW_TERMINAL], 1U); + KUNIT_EXPECT_EQ(test, f->t.terminal_messages, 1ULL); + KUNIT_EXPECT_EQ(test, f->t.terminal_bytes, 1ULL); + KUNIT_EXPECT_EQ(test, f->last_reg, last_reg); + KUNIT_EXPECT_EQ(test, ane_fw_transport_terminal(&f->t), 0); + KUNIT_EXPECT_EQ(test, f->t.terminal_messages, 1ULL); +} + +static void ane_transport_terminal_bounds_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + u8 *slot; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + slot = ane_test_slot(f, ANE_FW_TERMINAL); + put_unaligned_le64(0xdead0002, slot); + put_unaligned_le64(1, slot + 8); + KUNIT_EXPECT_EQ(test, ane_fw_transport_terminal(&f->t), -EPROTO); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot), 0xdead0002ULL); + KUNIT_EXPECT_EQ(test, f->t.head[ANE_FW_TERMINAL], 0U); +} + +static void ane_transport_terminal_wrap_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + u8 *slot; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + slot = ane_test_slot(f, ANE_FW_TERMINAL); + for (unsigned int i = 0; i < 2; i++) { + put_unaligned_le64(TEST_POOL_DVA | 2, slot + i * ANE_FW_SLOT_SIZE); + put_unaligned_le64(i + 1, slot + i * ANE_FW_SLOT_SIZE + 8); + } + KUNIT_EXPECT_EQ(test, ane_fw_transport_terminal(&f->t), 0); + KUNIT_EXPECT_EQ(test, f->t.head[ANE_FW_TERMINAL], 0U); + KUNIT_EXPECT_EQ(test, f->t.terminal_messages, 2ULL); + KUNIT_EXPECT_EQ(test, f->t.terminal_bytes, 3ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot), 1ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot + ANE_FW_SLOT_SIZE), 1ULL); +} + static struct kunit_case ane_transport_cases[] = { + KUNIT_CASE(ane_transport_terminal_ack_test), + KUNIT_CASE(ane_transport_terminal_bounds_test), + KUNIT_CASE(ane_transport_terminal_wrap_test), KUNIT_CASE(ane_transport_second_load_test), KUNIT_CASE(ane_transport_bar_context_test), KUNIT_CASE(ane_transport_bar_context_h16_test), diff --git a/drivers/accel/ane/ane_h16_backend.c b/drivers/accel/ane/ane_h16_backend.c index 4157cc4d4b787..b0d0a9f99f110 100644 --- a/drivers/accel/ane/ane_h16_backend.c +++ b/drivers/accel/ane/ane_h16_backend.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -787,6 +788,8 @@ static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, in struct ane_h16_mapping *m; u32 i, j, pages_per_block = SZ_16K / PAGE_SIZE; unsigned int order = get_order(SZ_16K); + struct sg_table sgt; + ssize_t mapped; int ret; if (!dev_is_dma_coherent(h->dev)) { @@ -818,15 +821,15 @@ static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, in m->cpu = vmap(m->pages, size / PAGE_SIZE, VM_MAP, PAGE_KERNEL); if (!m->cpu) return -ENOMEM; - memset(m->cpu, 0, size); - for (i = 0; i < m->blocks; i++) { - ret = iommu_map(h->domain, dva + i * SZ_16K, - page_to_phys(m->pages[i * pages_per_block]), SZ_16K, - prot, GFP_KERNEL); - if (ret) - return ret; - m->mapped += SZ_16K; - } + ret = sg_alloc_table_from_pages(&sgt, m->pages, size / PAGE_SIZE, 0, size, GFP_KERNEL); + if (ret) + return ret; + /* A failed call has already unmapped what it mapped. */ + mapped = iommu_map_sgtable(h->domain, dva, &sgt, prot); + sg_free_table(&sgt); + if (mapped < 0) + return mapped; + m->mapped = mapped; *out = (struct ane_fw_surface) { .cpu = m->cpu, .dva = dva, .size = size }; return 0; } diff --git a/drivers/accel/ane/ane_h16_boot.h b/drivers/accel/ane/ane_h16_boot.h index d8c9c8e6eb102..ccea3743fad1d 100644 --- a/drivers/accel/ane/ane_h16_boot.h +++ b/drivers/accel/ane/ane_h16_boot.h @@ -199,10 +199,11 @@ static inline int ane_h16_faults(struct ane_h16_boot *b, bool log) for (i = 0; i < 3; i++) { enum ane_h16_window w = ANE_H16_DART0 + i; u32 status = b->ops->read32(b->ctx, w, m->dart_fault); - u64 address = b->ops->read32(b->ctx, w, m->dart_address_lo); - address |= (u64)b->ops->read32(b->ctx, w, m->dart_address_hi) << 32; if (log || (status & BIT(31))) { + u64 address = b->ops->read32(b->ctx, w, m->dart_address_lo); + + address |= (u64)b->ops->read32(b->ctx, w, m->dart_address_hi) << 32; b->ops->log(b->ctx, "DART status", i, status); b->ops->log(b->ctx, "DART address", i, address); } diff --git a/drivers/accel/ane/ane_h16_boot_data.h b/drivers/accel/ane/ane_h16_boot_data.h index 4875abe800bb2..0dd4b1ea91b82 100644 --- a/drivers/accel/ane/ane_h16_boot_data.h +++ b/drivers/accel/ane/ane_h16_boot_data.h @@ -315,6 +315,20 @@ static const struct ane_h16_preload_rule ane_h16_preload_25g83 = { }, }; +/* 26.5.2 keeps the 25G83 DATA layout around a different resident image. */ +static const struct ane_h16_preload_rule ane_h16_preload_25f84 = { + .cells = ane_h16_patchbay_25g83, + .cell_count = ARRAY_SIZE(ane_h16_patchbay_25g83), + .zero_start = 0x1bc00, + .zero_end = 0x1c2a0, + .digest = { + 0x17, 0xc3, 0xef, 0x19, 0x6b, 0x89, 0xe4, 0xf5, + 0x94, 0x9a, 0xc7, 0x77, 0x76, 0x0e, 0x5b, 0x05, + 0x80, 0x1f, 0xf5, 0xaa, 0x3b, 0x9f, 0x93, 0x77, + 0x57, 0xfb, 0xd4, 0xee, 0xf5, 0x6b, 0x00, 0x45, + }, +}; + /* Every supported H16 firmware build, matched against the live preload. */ static const struct ane_h16_profile ane_h16_profiles[] __maybe_unused = { { @@ -327,6 +341,11 @@ static const struct ane_h16_profile ane_h16_profiles[] __maybe_unused = { .preload = &ane_h16_preload_25g83, .build = &ane_h16_build_26a428, }, + { + .identity = "H16 26.5.2 25F84", + .preload = &ane_h16_preload_25f84, + .build = &ane_h16_build_26a428, + }, }; #endif /* __ANE_H16_BOOT_DATA_H__ */ diff --git a/drivers/accel/ane/ane_h16_boot_test.c b/drivers/accel/ane/ane_h16_boot_test.c index 581bd4b41eee6..b29affe81c9ce 100644 --- a/drivers/accel/ane/ane_h16_boot_test.c +++ b/drivers/accel/ane/ane_h16_boot_test.c @@ -35,6 +35,7 @@ struct ane_h16_sim { u64 timeout_head; u32 lock; u32 fault; + u32 fault_address_reads; u32 channel_request; u32 ipc_request; u32 heap_request; @@ -99,6 +100,8 @@ static u32 sim_read32(void *ctx, enum ane_h16_window w, u32 off) return s->lock; if (off == s->mmio.dart_fault) return s->fault; + if (off == s->mmio.dart_address_lo || off == s->mmio.dart_address_hi) + s->fault_address_reads++; return 0; } if (w == ANE_H16_DAPF) { @@ -832,6 +835,21 @@ static void ane_h16_dart_fault_test(struct kunit *test) KUNIT_EXPECT_EQ(test, s->quarantines, 1U); } +static void ane_h16_fault_address_on_demand_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->boot.mmio = &s->mmio; + KUNIT_EXPECT_EQ(test, ane_h16_faults(&s->boot, false), 0); + KUNIT_EXPECT_EQ(test, s->fault_address_reads, 0U); + s->fault = BIT(31); + KUNIT_EXPECT_EQ(test, ane_h16_faults(&s->boot, false), -EIO); + KUNIT_EXPECT_EQ(test, s->fault_address_reads, 6U); + s->fault = 0; + KUNIT_EXPECT_EQ(test, ane_h16_faults(&s->boot, true), 0); + KUNIT_EXPECT_EQ(test, s->fault_address_reads, 12U); +} + static void ane_h16_map_failure_test(struct kunit *test) { struct ane_h16_sim *s = test->priv; @@ -972,6 +990,7 @@ static struct kunit_case ane_h16_boot_cases[] = { KUNIT_CASE(ane_h16_power_actual_test), KUNIT_CASE(ane_h16_rvbar_test), KUNIT_CASE(ane_h16_dart_fault_test), + KUNIT_CASE(ane_h16_fault_address_on_demand_test), KUNIT_CASE(ane_h16_map_failure_test), KUNIT_CASE(ane_h16_no_ready_test), KUNIT_CASE(ane_h16_wrong_table_test), diff --git a/drivers/accel/ane/ane_h16_program.c b/drivers/accel/ane/ane_h16_program.c index a663dbc59386c..5f6136f8b71a5 100644 --- a/drivers/accel/ane/ane_h16_program.c +++ b/drivers/accel/ane/ane_h16_program.c @@ -1046,6 +1046,8 @@ static int h16_wait_report(struct ane_h16_session *s, struct ane_h16_pending *re if (ret == -EAGAIN) ret = 0; } + if (!ret) + ret = ane_fw_transport_terminal(t); if (ret) break; now = h16_now(s);