Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion drivers/accel/ane/ane_fw_platform.c
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ const struct ane_fw_platform ane_h16_platform = {
.dapf_dart = 0,
.dapf_offset = 0x10000,
.dapf_size = SZ_16K,
/* The DT compatible is t6041; the physical M4 Pro is T6040. */
/* The DT compatible is t6041 on the M4 Max and the M4 Pro (T6040) alike. */
.soc = 0x6040,
.chip_family = 16,
.ane_type = 0x110,
Expand Down
42 changes: 42 additions & 0 deletions drivers/accel/ane/ane_fw_transport.h
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include "ane_fw_protocol.h"
#include "ane_h16_execute.h"

#define ANE_FW_TERMINAL 0
#define ANE_FW_IO 1
#define ANE_FW_SHAREDMALLOC 5
#define ANE_FW_IO_T2H 6
Expand Down Expand Up @@ -128,6 +129,8 @@ struct ane_fw_transport {
size_t pool_used;
size_t shared_bytes;
u32 shared_count;
u64 terminal_messages;
u64 terminal_bytes;
int error;
/* ane_fw_transport.h line that raised error, for diagnosis. */
unsigned int error_line;
Expand Down Expand Up @@ -809,6 +812,39 @@ static inline int ane_fw_transport_irq(struct ane_fw_transport *t, u32 *pending)
return 0;
}

/*
* The firmware's text log on a unidirectional type 2 ring: a slot is ready when
* bit 0 of its header is clear, and the consumer returns it by storing 1, with
* no doorbell. The text is discarded. An unserved ring fills and the firmware's
* printf blocks behind it.
*/
static inline int ane_fw_transport_terminal(struct ane_fw_transport *t)
{
const struct ane_fw_channel *c = &t->channels[ANE_FW_TERMINAL];
unsigned int i;

if (c->type != 2)
return 0;
for (i = 0; i < c->slots; i++) {
u8 *slot = ane_fw_transport_slot(t, ANE_FW_TERMINAL);
u64 header = t->ops->load64(t->ctx, slot), bytes;

t->ops->read_barrier(t->ctx);
if (header & 1)
break;
bytes = t->ops->load64(t->ctx, slot + 8);
if (bytes && !ane_fw_transport_report(t, header & ~3ULL, bytes))
return ane_fw_transport_fail(t, -EPROTO);
t->terminal_messages++;
t->terminal_bytes += bytes;
t->ops->write_barrier(t->ctx);
t->ops->store64(t->ctx, slot, 1);
t->ops->write_barrier(t->ctx);
ane_fw_transport_advance(t, ANE_FW_TERMINAL);
}
return 0;
}

/* The original absolute command deadline survives every provider/IRQ service. */
/*
* After a program is torn down the firmware posts FREE for its process pool,
Expand All @@ -835,6 +871,9 @@ static inline int ane_fw_transport_settle(struct ane_fw_transport *t, u64 quiet_
} else if (ret != -EAGAIN) {
return ret;
}
ret = ane_fw_transport_terminal(t);
if (ret)
return ret;
if (now - last >= quiet_ns || now - start >= limit_ns)
return served;
t->polls++;
Expand All @@ -858,6 +897,9 @@ static inline int ane_fw_transport_wait(struct ane_fw_transport *t, unsigned int
ret = ane_fw_transport_shared(t);
if (ret && ret != -EAGAIN)
return ret;
ret = ane_fw_transport_terminal(t);
if (ret)
return ret;
/* Recheck before sleeping, including time spent in alloc(). */
now = t->ops->now_ns(t->ctx);
if (now >= t->pending[ch].deadline_ns)
Expand Down
62 changes: 62 additions & 0 deletions drivers/accel/ane/ane_fw_transport_test.c
Original file line number Diff line number Diff line change
Expand Up @@ -1477,7 +1477,69 @@ static void ane_transport_poll_sleep_test(struct kunit *test)
KUNIT_EXPECT_EQ(test, ane_fw_poll_sleep_us(U64_MAX, &max_us), 1000U);
}

static void ane_transport_terminal_ack_test(struct kunit *test)
{
struct ane_transport_test *f = test->priv;
u8 *slot, *next;
u32 last_reg;

KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0);
slot = ane_test_slot(f, ANE_FW_TERMINAL);
next = slot + ANE_FW_SLOT_SIZE;
put_unaligned_le64(TEST_POOL_DVA | 2, slot);
put_unaligned_le64(1, slot + 8);
put_unaligned_le64(4, slot + 16);
put_unaligned_le64(3, next);
last_reg = f->last_reg;
KUNIT_EXPECT_EQ(test, ane_fw_transport_terminal(&f->t), 0);
KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot), 1ULL);
KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot + 8), 1ULL);
KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot + 16), 4ULL);
KUNIT_EXPECT_EQ(test, f->t.head[ANE_FW_TERMINAL], 1U);
KUNIT_EXPECT_EQ(test, f->t.terminal_messages, 1ULL);
KUNIT_EXPECT_EQ(test, f->t.terminal_bytes, 1ULL);
KUNIT_EXPECT_EQ(test, f->last_reg, last_reg);
KUNIT_EXPECT_EQ(test, ane_fw_transport_terminal(&f->t), 0);
KUNIT_EXPECT_EQ(test, f->t.terminal_messages, 1ULL);
}

static void ane_transport_terminal_bounds_test(struct kunit *test)
{
struct ane_transport_test *f = test->priv;
u8 *slot;

KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0);
slot = ane_test_slot(f, ANE_FW_TERMINAL);
put_unaligned_le64(0xdead0002, slot);
put_unaligned_le64(1, slot + 8);
KUNIT_EXPECT_EQ(test, ane_fw_transport_terminal(&f->t), -EPROTO);
KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot), 0xdead0002ULL);
KUNIT_EXPECT_EQ(test, f->t.head[ANE_FW_TERMINAL], 0U);
}

static void ane_transport_terminal_wrap_test(struct kunit *test)
{
struct ane_transport_test *f = test->priv;
u8 *slot;

KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0);
slot = ane_test_slot(f, ANE_FW_TERMINAL);
for (unsigned int i = 0; i < 2; i++) {
put_unaligned_le64(TEST_POOL_DVA | 2, slot + i * ANE_FW_SLOT_SIZE);
put_unaligned_le64(i + 1, slot + i * ANE_FW_SLOT_SIZE + 8);
}
KUNIT_EXPECT_EQ(test, ane_fw_transport_terminal(&f->t), 0);
KUNIT_EXPECT_EQ(test, f->t.head[ANE_FW_TERMINAL], 0U);
KUNIT_EXPECT_EQ(test, f->t.terminal_messages, 2ULL);
KUNIT_EXPECT_EQ(test, f->t.terminal_bytes, 3ULL);
KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot), 1ULL);
KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot + ANE_FW_SLOT_SIZE), 1ULL);
}

static struct kunit_case ane_transport_cases[] = {
KUNIT_CASE(ane_transport_terminal_ack_test),
KUNIT_CASE(ane_transport_terminal_bounds_test),
KUNIT_CASE(ane_transport_terminal_wrap_test),
KUNIT_CASE(ane_transport_second_load_test),
KUNIT_CASE(ane_transport_bar_context_test),
KUNIT_CASE(ane_transport_bar_context_h16_test),
Expand Down
21 changes: 12 additions & 9 deletions drivers/accel/ane/ane_h16_backend.c
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include <linux/platform_device.h>
#include <linux/pm_domain.h>
#include <linux/pm_runtime.h>
#include <linux/scatterlist.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
Expand Down Expand Up @@ -787,6 +788,8 @@ static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, in
struct ane_h16_mapping *m;
u32 i, j, pages_per_block = SZ_16K / PAGE_SIZE;
unsigned int order = get_order(SZ_16K);
struct sg_table sgt;
ssize_t mapped;
int ret;

if (!dev_is_dma_coherent(h->dev)) {
Expand Down Expand Up @@ -818,15 +821,15 @@ static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, in
m->cpu = vmap(m->pages, size / PAGE_SIZE, VM_MAP, PAGE_KERNEL);
if (!m->cpu)
return -ENOMEM;
memset(m->cpu, 0, size);
for (i = 0; i < m->blocks; i++) {
ret = iommu_map(h->domain, dva + i * SZ_16K,
page_to_phys(m->pages[i * pages_per_block]), SZ_16K,
prot, GFP_KERNEL);
if (ret)
return ret;
m->mapped += SZ_16K;
}
ret = sg_alloc_table_from_pages(&sgt, m->pages, size / PAGE_SIZE, 0, size, GFP_KERNEL);
if (ret)
return ret;
/* A failed call has already unmapped what it mapped. */
mapped = iommu_map_sgtable(h->domain, dva, &sgt, prot);
sg_free_table(&sgt);
if (mapped < 0)
return mapped;
m->mapped = mapped;
*out = (struct ane_fw_surface) { .cpu = m->cpu, .dva = dva, .size = size };
return 0;
}
Expand Down
5 changes: 3 additions & 2 deletions drivers/accel/ane/ane_h16_boot.h
Original file line number Diff line number Diff line change
Expand Up @@ -199,10 +199,11 @@ static inline int ane_h16_faults(struct ane_h16_boot *b, bool log)
for (i = 0; i < 3; i++) {
enum ane_h16_window w = ANE_H16_DART0 + i;
u32 status = b->ops->read32(b->ctx, w, m->dart_fault);
u64 address = b->ops->read32(b->ctx, w, m->dart_address_lo);

address |= (u64)b->ops->read32(b->ctx, w, m->dart_address_hi) << 32;
if (log || (status & BIT(31))) {
u64 address = b->ops->read32(b->ctx, w, m->dart_address_lo);

address |= (u64)b->ops->read32(b->ctx, w, m->dart_address_hi) << 32;
b->ops->log(b->ctx, "DART status", i, status);
b->ops->log(b->ctx, "DART address", i, address);
}
Expand Down
19 changes: 19 additions & 0 deletions drivers/accel/ane/ane_h16_boot_data.h
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,20 @@ static const struct ane_h16_preload_rule ane_h16_preload_25g83 = {
},
};

/* 26.5.2 keeps the 25G83 DATA layout around a different resident image. */
static const struct ane_h16_preload_rule ane_h16_preload_25f84 = {
.cells = ane_h16_patchbay_25g83,
.cell_count = ARRAY_SIZE(ane_h16_patchbay_25g83),
.zero_start = 0x1bc00,
.zero_end = 0x1c2a0,
.digest = {
0x17, 0xc3, 0xef, 0x19, 0x6b, 0x89, 0xe4, 0xf5,
0x94, 0x9a, 0xc7, 0x77, 0x76, 0x0e, 0x5b, 0x05,
0x80, 0x1f, 0xf5, 0xaa, 0x3b, 0x9f, 0x93, 0x77,
0x57, 0xfb, 0xd4, 0xee, 0xf5, 0x6b, 0x00, 0x45,
},
};

/* Every supported H16 firmware build, matched against the live preload. */
static const struct ane_h16_profile ane_h16_profiles[] __maybe_unused = {
{
Expand All @@ -327,6 +341,11 @@ static const struct ane_h16_profile ane_h16_profiles[] __maybe_unused = {
.preload = &ane_h16_preload_25g83,
.build = &ane_h16_build_26a428,
},
{
.identity = "H16 26.5.2 25F84",
.preload = &ane_h16_preload_25f84,
.build = &ane_h16_build_26a428,
},
};

#endif /* __ANE_H16_BOOT_DATA_H__ */
19 changes: 19 additions & 0 deletions drivers/accel/ane/ane_h16_boot_test.c
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ struct ane_h16_sim {
u64 timeout_head;
u32 lock;
u32 fault;
u32 fault_address_reads;
u32 channel_request;
u32 ipc_request;
u32 heap_request;
Expand Down Expand Up @@ -99,6 +100,8 @@ static u32 sim_read32(void *ctx, enum ane_h16_window w, u32 off)
return s->lock;
if (off == s->mmio.dart_fault)
return s->fault;
if (off == s->mmio.dart_address_lo || off == s->mmio.dart_address_hi)
s->fault_address_reads++;
return 0;
}
if (w == ANE_H16_DAPF) {
Expand Down Expand Up @@ -832,6 +835,21 @@ static void ane_h16_dart_fault_test(struct kunit *test)
KUNIT_EXPECT_EQ(test, s->quarantines, 1U);
}

static void ane_h16_fault_address_on_demand_test(struct kunit *test)
{
struct ane_h16_sim *s = test->priv;

s->boot.mmio = &s->mmio;
KUNIT_EXPECT_EQ(test, ane_h16_faults(&s->boot, false), 0);
KUNIT_EXPECT_EQ(test, s->fault_address_reads, 0U);
s->fault = BIT(31);
KUNIT_EXPECT_EQ(test, ane_h16_faults(&s->boot, false), -EIO);
KUNIT_EXPECT_EQ(test, s->fault_address_reads, 6U);
s->fault = 0;
KUNIT_EXPECT_EQ(test, ane_h16_faults(&s->boot, true), 0);
KUNIT_EXPECT_EQ(test, s->fault_address_reads, 12U);
}

static void ane_h16_map_failure_test(struct kunit *test)
{
struct ane_h16_sim *s = test->priv;
Expand Down Expand Up @@ -972,6 +990,7 @@ static struct kunit_case ane_h16_boot_cases[] = {
KUNIT_CASE(ane_h16_power_actual_test),
KUNIT_CASE(ane_h16_rvbar_test),
KUNIT_CASE(ane_h16_dart_fault_test),
KUNIT_CASE(ane_h16_fault_address_on_demand_test),
KUNIT_CASE(ane_h16_map_failure_test),
KUNIT_CASE(ane_h16_no_ready_test),
KUNIT_CASE(ane_h16_wrong_table_test),
Expand Down
2 changes: 2 additions & 0 deletions drivers/accel/ane/ane_h16_program.c
Original file line number Diff line number Diff line change
Expand Up @@ -1046,6 +1046,8 @@ static int h16_wait_report(struct ane_h16_session *s, struct ane_h16_pending *re
if (ret == -EAGAIN)
ret = 0;
}
if (!ret)
ret = ane_fw_transport_terminal(t);
if (ret)
break;
now = h16_now(s);
Expand Down