diff --git a/include/drm-uapi/pvr_drm.h b/include/drm-uapi/pvr_drm.h index ccf6c211246..f37485e9b92 100644 --- a/include/drm-uapi/pvr_drm.h +++ b/include/drm-uapi/pvr_drm.h @@ -109,6 +109,26 @@ struct drm_pvr_obj_array { #define DRM_IOCTL_PVR_DESTROY_HWRT_DATASET PVR_IOCTL(0x0c, DRM_IOW, destroy_hwrt_dataset) #define DRM_IOCTL_PVR_SUBMIT_JOBS PVR_IOCTL(0x0d, DRM_IOW, submit_jobs) +/* Proposed BO_MADVISE UAPI: replace with the merged kernel header. */ +#define DRM_IOCTL_PVR_BO_MADVISE PVR_IOCTL(0x0e, DRM_IOWR, bo_madvise) + +#define DRM_PVR_MADV_WILLNEED 0 +#define DRM_PVR_MADV_DONTNEED 1 + +/** + * struct drm_pvr_ioctl_bo_madvise_args - Mark an idle BO's contents purgeable + * @handle: GEM handle. + * @madv: DRM_PVR_MADV_WILLNEED or DRM_PVR_MADV_DONTNEED. + * @retained: Output, zero if purged; the caller must free and replace the BO. + * @_pad: Reserved, must be zero. + */ +struct drm_pvr_ioctl_bo_madvise_args { + __u32 handle; + __u32 madv; + __u32 retained; + __u32 _pad; +}; + /** * DOC: PowerVR IOCTL DEV_QUERY interface */ @@ -440,6 +460,13 @@ struct drm_pvr_dev_query_static_data_areas { * * Append only. Do not reorder. */ +/* Proposed capability query UAPI: replace with the merged kernel header. */ +#define DRM_PVR_CAP_RETRY_ENOMEM (1ULL << 0) +#define DRM_PVR_CAP_BO_MADVISE (1ULL << 1) +struct drm_pvr_dev_query_caps { + __u64 caps; +}; + enum drm_pvr_dev_query { /** * @DRM_PVR_DEV_QUERY_GPU_INFO_GET: The dev query args contain a pointer @@ -476,6 +503,7 @@ enum drm_pvr_dev_query { * a pointer to &struct drm_pvr_dev_query_static_data_areas. */ DRM_PVR_DEV_QUERY_STATIC_DATA_AREAS_GET, + DRM_PVR_DEV_QUERY_CAPS_GET, }; /** diff --git a/src/gallium/auxiliary/util/u_blitter.c b/src/gallium/auxiliary/util/u_blitter.c index f75065114a3..3e79c5a4935 100644 --- a/src/gallium/auxiliary/util/u_blitter.c +++ b/src/gallium/auxiliary/util/u_blitter.c @@ -154,6 +154,7 @@ struct blitter_context_priv bool has_stencil_export; bool has_texture_multisample; bool has_txf_txq; + bool blit_no_txf; /* util_blitter_set_txf(): single-sample blits sample, never fetch */ bool has_sample_shading; bool cube_as_2darray; bool has_texrect; @@ -551,6 +552,13 @@ void util_blitter_set_texture_multisample(struct blitter_context *blitter, ctx->has_texture_multisample = supported; } +void util_blitter_set_txf(struct blitter_context *blitter, bool supported) +{ + struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter; + + ctx->blit_no_txf = !supported; +} + void util_blitter_set_running_flag(struct blitter_context *blitter) { if (blitter->running) { @@ -2038,7 +2046,7 @@ bool util_blitter_blit_with_txf(struct blitter_context *blitter, bool use_txf = false; /* Don't support scaled blits. The TXF shader uses F2I for rounding. */ - if (ctx->has_txf_txq && + if (ctx->has_txf_txq && !ctx->blit_no_txf && !is_scaled && filter == PIPE_TEX_FILTER_NEAREST && src->target != PIPE_TEXTURE_CUBE && diff --git a/src/gallium/auxiliary/util/u_blitter.h b/src/gallium/auxiliary/util/u_blitter.h index 2f9928df952..cef926bf01d 100644 --- a/src/gallium/auxiliary/util/u_blitter.h +++ b/src/gallium/auxiliary/util/u_blitter.h @@ -176,6 +176,13 @@ struct pipe_context *util_blitter_get_pipe(struct blitter_context *blitter) void util_blitter_set_texture_multisample(struct blitter_context *blitter, bool supported); +/** + * Override whether single-sample blits may fetch texels (TXF) instead of + * sampling with NEAREST filtering, for a driver whose texel fetch is slower + * or missing. Multisample sources always fetch. + */ +void util_blitter_set_txf(struct blitter_context *blitter, bool supported); + /* The default function to draw a rectangle. This can only be used * inside of the draw_rectangle callback if the driver overrides it. */ void util_blitter_draw_rectangle(struct blitter_context *blitter, diff --git a/src/gallium/drivers/zink/zink_context.c b/src/gallium/drivers/zink/zink_context.c index 129a742d98b..efb44589eba 100644 --- a/src/gallium/drivers/zink/zink_context.c +++ b/src/gallium/drivers/zink/zink_context.c @@ -6361,6 +6361,10 @@ zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags) if (!ctx->blitter) goto fail; ctx->blitter->use_single_triangle = screen->driver_workarounds.inconsistent_interpolation; + /* the A8 (G5P): a 1:1 blit samples NEAREST at texel centres, the same texels a fetch reads; the A8's + * texel fetch is not captured, so a fetching blit shader is refused (glReadPixels of an XRGB8888 buffer) */ + if (screen->is_g5p_a8) + util_blitter_set_txf(ctx->blitter, false); } zink_set_last_vertex_key(ctx)->last_vertex_stage = true; diff --git a/src/gallium/drivers/zink/zink_kopper.c b/src/gallium/drivers/zink/zink_kopper.c index 9f6cee614c7..faabce381ad 100644 --- a/src/gallium/drivers/zink/zink_kopper.c +++ b/src/gallium/drivers/zink/zink_kopper.c @@ -23,6 +23,7 @@ */ #include "util/detect_os.h" +#include "util/u_debug.h" #include "zink_context.h" #include "zink_screen.h" @@ -30,6 +31,8 @@ #include "zink_resource.h" #include "zink_kopper.h" +DEBUG_GET_ONCE_NUM_OPTION(g5p_swapchain_images, "PVR_G5P_SWAPCHAIN_IMAGES", 0) + static void zink_kopper_set_present_mode_for_interval(struct kopper_displaytarget *cdt, int interval) { @@ -326,6 +329,14 @@ kopper_CreateSwapchain(struct zink_screen *screen, struct kopper_displaytarget * */ if (screen->frames_in_flight > 1) { uint32_t want = MAX2(cswap->scci.minImageCount, screen->frames_in_flight + 2); + if (screen->is_g5p) { + /* A/B knob for the drawable count (CAMetalLayer defaults to 3). + * Keep the default rule until measurements decide the image count. + */ + const int64_t images = debug_get_option_g5p_swapchain_images(); + if (images >= 2 && images <= 8) + want = MAX2(cswap->scci.minImageCount, (uint32_t)images); + } if (cdt->caps.maxImageCount) want = MIN2(want, cdt->caps.maxImageCount); cswap->scci.minImageCount = want; diff --git a/src/gallium/drivers/zink/zink_screen.c b/src/gallium/drivers/zink/zink_screen.c index ca3fc5286ed..5f62420c1ad 100644 --- a/src/gallium/drivers/zink/zink_screen.c +++ b/src/gallium/drivers/zink/zink_screen.c @@ -733,6 +733,11 @@ zink_init_screen_caps(struct zink_screen *screen) u_init_pipe_screen_caps(&screen->base, screen->is_cpu ? 0 : 1); caps->r11g11b10f_requires_color_buffer_float = screen->is_g5p; + /* The A8 (G5P): u_blitter's texture-fetch shaders iterate their coordinate with perspective. Its quads have + * w = 1 at every vertex, where perspective-correct and screen-linear interpolation give the same value (the + * 1/w plane is the constant 1.0). The A8 has no captured no-perspective iterator, and a refused blit shader + * failed every glReadPixels from an XRGB8888 (BGRX) buffer: the desktop's GPU probe. */ + caps->prefer_persp = screen->is_g5p_a8; caps->throttle_frames = screen->frames_in_flight; /* Vulkan spec says the builtins count, so make GL count them towards @@ -3889,6 +3894,7 @@ zink_internal_create_screen(const struct pipe_screen_config *config, int64_t dev /* Apple A9 (G5P, the PowerVR Vulkan driver): its memory, synchronization and lost-device handling */ screen->is_g5p = screen->info.driver_props.driverID == VK_DRIVER_ID_IMAGINATION_OPEN_SOURCE_MESA && !strcmp(screen->info.driver_props.driverName, "Mesa PowerVR G5P"); + screen->is_g5p_a8 = screen->is_g5p && !strcmp(screen->info.props.deviceName, "PowerVR Apple A8"); screen->precise_sync = screen->is_g5p; screen->frames_in_flight = zink_default_frames_in_flight(screen->info.driver_props.driverID); diff --git a/src/gallium/drivers/zink/zink_types.h b/src/gallium/drivers/zink/zink_types.h index 8533ae6ff6c..b2ee143b543 100644 --- a/src/gallium/drivers/zink/zink_types.h +++ b/src/gallium/drivers/zink/zink_types.h @@ -1506,6 +1506,7 @@ struct zink_screen { unsigned frames_in_flight; /* pipe_caps.throttle_frames */ bool precise_sync; bool is_g5p; + bool is_g5p_a8; /* the A8 on the G5P driver: no texel fetch (its form is not captured) */ const char *vendor_name; const char *device_name; diff --git a/src/imagination/common/device_info/apple-a8.h b/src/imagination/common/device_info/apple-a8.h new file mode 100644 index 00000000000..08d5d52392b --- /dev/null +++ b/src/imagination/common/device_info/apple-a8.h @@ -0,0 +1,127 @@ +/* + * SPDX-License-Identifier: MIT + */ + +/* + * Apple A8 GPU (T7000, iPod touch 6, iPhone 6, iPad mini 4): a PowerVR + * Rogue 6XT core (GX6450) run by Apple's own firmware (AGXG4P, firmware + * g4p_b0). BVNC 3.10.4.5 as the kernel decodes the legacy CORE_REVISION + * (0x00030a00, measured on an iPod touch 6 under Linux) and CORE_ID + * (0x011b0405) registers, the same rule that gives 9.15.6.2 for the A9 and + * 13.10.6.2 for the A10. Apple's kext also accepts minor 8; no entry is made + * for it until a device reports it. + * + * The A8 takes the A9's paths of the driver (has_apple_g5p): the same + * firmware family and job protocol, Rogue CDM control streams of seven words + * (A10: eight), and a shader ISA of the same Apple family. Apple's LLVM + * backend for these GPUs defines "g4", "g5" and "g9" instruction sets, the + * A8 (g4p) being the base the A9 (g5p) adds to. has_apple_g4p marks where + * the A8 differs from the A9. See g5p/README.md, "The A8". + * + * Values marked "A8" are established on the hardware or from Apple's + * drivers; the others are the A9 entry's placeholders (apple-a9.h) and are + * not verified for this GPU. + */ + +#ifndef APPLE_A8_H +#define APPLE_A8_H + +#include + +#include "pvr_device_info.h" + +#define PVR_DEVICE_IDENT_3_V_4_5 \ + .device_id = 0x7000, .series_name = "Apple", \ + .public_name = "A8", .arch = PVR_DEVICE_ARCH_ROGUE + +static const struct pvr_device_features pvr_device_features_3_V_4_5 = { + .has_apple_g5p = true, + .has_apple_g4p = true, + .has_apple_compute_dispatch = true, + /* A8: AGXMetalA8 (iOS 12.4) ComputeProgramVariant constructor + * (0x1a13eb034) -> +0x790, what -[AGXA8ComputePipeline + * maxTotalThreadsPerThreadgroup] (0x1a13c5028) returns: + * min(32 * floor((3585 - a) / a), 262144 / b), a = (4t + 28) & ~31, + * b = (64f + 448) & ~511, at most 512 and a multiple of 32; the A9's + * routine is the same with 3329 and 393216. threadExecutionWidth is 32 + * (0x1a13c5020). The shared formula's bank-1 term keeps one task in + * reserve (pvr_compute_limits.h), below Apple's for both chips. + */ + .apple_compute_dispatch = { + .bank0_budget = 3585U, + .bank1_budget = 262144U, + .register_granularity = 8U, + .bank0_scale = 4U, + .bank0_rounding_bias = 28U, + .bank0_granularity = 32U, + .bank1_scale = 64U, + .bank1_rounding_bias = 448U, + .bank1_granularity = 512U, + .task_size = 32U, + .max_workgroup_size = 512U, + }, + .has_num_clusters = true, /* A8 */ + + /* A9 placeholders (apple-a9.h) */ + .has_tile_size_x = true, + .has_tile_size_y = true, + .has_cluster_grouping = true, + .has_common_store_size_in_dwords = true, + .has_isp_max_tiles_in_flight = true, + .has_isp_samples_per_pixel = true, + .has_max_instances_per_pds_task = true, + .has_max_multisample = true, + .has_max_partitions = true, + .has_max_usc_tasks = true, + .has_num_raster_pipes = true, + .has_slc_cache_line_size_bits = true, + .has_slc_mcu_cache_controls = true, + .has_unified_store_depth = true, + .has_usc_min_output_registers_per_pix = true, + .has_usc_slots = true, + .has_uvs_banks = true, + .has_uvs_pba_entries = true, + .has_uvs_vtx_entries = true, + .has_vdm_cam_size = true, + .has_xt_top_infrastructure = true, + + .num_clusters = 4U, /* A8: CORE_ID N = 4; firmware chip info {2, 4, 0, 4} */ + + .tile_size_x = 32U, /* Rogue 6XT (gx6250.h); A8 unverified */ + .tile_size_y = 32U, + .common_store_size_in_dwords = 1280U * 4U * 4U, + .isp_max_tiles_in_flight = 8U, + .isp_samples_per_pixel = 2U, + .max_instances_per_pds_task = 32U, + .max_multisample = 8U, + .max_partitions = 8U, + .max_usc_tasks = 56U, + .num_raster_pipes = 2U, + .slc_cache_line_size_bits = 512U, + .unified_store_depth = 256U, + .usc_min_output_registers_per_pix = 2U, + .usc_slots = 32U, + .uvs_banks = 8U, + .uvs_pba_entries = 320U, + .uvs_vtx_entries = 288U, + .vdm_cam_size = 256U, +}; + +static const struct pvr_device_enhancements pvr_device_enhancements_3_10_4_5 = { 0 }; + +static const struct pvr_device_quirks pvr_device_quirks_3_10_4_5 = { 0 }; + +static const struct pvr_device_info pvr_device_info_3_10_4_5 = { + .ident = { + PVR_DEVICE_IDENT_3_V_4_5, + .b = 3, + .v = 10, + .n = 4, + .c = 5, + }, + .features = pvr_device_features_3_V_4_5, + .enhancements = pvr_device_enhancements_3_10_4_5, + .quirks = pvr_device_quirks_3_10_4_5, +}; + +#endif /* APPLE_A8_H */ diff --git a/src/imagination/common/pvr_device_info.c b/src/imagination/common/pvr_device_info.c index 081ca98dcb8..34908a1bac1 100644 --- a/src/imagination/common/pvr_device_info.c +++ b/src/imagination/common/pvr_device_info.c @@ -43,6 +43,7 @@ #include "device_info/bxs-4-64.h" #include "device_info/apple-a10.h" #include "device_info/apple-a9.h" +#include "device_info/apple-a8.h" #include "util/u_string.h" @@ -84,6 +85,7 @@ int pvr_device_info_init(struct pvr_device_info *info, uint64_t bvnc) return 0 switch (bvnc) { + CASE_PACKED_BVNC_DEVICE_INFO(3, 10, 4, 5); CASE_PACKED_BVNC_DEVICE_INFO(4, 40, 2, 51); CASE_PACKED_BVNC_DEVICE_INFO(4, 45, 2, 58); CASE_PACKED_BVNC_DEVICE_INFO(4, 46, 6, 62); diff --git a/src/imagination/common/pvr_device_info.h b/src/imagination/common/pvr_device_info.h index ea6b3471f27..875a8e1985c 100644 --- a/src/imagination/common/pvr_device_info.h +++ b/src/imagination/common/pvr_device_info.h @@ -376,6 +376,12 @@ struct pvr_device_features { */ bool has_apple_g5p : 1; + /* + * Apple A8 GPU (G4P): set together with has_apple_g5p by the A8 entry, + * which takes the A9's paths; marks where the A8 differs from the A9. + */ + bool has_apple_g4p : 1; + /* Only set where Apple's compute residency parameters are established. */ bool has_apple_compute_dispatch : 1; struct pvr_apple_compute_dispatch_limits apple_compute_dispatch; diff --git a/src/imagination/g5p/README.md b/src/imagination/g5p/README.md index 85d5688caed..1b62c5935c5 100644 --- a/src/imagination/g5p/README.md +++ b/src/imagination/g5p/README.md @@ -21,6 +21,43 @@ its A9 mode (`g9p/g9p_a9*.h`). None of this is proposed upstream. It was written by AI agents (every commit carries `Generated-by: LLM`). +## The A8 + +The Apple A8's GPU (T7000, BVNC 3.10.4.5, `common/device_info/apple-a8.h`) +takes these paths too: its entry sets `apple_g5p` and `apple_g4p`. The +evidence for sharing them: + +- Apple's LLVM backend for these GPUs (`libLLVM.dylib`, iOS 16) has one + processor table with `g4p-b1` (A8), `g5p-b0` (A9) and `g9p-a0`/`g9p-b0` + (A10), and the features "Enable G4 instructions", "Enable G5 + instructions" and "Enable G9 instructions". The A8 is the base set; the + A9 adds to it. The A8 has its own scheduling model, the A9 and A10 share + one. +- On an A8 (iPadOS 15) Metal's compute kernels end the A10 way (`0x8000` at + a 64-byte boundary after `0xc000` padding), its empty kernel is the A10's, + and its CDM control stream has the seven-word records this driver writes + for the A9 (`g5p_cdm.h`). + +Where the A8 is known to differ, `apple_g4p` selects its form: + +- PDS programs encode DOUT the Rogue way, not in Apple's extended form + (`PVR_G5P_PDS_FEATURES()`, `vulkan/pvr_g5p.h`): Metal's compute PDS + programs on an A8 decode to the A10's operands in Rogue's field layout. +- The end of tile's EVENT_PIXEL_PDS_INFO counts the USC shared size in + Rogue's unit: 0x202 for one target on the A8, 0x402 on the A9 + (`g5p_mrt_eot_info_g4p()`). + +- Compute: Apple's A8 residency limits (`apple-a8.h`), the PDS in Rogue's + layout (`G5P_PDS_DOUT()`), the kernel launch and KERNEL8 max_instances + as AGXMetalA8 builds them (`pvr_arch_g5p_compute.c`, `g5p_cdm.h`). +- USC (`g5p_usc_g4p`, from Apple's A8 compiler output): the long emitpix, + the A8 varying iterator, the A8 end of tile (`1e010070`, fed the target's + address); the background program is the A9's. + +Where the A8 differs and this driver has no A8 form yet, it refuses by +name: texture samples other than the narrow implicit-LOD 2D one (`sample_g4p()`), +several colour targets, integer targets, 4x, LOAD. + ## Origin The A9 driver is Berkay Kösebay's: the shader backend, the state packing, the diff --git a/src/imagination/g5p/g5p_cdm.h b/src/imagination/g5p/g5p_cdm.h index 9fcea021a73..421d189e837 100644 --- a/src/imagination/g5p/g5p_cdm.h +++ b/src/imagination/g5p/g5p_cdm.h @@ -43,4 +43,58 @@ g5p_cdm_indirect(uint32_t sizes, uint32_t data, uint32_t code, (uint32_t)(address >> 32), (uint32_t)address, (threads - 1) << 18, } }; } + +/* + * The A8 (G4P): PDS DOUT in Rogue's field layout, src1 (the control word's + * data dword) at bits 16-23 and src0 (the 64-bit source: space << 5 | index) + * at bits 8-14, where the A9 and A10 use Apple's extended form + * (g9p_pds_dout(): src1 at 10-17, src0 at 3-9). AGXMetalA8's compute PDS + * encoder (iOS 12.4, 0x1a1453d44) builds DOUTW as 0xf0004001 | src1 << 16 + * and DOUTU as 0xfc000002 | src0 << 8; Metal's A8 programs decode the same + * (DOUTW q0, c2 = 0xf0024001). Arguments as g9p_pds_dout(). + */ +static inline uint32_t g5p_pds_dout_g4p(uint32_t type, uint32_t src64, uint32_t space, + uint32_t ctrl, uint32_t flags) +{ + return 0xf0000000u | flags | ctrl << 16 | (space << 5 | src64) << 8 | type; +} + +/* A DOUT for the chip: Rogue's layout on the A8 (g4p), Apple's extended one otherwise */ +#define G5P_PDS_DOUT(g4p, ...) ((g4p) ? g5p_pds_dout_g4p(__VA_ARGS__) : g9p_pds_dout(__VA_ARGS__)) + +/* KERNEL0's PDS data size in 16-byte units: the A8 counts them directly (every captured A8 record: one unit + * = 1, k_store_one's 0x2106, its binding DMA's two = 2, 0x00040008), G9P/A9 minus one */ +#define G5P_CDM_DATA_UNITS(g4p, n) ((g4p) ? (uint32_t)(n) << 2 : G9P_CDM_DATA_BLOCKS(n)) + +/* + * The A8's CDM KERNEL8 max_instances (bits 27-31; 32 wraps to 0), as + * AGXMetalA8 computes it (iOS 12.4, 0x1a1428938): a workgroup of 32 or more + * threads gets 32; a smaller one gets one workgroup per task when the kernel + * asks for it (variant +0x79b: its compute metadata's field 30 at vtable + * offset 0x40, which Apple's compiler sets, of 25 test kernels, only for the one + * with a device atomic and not for the one with a barrier; Mesa passes "the + * kernel does device atomics"), else 8 workgroups + * (under 5 threads) or 32 / threads, at most (m + 319) / (2m - 1) with m the + * threadgroup memory in 64-byte units; max_instances is workgroups * threads. + * Apple writes 8 for a 1x1x1 kernel (0x40000000, OrchardLinux/gpu a8/M2). + */ +static inline unsigned g5p_cdm_max_instances_g4p(unsigned threads, bool one_wg_per_task, + unsigned shared_bytes) +{ + if (threads >= 32) + return 0; + unsigned wgs; + if (one_wg_per_task) { + wgs = 1; + } else { + wgs = threads < 5 ? 8 : 32 / threads; + const unsigned m = (shared_bytes + 63) / 64; + if (m) { + const unsigned fit = (m + 0x13f) / (2 * m - 1); + if (fit < wgs) + wgs = fit; + } + } + return (wgs * threads) & 31; +} #endif diff --git a/src/imagination/g5p/g5p_fs_common.h b/src/imagination/g5p/g5p_fs_common.h index eab1f4661df..d973ee6e1ab 100644 --- a/src/imagination/g5p/g5p_fs_common.h +++ b/src/imagination/g5p/g5p_fs_common.h @@ -17,12 +17,16 @@ static inline unsigned g5p_common_align(unsigned n, unsigned a) return (n + a - 1) & ~(a - 1); } +/* image_stride: the compact layout's dwords per image (0: the image state rounded up to eight dwords) */ static inline bool -g5p_stage_common_plan(unsigned start, unsigned textures, unsigned samplers, +g5p_stage_common_plan_stride(unsigned start, unsigned textures, unsigned samplers, unsigned image_dwords, + unsigned sampler_dwords, unsigned image_stride, bool compact, unsigned dynamic_ptrs, unsigned size_queries, unsigned ubos, uint64_t ubo_dwords, uint64_t push_dwords, struct g5p_fs_common_plan *p) { + if (!image_stride) + image_stride = g5p_common_align(image_dwords, 8); if (start > 4 || textures > 16 || samplers > 17 || dynamic_ptrs > 16 || size_queries > 4 || ubos > 13) return false; @@ -39,10 +43,12 @@ g5p_stage_common_plan(unsigned start, unsigned textures, unsigned samplers, p->pointer_end = next; } if (textures && p->compact) { - next = g5p_common_align(next, 8) + 8 * textures + 2 * samplers; + /* compact images sit eight dwords apart (pco_g5p_general.c plan_common: the A9's eight-dword state, + * the A8's four-dword state padded) */ + next = g5p_common_align(next, 8) + image_stride * textures + sampler_dwords * samplers; } else { for (unsigned i = 0; i < textures; i++) - next = g5p_common_align(next, 8) + 10; + next = g5p_common_align(next, 8) + image_dwords + sampler_dwords; } unsigned dynamic_end = next; if (!p->low_pointers) @@ -70,12 +76,23 @@ g5p_stage_common_plan(unsigned start, unsigned textures, unsigned samplers, return false; } +static inline bool +g5p_stage_common_plan(unsigned start, unsigned textures, unsigned samplers, unsigned image_dwords, + unsigned sampler_dwords, + bool compact, unsigned dynamic_ptrs, + unsigned size_queries, unsigned ubos, uint64_t ubo_dwords, + uint64_t push_dwords, struct g5p_fs_common_plan *p) +{ + return g5p_stage_common_plan_stride(start, textures, samplers, image_dwords, sampler_dwords, 0, compact, + dynamic_ptrs, size_queries, ubos, ubo_dwords, push_dwords, p); +} + static inline bool g5p_fs_common_plan(unsigned start, unsigned textures, unsigned dynamic_ptrs, unsigned size_queries, unsigned ubos, uint64_t ubo_dwords, uint64_t push_dwords, struct g5p_fs_common_plan *p) { - return g5p_stage_common_plan(start, textures, textures, textures > 4, + return g5p_stage_common_plan(start, textures, textures, 8, 2, textures > 4, dynamic_ptrs, size_queries, ubos, ubo_dwords, push_dwords, p); } diff --git a/src/imagination/g5p/g5p_mrt_programs.h b/src/imagination/g5p/g5p_mrt_programs.h index 73810cb749f..79ebeae50b0 100644 --- a/src/imagination/g5p/g5p_mrt_programs.h +++ b/src/imagination/g5p/g5p_mrt_programs.h @@ -66,6 +66,15 @@ static inline uint32_t g5p_mrt_eot_info(unsigned n) return ((g5p_mrt_pbe_dwords(n) + 15) / 16) * 0x400 | 2; } +/* The A8's EVENT_PIXEL_PDS_INFO: usc_sr_size in Rogue's unit, DIV_ROUND_UP(dwords, + * 16) at bit 9, not twice that. Captured 0x202 for a one-target 64x64 BGRA8 pass + * (iPod touch 6, 3D record +0x3c4, from a live iOS dump), + * where the A9 writes 0x402. Two or more targets are not captured on the A8. */ +static inline uint32_t g5p_mrt_eot_info_g4p(unsigned n) +{ + return ((g5p_mrt_pbe_dwords(n) + 15) / 16) << 9 | 2; +} + /* Partial-render restore SIZEINFO (restore[1], record+0x51c): usc_sharedsize * of Apple's N-image restore loader, 0x800000 per target (1, 2, 4 units). */ static inline uint32_t g5p_mrt_restore1(unsigned n) @@ -126,20 +135,121 @@ static inline unsigned g5p_mrt_clear_compact(uint8_t *out, unsigned n, bool bgra } /* The clear program. Returns its size, 16 * 4N + 6. */ +/* + * The A8's background program for one R8 (halves 1) or RG8 (halves 2) target: AGXMetalA8's (iOS 12.4, a8-passcap): + * the clear colour's low channels converted as for four (g5p_mrt_clear_usc()), for R8 a zero into half 1 + * (0188 0101 0101 0120 0128 0039), then the one-word emitpix 111c 0900 8200 and END + */ +static inline unsigned g5p_mrt_clear_usc_g4p_narrow(uint8_t *out, unsigned halves) +{ + unsigned o = 0; + for (int c = (int)halves - 1; c >= 0; c--) { + const uint8_t h = (uint8_t)c; + const uint8_t ins[16] = {0x8a, 0x01, h, 0x01, 0x01, 0x01, 0xa0, 0x01, + (uint8_t)(4 * h + 1), 0x01, 0x40, 0x41, 0x04, 0x01, 0xb0, 0x00}; + memcpy(out + o, ins, sizeof(ins)); + o += sizeof(ins); + } + if (halves == 1) { + static const uint8_t zero_g[12] = {0x88, 0x01, 0x01, 0x01, 0x01, 0x01, 0x20, 0x01, 0x28, 0x01, 0x39, 0x00}; + memcpy(out + o, zero_g, sizeof(zero_g)); + o += sizeof(zero_g); + } + const unsigned len = g5p_usc_emit(out + o, 1, 0); + if (!len) + return 0; + return o + len + g5p_usc_special(out + o + len, true); +} + +/* The A8's RGBA8UI background program (AGXMetalA8, a8-passcap rgba8u): the four clear uints in common-store + * dwords 0..3 clamped to 255 and packed into one output word R | G << 8 | B << 16 | A << 24, high half first: + * r0 = umin(255, B), r1 = umin(255, A), h1 = h0 | h2 << 8; r1 = umin(255, R), r2 = umin(255, G), + * h0 = h2 | h4 << 8; emitpix one word. The second clamp carries Apple's destination control 0x80. */ +static inline unsigned g5p_mrt_clear_usc_g4p_rgba8ui(uint8_t *out) +{ + static const struct { uint8_t dst, ctl, dword; } clamp[2][2] = { + { { 0, 0, 2 }, { 2, 0x80, 3 } }, + { { 2, 0, 0 }, { 4, 0, 1 } }, + }; + unsigned o = 0, len; + for (unsigned i = 0; i < 2; i++) { + for (unsigned k = 0; k < 2; k++) { + len = g5p_usc_minmax_literal_shared(out + o, G5P_UMIN, clamp[i][k].dst, clamp[i][k].ctl, 255, + clamp[i][k].dword); + if (!len) + return 0; + o += len; + } + len = g5p_usc_pack_byte_half(out + o, 1 - i, clamp[i][0].dst, clamp[i][1].dst, 8); + if (!len) + return 0; + o += len; + } + len = g5p_usc_emit(out + o, 1, 0); + if (!len) + return 0; + return o + len + g5p_usc_special(out + o + len, true); +} + +/* The A8's R32F background program (AGXMetalA8, a8-passcap r32f): the clear colour's red f32 (common-store + * dword 0) moved into output word 0, emitpix one word. */ +static inline unsigned g5p_mrt_clear_usc_g4p_r32f(uint8_t *out) +{ + unsigned o = g5p_usc_mov_shared(out, 0, 0, 0), len; + if (!o) + return 0; + len = g5p_usc_emit(out + o, 1, 0); + if (!len) + return 0; + return o + len + g5p_usc_special(out + o + len, true); +} + +/* The A8's colour LOAD background (AGXMetalA8, a8-passcap load: passcap USC heap 0x2180, the same bytes before and + * after commit): the fragment position converted and transformed by the 2x3 matrix in common-store dwords 8..13, the A8 + * narrow 2D sample of the texture state in dwords 0..3 with the sampler in dwords 4..7, emitpix of two words (the f16 + * pair BGRA8's end of tile takes). Its inputs are the three DOUTDs of the background data (pvr_arch_g5p.c). The bytes are + * Apple's: the conversion and FMA forms with common-store operands have no encoder here yet. */ +#define G5P_A8_LOAD_USC_BYTES 124 +static inline unsigned g5p_mrt_load_usc_g4p(uint8_t *out) +{ + static const uint8_t usc[G5P_A8_LOAD_USC_BYTES] = { + 0x1c, 0x21, 0x00, 0x68, 0x4d, 0x00, 0x99, 0x01, 0x00, 0x01, 0x15, 0x81, 0x20, 0x01, 0x68, 0x04, 0x99, 0x01, + 0x04, 0x81, 0x01, 0x81, 0x30, 0x51, 0x48, 0x09, 0x70, 0x00, 0x99, 0x01, 0x00, 0x01, 0x15, 0x85, 0x20, 0x01, + 0x68, 0x04, 0x99, 0x01, 0x00, 0x81, 0x01, 0x81, 0x30, 0x51, 0x48, 0x09, 0x70, 0x00, 0x9e, 0x41, 0x02, 0x01, + 0xa5, 0x09, 0x00, 0x41, 0xb5, 0x01, 0x00, 0x41, 0x4c, 0x00, 0x9e, 0x41, 0x02, 0x01, 0xad, 0x01, 0x00, 0x41, + 0x84, 0x01, 0x4c, 0x00, 0x9a, 0x01, 0x04, 0x01, 0xa1, 0x89, 0x00, 0x41, 0xb1, 0x01, 0x00, 0x41, 0x4c, 0x00, + 0x9a, 0x01, 0x00, 0x01, 0xa9, 0x81, 0x00, 0x41, 0x88, 0x01, 0x4c, 0x00, 0x1a, 0x59, 0x00, 0x41, 0x00, 0x01, + 0x88, 0x01, 0x00, 0x01, 0x00, 0x80, 0x4d, 0x00, 0x1c, 0x21, 0x00, 0x09, 0x00, 0x82, 0x00, 0x80 + }; + memcpy(out, usc, sizeof(usc)); + return sizeof(usc); +} + static inline unsigned g5p_mrt_clear_usc(uint8_t *out, unsigned n) { if (n > 4) return g5p_mrt_clear_compact(out, n, true); unsigned o = 0; - for (unsigned t = 0; t < n; t++) + /* The A8 (AGXMetalA8, a8-passcap mrt2) converts the last target first and reads target t's clear colour from + * slot n - 1 - t of the clear DMA (its colours in reverse target order); the A9 in order */ + for (unsigned i = 0; i < n; i++) for (int c = 3; c >= 0; c--) { + const unsigned t = g5p_usc_g4p ? n - 1 - i : i, slot = g5p_usc_g4p ? n - 1 - t : t; uint8_t h = 4 * t + c; + const uint8_t src = (uint8_t)(4 * slot + c); const uint8_t ins[16] = {0x8a, 0x01, h, 0x01, 0x01, 0x01, 0xa0, 0x01, - (uint8_t)(4 * h + 1), 0x01, 0x40, 0x41, 0x04, 0x01, 0xb0, 0x00}; + (uint8_t)(4 * src + 1), 0x01, 0x40, 0x41, 0x04, 0x01, 0xb0, 0x00}; memcpy(out + o, ins, sizeof(ins)); o += sizeof(ins); } - /* emitpix of 2N words (the 3-bit field wraps: 8 is 0), then END */ + /* emitpix of 2N words (the 3-bit field wraps: 8 is 0), then END. The A8 takes the long emitpix + * (g5p_usc_emit()); its background program for one target is otherwise these bytes (Apple's, iOS 12.4) */ + if (g5p_usc_g4p) { + const unsigned len = g5p_usc_emit(out + o, 2 * n, 0); + if (!len) + return 0; + return o + len + g5p_usc_special(out + o + len, true); + } const uint8_t tail[6] = {0x1c, (uint8_t)(((2 * n) & 7) << 4 | 1), 0x00, 0x08, 0x00, 0x80}; memcpy(out + o, tail, sizeof(tail)); return o + sizeof(tail); @@ -150,6 +260,31 @@ static inline unsigned g5p_mrt_clear_usc(uint8_t *out, unsigned n) static inline unsigned g5p_mrt_eot_usc_mode(uint8_t *out, unsigned n, bool integer) { unsigned o = 0; + if (g5p_usc_g4p) { + /* The A8's end of tile for one target: 1e 01 00 70, which takes the target from the common store + * (its address and 2, 0: the EOT DMA), then the A9's branch to an END at the next 64-byte boundary + * (Apple's program, iOS 12.4, a8-shadercap). Other target counts and the integer form are not + * captured. */ + if ((n != 1 && n != 2) || integer) { + g5p_usc_error = "A8 end of tile captured for one or two float targets only"; + return 0; + } + /* two targets (Apple's, a8-passcap mrt2): 1c 01 00 70 (target 0), 4c 04, 1e 01 00 71 08 00 (target 1, its + * state from the second EOT DMA block), then the branch to the END at byte 64 */ + static const uint8_t g4_1[10] = {0x1e, 0x01, 0x00, 0x70, 0x00, 0x01, 0x3c, 0x01, 0x00, 0x00}; + static const uint8_t g4_2[18] = {0x1c, 0x01, 0x00, 0x70, 0x4c, 0x04, 0x1e, 0x01, 0x00, 0x71, + 0x08, 0x00, 0x00, 0x01, 0x34, 0x01, 0x00, 0x00}; + const uint8_t *g4 = n == 2 ? g4_2 : g4_1; + const unsigned g4_len = n == 2 ? sizeof(g4_2) : sizeof(g4_1); + memcpy(out, g4, g4_len); + for (o = g4_len; o < 64; o += 2) { + out[o] = 0x00; + out[o + 1] = 0xc0; + } + out[o] = 0x00; + out[o + 1] = 0x80; + return o + 2; + } for (unsigned k = 0; k < n; k++) { /* output half 2 * (n - 1 - k) = target k's output-half offset (in its PBE block) */ const unsigned len = g5p_usc_mov_shared_half(out + o, 2 * (n - 1 - k), diff --git a/src/imagination/g5p/g5p_pbe.h b/src/imagination/g5p/g5p_pbe.h index 86daf6f40c6..69c937ebf52 100644 --- a/src/imagination/g5p/g5p_pbe.h +++ b/src/imagination/g5p/g5p_pbe.h @@ -14,4 +14,40 @@ static inline void g5p_pbe_2d(uint64_t q[3],uint32_t key,bool srgb,uint64_t va, q[1]=va>>1|(uint64_t)(s&0x3fffc)<<45|(uint64_t)srgb<<45|1ull<<63; q[2]=(uint64_t)(s&3)<<62; } + +/* + * The A8's colour target words: Rogue's PBESTATE_REG_WORD0/1, as AGXMetalA8 packs them (iOS 12.4, + * sub_1a13fae24). The kernel copies them to + * the 3D record's W+0x118 + 16 i (and +0x4c0, +0x540). Word 0: tilerelative (bit 57), the swizzle (three bits + * per channel from bit 20; RGBA8 0o3210, BGRA8 0o3012), memlayout TWIDDLE_2D (bit 32) or the linear row stride + * in two-pixel units (bits 34-49). Word 1: maxclip x (bits 32-45) and y (0-13), and for a twiddled target + * the log2 sizes (x at 60, y at 28). Captured on the A8 for 64x64 BGRA8 and RGBA8 twiddled targets: + * 0x0200000160a00000 / 0x0200000168800000 and 0x6000003f6000003f. + */ +#define G5P_PBE_A8_SWIZZLE_RGBA 03210u +#define G5P_PBE_A8_SWIZZLE_BGRA 03012u +/* An unused target: an empty clip rectangle, minclip (1, 1) and maxclip (0, 0) */ +#define G5P_PBE_A8_EMPTY0 0x40ull +#define G5P_PBE_A8_EMPTY1 (0x4000ull << 32) + +static inline unsigned g5p_pbe_a8_log2_ceil(uint32_t n) +{ + unsigned l = 0; + while ((1u << l) < n) + l++; + return l; +} + +static inline void g5p_pbe_a8_reg_words(uint64_t w[2], uint32_t width, uint32_t height, unsigned swizzle, + bool twiddled, uint32_t stride_px) +{ + w[0] = 1ull << 57 | (uint64_t)(swizzle & 07777) << 20; + if (twiddled) + w[0] |= 1ull << 32; + else + w[0] |= (uint64_t)(((stride_px - 1) >> 1) & 0xffff) << 34; + w[1] = (uint64_t)((width - 1) & 0x3fff) << 32 | ((height - 1) & 0x3fff); + if (twiddled) + w[1] |= (uint64_t)(g5p_pbe_a8_log2_ceil(width) & 0xf) << 60 | (uint64_t)(g5p_pbe_a8_log2_ceil(height) & 0xf) << 28; +} #endif diff --git a/src/imagination/g5p/g5p_usc.c b/src/imagination/g5p/g5p_usc.c index 9431b87526d..5271b395eef 100644 --- a/src/imagination/g5p/g5p_usc.c +++ b/src/imagination/g5p/g5p_usc.c @@ -9,6 +9,13 @@ #include _Thread_local const char *g5p_usc_error; +bool g5p_usc_g4p; +bool g5p_usc_g4p_a9_forms; +int g5p_usc_g4p_texfetch; +int g5p_usc_g4p_texfetch_level; +bool g5p_usc_g4p_vtex; +bool g5p_usc_g4p_vtex_apple; +bool g5p_usc_g4p_tbo; const char *const g5p_binop_name[G5P_BIN_COUNT] = {"fadd", "fmul", "iadd", "imul", "and", "or", "xor", "ishr", "isub"}; @@ -56,6 +63,42 @@ static unsigned finish(uint8_t *out, const uint8_t *raw, unsigned n) return n; } +/* An A8 form: the A9's one-instruction length rule is not established for it */ +static unsigned finish_g4p(uint8_t *out, const uint8_t *raw, unsigned n) +{ + memcpy(out, raw, n); + g5p_usc_error = NULL; + return n; +} + +/* + * The A8 varying iterator: 0x18, (count % 4) << 3 | 1, the destination half, + * 0x81 (0xa1 for four components), slot << 2, then 0 (32-bit) or, for the + * narrow (16-bit) form, 01 04 00. Apple's A8 output (a8-shadercap, iOS 12.4): + * 18 09 04 81 10 00 (one f32 component at slot 4 into half 4), 18 11 00 81 04 + * 00 (two f32), 18 19 00 81 04 01 04 00 (three f16), 18 01 00 a1 04 01 04 00 + * (four f16), where the A9 writes 08 0b 04 08, 08 13 00 03 00 80, 0a 1b 00 02 + * and 0a 03 00 02. Only those shapes are accepted; the narrow form only at + * slot 1, the one captured (its 0x04 may be the slot or a constant). + */ +static unsigned iterator_g4p(uint8_t *out, unsigned dst_half, unsigned slot, unsigned count, bool narrow, + bool saturate, unsigned flags) +{ + /* by name: the mode a refused shader needs */ + CHECK(!saturate, "A8 saturating iterator not captured"); + CHECK(!(flags & G5P_ITER_LINEAR), "A8 no-perspective iterator not captured"); + CHECK(!(flags & G5P_ITER_CENTROID), "A8 centroid iterator not captured"); + CHECK(!(flags & G5P_ITER_DRC1), "A8 iterator on the second dependent-read counter not captured"); + CHECK(narrow ? (count == 3 || count == 4) : (count == 1 || count == 2), + "A8 iterator: only 1-2 f32 or 3-4 f16 components are captured"); + CHECK(!narrow || slot == 1, "A8 narrow iterator captured at slot 1 only"); + CHECK(dst_half < 256 && slot < 64, "A8 iterator destination/slot range"); + const uint8_t raw[8] = {0x18, (uint8_t)(((count % 4) << 3) | 1), (uint8_t)dst_half, + (uint8_t)(0x81 | (count == 4 ? 0x20 : 0)), (uint8_t)(slot << 2), + narrow ? 1 : 0, 0x04, 0x00}; + return finish_g4p(out, raw, narrow ? 8 : 6); +} + /* _destination / _selector / _header of the reference. */ static bool destination(uint8_t *raw, unsigned dst_half, unsigned bank) { @@ -567,6 +610,8 @@ unsigned g5p_usc_branch(uint8_t *out, int delta, bool conditional) unsigned g5p_usc_iterator(uint8_t *out, unsigned dst_half, unsigned slot, unsigned count, bool narrow, bool saturate) { + if (g5p_usc_g4p) + return iterator_g4p(out, dst_half, slot, count, narrow, saturate, 0); CHECK(count >= 1 && count <= 4 && !(dst_half % 2), "iterator count/alignment"); CHECK(dst_half < 256, "iterator destination must be u8"); CHECK(slot < 128, "iterator slot must be u7"); @@ -579,6 +624,8 @@ unsigned g5p_usc_iterator_flags(uint8_t *out, unsigned dst_half, unsigned slot, { CHECK(!(flags & ~(G5P_ITER_LINEAR | G5P_ITER_CENTROID | G5P_ITER_DRC1)), "unknown iterator flag"); CHECK(!(narrow && (flags & (G5P_ITER_LINEAR | G5P_ITER_CENTROID))), "narrow iterator modes not witnessed"); + if (g5p_usc_g4p) + return iterator_g4p(out, dst_half, slot, count, narrow, saturate, flags); if (!g5p_usc_iterator(out, dst_half, slot, count, narrow, saturate)) return 0; if (flags & G5P_ITER_DRC1) @@ -600,6 +647,8 @@ unsigned g5p_usc_coeff_load(uint8_t *out, unsigned dst_half, unsigned coeff_dwor unsigned g5p_usc_coord_iterator(uint8_t *out, unsigned dst_half, unsigned slot, unsigned count) { + if (g5p_usc_g4p) /* the A8 iterates a texture coordinate with its plain iterator (Apple's f_tex) */ + return iterator_g4p(out, dst_half, slot, count, false, false, 0); CHECK((count == 2 || count == 3) && !(dst_half % 2), "coordinate iterator count/alignment"); CHECK(dst_half < 64, "coordinate destination half must be u6"); CHECK(slot < 128, "iterator slot must be u7"); @@ -609,8 +658,15 @@ unsigned g5p_usc_coord_iterator(uint8_t *out, unsigned dst_half, unsigned slot, unsigned g5p_usc_emit(uint8_t *out, unsigned words, unsigned source_half) { - CHECK(words == 2 || words == 4, "only two/four output words characterized"); CHECK(source_half < 256, "output source half must be u8"); + if (g5p_usc_g4p) { + /* A8: byte 3 bit 0 and the extension halfword 0x8200 (Apple's 1c 21 00 09 00 82 in every captured + * fragment program; the A9's 1c 21 00 08) */ + CHECK(words == 1 || words == 2 || words == 4, "A8 emitpix captured for 1, 2 and 4 words (111c, 211c, 411c)"); + const uint8_t g4[6] = {0x1c, (uint8_t)((words << 4) | 1), (uint8_t)source_half, 9, 0x00, 0x82}; + return finish_g4p(out, g4, 6); + } + CHECK(words == 2 || words == 4, "only two/four output words characterized"); uint8_t raw[4] = {0x1c, (words << 4) | 1, source_half, 8}; return finish(out, raw, 4); } @@ -624,6 +680,8 @@ unsigned g5p_usc_emit_mrt(uint8_t *out, unsigned words, unsigned source_half) { CHECK(g5p_mrt_emit_words_witnessed(words), "only 1/2/4/8 terminal source words have an Apple witness"); CHECK(source_half < 256, "output source half must be u8"); + if (g5p_usc_g4p) + return g5p_usc_emit(out, words, source_half); uint8_t raw[4] = {0x1c, (g5p_mrt_emit_field(words) << 4) | 1, source_half, 8}; return finish(out, raw, 4); } @@ -840,6 +898,67 @@ unsigned g5p_usc_unorm8_byte(uint8_t *out, unsigned dst_half, unsigned byte, uns return finish(out, raw, n); } +/* Integer min/max of a short literal A and a common-store dword C, the 32-bit result in bank 0: AGXMetalA8's + * RGBA8UI clear (iPod touch 6, iOS 12.4, a8-passcap rgba8u) clamps each clear channel with + * 9a 01 DD CC 7f 01 26 01 SS 01 00 41 b2 21 b0 00 (umin(255, common dword SS)) + * laid out as g5p_usc_fma_literal(): literal_prefix() in the three-source class 0x9a, B unused (selector 0), + * then C = the common-store dword (shared_src(), group 1), the min/max code (minmax_code[], as + * g5p_usc_minmax_literal()), control 0x21 and the routing byte 0xb0 (literal A, common-store C; the register + * forms route 0x08 and 0x7c), no source control. CC carries the destination control: Apple sets 0x80 on one of + * its four clamps (the second), on a register its next clamp writes again. Only the short literal is witnessed. */ +#define G5P_MINMAX_CTL 0x21 +#define G5P_ROUTING_LITERAL_A_SHARED_C 0xb0 +unsigned g5p_usc_minmax_literal_shared(uint8_t *out, enum g5p_minmax op, unsigned dst_half, unsigned dst_control, + uint32_t value, unsigned c_dword) +{ + CHECK(op < G5P_MINMAX_COUNT, "unsupported integer min/max"); + CHECK(value <= 0xffff, "only the short literal is witnessed"); + CHECK(half_register(dst_half, "dst_half requires a measured 32-bit aligned half index"), g5p_usc_error); + uint8_t raw[G5P_USC_MAX]; + unsigned n = literal_prefix(raw, 0x9a, dst_half, 0, value, 0, 0, G5P_HEADER_FRAGMENT, 0); + if (!n) + return 0; + if (!cdestination(raw + 2, dst_half, 0, dst_control)) + return 0; + if (!shared_src(raw + n, c_dword, 1)) + return 0; + n += 4; + raw[n++] = minmax_code[op]; + raw[n++] = G5P_MINMAX_CTL; + raw[n++] = G5P_ROUTING_LITERAL_A_SHARED_C; + raw[n++] = G5P_SOURCE_CONTROL_NONE; + return finish(out, raw, n); +} + +/* Two bytes into one 16-bit half of bank 0, dst = B | C << shift: AGXMetalA8's RGBA8UI clear (a8-passcap rgba8u) + * packs its clamped channels (each 0..255) pairwise with + * 8a 01 DH 01 11 BB 20 01 CC 01 f1 1f 58 00 (dst half DH = half BB | half CC << 8) + * literal_prefix() in the three-source class with the 16-bit result (0x8a) and the shift as A's short literal, + * B and C register halves of group 1 (C's selector without the continuation, then the 0x01 parcel byte, as + * g5p_usc_fma_literal()), opcode 0xf1, control 0x1f, routing 0x58, no source control. The captured operands are + * 0..255, so whether the opcode ors, adds or inserts the shifted C is not distinguished; only shift 8 is witnessed. */ +#define G5P_PACK_BYTE_OP 0xf1 +#define G5P_PACK_BYTE_CTL 0x1f +#define G5P_PACK_BYTE_ROUTING 0x58 +unsigned g5p_usc_pack_byte_half(uint8_t *out, unsigned dst_half, unsigned b_half, unsigned c_half, + unsigned shift) +{ + CHECK(shift == 8, "only the byte pack (shift 8) is witnessed"); + uint8_t raw[G5P_USC_MAX]; + unsigned n = literal_prefix(raw, G5P_CLASS_TERNARY_RESULT_HALF, dst_half, 0, shift, b_half, 1, + G5P_HEADER_FRAGMENT, 0); + if (!n) + return 0; + SEL(c, c_half, 1, 0); + raw[n++] = c; + raw[n++] = G5P_PARCEL_CONTINUES; + raw[n++] = G5P_PACK_BYTE_OP; + raw[n++] = G5P_PACK_BYTE_CTL; + raw[n++] = G5P_PACK_BYTE_ROUTING; + raw[n++] = G5P_SOURCE_CONTROL_NONE; + return finish(out, raw, n); +} + /* A move of a common-store dword into a 16-bit half of bank 0 (the end-of-tile program's state moves, * g5p_mrt_programs.h): alu_header() with the 16-bit move class (G5P_CLASS_MOVE_HALF, as g5p_usc_half_immediate()) * and the fragment header, A = the common-store dword (shared_src(), group 1), then the half move's opcode and @@ -999,8 +1118,106 @@ static unsigned texture_base(uint8_t *raw, const struct g5p_tex *t) return 6; } +/* + * The A8's 2D sample with an implicit LOD and a packed-f16 (narrow) result, as Apple's A8 compiler writes + * f_tex's (iPod touch 6, iOS 12.4, a8-shadercap): 1a 59 00 41 00 01 88 01 00 01 00 80, where the A9 writes + * 18 f5 00 01 08 80 for the same source (image qword 0, sampler qword 4, coordinate dword 0 under coordinate + * control). Byte 2 is the destination half and byte 8 the coordinate dword << 1 (Metal's blit program has + * eight samples, 1a 79 DD 41 00 01 ac 01 CC 01 14 80, with eight destinations and coordinates); byte 6 is + * 0x80 | mode << 4 | sampler dword << 1 (sampler 4 here, 6 in the blit, whose mode 2 is an explicit LOD). Byte 1 bit 5 is a 32-bit result there, which + * also carries an operand byte 10 not decoded, so only the narrow form is taken; the image qword is 0 in + * every witness (bytes 3-4 41 00), so only image qword 0. + */ +static unsigned sample_g4p(uint8_t *out, const struct g5p_tex *t) +{ + /* Two result forms, both Apple's: narrow (byte 1 0x59: the f16 results f_tex writes straight to the output) and + * 32-bit (byte 1 0x79: one 32-bit register per channel, read by ALU after the wait; AGXMetalA8's R32F and RGBA8UI + * load backgrounds, passcap USC 0x4826 and 0x4da6, the latter feeding umin and byte packs). Metal's blit program + * has the 32-bit form with byte 10 = 0x14, an operand not decoded and not used here. */ + CHECK((t->dim == G5P_TEX_2D || t->dim == G5P_TEX_CUBE) && !t->read && !t->class_variant && t->coord_control && + !t->gradient && !t->lod_control && !t->length, + "A8 sample captured only as a 2D or cube sample with an implicit LOD"); + /* the cube sample: mode 4 (byte 6 0xc8 with the sampler at dword 4), the direction in three temps from the + * coordinate dword, no cube-coordinate instruction (AGXMetalA8's f_tex_cube, passcap USC 0x6076: two FMAs and a + * move build r0..r2, then 1a 59 00 41 00 01 c8 01 00 01 00 80). Only the narrow form is captured. */ + CHECK(t->dim != G5P_TEX_CUBE || t->narrow, "A8 cube sample captured only in the narrow form (result straight to the output)"); + /* the explicit level in the register after the coordinate (Apple's sample at a varying level, iOS capture, USC + * heap 0x1_02328000 + 0x8814: the iterated coordinate in r0..r1, r2 = r0 * 4.0, then + * 1a 59 00 41 00 01 a8 09 00 01 00 80): byte 6 mode 2, byte 7 0x09, byte 10 0. 2D only. */ + CHECK(!t->lod_reg || t->dim == G5P_TEX_2D, "A8 explicit-level sample captured for 2D only"); + const uint8_t lod_mode = t->lod_reg ? 2 << 4 : 0, lod_byte7 = t->lod_reg ? 0x09 : 0x01; + /* The A8's multi-image sample, Apple's form (iOS 12.4, AGXMetalA8's programs sampling two to seven textures + * with one shared sampler or one per texture, captured with a8-passcap; USC heap 0x1_002cc000 + + * 0x8a08..0x97dc): byte 4 is the image's common dword * 2, and the sampler's four-dword common slot S + * (dword 4S) is byte 1 bits 7:6 | byte 5 bits 2:1 << 2; byte 6 is 0x88 in every sample. Apple's four-texture + * programs with one sampler and with four differ only in byte 1 bits 7:6 (S = 4 shared; 4, 5, 6, 7 per + * texture); five textures with five samplers have S = 5..9 (byte 5 03, 05), and seven textures' shared + * sampler is S = 7 (b1 c1/e9/f1, byte 5 03) at dword 28, where its PDS DMAs it. The single-image forms agree: + * 0x59/0x79 are S = 1 (dword 4). Byte 1 bit 5 is the 32-bit result and bits 4:3 the components written minus + * one (Apple writes .r as 0, .g as 1, .b as 2); this writes four. On the iPod touch 6 every multi-texture + * test passed with it. */ + if (t->a8_slots) { + const unsigned image_dword = t->image_qword, sampler_dword = t->sampler_qword; /* dwords (g5p_tex) */ + CHECK(t->dim == G5P_TEX_2D, "A8 multi-image sample: 2D only (Apple's form is captured for 2D)"); + CHECK(!(image_dword % 4) && image_dword <= 124, "A8 multi-image sample: image dword a multiple of 4 below 128"); + CHECK(!(sampler_dword % 4) && sampler_dword / 4 <= 11, "A8 multi-image sample: sampler slot 0..11"); + CHECK(t->coord_dword <= 31 && !(t->dst_half & 1) && t->dst_half < 64, "A8 sample: coordinate/destination range"); + const unsigned slot = sampler_dword / 4; + if (t->vtx) { + /* Apple's vertex form (g5p_tex.vtx) */ + CHECK(!t->narrow && !t->lod_reg && t->coord_dword <= 31, "A8 vertex sample: 32-bit, Apple's constant level"); + const uint8_t b6 = 0xa0 | (!t->read_flag ? 0x08 : t->int_result ? 0x04 : 0x0c); + const uint8_t vraw[12] = {0x1a, (uint8_t)((slot & 3) << 6 | 0x3d), (uint8_t)t->dst_half, 0x41, + (uint8_t)(2 * image_dword), (uint8_t)(1 | (slot >> 2) << 1), b6, 0x05, + (uint8_t)(t->coord_dword << 1), 0x01, 0x1c, 0x80}; + return finish_g4p(out, vraw, sizeof(vraw)); + } + const uint8_t raw[12] = {0x1a, (uint8_t)((slot & 3) << 6 | (t->narrow ? 0x19 : 0x39)), (uint8_t)t->dst_half, + 0x41, (uint8_t)(2 * image_dword), (uint8_t)(1 | (slot >> 2) << 1), + (uint8_t)(0x88 | lod_mode | (t->read_flag ? 4 : 0)), lod_byte7, + (uint8_t)(t->coord_dword << 1), 0x01, 0x00, 0x80}; + return finish_g4p(out, raw, sizeof(raw)); + } + /* Another image or sampler slot: the A8 form's image field has one witnessed value (bytes 3-4 41 00 at qword 0) + * and its sampler field three bits of dwords. With PVR_A8_A9_FORMS the sample is the A9's own six-byte + * instruction (g5p_usc_sample()'s A9 branch: image and sampler qwords in bytes 3 and 4, the coordinate from + * registers), over the A8's state words at the same common-store positions. */ + if (t->image_qword != 0 || t->sampler_qword > 7 || (t->sampler_qword & 1)) { + CHECK(g5p_usc_g4p_a9_forms, "A8 sample captured only with the image descriptor at common qword 0"); + CHECK(!t->lod_reg, "A8 explicit-level sample: no A9 form"); + CHECK(t->dim == G5P_TEX_2D, "A8 sample in another image slot: the A9 form is taken for 2D only"); + CHECK(!(t->sampler_qword & 1), "A8 sample in another image slot: the sampler must start a qword"); + struct g5p_tex a9 = *t; + a9.sampler_qword = t->sampler_qword / 2; /* the A8 callers pass the sampler's dword */ + uint8_t raw[G5P_USC_MAX]; + if (!texture_base(raw, &a9)) + return 0; + if (t->narrow) + raw[0] &= ~2; + return finish(out, raw, 6); + } + /* the A8 names the sampler by its common DWORD (Apple's 0x88 reads the sampler at dword 4 in tex_plain and in + * the load background): the caller passes the dword in sampler_qword */ + /* byte 6 is 0x80 | mode << 4 | sampler dword << 1: Apple's tex_lod, tex_bias, tex_grad and tex_cube samples + * (passcap USC 0x5388, 0x5588, 0x57bc, 0x6076) read the same sampler (dword 4) with bytes a8, 98, b8 and c8, + * and the blit's explicit-LOD samples are ac (sampler dword 6). Bits 4-6 are the sample mode, so a sampler at + * dword 8 or above would encode another mode: the field is three bits. */ + CHECK(t->sampler_qword <= 7, "A8 sample: sampler dword above 7 (byte 6 bits 4-6 are the sample mode)"); + /* witnessed sampler dwords are 0, 4 and 6; byte 6 bit 1 (an odd dword) is also set by Apple's array sample (8a) + * and bit 0 by its 0x3000 sample (89), neither decoded, so an odd dword could alias a flag */ + CHECK(!(t->sampler_qword & 1), "A8 sample: odd sampler dword (byte 6 bit 1 is set by Apple's undecoded array sample)"); + CHECK(t->coord_dword <= 31 && !(t->dst_half & 1) && t->dst_half < 64, "A8 sample: coordinate/destination range"); + const uint8_t mode = t->dim == G5P_TEX_CUBE ? 4 : t->lod_reg ? 2 : 0; + const uint8_t raw[12] = {0x1a, t->narrow ? 0x59 : 0x79, (uint8_t)t->dst_half, 0x41, 0x00, 0x01, + (uint8_t)(0x80 | mode << 4 | t->sampler_qword << 1 | (t->read_flag ? 4 : 0)), + lod_byte7, (uint8_t)(t->coord_dword << 1), 0x01, 0x00, 0x80}; + return finish_g4p(out, raw, sizeof(raw)); +} + unsigned g5p_usc_sample(uint8_t *out, const struct g5p_tex *t) { + if (g5p_usc_g4p) + return sample_g4p(out, t); CHECK(!t->read, "mode must be sample"); uint8_t raw[G5P_USC_MAX]; if (!texture_base(raw, t)) @@ -1054,6 +1271,8 @@ unsigned g5p_usc_ms_own_sample(uint8_t *out, unsigned dst_half) unsigned g5p_usc_texture_lod(uint8_t *out, const struct g5p_tex *t) { + /* The A8 takes only the narrow implicit-LOD 2D sample (sample_g4p()) */ + CHECK(!g5p_usc_g4p, "A8 texture sample form not captured"); CHECK(t->lod_bank == 1 || t->lod_bank == 3, "only LOD source banks 1 (shared) and 3 (temp) are measured"); CHECK(t->lod_dword <= 95, "lod_dword must be an integer in [0,95]"); CHECK(!t->narrow, "narrow result is not witnessed with explicit LOD"); @@ -1087,6 +1306,8 @@ unsigned g5p_usc_texture_lod(uint8_t *out, const struct g5p_tex *t) unsigned g5p_usc_gather(uint8_t *out, const struct g5p_tex *t) { + /* The A8 takes only the narrow implicit-LOD 2D sample (sample_g4p()) */ + CHECK(!g5p_usc_g4p, "A8 texture sample form not captured"); CHECK(t->dim == G5P_TEX_2D, "only 2d gather is measured"); CHECK(!t->read && !t->narrow, "gather is a 2d sample form"); uint8_t raw[G5P_USC_MAX]; @@ -1100,6 +1321,9 @@ unsigned g5p_usc_gather(uint8_t *out, const struct g5p_tex *t) unsigned g5p_usc_sample_ext(uint8_t *out, const struct g5p_tex_ext *t) { + /* The A8 takes only the narrow implicit-LOD 2D sample (sample_g4p()) */ + /* a vertex sample takes the A9's recipe on the A8 (g5p_usc.h g5p_usc_g4p_vtex) */ + CHECK(!g5p_usc_g4p || (t->vertex && g5p_usc_g4p_vtex), "A8 texture sample form not captured"); const bool coupled = t->tuple != G5P_TEX_TUPLE_ORDINARY; CHECK((unsigned)t->tuple <= G5P_TEX_TUPLE_COUPLED_TEMP_LEVEL_COMPARE_OFFSET, "unqualified texture recipe"); CHECK(!coupled || t->lod_shared == (t->tuple == G5P_TEX_TUPLE_COUPLED_SHARED_LEVEL), @@ -1187,6 +1411,8 @@ unsigned g5p_usc_sample_ext(uint8_t *out, const struct g5p_tex_ext *t) #define G5P_TEX_COORD_LAST_READ 0x80 unsigned g5p_usc_sample_register_lod(uint8_t *out, const struct g5p_tex_ext *t) { + /* The A8 takes only the narrow implicit-LOD 2D sample (sample_g4p()) */ + CHECK(!g5p_usc_g4p, "A8 texture sample form not captured"); CHECK(t->tuple == G5P_TEX_TUPLE_COUPLED_TEMP_LEVEL && t->lod == G5P_TEX_LOD_LEVEL && !t->lod_shared && (t->dim == G5P_TEX_2D_ARRAY || t->dim == G5P_TEX_CUBE), "the register-LOD form is an array or cube sample at a level"); @@ -1314,6 +1540,13 @@ unsigned g5p_usc_address_load(uint8_t *out, unsigned dst_half, unsigned byte_cou return finish(out, raw, length); } +/* The A8 has no 64-bit address add: AGXMetalA8 never emits class 0x89 (none in the iPod touch 6 shader and pass + * captures), the A10/A9 form faults in A8 compute, and Apple's A8 builds base + offset from 32-bit adds + * with the carry's high half (g9p_addr()). That build needs a scratch register the fragment/vertex allocator does + * not provide here yet, so graphics programs that index a buffer are refused on the A8. */ +#define G5P_A8_NO_ADDR_ADD "the A8 has no 64-bit address add (Apple's 32-bit carry build is not ported to " \ + "graphics programs)" + /* `89 11` (vertex) / `89 01` (fragment): pair = pointer pair at common dword * `base` + (index << shift). Bytes from Apple's own compiles of a constant * buffer indexed by a register (tools/mesa-a9-gles3tex/dynubo.metal: v_cexpr, @@ -1329,6 +1562,7 @@ unsigned g5p_usc_address_add_shift(uint8_t *out, unsigned pair_half, unsigned ba CHECK(half_register(index_half, "index_half requires a measured 32-bit register"), g5p_usc_error); CHECK(base_common_dword < 256 && !(base_common_dword & 1), "pointer base must be an even common dword below 256"); CHECK(shift <= 4, "only shifts 0-4 are corpus-verified"); + CHECK(!g5p_usc_g4p, G5P_A8_NO_ADDR_ADD); /* The layout of g5p_usc_alu_shared(): alu_header() (vertex code: destination bank 1, and B's bank bit in * the header), A = the pointer base (shared_src(), group 1; above dword 31 its high bits are the field that * the vertex XFB record's address add and every shared ALU operand use), B = the index with its last read, @@ -2086,6 +2320,7 @@ unsigned g5p_usc_address_add(uint8_t *out, unsigned dst_half, unsigned ptr_dword CHECK(!(ptr_dword & 1), "64-bit pointer base must be pair aligned"); CHECK(off_group <= 1, "off_group must be an integer in [0,1]"); CHECK(shift <= 4, "address shifts 0..4 (uchar..uint4) are witnessed"); + CHECK(!g5p_usc_g4p, G5P_A8_NO_ADDR_ADD); uint8_t raw[G5P_USC_MAX], a[4]; if (!alu_header(raw, 0x89, dst_half, 1, 0x11, 0, 0) || !shared_src(a, ptr_dword, ptr_group)) return 0; diff --git a/src/imagination/g5p/g5p_usc.h b/src/imagination/g5p/g5p_usc.h index 628698d7d81..4cb8c12a9c4 100644 --- a/src/imagination/g5p/g5p_usc.h +++ b/src/imagination/g5p/g5p_usc.h @@ -22,6 +22,55 @@ extern _Thread_local const char *g5p_usc_error; +/* + * Encode for the A8 (G4P) instead of the A9: set once by the driver for an A8 + * device (one process drives one Apple GPU). The forms that differ, from + * Apple's A8 compiler output (iPod touch 6, iOS 12.4, a8-shadercap): emitpix + * takes an extension halfword, varyings iterate with opcode 0x18, and texture + * samples have another form (refused until decoded). Everything else Apple's + * A8 compiler emits the A9's way (the vertex programs byte for byte). + */ +extern bool g5p_usc_g4p; +/* + * On the A8, the A9's instruction where the A8 form captured from Apple cannot express the operands (today: a + * texture sample whose image is not at common qword 0, or whose sampler is beyond common dword 7). Set by the + * driver for an A8 (PVR_A8_A9_FORMS, default on). These are A9 instructions no A8 capture has witnessed. + */ +extern bool g5p_usc_g4p_a9_forms; +/* + * texelFetch on the A8: a fragment 2D fetch is drawn as the A8's sample with a point, non-normalized, clamped copy of + * the texture's sampler, as Apple's read (an iOS capture of a Metal read()): the integer coordinate as a float, + * byte 6 bit 2 set (g5p_usc_g4p_texfetch = 2; 1, the texel's centre without the bit, reads the same on the iPod + * touch 6). A constant level above 0 (the sampler's LOD range clamped to it) only with PVR_A8_TEXFETCH_LEVEL: + * 1 ("level"): the coordinate in the level's texels; 2 ("scale"): in level 0's texels (times 2^level); + * 3 ("apple"): every fetch in Apple's read form (AGXCompilerCore's read(): an explicit level, 0 when the source + * names none): byte 6 0xac (mode 2 + bit 2), the level as an f32 in the register after the coordinate (byte 7 + * 0x09, as in an iOS capture of a Metal sample at a varying level), through Apple's constant read sampler + * (normalized, clamped to the edge, point, LOD 0..max; PCO_G5P_UPLOAD_SAMPLER_READ), the coordinate unscaled. + * Not yet run on the hardware. + */ +extern int g5p_usc_g4p_texfetch; +extern int g5p_usc_g4p_texfetch_level; +/* + * Vertex-stage textures on the A8, refused by default. PVR_A8_VTEX=1 takes the A9's qualified vertex recipe over the + * A8's four-dword image and sampler words. It does not work yet: on the iPod touch 6 a vertex texelFetch read zeros + * and a vertex textureLod locked the TA. No capture shows an A8 vertex texture; the A8's vertex programs are + * otherwise the A9's. + */ +extern bool g5p_usc_g4p_vtex; +/* + * PVR_A8_VTEX_APPLE=1 (not yet run on the hardware): vertex textures in Apple's form instead (it enables them on its + * own), as an iOS capture shows it for AGXMetalA8's vertex programs (a sample, a read of RGBA32Float and of + * RGBA32Sint): the A8's 12-byte sample with the coordinate and the result in bank 1 and Apple's constant level 0, + * 1a 7d 00 41 00 01 a8|ac|a4 05 00 01 1c 80 (g5p_tex.vtx), the image and sampler at common dwords 0 and 4; a read's + * sampler is the fetch sampler's words, as Apple's (0x0202240000000fff). The A9's recipe (PVR_A8_VTEX=1) uses the A9's + * own sample instruction (1a f5 .. 01 08 85 c6 01 80 81, a coupled f16 level), which no A8 program contains. + */ +extern bool g5p_usc_g4p_vtex_apple; +/* A fragment texel-buffer read on the A8 (st/mesa's pixel-buffer uploads) as a 2D texelFetch of the buffer's linear + * state (PVR_A8_TBO, on by default; 0 refuses it) */ +extern bool g5p_usc_g4p_tbo; + enum g5p_binop { G5P_BIN_FADD, G5P_BIN_FMUL, G5P_BIN_IADD, G5P_BIN_IMUL, G5P_BIN_AND, G5P_BIN_OR, G5P_BIN_XOR, G5P_BIN_ISHR, @@ -158,6 +207,22 @@ struct g5p_tex { unsigned lod_bank, lod_dword; bool gradient, lod_control; unsigned length; /* 0 = minimum */ + /* The A8's multi-image form (Apple's, from an iOS capture): byte 4 = the image's common dword * 2, the + * sampler's four-dword slot in byte 1 bits 7:6 and byte 5 bits 2:1. image_qword and sampler_qword then hold + * dwords. */ + bool a8_slots; + /* g5p_usc_g4p_texfetch = 2: Apple's read flag, byte 6 bit 2 */ + bool read_flag; + /* Apple's explicit level in a register (an iOS capture of a Metal sample at a varying level: + * 1a 59 00 41 00 01 a8 09 00 01 00 80): mode 2, byte 7 0x09, the level an f32 in the register after the + * coordinate's two */ + bool lod_reg; + /* Apple's vertex-stage sample (an iOS capture, AGXMetalA8's v_sample / v_read_f32 / v_read_i32: + * 1a 7d 00 41 00 01 a8|ac|a4 05 00 01 1c 80): the coordinate and the result in bank 1 (byte 1 bit 2, byte 7 + * bit 2), mode 2 with Apple's constant level 0 (byte 10 0x1c); byte 6 0xa8 for a sample, 0xac for a read of a float + * or normalized texture, 0xa4 (int_result) for a read of an integer one */ + bool vtx; + bool int_result; }; unsigned g5p_usc_sample(uint8_t *out, const struct g5p_tex *t); /* The 2D-multisample fetch's sample operand "the task's own sample" (a sample-rate program reads the @@ -201,6 +266,7 @@ struct g5p_tex_ext { unsigned extras_half; enum g5p_tex_tuple tuple; bool lod_shared; /* prefix level source: common store instead of register */ + bool vertex; /* a vertex-stage sample (the A8 takes the A9's recipe there, g5p_usc_g4p_vtex) */ }; /* A9 compute's register-level array or cube sample and fetch: the coupled temporary-level tuple, the fetch's * coordinate kept live (fields at the encoder). */ @@ -236,6 +302,14 @@ unsigned g5p_usc_f2u16_half(uint8_t *out, unsigned dst_half, unsigned src_half, * end-of-tile program's state moves (g5p_mrt_programs.h). */ unsigned g5p_usc_mov_shared_half(uint8_t *out, unsigned dst_half, unsigned src_dword); unsigned g5p_usc_unorm8_byte(uint8_t *out, unsigned dst_half, unsigned byte, unsigned src_dword); +/* Integer min/max of a short literal and common-store dword c_dword (group 1) into the 32-bit register at + * dst_half, bank 0 (fields at the encoder): the A8's RGBA8UI clear clamps each channel with umin(255, c). */ +unsigned g5p_usc_minmax_literal_shared(uint8_t *out, enum g5p_minmax op, unsigned dst_half, unsigned dst_control, + uint32_t value, unsigned c_dword); +/* The 16-bit dst_half of bank 0 = half b_half | half c_half << 8 (fields at the encoder): the A8's RGBA8UI clear + * packs two clamped channels into one half with it. */ +unsigned g5p_usc_pack_byte_half(uint8_t *out, unsigned dst_half, unsigned b_half, unsigned c_half, + unsigned shift); /* length 0 = minimum */ unsigned g5p_usc_buffer_load(uint8_t *out, unsigned dst_half, unsigned byte_count, unsigned base_common_dword, unsigned index, unsigned length); diff --git a/src/imagination/g5p/pco_g5p_general.c b/src/imagination/g5p/pco_g5p_general.c index 7436e270c17..7ca4ad9f398 100644 --- a/src/imagination/g5p/pco_g5p_general.c +++ b/src/imagination/g5p/pco_g5p_general.c @@ -100,6 +100,9 @@ struct tex_bind { unsigned element, selement; /* array elements of the texture / sampler binding */ unsigned image, sampler; /* common dwords */ bool needs_sampler, needs_fetch_sampler; + bool fetch; /* A8: a texelFetch drawn as a sample (G5P_TEX_FETCH): its own sampler slot, the fetch sampler */ + unsigned fetch_level; /* the level that fetch reads */ + bool fetch_read; /* that fetch in Apple's read form (G5P_TEX_LOD_REG): Apple's read sampler */ unsigned image_flags; /* PCO_G5P_UPLOAD_IMAGE_AS_2D / _ARRAY: how its samples read the state */ }; @@ -113,6 +116,13 @@ struct range { enum { BLEND_NONE, BLEND_PREMULT, BLEND_STRAIGHT_OVER, BLEND_REPLACE, BLEND_GENERAL }; /* nir_tex_instr.backend_flags: an array sample of layer 0 drawn as 2D (layer0_as_2d()) */ #define G5P_TEX_LAYER0_AS_2D (1u << 30) +/* nir_tex_instr.backend_flags: an A8 texelFetch drawn as a sample (a8_fetch_as_sample()) */ +#define G5P_TEX_FETCH (1u << 27) +/* and the level it reads, bits 23-26 */ +#define G5P_TEX_FETCH_LEVEL_SHIFT 23 +/* nir_tex_instr.backend_flags: the A8 sample in Apple's explicit-level form (g5p_tex.lod_reg), a fetch through Apple's + * read sampler (PVR_A8_TEXFETCH_LEVEL=apple, PVR_A8_VTEX_APPLE) */ +#define G5P_TEX_LOD_REG (1u << 22) /* The general blend epilogue (g5p_blend.c, hyprland-gpu.md §11.2) for * settings without a measured epilogue. Premultiplied, straight-over and @@ -180,6 +190,8 @@ struct gctx { unsigned hi_inline; uint64_t tex_pending, tex_coord_pending; unsigned tex_batch; + bool a8_slots; /* the A8's multi-image layout (plan_common) */ + unsigned tex_batch_image; /* a8_slots: the common dword of the batch's last sampled image */ nir_shader *nir; nir_function_impl *impl; bool vs, half_output; @@ -369,6 +381,18 @@ static void tex_flush(struct gctx *c) c->tex_batch = 0; } +/* The A8 waits `4d 04` before a sample of another image while samples are outstanding: Apple's two-texture + * program (iOS capture, USC heap 0x1_02328000 + 0x6210) is sample t0, 4d 04, sample t1, 4d 00, and its 3- and + * 4-texture programs (+0x73ea, +0x7c22) put 4d 04 between every two samples of different images, while its blit + * (+0xf66) samples one image eight times back to back. Only the multi-image programs. */ +static void a8_image_order(struct gctx *c, unsigned image) +{ + if (!c->a8_slots || !c->tex_batch || image == c->tex_batch_image) + return; + uint8_t raw[G5P_USC_MAX]; + put(c, raw, g5p_usc_wait(raw, 4), "wait (another image, Apple's 4d 04)"); +} + /* ---- vertex register banks (docs/gpu/vs-two-bank-allocator.md) ---- * A vertex register below VS_BANK0 is bank 1 (the attributes, outputs and * unified store; every vertex temp so far); VS_BANK0 + i is bank 0 r, the @@ -826,8 +850,23 @@ static bool offset_only(nir_def *d) /* The round-18 sample: 2D float, implicit LOD, coordinates only. */ static bool plain_tex(const nir_tex_instr *t) { + /* an A8 texelFetch drawn as a sample reads integer textures too: the 32-bit result is the texel's bits */ + const bool fetch = t->backend_flags & G5P_TEX_FETCH; if (t->op != nir_texop_tex || t->sampler_dim != GLSL_SAMPLER_DIM_2D || t->is_array || t->is_shadow || - t->def.num_components != 4 || nir_alu_type_get_base_type(t->dest_type) != nir_type_float) + t->def.num_components != 4 || (nir_alu_type_get_base_type(t->dest_type) != nir_type_float && !fetch)) + return false; + for (unsigned i = 0; i < t->num_srcs; i++) + if (t->src[i].src_type != nir_tex_src_coord && t->src[i].src_type != nir_tex_src_texture_deref && + t->src[i].src_type != nir_tex_src_sampler_deref) + return false; + return true; +} + +/* The A8's cube sample (sample_g4p()): an implicit-LOD float sample of a cube, not an array, no compare */ +static bool a8_cube_tex(const nir_tex_instr *t) +{ + if (!g5p_usc_g4p || t->op != nir_texop_tex || t->sampler_dim != GLSL_SAMPLER_DIM_CUBE || t->is_array || + t->is_shadow || t->def.num_components != 4 || nir_alu_type_get_base_type(t->dest_type) != nir_type_float) return false; for (unsigned i = 0; i < t->num_srcs; i++) if (t->src[i].src_type != nir_tex_src_coord && t->src[i].src_type != nir_tex_src_texture_deref && @@ -963,6 +1002,29 @@ static const char *vs_tex_refusal(const nir_tex_instr *t) return NULL; } +/* The shapes emit_vertex_tex_g4p() takes (PVR_A8_VTEX_APPLE): a 2D sample at an implicit or explicit level, or a texelFetch + * drawn as Apple's read (a8_fetch_as_sample()) */ +static const char *a8_vs_tex_refusal(const nir_tex_instr *t) +{ + const bool fetch = t->backend_flags & G5P_TEX_FETCH; + if (t->op == nir_texop_txf) + return "A8 vertex texelFetch: 2D, 4 x 32-bit, at a constant level (Apple's read form)"; + if ((t->op != nir_texop_tex && t->op != nir_texop_txl) || t->sampler_dim != GLSL_SAMPLER_DIM_2D || t->is_array || + t->is_shadow || t->def.num_components != 4 || t->def.bit_size != 32 || + (nir_alu_type_get_base_type(t->dest_type) != nir_type_float && !fetch)) + return "A8 vertex texture: only Apple's 2D sample and read"; + for (unsigned i = 0; i < t->num_srcs; i++) + if (t->src[i].src_type != nir_tex_src_coord && t->src[i].src_type != nir_tex_src_lod && + t->src[i].src_type != nir_tex_src_texture_deref && t->src[i].src_type != nir_tex_src_sampler_deref) + return "A8 vertex texture: a source other than the coordinate and level"; + /* Apple's vertex form samples at its constant level 0 (byte 10 0x1c) */ + const int li = nir_tex_instr_src_index(t, nir_tex_src_lod); + if ((li >= 0 && (!nir_src_is_const(t->src[li].src) || nir_src_as_float(t->src[li].src) != 0.0f)) || + (fetch && ((t->backend_flags >> G5P_TEX_FETCH_LEVEL_SHIFT) & 0xf))) + return "A8 vertex texture: Apple's vertex form is captured at level 0 only"; + return NULL; +} + static const char *vs_qualified_tex_refusal(const nir_tex_instr *t) { if (t->sampler_dim == GLSL_SAMPLER_DIM_BUF) @@ -1255,7 +1317,8 @@ static void scan_cf(struct gctx *c, struct exec_list *list, struct unsupported * } case nir_instr_type_tex: { nir_tex_instr *t = nir_instr_as_tex(ins); - const char *why = c->vs ? (g5p_vtex_qualified() ? vs_qualified_tex_refusal(t) : !g5p_vtex_probe() ? "vertex stage" : vs_tex_refusal(t)) : + const char *why = c->vs ? (g5p_usc_g4p && g5p_usc_g4p_vtex_apple && !size_query(t) ? a8_vs_tex_refusal(t) : + g5p_vtex_qualified() ? vs_qualified_tex_refusal(t) : !g5p_vtex_probe() ? "vertex stage" : vs_tex_refusal(t)) : plain_tex(t) ? NULL : tex_shape_refusal(t, true, t->sampler_dim == GLSL_SAMPLER_DIM_3D, true); if (why) { snprintf(buf, sizeof(buf), "tex(op %u, dim %u%s%s): %s", t->op, t->sampler_dim, @@ -2086,6 +2149,15 @@ static void tex_bindings(struct gctx *c, nir_tex_instr *t, struct tex_bind *b) sv->data.binding, element, selement}; b->needs_sampler = t->op != nir_texop_txf; b->needs_fetch_sampler = t->op == nir_texop_txf; + b->fetch = t->backend_flags & G5P_TEX_FETCH; + b->fetch_level = b->fetch ? (t->backend_flags >> G5P_TEX_FETCH_LEVEL_SHIFT) & 0xf : 0; + b->fetch_read = b->fetch && (t->backend_flags & G5P_TEX_LOD_REG); + /* the A8's vertex reads in Apple's form (PVR_A8_VTEX_APPLE) share one sampler, Apple's constant read sampler (one per program in Apple's + * constant data), in the fetch sampler's place (PCO_G5P_UPLOAD_FETCH_SAMPLER) */ + if (c->vs && b->fetch_read) { + b->needs_sampler = false; + b->needs_fetch_sampler = true; + } b->image_flags = (t->backend_flags & PCO_G5P_TEX_ONE_LAYER_AS_2D) ? PCO_G5P_UPLOAD_IMAGE_ONE_LAYER : (t->backend_flags & G5P_TEX_LAYER0_AS_2D) ? PCO_G5P_UPLOAD_IMAGE_AS_2D : t->is_array ? PCO_G5P_UPLOAD_IMAGE_ARRAY : 0; @@ -2098,7 +2170,8 @@ static struct tex_bind *tex_for(struct gctx *c, nir_tex_instr *t) for (unsigned i = 0; i < c->ntex; i++) if (c->tex[i].set == b.set && c->tex[i].binding == b.binding && c->tex[i].sset == b.sset && c->tex[i].sbinding == b.sbinding && c->tex[i].element == b.element && c->tex[i].selement == b.selement && - c->tex[i].image_flags == b.image_flags) + c->tex[i].image_flags == b.image_flags && c->tex[i].fetch == b.fetch && c->tex[i].fetch_level == b.fetch_level && + c->tex[i].fetch_read == b.fetch_read) { c->tex[i].needs_sampler |= b.needs_sampler; c->tex[i].needs_fetch_sampler |= b.needs_fetch_sampler; return &c->tex[i]; } @@ -2290,12 +2363,25 @@ static void plan_common(struct gctx *c) * Apple's f16 capacity layout: all eight-dword images, then samplers. * Interleaving wastes six dwords per binding and exceeds the selectors. */ bool compact = qualified || (!c->vs && c->ntex > 4); + /* The texture and sampler state each texture takes in the common store: the A9's three-qword image state + * (eight dwords with its metadata) and one-qword sampler; the A8's Rogue TEXSTATE_IMAGE_WORD0/1 and + * TEXSTATE_SAMPLER_WORD0/1, four dwords each, the sampler right after the image (AGXMetalA8's tex_plain and load: + * texture state then sampler in one 8-dword block, the sample's sampler field naming dword 4) */ + const unsigned sdw = g5p_usc_g4p ? 4 : 2, idw = g5p_usc_g4p ? 4 : 8; + /* vertex textures on the A8 (g5p_usc_g4p_vtex): the compact layout */ + if (g5p_usc_g4p && !g5p_usc_g4p_a9_forms && compact && c->ntex && !(c->vs && g5p_usc_g4p_vtex)) + refuse(c, "the A8 samples one image slot (common dword 0); more than four textures or vertex textures are " + "not captured"); unsigned sampler_count = 0; bool fetch_sampler = false; for (unsigned i = 0; i < c->ntex; i++) { sampler_count += !qualified || c->tex[i].needs_sampler; fetch_sampler |= qualified && c->tex[i].needs_fetch_sampler; } + /* the samplerless read's sampler policy is the A9's one-qword word (pvr_g5p_fetch_sampler_word()); the A8's, + * with vertex textures, four dwords of Rogue sampler words (point, non-normalized, clamped) */ + if (g5p_usc_g4p && fetch_sampler && !g5p_usc_g4p_vtex) + refuse(c, "a samplerless texture read on the A8 (its fetch sampler is the A9's word, not Rogue's)"); struct g5p_fs_common_plan common_plan = {0}; bool planned = !c->vs || qualified; if (planned) { @@ -2310,7 +2396,8 @@ static void plan_common(struct gctx *c) push_dwords += (r->hi - r->lo) / 4; } } - if (!g5p_stage_common_plan(next, c->ntex, sampler_count + fetch_sampler, + if (!g5p_stage_common_plan_stride(next, c->ntex, sampler_count + fetch_sampler, idw, sdw, + g5p_usc_g4p && c->vs && g5p_usc_g4p_vtex_apple ? 4 : 0, compact, c->nuptr, c->nsize, ubos, ubo_dwords, push_dwords + ssbo_reserve, &common_plan)) refuse(c, "common layout cannot fit pointers below 32 and data below 256 dwords"); @@ -2336,23 +2423,61 @@ static void plan_common(struct gctx *c) assert(next == common_plan.pointer_end); } } - unsigned image_base = align(next, 8), sampler_base = image_base + 8 * c->ntex; + /* the A8's vertex textures in Apple's form (PVR_A8_VTEX_APPLE) in Apple's spacing: four-dword images, then the + * samplers */ + const unsigned istride = g5p_usc_g4p && c->vs && g5p_usc_g4p_vtex_apple ? 4 : 8; + unsigned image_base = align(next, 8), sampler_base = image_base + istride * c->ntex; unsigned sampler_cursor = sampler_base; - c->fetch_sampler_common = sampler_base + 2 * sampler_count; - for (unsigned i = 0; i < c->ntex; i++) { + /* The A8's programs sampling two or more images, or one after other common data (g5p_usc.h g5p_tex.a8_slots), + * in Apple's layout (iOS 12.4, AGXMetalA8's programs sampling two to seven textures, a8-passcap): the images in + * four-dword slots from the base (the next four-dword boundary after the blend colour or storage pointers), then + * the samplers, one slot each; the sample names its sampler's slot in byte 1 bits 7:6 and + * byte 5 bits 2:1 (g5p_usc.c sample_g4p). With one shared sampler Apple puts it at slot N after N images, + * with a sampler per texture texture j's at slot N + j; the seven-texture program's PDS DMAs 28 dwords of + * images to common 0 and its sampler to 28. A sampler per texture while the last one's slot is at most 11 + * (slots up to 9 are witnessed; 10 and 11 combine byte-5 05 with byte-1 values witnessed at other slots), else + * Apple's one shared sampler, texture 0's (seven images; byte 5 07 for slots 12-15 is not captured). On the + * iPod every multi-texture test passed this way: 2-5 images, a sampler each (own-filter checkers) and four + * images after the blend colour (images at 4..16). Shapes past slot 11 keep the per-texture eight-dword layout + * and the A9 form. */ + const unsigned slot_base = next ? align(next, 4) : 0, base_slot = slot_base / 4; + const bool a8_multi = g5p_usc_g4p && !c->vs && !qualified && c->ntex && (next || c->ntex >= 2); + const bool a8_each = base_slot + 2 * c->ntex - 1 <= 11, a8_shared = base_slot + c->ntex <= 11; + c->a8_slots = a8_multi && (a8_each || a8_shared); + c->fetch_sampler_common = sampler_base + sdw * sampler_count; + for (unsigned i = 0; c->a8_slots && i < c->ntex; i++) { struct tex_bind *t = &c->tex[i]; - t->image = compact ? image_base + 8 * i : align(next, 8); - t->sampler = compact ? ((!qualified || t->needs_sampler) ? sampler_cursor : c->fetch_sampler_common) : t->image + 8; - sampler_cursor += compact && (!qualified || t->needs_sampler) ? 2 : 0; - next = compact ? c->fetch_sampler_common + 2 * (qualified && fetch_sampler) : t->image + 10; + t->image = slot_base + 4 * i; + t->sampler = slot_base + 4 * c->ntex + (a8_each ? 4 * i : 0); + add_upload(c, PCO_G5P_UPLOAD_IMAGE, t->set, t->binding, t->element, t->image_flags, idw, t->image); + if (!i || a8_each) + add_upload(c, PCO_G5P_UPLOAD_SAMPLER, t->sset, t->sbinding, t->selement, + 32 | (t->fetch ? PCO_G5P_UPLOAD_SAMPLER_FETCH | + t->fetch_level << PCO_G5P_UPLOAD_SAMPLER_FETCH_LEVEL_SHIFT | + (t->fetch_read ? PCO_G5P_UPLOAD_SAMPLER_READ : 0) : 0), + sdw, t->sampler); + } + if (c->a8_slots) /* the common plan's span of the textures: eight dwords each, the compact plan's samplers after */ + next = align(next, 8) + 8 * c->ntex + (compact ? sdw * sampler_count : 0); + for (unsigned i = 0; !c->a8_slots && i < c->ntex; i++) { + struct tex_bind *t = &c->tex[i]; + t->image = compact ? image_base + istride * i : align(next, 8); + t->sampler = compact ? ((!qualified || t->needs_sampler) ? sampler_cursor : c->fetch_sampler_common) : t->image + idw; + sampler_cursor += compact && (!qualified || t->needs_sampler) ? sdw : 0; + next = compact ? c->fetch_sampler_common + (g5p_usc_g4p ? sdw : 2) * (qualified && fetch_sampler) + : t->image + idw + sdw; if (t->image / 2 > (compact ? 120 : 28) || t->sampler / 2 > (compact ? 124 : 32)) refuse(c, "texture descriptors beyond the bounded qualified selectors"); - add_upload(c, PCO_G5P_UPLOAD_IMAGE, t->set, t->binding, t->element, t->image_flags, 8, t->image); + add_upload(c, PCO_G5P_UPLOAD_IMAGE, t->set, t->binding, t->element, t->image_flags, idw, t->image); if (!qualified || t->needs_sampler) - add_upload(c, PCO_G5P_UPLOAD_SAMPLER, t->sset, t->sbinding, t->selement, 32, 2, t->sampler); + add_upload(c, PCO_G5P_UPLOAD_SAMPLER, t->sset, t->sbinding, t->selement, + 32 | (t->fetch ? PCO_G5P_UPLOAD_SAMPLER_FETCH | + t->fetch_level << PCO_G5P_UPLOAD_SAMPLER_FETCH_LEVEL_SHIFT | + (t->fetch_read ? PCO_G5P_UPLOAD_SAMPLER_READ : 0) : 0), + sdw, t->sampler); } if (qualified && fetch_sampler) - add_upload(c, PCO_G5P_UPLOAD_FETCH_SAMPLER, 0, 0, 0, 0, 2, c->fetch_sampler_common); + add_upload(c, PCO_G5P_UPLOAD_FETCH_SAMPLER, 0, 0, 0, 0, g5p_usc_g4p ? sdw : 2, c->fetch_sampler_common); /* Pointer pairs of register-indexed UBO arrays: even dwords below 32 (the * addr.add_shift base field, 0x81 | 4 * dword). */ for (unsigned i = 0; i < c->nuptr; i++) { @@ -3592,6 +3717,41 @@ static void emit_ibin(struct gctx *c, nir_alu_instr *a) static void emit_isub(struct gctx *c, nir_alu_instr *a) { bool neg = a->op == nir_op_ineg; + if (g5p_usc_g4p && !c->vs) { + /* The A8: the A9's register subtract (op a5) gives x - y = x on the iPod touch 6 (a remainder a - 3q read + * back a while 3q and the quotient read right), so x - y is x + y * -1, with the integer multiply and add the + * A8 runs right; -1 is a MOVI */ + uint8_t raw[G5P_USC_MAX]; + struct ref x = neg ? (struct ref){0} : alu_src(c, a, 0), y = alu_src(c, a, neg ? 0 : 1); + const unsigned m1 = scratch_imm(c, 0xffffffffu); + const unsigned xr = neg ? 0 : reg_of(c, x, "isub"); + const unsigned yr = reg_of(c, y, "isub"); + const unsigned gy = vgrp(c, y); + const bool dy = dies(c, y); + free_dying(c, &y, 1); + const unsigned p = alloc_reg(c, dy ? (int)yr : -1); + struct g5p_alu f = G5P_ALU(2 * p); + f.a = gsrc(c, yr, (struct ref){0}, gy); + f.b = gsrc(c, m1, (struct ref){0}, 1); + put(c, raw, g5p_usc_binary(raw, G5P_BIN_IMUL, &f), "isub: y * -1"); + if (neg) { + c->p->evidence |= PCO_G5P_EV_FS_ALU; + define(c, &a->def, p); + return; + } + const unsigned gx = vgrp(c, x); + const bool dx = dies(c, x); + free_dying(c, &x, 1); + c->scratch |= BITFIELD64_BIT(p); + const unsigned dst = alloc_reg(c, dx ? (int)xr : -1); + struct g5p_alu g = G5P_ALU(2 * dst); + g.a = gsrc(c, xr, (struct ref){0}, gx); + g.b = gsrc(c, p, (struct ref){0}, 1); + put(c, raw, g5p_usc_binary(raw, G5P_BIN_IADD, &g), "isub: x + (-y)"); + c->p->evidence |= PCO_G5P_EV_FS_ALU; + define(c, &a->def, dst); + return; + } struct ref r[2] = {neg ? (struct ref){0} : alu_src(c, a, 0), alu_src(c, a, neg ? 0 : 1)}; unsigned ra = neg ? zero_reg(c) : reg_of(c, r[0], "isub"); unsigned rb = reg_of(c, r[1], "isub"); @@ -4276,6 +4436,14 @@ static void drc_flush(struct gctx *c, struct drc *d) static bool drc_take(struct gctx *c, struct drc *d) { tex_flush(c); + /* The A8 has no captured counter-1 iterator (g5p_usc.c iterator_g4p()): its reads stay on counter 0, a full + * batch of eight waited for before the next, as AGXMetalA8's programs wait `4d 00` */ + if (g5p_usc_g4p) { + if (d->n[0] == 8) + drc_flush(c, d); + d->n[0]++; + return false; + } if (d->n[0] == 8 && d->n[1] == 8) drc_flush(c, d); if (d->n[0] < 8) { @@ -4322,10 +4490,17 @@ static void read_varying(struct gctx *c, nir_intrinsic_instr *in, struct drc *d) } unsigned flags = (v->interp == PCO_G5P_INTERP_NOPERSPECTIVE || flat_iterated ? G5P_ITER_LINEAR : 0) | (v->centroid && !flat_iterated ? G5P_ITER_CENTROID : 0); - if (drc_take(c, d)) - flags |= G5P_ITER_DRC1; unsigned dst = alloc_run(c, n, "varying"); - put(c, raw, g5p_usc_iterator_flags(raw, 2 * dst, slot, n, false, false, flags), "iterator"); + /* The A8's f32 iterator is captured for one and two components (AGXMetalA8's f_discard and f_tex: `18 09 ..`, + * `18 11 ..`), whose slot counts components (f_discard's color.a is slot 4). A three- or four-component f32 + * varying is iterated as those pieces: components 0-1, then 2(-3), each its own dependent read. */ + for (unsigned j0 = 0; j0 < n; j0 += g5p_usc_g4p ? 2 : n) { + const unsigned cnt = g5p_usc_g4p ? MIN2(2, n - j0) : n; + unsigned f = flags; + if (drc_take(c, d)) + f |= G5P_ITER_DRC1; + put(c, raw, g5p_usc_iterator_flags(raw, 2 * (dst + j0), slot + j0, cnt, false, false, f), "iterator"); + } for (unsigned j = 0; j < n; j++) { struct val *val = V(c, &in->def, j); val->state = VAL_REG; @@ -4929,7 +5104,7 @@ static void emit_fragment_cube_gradient(struct gctx *c, nir_tex_instr *t) unsigned dst = alloc_run(c, 4, "cube gradient result"); struct g5p_tex_ext x = { .dst_half = 2 * dst, .image_qword = b->image / 2, - .sampler_qword = b->sampler / 2, .coord_dword = 0, + .sampler_qword = g5p_usc_g4p ? b->sampler : b->sampler / 2, .coord_dword = 0, .dim = G5P_TEX_CUBE, .mask = 15, .lod = G5P_TEX_LOD_GRAD, .lod_dword = 3, }; @@ -4948,6 +5123,23 @@ static void emit_tex_ext(struct gctx *c, nir_tex_instr *t) emit_fragment_cube_gradient(c, t); return; } + /* the A8 draws fragment textures only as its own sample (sample_g4p()): name what this one is */ + if (g5p_usc_g4p && !c->vs) { + const int li = nir_tex_instr_src_index(t, nir_tex_src_lod); + refuse(c, "A8 fragment %s not in a captured form: op %d, dim %d%s%s%s, %s result%s%s%s%s%s%s", + t->op == nir_texop_txf ? "texel read" : "texture sample", (int)t->op, (int)t->sampler_dim, + t->is_array ? " array" : "", t->is_shadow ? " shadow" : "", + (t->backend_flags & PVR_G5P_TBO_NIR_FOLDED) ? " (texel buffer)" : "", + nir_alu_type_get_base_type(t->dest_type) == nir_type_float ? "float" : + nir_alu_type_get_base_type(t->dest_type) == nir_type_int ? "int" : "uint", + li < 0 ? "" : nir_src_is_const(t->src[li].src) ? ", constant level" : ", variable level", + nir_tex_instr_src_index(t, nir_tex_src_offset) >= 0 ? ", offset" : "", + nir_tex_instr_src_index(t, nir_tex_src_bias) >= 0 ? ", bias" : "", + nir_tex_instr_src_index(t, nir_tex_src_ddx) >= 0 ? ", gradient" : "", + nir_tex_instr_src_index(t, nir_tex_src_ms_index) >= 0 ? ", sample index" : "", + t->def.num_components != 4 ? ", not four components" : ""); + return; + } bool saved_bank = c->tex_bank0; c->tex_bank0 = c->vs; uint8_t raw[G5P_USC_MAX]; @@ -4957,7 +5149,7 @@ static void emit_tex_ext(struct gctx *c, nir_tex_instr *t) bool third = array || sd == GLSL_SAMPLER_DIM_3D || sd == GLSL_SAMPLER_DIM_CUBE; int packed = packed_offset_components(t, sd == GLSL_SAMPLER_DIM_3D ? 3 : 2); struct g5p_tex_ext x = { - .image_qword = b->image / 2, .sampler_qword = b->sampler / 2, .read = read, + .image_qword = b->image / 2, .sampler_qword = g5p_usc_g4p ? b->sampler : b->sampler / 2, .read = read, .dim = sd == GLSL_SAMPLER_DIM_CUBE ? G5P_TEX_CUBE : sd == GLSL_SAMPLER_DIM_3D ? G5P_TEX_3D : array ? G5P_TEX_2D_ARRAY : G5P_TEX_2D, .mask = t->is_shadow ? 1 : 0xf, .compare = t->is_shadow, .offset = packed >= 0, @@ -5195,7 +5387,7 @@ static void emit_vertex_qualified_texture(struct gctx *c, nir_tex_instr *t) .sampler_qword=(read ? c->fetch_sampler_common : b->sampler)/2, .coord_dword=H(c,block)/2, .dim=cube ? G5P_TEX_CUBE : t->sampler_dim==GLSL_SAMPLER_DIM_3D ? G5P_TEX_3D : array ? G5P_TEX_2D_ARRAY : G5P_TEX_2D, - .mask=compare ? 1 : 15, .read=read, .compare=compare, + .mask=compare ? 1 : 15, .read=read, .compare=compare, .vertex=true, .lod=gradient ? G5P_TEX_LOD_GRAD : G5P_TEX_LOD_LEVEL, .lod_dword=buffer ? c->tsize[size_for(c,t)].common + 1 : H(c,level)/2, .lod_shared=buffer, .offset=packed >= 0, .extras_half=compare || packed >= 0 ? extras : 0, @@ -5275,7 +5467,7 @@ static void emit_tex_ms_own_sample(struct gctx *c, nir_tex_instr *t, struct tex_ put(c, raw, g5p_usc_move(raw, 2 * coord + 2, 2 * yr, 0, vgrp(c, y), 1), "coordinate y"); put(c, raw, g5p_usc_ms_own_sample(raw, 2 * coord + 4), "the task's own sample"); c->scratch |= BITFIELD64_RANGE(coord, 3); - struct g5p_tex s = {.image_qword = b->image / 2, .sampler_qword = b->sampler / 2, .coord_dword = coord, + struct g5p_tex s = {.image_qword = c->a8_slots ? b->image : b->image / 2, .a8_slots = c->a8_slots, .sampler_qword = g5p_usc_g4p ? b->sampler : b->sampler / 2, .coord_dword = coord, .coord_control = true, .dim = G5P_TEX_2D_MS, .narrow = true, .dst_half = 0}; put(c, raw, g5p_usc_sample(raw, &s), "texture sample (2D multisample)"); c->tex_coord_pending |= BITFIELD64_RANGE(coord, 3); @@ -5284,6 +5476,98 @@ static void emit_tex_ms_own_sample(struct gctx *c, nir_tex_instr *t, struct tex_ c->p->evidence |= PCO_G5P_EV_TEX_CAPTURE; } +/* The A8's cube sample, as AGXMetalA8's f_tex_cube: the direction in three consecutive temps (computed: FMAs and a + * move, no cube-coordinate instruction; the hardware selects the face), then the narrow sample in mode 4 */ +static void emit_tex_cube_g4p(struct gctx *c, nir_tex_instr *t) +{ + uint8_t raw[G5P_USC_MAX]; + struct tex_bind *b = tex_for(c, t); + const bool narrow = c->narrow_tex == t; + if (!narrow) + refuse(c, "A8 cube sample whose result feeds arithmetic (only the narrow form, straight to the output, is " + "captured)"); + nir_def *cd = nir_get_tex_src(t, nir_tex_src_coord); + struct ref r[3]; + unsigned reg[3]; + for (unsigned k = 0; k < 3; k++) { + r[k] = resolve_value(c, cd, k); + reg[k] = reg_of(c, r[k], "cube direction"); + } + unsigned coord; + if (reg[1] == reg[0] + 1 && reg[2] == reg[0] + 2) { + coord = reg[0]; + for (unsigned k = 0; k < 3; k++) + vgrp(c, r[k]); /* read by the sample instruction, the next one put */ + } else { + coord = alloc_run(c, 3, "cube direction"); + for (unsigned k = 0; k < 3; k++) + put(c, raw, g5p_usc_move(raw, 2 * (coord + k), 2 * reg[k], 0, vgrp(c, r[k]), 1), "cube direction"); + c->scratch |= BITFIELD64_RANGE(coord, 3); + } + struct g5p_tex s = {.image_qword = c->a8_slots ? b->image : b->image / 2, .a8_slots = c->a8_slots, .sampler_qword = b->sampler, .coord_dword = coord, + .coord_control = true, .dim = G5P_TEX_CUBE, .narrow = true}; + put(c, raw, g5p_usc_sample(raw, &s), "cube sample"); + c->tex_coord_pending |= BITFIELD64_RANGE(coord, 3); + c->tex_batch++; + tex_flush(c); + c->p->evidence |= PCO_G5P_EV_TEX_CAPTURE; +} + +/* The A8's vertex-stage sample in Apple's form (PVR_A8_VTEX_APPLE), as AGXMetalA8 compiles v_sample, v_read_f32 and + * v_read_i32 (an iOS capture, USC heap 0x1_007c8000 + 0xa166 / 0xa40a / 0xa8a6): the coordinate as two f32 in + * bank-1 temps (a read's integer coordinate converted to float first, as in the fragment read), then + * 1a 7d 00 41 00 01 a8|ac|a4 05 00 01 1c 80 (g5p_tex.vtx): the result over the coordinate in bank 1, mode 2 at + * Apple's constant level 0, then the wait. The image and sampler are named by their common dwords (Apple: image at 0, + * sampler at 4, slot 1); a read's sampler is Apple's read sampler, the fetch sampler's words (point, non-normalized, + * clamped, level 0: the capture's constant 0x0202240000000fff). 2D at level 0 only; no array, compare, offset or + * gradient. */ +static void emit_vertex_tex_g4p(struct gctx *c, nir_tex_instr *t) +{ + uint8_t raw[G5P_USC_MAX]; + const bool fetch = t->backend_flags & G5P_TEX_FETCH; + const char *why = a8_vs_tex_refusal(t); + if (why) { + refuse(c, "%s", why); + return; + } + struct tex_bind *b = tex_for(c, t); + nir_def *cd = nir_get_tex_src(t, nir_tex_src_coord); + tex_flush(c); + /* bank 1, as Apple's: the coordinate in the first two registers, the result over them */ + const bool saved_bank0 = c->tex_bank0; + c->tex_bank0 = false; + const unsigned coord = alloc_run(c, 4, "A8 vertex sample coordinate and result"), dst = coord; + c->tex_bank0 = saved_bank0; + if (vbank(c, coord) != 1) + refuse(c, "A8 vertex sample: Apple's form takes bank-1 registers"); + vertex_qualified_copy(c, coord, cd, 0); + vertex_qualified_copy(c, coord + 1, cd, 1); + struct g5p_tex s = {.image_qword = b->image, .a8_slots = true, .sampler_qword = b->sampler, + .coord_dword = H(c, coord) / 2, .coord_control = true, .dim = G5P_TEX_2D, .narrow = false, + .read_flag = fetch, .vtx = true, + .int_result = nir_alu_type_get_base_type(t->dest_type) != nir_type_float, + .dst_half = H(c, dst)}; + const unsigned n = g5p_usc_sample(raw, &s); + if (!n) + refuse(c, "A8 vertex sample: encoder rejects the fields (%s) [image at common dword %u, sampler at %u]", + g5p_usc_error ? g5p_usc_error : "?", b->image, b->sampler); + put(c, raw, n, "A8 vertex sample (Apple's form)"); + c->tex_pending |= BITFIELD64_RANGE(dst, 4); + c->tex_batch++; + tex_flush(c); + for (unsigned k = 0; k < 4; k++) { + struct val *v = V(c, &t->def, k); + if (!v->uses) { + c->busy &= ~BITFIELD64_BIT(dst + k); + continue; + } + v->state = VAL_REG; + v->held = true; + v->reg = dst + k; + } + c->p->evidence |= PCO_G5P_EV_TEX_F32; +} + static void emit_tex(struct gctx *c, nir_tex_instr *t) { if (c->tex_batch == 4 || c->vs || !plain_tex(t) || size_query(t) || c->narrow_tex == t) @@ -5292,10 +5576,18 @@ static void emit_tex(struct gctx *c, nir_tex_instr *t) emit_size(c, t); return; } + if (c->vs && g5p_usc_g4p && g5p_usc_g4p_vtex_apple) { + emit_vertex_tex_g4p(c, t); + return; + } if (c->vs && g5p_vtex_qualified()) { emit_vertex_qualified_texture(c, t); return; } + if (!c->vs && a8_cube_tex(t)) { + emit_tex_cube_g4p(c, t); + return; + } if (c->vs || !plain_tex(t)) { emit_tex_ext(c, t); return; @@ -5309,10 +5601,30 @@ static void emit_tex(struct gctx *c, nir_tex_instr *t) emit_tex_ms_own_sample(c, t, b, narrow); return; } - if (slot >= 0) { + const bool lod_reg = t->backend_flags & G5P_TEX_LOD_REG; + if (lod_reg) { + /* Apple's explicit level (g5p_tex.lod_reg): the coordinate and the f32 level in a run of three */ + nir_def *cd = nir_get_tex_src(t, nir_tex_src_coord); + struct ref x = resolve_value(c, cd, 0), y = resolve_value(c, cd, 1); + unsigned xr = reg_of(c, x, "texture coordinate"), yr = reg_of(c, y, "texture coordinate"); + coord = alloc_run(c, 3, "texture coordinate and level"); + put(c, raw, g5p_usc_move(raw, 2 * coord, 2 * xr, 0, vgrp(c, x), 1), "coordinate x"); + put(c, raw, g5p_usc_move(raw, 2 * coord + 2, 2 * yr, 0, vgrp(c, y), 1), "coordinate y"); + const float level = (float)((t->backend_flags >> G5P_TEX_FETCH_LEVEL_SHIFT) & 0xf); + put(c, raw, g5p_usc_immediate(raw, 2 * coord + 4, fui(level), 0, -1), "level (f32)"); + c->scratch |= BITFIELD64_RANGE(coord, 3); + c->p->evidence |= PCO_G5P_EV_TEX_COMPUTED; + } else if (slot >= 0) { tex_flush(c); coord = narrow ? 0 : alloc_run(c, 2, "texture coordinate"); put(c, raw, g5p_usc_coord_iterator(raw, 2 * coord, slot, 2), "coordinate iterator"); + /* The A8's coordinate is the plain iterator's result, a dependent read on counter 0 that the sample must + * wait for: AGXMetalA8's f_tex and every tex_* fragment program are `18 11 00 81 04 00`, `4d 00`, then the + * sample (the A9's coordinate iterator feeds its sample directly). Without the wait the sample read the + * coordinate register before the iteration landed (the live ladder's texture rungs: clamped, run-dependent + * texels). */ + if (g5p_usc_g4p) + put(c, raw, g5p_usc_wait(raw, 0), "wait (coordinate)"); } else { /* ISA round 2 §4: an ALU-produced coordinate uses the same sample * encoding with the temp base (21 Apple computed-coordinate probes). */ @@ -5332,17 +5644,34 @@ static void emit_tex(struct gctx *c, nir_tex_instr *t) } c->p->evidence |= PCO_G5P_EV_TEX_COMPUTED; } - struct g5p_tex s = {.image_qword = b->image / 2, .sampler_qword = b->sampler / 2, .coord_dword = coord, - .coord_control = true, .dim = G5P_TEX_2D, .narrow = narrow}; + struct g5p_tex s = {.image_qword = c->a8_slots ? b->image : b->image / 2, .a8_slots = c->a8_slots, .sampler_qword = g5p_usc_g4p ? b->sampler : b->sampler / 2, .coord_dword = coord, + .coord_control = true, .dim = G5P_TEX_2D, .narrow = narrow, + .read_flag = (t->backend_flags & G5P_TEX_FETCH) && (g5p_usc_g4p_texfetch == 2 || lod_reg), + .lod_reg = lod_reg}; unsigned dst = 0; if (!narrow) { dst = alloc_run(c, 4, "texture result"); } s.dst_half = 2 * dst; - put(c, raw, g5p_usc_sample(raw, &s), "texture sample"); - c->tex_coord_pending |= BITFIELD64_RANGE(coord, 2); + a8_image_order(c, b->image); + const unsigned tex_n = g5p_usc_sample(raw, &s); + if (!tex_n && g5p_usc_g4p) + refuse(c, "texture sample: encoder rejects the fields (%s) [A8: image set %u binding %u[%u] at common dword " + "%u, sampler set %u binding %u at %u, %u images, %s layout, common store starts at %u; sampler dim %d%s, " + "texop %d%s%s%s%s%s, %s result]", + g5p_usc_error ? g5p_usc_error : "?", b->set, b->binding, b->element, b->image, b->sset, b->sbinding, + b->sampler, c->ntex, c->a8_slots ? "Apple multi-image" : "per-texture", c->ntex ? c->tex[0].image : 0, + (int)t->sampler_dim, t->is_array ? " array" : "", (int)t->op, + nir_tex_instr_src_index(t, nir_tex_src_lod) >= 0 ? " lod" : "", + nir_tex_instr_src_index(t, nir_tex_src_bias) >= 0 ? " bias" : "", + nir_tex_instr_src_index(t, nir_tex_src_ddx) >= 0 ? " grad" : "", + nir_tex_instr_src_index(t, nir_tex_src_offset) >= 0 ? " offset" : "", + nir_tex_instr_src_index(t, nir_tex_src_comparator) >= 0 ? " compare" : "", narrow ? "narrow" : "32-bit"); + put(c, raw, tex_n, "texture sample"); + c->tex_coord_pending |= BITFIELD64_RANGE(coord, lod_reg ? 3 : 2); c->tex_pending |= BITFIELD64_RANGE(dst, 4); c->tex_batch++; + c->tex_batch_image = b->image; if (narrow) tex_flush(c); c->p->evidence |= narrow ? PCO_G5P_EV_TEX_CAPTURE : PCO_G5P_EV_TEX_F32; @@ -5693,7 +6022,9 @@ static void plan_fs_output(struct gctx *c) if (mr != G5P_MRT_OK) refuse(c, "%s", g5p_mrt_refusal_str(mr)); } - c->mrt.words = (c->key->packed_scalar8 ? 1 : c->key->packed_uint_bits ? 2 : 4) * c->mrt.targets; + const bool one_word = c->key->packed_scalar8 || c->key->a8_packed_word || + c->key->color_output == PCO_G5P_COLOR_F32; + c->mrt.words = (one_word ? 1 : c->key->packed_uint_bits ? 2 : 4) * c->mrt.targets; c->mrt.pixel_output_width = c->key->packed_scalar8 ? 2 : c->mrt.words; c->mrt.emit_field = g5p_mrt_emit_field(c->mrt.words); return; /* No narrow f16 loads, output ALU fusion, or f16 constants. */ @@ -5701,6 +6032,12 @@ static void plan_fs_output(struct gctx *c) enum g5p_mrt_refusal mr = g5p_mrt_check_writes(&c->mrt, c->slot_written); if (mr != G5P_MRT_OK) refuse(c, "%s", g5p_mrt_refusal_str(mr)); + if (c->key->a8_narrow_halves) { + /* the A8's R8/RG8: one output word (emitpix 111c), no narrow tex/iterator form */ + c->mrt.words = 1; + c->mrt.emit_field = g5p_mrt_emit_field(1); + return; + } /* The narrow tex/iterator forms write output halves 0..3 directly: one target only. */ bool all = c->mrt.targets == 1 && c->out_written[0] && c->out_written[1] && c->out_written[2] && c->out_written[3]; @@ -5710,7 +6047,8 @@ static void plan_fs_output(struct gctx *c) V(c, c->out[k].def, k)->uses == 1; if (plain) { nir_instr *p = nir_def_instr(c->out[0].def); - if (p->type == nir_instr_type_tex && plain_tex(nir_instr_as_tex(p))) { + if (p->type == nir_instr_type_tex && (plain_tex(nir_instr_as_tex(p)) || a8_cube_tex(nir_instr_as_tex(p))) && + nir_alu_type_get_base_type(nir_instr_as_tex(p)->dest_type) == nir_type_float) { c->narrow_tex = nir_instr_as_tex(p); return; } @@ -5738,7 +6076,9 @@ static void plan_fs_output(struct gctx *c) src = in; c->narrow_sat[k] = a; } - if (sat4) { + /* The A8 has no captured saturating narrow iterator: there the A9's general path iterates the varying and + * saturates it in the ALU (the shader's own fsat), as the A9 does whenever the narrow form does not apply. */ + if (sat4 && !g5p_usc_g4p) { c->narrow_iter = src; return; } @@ -5769,9 +6109,17 @@ static void fs_prologue(struct gctx *c) uint8_t raw[G5P_USC_MAX]; if (c->narrow_tex || c->narrow_iter || !any_color(c)) return; - for (unsigned d = 0; d < c->mrt.words; d++) { + for (unsigned i = 0; i < c->mrt.words; i++) { + /* the A8 writes the last target's output dwords first (AGXMetalA8's two-target f_mrt2: r2, r3, r0, r1) */ + const unsigned per = c->mrt.targets > 1 ? c->mrt.words / c->mrt.targets : c->mrt.words; + const unsigned d = (g5p_usc_g4p && c->mrt.targets > 1) ? (c->mrt.targets - 1 - i / per) * per + i % per : i; uint16_t h0 = 0, h1 = 0; bool c0 = out_const(c, 2 * d, &h0), c1 = out_const(c, 2 * d + 1, &h1); + /* the A8's R8 output: half 1 is zero whatever the shader writes (Apple's 7130 literal) */ + if (c->key->a8_narrow_halves == 1 && d == 0) { + c1 = true; + h1 = 0; + } if (!c0 && !c1) continue; if (!c0) @@ -6347,6 +6695,42 @@ static bool sink_vs_stores(nir_shader *nir) return nir_progress(any, impl, nir_metadata_control_flow); } +/* The A8: a remainder from the quotient (umod/irem/imod -> a - (a / b) * b, imod's sign then that of b), the form + * that reads right on the iPod touch 6 together with the subtract below (emit_isub()). */ +static bool a8_mod_from_div(nir_builder *b, nir_alu_instr *alu, void *data) +{ + if (alu->op != nir_op_umod && alu->op != nir_op_imod && alu->op != nir_op_irem) + return false; + b->cursor = nir_before_instr(&alu->instr); + nir_def *x = nir_ssa_for_alu_src(b, alu, 0), *y = nir_ssa_for_alu_src(b, alu, 1); + nir_def *q = alu->op == nir_op_umod ? nir_udiv(b, x, y) : nir_idiv(b, x, y); + /* q * y: a constant divisor's product in shifts and adds, a register divisor's by the multiply */ + nir_def *qy = NULL; + if (nir_src_is_const(alu->src[1].src) && y->num_components == 1 && y->bit_size == 32) { + const uint32_t k = (uint32_t)nir_alu_src_as_uint(alu->src[1]); + const uint32_t m = (int32_t)k < 0 && alu->op != nir_op_umod ? -k : k; + for (unsigned bit = 0; bit < 32; bit++) + if (m >> bit & 1) { + nir_def *t = bit ? nir_ishl_imm(b, q, bit) : q; + qy = qy ? nir_iadd(b, qy, t) : t; + } + if (!qy) + qy = nir_imm_int(b, 0); + if (m != k) + qy = nir_ineg(b, qy); + } else { + qy = nir_imul(b, q, y); + } + nir_def *r = nir_isub(b, x, qy); + if (alu->op == nir_op_imod) { + /* GLSL/SPIR-V SMod: the result takes the divisor's sign */ + nir_def *fix = nir_iand(b, nir_ine_imm(b, r, 0), nir_ine(b, nir_ilt_imm(b, r, 0), nir_ilt_imm(b, y, 0))); + r = nir_bcsel(b, fix, nir_iadd(b, r, y), r); + } + nir_def_replace(&alu->def, r); + return true; +} + /* iabs as imax(x, -x): lower_iabs's rule is in nir_opt_algebraic_late, * which this backend does not run. */ static bool lower_iabs_instr(nir_builder *b, nir_alu_instr *a, void *data) @@ -6514,6 +6898,64 @@ static bool layer0_as_2d(nir_builder *b, nir_tex_instr *t, void *data) return true; } +/* The A8's texelFetch (g5p_usc.h): a fragment texelFetch of a 2D texture at a constant level drawn as the A8's + * sample. The coordinate becomes a float (the texel's centre in mode 1; the integer itself in mode 2, as Apple's + * read converts it), the op a sample through the binding with a fetch sampler in its own slot (point, + * non-normalized, clamped, level 0: PCO_G5P_UPLOAD_SAMPLER_FETCH), marked G5P_TEX_FETCH. */ +static bool a8_fetch_as_sample(nir_builder *b, nir_tex_instr *t, void *data) +{ + /* a uniform texel buffer's fetch, folded to 2D (g5p_texel_nir.h: the 4096-wide linear state over the buffer, the + * folded coordinate in backend1): the same 2D fetch of that state (st/mesa's PBO upload reads one) */ + const bool buf = t->sampler_dim == GLSL_SAMPLER_DIM_BUF && (t->backend_flags & PVR_G5P_TBO_NIR_FOLDED) && + g5p_usc_g4p_tbo; + if (t->op != nir_texop_txf || (t->sampler_dim != GLSL_SAMPLER_DIM_2D && !buf) || t->is_array || t->is_shadow || + t->def.num_components != 4 || t->def.bit_size != 32) + return false; + const int li = nir_tex_instr_src_index(t, nir_tex_src_lod); + const int ci = nir_tex_instr_src_index(t, buf ? nir_tex_src_backend1 : nir_tex_src_coord); + const int ti = nir_tex_instr_src_index(t, nir_tex_src_texture_deref); + if (ci < 0 || ti < 0 || t->num_srcs != 2 + (li >= 0) + buf) + return false; + if (buf && b->shader->info.stage != MESA_SHADER_FRAGMENT) + return false; + if (li >= 0 && !nir_src_is_const(t->src[li].src)) + return false; + /* Apple's read form (G5P_TEX_LOD_REG): PVR_A8_TEXFETCH_LEVEL=apple, and every vertex-stage fetch (this pass runs + * on vertex programs only with PVR_A8_VTEX_APPLE) */ + const bool apple = b->shader->info.stage == MESA_SHADER_VERTEX || g5p_usc_g4p_texfetch_level == 3; + /* a constant level above 0 only with PVR_A8_TEXFETCH_LEVEL or in Apple's form */ + const unsigned level = li >= 0 ? nir_src_as_uint(t->src[li].src) : 0; + if (level > 14 || (level && !g5p_usc_g4p_texfetch_level && !apple)) + return false; + b->cursor = nir_before_instr(&t->instr); + nir_def *coord = nir_i2f32(b, t->src[ci].src.ssa); + if (g5p_usc_g4p_texfetch != 2 && !apple) + coord = nir_fadd_imm(b, coord, 0.5); + if (level && g5p_usc_g4p_texfetch_level == 2 && !apple) + coord = nir_fmul_imm(b, coord, (double)(1u << level)); + nir_deref_instr *deref = nir_src_as_deref(t->src[ti].src); + /* a texture without a sampler (a texel buffer's textureBuffer): the sample names the texture alone, its sampler + * slot the fetch sampler (tex_bindings(): the texture's binding) */ + const bool sampler = glsl_type_is_sampler(glsl_without_array(deref->type)); + nir_tex_instr *s = nir_tex_instr_create(b->shader, 2 + sampler); + s->op = nir_texop_tex; + s->sampler_dim = GLSL_SAMPLER_DIM_2D; + s->dest_type = t->dest_type; + s->coord_components = 2; + s->texture_index = t->texture_index; + s->sampler_index = t->texture_index; + s->backend_flags = (t->backend_flags & ~PVR_G5P_TBO_NIR_FOLDED) | G5P_TEX_FETCH | level << G5P_TEX_FETCH_LEVEL_SHIFT | + (apple ? G5P_TEX_LOD_REG : 0); + s->src[0] = nir_tex_src_for_ssa(nir_tex_src_coord, coord); + s->src[1] = nir_tex_src_for_ssa(nir_tex_src_texture_deref, &deref->def); + if (sampler) + s->src[2] = nir_tex_src_for_ssa(nir_tex_src_sampler_deref, &deref->def); + nir_def_init(&s->instr, &s->def, 4, 32); + nir_builder_instr_insert(b, &s->instr); + nir_def_replace(&t->def, &s->def); + return true; +} + /* A 4-sample texture (texture_multisampled_2d, sampler2DMS) is sampled as the 2W x 2H 2D * texture of its samples (pvr_arch_g5p_tex.c): sample s of texel p is texel * 2p + (s >> 1, s & 1); its size is half that texture's; it has 4 samples. */ @@ -6638,6 +7080,10 @@ static bool lower(nir_shader *nir, bool preserve_cf) NIR_PASS(_, nir, nir_shader_tex_pass, ms_as_2d, nir_metadata_control_flow, NULL); NIR_PASS(_, nir, nir_shader_tex_pass, layer0_as_2d, nir_metadata_control_flow, NULL); pvr_g5p_tbo_lower(nir); + /* after the texel-buffer fold, which it also draws as a sample */ + if (g5p_usc_g4p && ((g5p_usc_g4p_texfetch && nir->info.stage == MESA_SHADER_FRAGMENT) || + (g5p_usc_g4p_vtex_apple && nir->info.stage == MESA_SHADER_VERTEX))) + NIR_PASS(_, nir, nir_shader_tex_pass, a8_fetch_as_sample, nir_metadata_control_flow, NULL); NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_push_const, nir_address_format_32bit_offset); NIR_PASS(_, nir, nir_lower_flrp, 32, false); bool flattened = flatten_limit(nir, preserve_cf ? 0 : UINT_MAX); @@ -6656,7 +7102,13 @@ static bool lower(nir_shader *nir, bool preserve_cf) * register divisor and a magic multiply for a constant * (tools/mesa-a9-hypr-re f_udiv_reg, f_udiv_const), with forms we have * not decoded; these use ops we encode. */ - NIR_PASS(_, nir, nir_opt_idiv_const, 32); + /* The A8 divides by a constant as by a register (the float reciprocal and two integer corrections, + * nir_lower_idiv) and builds remainders from the quotient (a8_mod_from_div()): the configuration that reads + * right on the iPod touch 6. */ + if (!g5p_usc_g4p) + NIR_PASS(_, nir, nir_opt_idiv_const, 32); + else + NIR_PASS(_, nir, nir_shader_alu_pass, a8_mod_from_div, nir_metadata_control_flow, NULL); NIR_PASS(_, nir, nir_lower_idiv, &(nir_lower_idiv_options){.allow_fp16 = false}); /* PCO already lowers signed MSB and LSB to unsigned MSB. Finish that * graph with integer fill/count instead of an unqualified native CLZ. */ @@ -8431,7 +8883,7 @@ static bool compile_general_link(nir_shader *nir, bool half_output, const uint8_ if (c->key->mrt_blend_state) refuse(c, "raw32 MRT blend or masks are not qualified"); if (c->key->color_output != PCO_G5P_COLOR_UINT32 && - c->key->color_output != PCO_G5P_COLOR_SINT32) + c->key->color_output != PCO_G5P_COLOR_SINT32 && c->key->color_output != PCO_G5P_COLOR_F32) refuse(c, "unknown colour output format"); if (c->key->packed_scalar8 && c->key->packed_uint_bits != 8) refuse(c, "scalar R8 output requires eight-bit channels"); diff --git a/src/imagination/g5p/pco_g5p_general.h b/src/imagination/g5p/pco_g5p_general.h index 867c30dfc81..c48254ec65e 100644 --- a/src/imagination/g5p/pco_g5p_general.h +++ b/src/imagination/g5p/pco_g5p_general.h @@ -86,6 +86,9 @@ enum pco_g5p_color_output { * bytes B,G,R,A), stored through a physical R32UI PBE */ PCO_G5P_COLOR_PACKED_BGRA8, PCO_G5P_COLOR_PACKED_RGBA8, + /* The A8's R32F target: the red channel's 32-bit float bits as the only output word (emitpix 111c, + * AGXMetalA8's r32f fragment program, a8-passcap) */ + PCO_G5P_COLOR_F32, }; struct pco_g5p_rt_blend { @@ -120,6 +123,12 @@ struct pco_g5p_fs_key { bool packed_sint; /* saturate signed channels, then preserve low bits */ bool packed_scalar8; /* R8: one scalar source; never pack unused G/B/A */ uint8_t packed_uint_bits; /* 8/10/16: packed output, scalar8 is one word; full mask only */ + /* The A8, one R8 (1) or RG8 (2) UNORM target: the f16 output is one word, halves 0..1, R8's half 1 zero + * (AGXMetalA8's 111c fragment programs, a8-passcap) */ + uint8_t a8_narrow_halves; + /* The A8, one RGBA8UI target: the packed word (R | G << 8 | B << 16 | A << 24) is the only output word, + * emitpix 111c (AGXMetalA8's rgba8u fragment program, a8-passcap); the A9 pads it to two */ + bool a8_packed_word; /* Apple's masked form (enabled): no blending, write mask RGB, one 4-byte UNORM8 target, one * sample. AGXMetalA9 runs it OPAQUE (FAST_PUNCH_THROUGH with a discard) with ISPCTL upass 7, the program * keeping the destination alpha from a tile read issued before its kill. 1: the program may discard; @@ -170,6 +179,16 @@ enum pco_g5p_upload_kind { /* PCO_G5P_UPLOAD_SSBO_ADDR at this offset: not the buffer's address but its * bound size in bytes (one dword; get_ssbo_size, WGSL arrayLength) */ #define PCO_G5P_SSBO_SIZE_OFFSET 0xfffcu +/* PCO_G5P_UPLOAD_SAMPLER offset flag (the A8's texelFetch): the binding's sampler made a point, non-normalized, + * clamped sampler at level 0 for a texelFetch drawn as a sample (the offset's other bits: the sampler's place) */ +#define PCO_G5P_UPLOAD_SAMPLER_FETCH 0x4000u +/* with it, the level the fetch reads in offset bits 8-11 (the sampler's LOD range clamped to it) */ +#define PCO_G5P_UPLOAD_SAMPLER_FETCH_LEVEL_SHIFT 8 +/* with it (PVR_A8_TEXFETCH_LEVEL=apple, PVR_A8_VTEX_APPLE): not the binding's sampler but Apple's constant read() + * sampler (AGXCompilerCore's constant sampler 0x807bff0000002049, encoded by AGXMetalA8 into the program's constant + * data: clamp to edge, point, nearest mip, normalized, LOD 0..max; vulkan/pvr_g5p_a8_fetch.h); the sample takes its + * level in a register */ +#define PCO_G5P_UPLOAD_SAMPLER_READ 0x8000u #define PCO_G5P_UPLOAD_IMAGE_AS_2D 1u #define PCO_G5P_UPLOAD_IMAGE_ARRAY 2u /* ONE_LAYER: an array sample at any layer of a one-layer view, drawn as a 2D diff --git a/src/imagination/g9p/g9p_compile.c b/src/imagination/g9p/g9p_compile.c index 2d603df2a60..b6c19884cf5 100644 --- a/src/imagination/g9p/g9p_compile.c +++ b/src/imagination/g9p/g9p_compile.c @@ -1654,7 +1654,11 @@ emit_alu_op(struct g9p_ctx *ctx, nir_alu_instr *alu, const int *r, int dst) if (a < 0) return false; - g9p_shift(ctx->c, dst, a, r[1], false); + /* the A8: a constant amount in Apple's literal form (g9p_shift_imm()) */ + if (g5p_usc_g4p && !ctx->c->fragment && nir_src_is_const(alu->src[1].src)) + g9p_shift_imm(ctx->c, dst, a, nir_alu_src_as_uint(alu->src[1]) & 31, false); + else + g9p_shift(ctx->c, dst, a, r[1], false); return true; } case nir_op_ushr: { @@ -1662,7 +1666,11 @@ emit_alu_op(struct g9p_ctx *ctx, nir_alu_instr *alu, const int *r, int dst) if (a < 0) return false; - g9p_shift(ctx->c, dst, a, r[1], true); + /* the A8: a constant amount in Apple's literal form (g9p_shift_imm()) */ + if (g5p_usc_g4p && !ctx->c->fragment && nir_src_is_const(alu->src[1].src)) + g9p_shift_imm(ctx->c, dst, a, nir_alu_src_as_uint(alu->src[1]) & 31, true); + else + g9p_shift(ctx->c, dst, a, r[1], true); return true; } case nir_op_ishr: @@ -4184,7 +4192,7 @@ generate(struct g9p_ctx *ctx, nir_function_impl *impl, unsigned st) memcpy(body, ctx->c->w, body_n * sizeof(uint16_t)); ctx->c->n = 0; if (ctx->stage == MESA_SHADER_COMPUTE) - g9p_prologue(ctx->c, 4 * ctx->out->num_buffers); + g9p_prologue(ctx->c, 4 * ctx->out->num_buffers, ctx->out->uses_workgroup_id || ctx->out->shared_size); if (stack) { /* * Fragment code: the stack at level 0, all lanes on, as Apple's first level starts diff --git a/src/imagination/g9p/g9p_isa.h b/src/imagination/g9p/g9p_isa.h index 9eef9e1535e..9c39840acc1 100644 --- a/src/imagination/g9p/g9p_isa.h +++ b/src/imagination/g9p/g9p_isa.h @@ -140,10 +140,17 @@ g9p_code_word(struct g9p_code *c, unsigned at) * half @count_half, i.e. dword @count_half / 2, both halves = threads per * group) with r0 and disables the lanes past it. */ +extern bool g5p_usc_g4p; + +/* + * @common: the kernel's record has a common store (threadgroup words: it reads the workgroup ID or threadgroup + * memory). The A8 clears bit 15 of the second word then: AGXMetalA8's 25 kernels have 8189 (+8 per buffer) + * under common size 0 (k_store_one, k_atomic) and 0189.. under common size 1 or more (the other 23) + */ static inline void -g9p_prologue(struct g9p_code *c, unsigned count_half) +g9p_prologue(struct g9p_code *c, unsigned count_half, bool common) { - G9P_EMIT(c, 0x1181, 0x8181 | 2 * count_half, 0x4100, 0xacb7); + G9P_EMIT(c, 0x1181, ((g5p_usc_g4p && common) ? 0x0181 : 0x8181) | 2 * count_half, 0x4100, 0xacb7); } /* @@ -318,6 +325,20 @@ g9p_add_imm(struct g9p_code *c, unsigned dst, unsigned src, unsigned imm) static inline void g9p_tg_load(struct g9p_code *c, unsigned dst, unsigned word) { + if (g5p_usc_g4p && !c->fragment) { + /* + * The A8's load of threadgroup word 0, where the kernel task's PDS writes the workgroup ID: AGXMetalA8's + * every ID-reading kernel starts with it (k_store_const/tid 1516 0004 into r2, k_threadgroup 1516 0002 + * into r1; then 004d). Only offset 0 has a captured form; a read of y or z is refused by name. + */ + if (word) { + if (!c->error) + c->error = "A8: a workgroup-ID y/z read (only the threadgroup word-0 load is captured)"; + return; + } + G9P_EMIT(c, 0x1516, 2 * dst); + return; + } G9P_EMIT(c, 0x2314, 0x0100 | 2 * dst, 0x8100 | (word & 0xff), 0x0200 | (word >> 8)); } @@ -402,6 +423,34 @@ static inline void g9p_addr(struct g9p_code *c, unsigned dst, unsigned base_dword, unsigned idx, unsigned shift, bool kill) { + if (g5p_usc_g4p && !c->fragment && !shift) { + /* + * The A8 has no 64-bit add of a uniform pair: AGXMetalA8 (k_store_tid, iPod touch 6 capture, USC heap 0x1ec0; + * also 0x1700) forms base + offset from 32-bit adds: + * 9911 0203 8585 4100 00a1 r1 = s1 + r1 (the offset's high word; ours is 32-bit: s1 alone) + * 9d11 0403 0101 4100 00a5 r2 = carry(s0 + r0) the carry is the result's HIGH half, r2.hi + * 9931 0203 8b84 08a1 r1 = r1 + r2.hi register A and B (header 0x31), B = half 2t + 1 + * 9911 0003 8181 4100 00a1 r0 = s0 + r0 + * Our offset is a 32-bit byte offset, so the high word starts as s1 alone, moved with Apple's common-store + * move (98 01 DD 03 85 01 00 41 54 01 38 00, the base-only address of the same capture). Order: the carry + * into r[dst + 1], s1 into r[dst], r[dst + 1] = r[dst] + r[dst + 1].hi, then the low add, which reads the + * offset last. Adding the carry's low half (half 2t, the previous form) put s0[31:16] into the high word: on + * the device, stores landed at 0x11_00100000 for 0x1_00100000. The A9 form below + * (1189 .. 01a1 0600) faults on the A8. + */ + const unsigned u = base_dword, hi = dst + 1; + const uint16_t u_ext = 0x4100 | ((u >> 5) & 7) << 1, u1_ext = 0x4100 | (((u + 1) >> 5) & 7) << 1; + if (idx == dst || idx == hi) { + c->error = "A8 address build: the offset register is in the destination pair"; + return; + } + G9P_EMIT(c, 0x119d, g9p_dst(c, hi), 0x0101 | (u & 31) << 2 | idx << 10, u_ext, 0x00a5); + G9P_EMIT(c, 0x0198, g9p_dst(c, dst), 0x0181 | ((u + 1) & 31) << 2, u1_ext, 0x0154, 0x0038); + G9P_EMIT(c, 0x3199, g9p_dst(c, hi), (0x81 | (2 * hi + 1) << 1) << 8 | 0x80 | (2 * dst) << 1, 0x08a1); + G9P_EMIT(c, 0x1199, g9p_dst(c, dst), 0x0181 | (u & 31) << 2 | idx << 10 | (kill ? 0x8000 : 0), u_ext, + G9P_IADD); + return; + } G9P_EMIT(c, g9p_w0(c, 0x1189), g9p_dst(c, dst), (kill ? 0x8000 : 0) | (1 | idx << 2) << 8 | 0x81 | 4 * (base_dword & 31), 0x4100 | (base_dword >> 5) << 1, 0x01a1, 0x0600 | shift << 13); @@ -412,10 +461,25 @@ g9p_addr(struct g9p_code *c, unsigned dst, unsigned base_dword, unsigned idx, * bits 8-9 are the load's bank bits: fragment code clears them (Apple's * fragment loads are 2012/3812 where compute has 2312, probes/fsload.metal) */ +/* the A8 (set by the device for the A8's G5P path, g5p_usc.c) */ +extern bool g5p_usc_g4p; + static inline void g9p_mem_at(struct g9p_code *c, bool store, unsigned size, unsigned reg, unsigned pair) { + if (g5p_usc_g4p && !c->fragment) { + /* + * The A8's compute load/store (AGXMetalA8's output, iPod touch 6 capture, every load and store of the + * 25 kernels): W0 size << 11 | 0x0510; W1 0x0500 | data register << 1; W2 the address pair's first + * register << 1 | 0x0100, with 0x8000 for a store; W3 0x0020, 0x4000 for a store. k_store_one's store + * of r2 to [r0:r1] is 2510 0504 8100 4020. The A9 form (2310, store bit 7 in W1, 8800) stores nothing + * on the A8 (Mesa's k_store_lid completes inside Apple's job without a store). + */ + G9P_EMIT(c, size << 11 | 0x0510, 0x0500 | reg << 1, (store ? 0x8100 : 0x0100) | pair << 1, + store ? 0x4020 : 0x0020); + return; + } G9P_EMIT(c, size << 11 | (c->fragment ? 0x0010 : 0x0310), (1 | pair << 2) << 8 | (store ? 0x80 : 0) | reg << 1, 0x8800); } @@ -510,6 +574,18 @@ g9p_convert(struct g9p_code *c, unsigned dst, unsigned src, unsigned op) (op & 4) ? 0x0468 : 0x0460); } +/* + * dst = a << n (left) or a >> n (logical), n a literal: the A8's shifts (AGXMetalA8 k_store_const/tid, + * k_threadgroup, k_iadd: 119a 8300 8101 0120 0105 0120 14f0 is r0 = r0 << 2, 519e 0302 0101 0120 013d 0120 + * 14f4 is r1 = r0 >> 30). Apple never shifts by a register on the A8; its left shift sets W1 bit 15. + */ +static inline void +g9p_shift_imm(struct g9p_code *c, unsigned dst, unsigned a, unsigned n, bool right) +{ + G9P_EMIT(c, right ? 0x519e : 0x119a, (right ? 0 : 0x8000) | g9p_dst(c, dst), 0x0101 | a << 10, 0x0120, + 0x0101 | (n & 31) << 1, 0x0120, right ? 0x14f4 : 0x14f0); +} + /* dst = a << b (left) or a >> b (logical), b a register */ static inline void g9p_shift(struct g9p_code *c, unsigned dst, unsigned a, unsigned b, bool right) diff --git a/src/imagination/vulkan/pvr_arch_cmd_buffer.c b/src/imagination/vulkan/pvr_arch_cmd_buffer.c index 221e1a0848d..494b6a1bafa 100644 --- a/src/imagination/vulkan/pvr_arch_cmd_buffer.c +++ b/src/imagination/vulkan/pvr_arch_cmd_buffer.c @@ -119,7 +119,9 @@ static VkResult g5p_upload_alloc(struct pvr_cmd_buffer *cmd_buffer, unsigned idx offset = cmd_buffer->g5p_upload[idx].block ? ALIGN_POT(cmd_buffer->g5p_upload[idx].used, align) : 0; if (!cmd_buffer->g5p_upload[idx].block || offset + aligned_size > cmd_buffer->g5p_upload[idx].block->size) { - struct pvr_suballocator *allocator = idx ? &device->suballoc_pds : &device->suballoc_general; + struct pvr_suballocator *allocator = + idx ? pvr_cmd_suballocator(device, &device->suballoc_pds) + : pvr_cmd_suballocator(device, &device->suballoc_general); struct pvr_suballoc_bo *block; VkResult result = pvr_bo_suballoc(allocator, G5P_UPLOAD_BLOCK_SIZE, line, false, &block); if (result != VK_SUCCESS) @@ -3076,6 +3078,7 @@ pvr_arch_cmd_buffer_alloc_mem(struct pvr_cmd_buffer *cmd_buffer, else UNREACHABLE("Unknown heap type"); + allocator = pvr_cmd_suballocator(cmd_buffer->device, allocator); result = pvr_bo_suballoc(allocator, size, cache_line_size, false, &suballoc_bo); if (result != VK_SUCCESS) diff --git a/src/imagination/vulkan/pvr_arch_device.c b/src/imagination/vulkan/pvr_arch_device.c index 9f8a933db92..a6597ba27cb 100644 --- a/src/imagination/vulkan/pvr_arch_device.c +++ b/src/imagination/vulkan/pvr_arch_device.c @@ -38,6 +38,7 @@ #include "pvr_utrace.h" #include "pvr_errlog.h" #include "util/os_misc.h" +#include "util/u_debug.h" #define PVR_GLOBAL_FREE_LIST_INITIAL_SIZE (2U * 1024U * 1024U) #define PVR_GLOBAL_FREE_LIST_MAX_SIZE (256U * 1024U * 1024U) @@ -796,7 +797,7 @@ VkResult PVR_PER_ARCH(create_device)(struct pvr_physical_device *pdevice, assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO); result = pvr_winsys_create(pdevice->render_path, - pdevice->display_path, false, + pdevice->display_path, false, false, pAllocator ? pAllocator : &instance->vk.alloc, &ws); if (result != VK_SUCCESS) @@ -886,6 +887,20 @@ VkResult PVR_PER_ARCH(create_device)(struct pvr_physical_device *pdevice, device, PVR_SUBALLOCATOR_VIS_TEST_SIZE); + device->cmd_suballoc = debug_get_bool_option("PVR_CMD_SUBALLOC", true); + pvr_bo_suballocator_init(&device->suballoc_cmd_general, + device->heaps.general_heap, + device, + device->suballoc_general.default_size); + pvr_bo_suballocator_init(&device->suballoc_cmd_pds, + device->heaps.pds_heap, + device, + device->suballoc_pds.default_size); + pvr_bo_suballocator_init(&device->suballoc_cmd_usc, + device->heaps.usc_heap, + device, + device->suballoc_usc.default_size); + if (p_atomic_inc_return(&instance->active_device_count) > PVR_SECONDARY_DEVICE_THRESHOLD) { initial_free_list_size = PVR_SECONDARY_DEVICE_FREE_LIST_INITAL_SIZE; @@ -1064,6 +1079,9 @@ VkResult PVR_PER_ARCH(create_device)(struct pvr_physical_device *pdevice, pvr_bo_log_summary(device, "destroy"); pvr_bo_log_device_finish(device); } + pvr_bo_suballocator_fini(&device->suballoc_cmd_usc); + pvr_bo_suballocator_fini(&device->suballoc_cmd_pds); + pvr_bo_suballocator_fini(&device->suballoc_cmd_general); pvr_bo_suballocator_fini(&device->suballoc_vis_test); pvr_bo_suballocator_fini(&device->suballoc_usc); pvr_bo_suballocator_fini(&device->suballoc_transfer); @@ -1140,6 +1158,9 @@ void PVR_PER_ARCH(destroy_device)(struct pvr_device *device, pvr_bo_log_summary(device, "destroy"); pvr_bo_log_device_finish(device); } + pvr_bo_suballocator_fini(&device->suballoc_cmd_usc); + pvr_bo_suballocator_fini(&device->suballoc_cmd_pds); + pvr_bo_suballocator_fini(&device->suballoc_cmd_general); pvr_bo_suballocator_fini(&device->suballoc_vis_test); pvr_bo_suballocator_fini(&device->suballoc_usc); pvr_bo_suballocator_fini(&device->suballoc_transfer); diff --git a/src/imagination/vulkan/pvr_arch_g5p.c b/src/imagination/vulkan/pvr_arch_g5p.c index 3678eb55f38..654f5f2b2ec 100644 --- a/src/imagination/vulkan/pvr_arch_g5p.c +++ b/src/imagination/vulkan/pvr_arch_g5p.c @@ -22,6 +22,7 @@ #include "vk_log.h" #include "util/ralloc.h" +#include "util/u_debug.h" #include "util/u_dynarray.h" #include "util/u_math.h" @@ -36,10 +37,12 @@ #include "pdsc_internal.h" #include "pdsc_isa_pack.h" #include "pdsc_ops.h" +#include "pvr_bo_log.h" #include "pvr_bo.h" #include "pvr_cmd_buffer.h" #include "pvr_csb.h" #include "pvr_device.h" +#include "pvr_physical_device.h" #include "pvr_free_list.h" #include "pvr_g5p.h" #include "pvr_g5p_dgl.h" @@ -102,10 +105,10 @@ g5p_pds_pack(pdsc_program *p, uint32_t *dest) /* DOUTU from data dwords 0-1, then HALT: the code of a kick */ static unsigned -g5p_pds_kick_code(uint32_t *code) +g5p_pds_kick_code(uint32_t *code, enum pdsc_feature features) { pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-kick"); + pdsc_program *p = pdsc_program_create(NULL, features, "g5p-kick"); pdsc_dout(p, pdsc_ref_const64(0), pdsc_ref_null(), pdsc_ref_null(), .dstdout = PDSC_DSTDOUT_U, .end = true); pdsc_halt(p); @@ -117,10 +120,10 @@ g5p_pds_kick_code(uint32_t *code) /* DOUTD from data dwords 0-1 with the control in dword 2, then (eot) DOUTU * from dwords 4-5; HALT (pdsc's const indices count dwords) */ static unsigned -g5p_pds_dma_code(uint32_t *code, bool eot) +g5p_pds_dma_code(uint32_t *code, bool eot, enum pdsc_feature features) { pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, eot ? "g5p-eot" : "g5p-clear-dma"); + pdsc_program *p = pdsc_program_create(NULL, features, eot ? "g5p-eot" : "g5p-clear-dma"); pdsc_dout(p, pdsc_ref_const64(0), pdsc_ref_const32(2), pdsc_ref_null(), .dstdout = PDSC_DSTDOUT_D, .end = !eot); if (eot) /* pdsc's const64 index counts dwords: dwords 4-5 */ @@ -135,10 +138,10 @@ g5p_pds_dma_code(uint32_t *code, bool eot) /* n DOUTDs, source i from data dwords 2i..2i+1 and its control in dword * 2n + 2i, the last one ending; HALT (the A9 ICD's uniform loader) */ static unsigned -g5p_pds_loader_code(uint32_t *code, unsigned n) +g5p_pds_loader_code(uint32_t *code, unsigned n, enum pdsc_feature features) { pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-loader"); + pdsc_program *p = pdsc_program_create(NULL, features, "g5p-loader"); for (unsigned i = 0; i < n; i++) pdsc_dout(p, pdsc_ref_const64(2 * i), pdsc_ref_const32(2 * n + 2 * i), pdsc_ref_null(), .dstdout = PDSC_DSTDOUT_D, .end = i == n - 1); @@ -151,35 +154,35 @@ g5p_pds_loader_code(uint32_t *code, unsigned n) static uint64_t g5p_doutu_src0(uint32_t exe_off, unsigned temps) { - pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-doutu"); + /* packing a field reads only the feature set: no program allocation per draw */ + static const pdsc_program pack = { .name = "g5p-doutu", .features = PDSC_FEATURE_DOUT_EXT }; + const pdsc_program *p = &pack; struct pdsc_doutu_src0 s = { .exe_off = exe_off, .temps = temps }; const uint64_t v = pdsc_doutu_src0_pack(p, &s); - ralloc_free(p); return v; } static uint32_t g5p_doutd_control_at(unsigned common, unsigned dwords, bool last) { - pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-doutd"); + /* packing a field reads only the feature set: no program allocation per draw */ + static const pdsc_program pack = { .name = "g5p-doutd", .features = PDSC_FEATURE_DOUT_EXT }; + const pdsc_program *p = &pack; struct pdsc_doutd_src1 s = { .last_issue = last, .store_dest = PDSC_STORE_DEST_COMMON, .ao = common, .bsize = dwords }; const uint32_t v = pdsc_doutd_src1_pack(p, &s); - ralloc_free(p); return v; } static uint32_t g5p_doutd_control(unsigned dwords) { - pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-doutd"); + /* packing a field reads only the feature set: no program allocation per draw */ + static const pdsc_program pack = { .name = "g5p-doutd", .features = PDSC_FEATURE_DOUT_EXT }; + const pdsc_program *p = &pack; struct pdsc_doutd_src1 s = { .last_issue = true, .store_dest = PDSC_STORE_DEST_COMMON, .ao = 0, .bsize = dwords }; const uint32_t v = pdsc_doutd_src1_pack(p, &s); - ralloc_free(p); return v; } @@ -187,7 +190,7 @@ g5p_doutd_control(unsigned dwords) static uint32_t g5p_usc_off(const struct pvr_device *device, const struct pvr_suballoc_bo *bo) { - return (uint32_t)(bo->dev_addr.addr - device->heaps.usc_heap->base_addr.addr); + return (uint32_t)(bo->dev_addr.addr - PVR_G5P_USC_EXEC_BASE(device)); } /* A kick of a USC program, its code at data + 0x20 as the registers expect; @@ -199,7 +202,7 @@ g5p_upload_kick_temps(struct pvr_device *device, const struct pvr_suballoc_bo *u const uint64_t doutu = pvr_g5p_msaa_doutu(g5p_doutu_src0(g5p_usc_off(device, usc), temps), samples); const uint32_t data[4] = { (uint32_t)doutu, (uint32_t)(doutu >> 32), 0, 0 }; uint32_t code[8]; - const unsigned n = g5p_pds_kick_code(code); + const unsigned n = g5p_pds_kick_code(code, PVR_G5P_PDS_FEATURES(&device->pdevice->dev_info)); VkResult result = pvr_gpu_upload_pds(device, data, ARRAY_SIZE(data), 16, code, n, G5P_KICK_CODE, 64, out); if (result == VK_SUCCESS && out->code_offset != out->data_offset + G5P_KICK_CODE) @@ -224,6 +227,17 @@ g5p_free_programs(struct pvr_device *device, struct pvr_g5p_programs *p) pvr_bo_suballoc_free(p->clear_usc[i]); pvr_bo_suballoc_free(p->clear_kick[i].pvr_bo); } + for (unsigned i = 0; i < 2; i++) { + pvr_bo_suballoc_free(p->clear_usc_narrow[i]); + pvr_bo_suballoc_free(p->clear_kick_narrow[i].pvr_bo); + } + pvr_bo_suballoc_free(p->clear_usc_rgba8ui); + pvr_bo_suballoc_free(p->clear_kick_rgba8ui.pvr_bo); + pvr_bo_suballoc_free(p->clear_usc_r32f); + pvr_bo_suballoc_free(p->clear_kick_r32f.pvr_bo); + pvr_bo_suballoc_free(p->a8_load_usc); + pvr_bo_suballoc_free(p->a8_load_kick.pvr_bo); + pvr_bo_suballoc_free(p->a8_load_dma.pvr_bo); for (unsigned i = 0; i < ARRAY_SIZE(p->raw_load); i++) { if (p->raw_load[i]) { g5p_free_programs(device, p->raw_load[i]); @@ -317,7 +331,7 @@ g5p_upload_load(struct pvr_device *device, struct pvr_g5p_programs *p, unsigned if (result != VK_SUCCESS) return result; uint32_t code[2 * ARRAY_SIZE(p->load_upload) + 2]; - const unsigned n = g5p_pds_loader_code(code, p->load_uploads); + const unsigned n = g5p_pds_loader_code(code, p->load_uploads, PVR_G5P_PDS_FEATURES(&device->pdevice->dev_info)); return pvr_gpu_upload_pds(device, NULL, 0, 0, code, n, 16, 64, &p->load_dma); } @@ -349,6 +363,9 @@ g5p_raw_programs_unlocked(struct pvr_device *device, unsigned raw, unsigned word { struct pvr_g5p_programs *p = &device->g5p->programs; VkResult result; + if (PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p)) + return vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, + "G5P: an integer colour pass on the A8 (its end of tile is captured for float targets only)"); const unsigned variant = g5p_raw_variant(words); if (!p->raw_clear_usc[variant]) { uint8_t code[6 * G5P_USC_MAX]; @@ -413,8 +430,13 @@ g5p_upload_programs(struct pvr_device *device, struct pvr_g5p_programs *p) return vk_errorf(device, VK_ERROR_INITIALIZATION_FAILED, "G5P: the HALT's code is at data + %#x, not + 0x10", p->halt.code_offset - p->halt.data_offset); + /* The A8 builds the forms captured on it: one colour target, one sample, no LOAD (its texture sample + * is not decoded); the others stay unbuilt and their passes are refused by name */ + const bool g4p = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p); for (unsigned i = 0; i < PVR_G5P_MRT_VARIANTS; i++) { const unsigned n = g5p_mrt_variant_targets[i]; + if (g4p && ((n != 1 && n != 2) || i >= 7)) + continue; uint8_t code[MAX2(G5P_MRT_CLEAR_USC_MAX, G5P_MRT_EOT_USC_MAX)]; /* variants 7..10: the RGBA8 byte order of the compact clear */ unsigned size = i >= 7 ? g5p_mrt_clear_compact(code, n, false) : g5p_mrt_clear_usc(code, n); @@ -430,9 +452,64 @@ g5p_upload_programs(struct pvr_device *device, struct pvr_g5p_programs *p) if (result != VK_SUCCESS) return result; } + /* the A8's R8 and RG8 background programs */ + for (unsigned i = 0; g4p && i < 2; i++) { + uint8_t code[G5P_MRT_CLEAR_USC_MAX]; + const unsigned size = g5p_mrt_clear_usc_g4p_narrow(code, i + 1); + if (!size) + return vk_errorf(device, VK_ERROR_UNKNOWN, "G5P: A8 narrow clear: %s", g5p_usc_error); + result = pvr_gpu_upload_usc(device, code, size, 64, &p->clear_usc_narrow[i]); + if (result != VK_SUCCESS) + return result; + result = g5p_upload_kick(device, p->clear_usc_narrow[i], 1, &p->clear_kick_narrow[i]); + if (result != VK_SUCCESS) + return result; + } + /* the A8's RGBA8UI background program */ + if (g4p) { + uint8_t code[G5P_MRT_CLEAR_USC_MAX]; + const unsigned size = g5p_mrt_clear_usc_g4p_rgba8ui(code); + if (!size) + return vk_errorf(device, VK_ERROR_UNKNOWN, "G5P: A8 RGBA8UI clear: %s", g5p_usc_error); + result = pvr_gpu_upload_usc(device, code, size, 64, &p->clear_usc_rgba8ui); + if (result != VK_SUCCESS) + return result; + result = g5p_upload_kick(device, p->clear_usc_rgba8ui, 1, &p->clear_kick_rgba8ui); + if (result != VK_SUCCESS) + return result; + } + /* the A8's colour LOAD background: Apple's program, its kick, and the loader of its three uploads */ + if (g4p) { + uint8_t code[G5P_A8_LOAD_USC_BYTES]; + const unsigned size = g5p_mrt_load_usc_g4p(code); + result = pvr_gpu_upload_usc(device, code, size, 64, &p->a8_load_usc); + if (result != VK_SUCCESS) + return result; + result = g5p_upload_kick(device, p->a8_load_usc, 1, &p->a8_load_kick); + if (result != VK_SUCCESS) + return result; + uint32_t pds[8]; + const unsigned n = g5p_pds_loader_code(pds, 3, PVR_G5P_PDS_FEATURES(&device->pdevice->dev_info)); + result = pvr_gpu_upload_pds(device, NULL, 0, 0, pds, n, 16, 64, &p->a8_load_dma); + if (result != VK_SUCCESS) + return result; + } + /* the A8's R32F background program */ + if (g4p) { + uint8_t code[G5P_MRT_CLEAR_USC_MAX]; + const unsigned size = g5p_mrt_clear_usc_g4p_r32f(code); + if (!size) + return vk_errorf(device, VK_ERROR_UNKNOWN, "G5P: A8 R32F clear: %s", g5p_usc_error); + result = pvr_gpu_upload_usc(device, code, size, 64, &p->clear_usc_r32f); + if (result != VK_SUCCESS) + return result; + result = g5p_upload_kick(device, p->clear_usc_r32f, 1, &p->clear_kick_r32f); + if (result != VK_SUCCESS) + return result; + } /* 4x (one target): the multisample store's end of tile, and the clear's * kick at sample rate */ - { + if (!g4p) { uint8_t code[G5P_MRT_EOT_USC_MAX]; const unsigned size = pvr_g5p_msaa_eot_usc(code); result = pvr_gpu_upload_usc(device, code, size, 64, &p->eot_usc_ms); @@ -444,11 +521,11 @@ g5p_upload_programs(struct pvr_device *device, struct pvr_g5p_programs *p) } uint32_t pds[8]; - unsigned n = g5p_pds_dma_code(pds, false); + unsigned n = g5p_pds_dma_code(pds, false, PVR_G5P_PDS_FEATURES(&device->pdevice->dev_info)); result = pvr_gpu_upload_pds(device, NULL, 0, 0, pds, n, 16, 64, &p->clear_dma); if (result != VK_SUCCESS) return result; - n = g5p_pds_dma_code(pds, true); + n = g5p_pds_dma_code(pds, true, PVR_G5P_PDS_FEATURES(&device->pdevice->dev_info)); result = pvr_gpu_upload_pds(device, NULL, 0, 0, pds, n, 16, 64, &p->eot); if (result != VK_SUCCESS) return result; @@ -476,7 +553,7 @@ g5p_upload_programs(struct pvr_device *device, struct pvr_g5p_programs *p) result = g5p_upload_zeroed(device, G5P_SCISSOR_POOL_BYTES, &p->scissor_pool); if (result == VK_SUCCESS) result = g5p_upload_zeroed(device, G5P_DBIAS_POOL_BYTES, &p->dbias_pool); - if (result == VK_SUCCESS) + if (result == VK_SUCCESS && !g4p) result = g5p_upload_load(device, p, 0); return result; } @@ -496,9 +573,17 @@ PVR_PER_ARCH(g5p_device_init)(struct pvr_device *device) /* vkCmdClearAttachments' clear quad: its pipelines are made at first use (pvr_arch_g5p_gfx.c) */ simple_mtx_init(&device->g5p->clear_lock, mtx_plain); + /* The parameter manager pool every render asks for (pvr_g5p.h); the + * kernel builds at most 16 MiB */ + /* The A8's kernel sizes its pool itself (powervr.a8_pm_blocks) and does not read this list: it keeps + * the 4 MiB the iPod's renders ran with. */ + const bool a8 = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p); + const int64_t fl_mb = debug_get_num_option("PVR_G5P_FREE_LIST_MB", a8 ? 4 : PVR_G5P_FREE_LIST_SIZE >> 20); + const uint32_t fl_size = (uint32_t)CLAMP(fl_mb, 1, 16) << 20; + VkResult result = g5p_upload_programs(device, &device->g5p->programs); if (result == VK_SUCCESS) - result = pvr_free_list_create(device, PVR_G5P_FREE_LIST_SIZE, PVR_G5P_FREE_LIST_SIZE, 0, 0, + result = pvr_free_list_create(device, fl_size, fl_size, 0, 0, device->global_free_list, &device->g5p->free_list); if (result == VK_SUCCESS) result = PVR_PER_ARCH(g5p_meta_init)(device); @@ -632,6 +717,51 @@ g5p_load_background(struct pvr_cmd_buffer *cmd_buffer, const uint64_t state[][4] return VK_SUCCESS; } +/* + * The A8's colour LOAD background (AGXMetalA8, a8-passcap load): Apple's program (g5p_mrt_load_usc_g4p()) reads, by + * three DOUTDs of 4 + 4 + 6 dwords into common dwords 0, 4 and 8, the target's texture state (its first two qwords), + * Apple's load sampler (00000fff 02022400 0 0: the passcap's words) and the 2x3 matrix from the fragment position to the + * sample coordinate (identity: 1, 0, 0, 1, 0, 0). Data controls 10000004, 10008004, 90010006; uniform size 3 units, + * common-store size 14 dwords, as Apple's record has them. + */ +static VkResult +g5p_load_background_g4p(struct pvr_cmd_buffer *cmd_buffer, const uint64_t state[4], uint64_t bgnd[4], + const struct pvr_g5p_programs *p) +{ + static const uint32_t sampler[4] = { 0x00000fff, 0x02022400, 0, 0 }; + static const float matrix[6] = { 1.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f }; + static const struct { unsigned common, dwords; } up[3] = { { 0, 4 }, { 4, 4 }, { 8, 6 } }; + struct pvr_suballoc_bo *bo[3]; + const void *src[3] = { state, sampler, matrix }; + const size_t bytes[3] = { 16, sizeof(sampler), sizeof(matrix) }; + uint32_t data[12] = { 0 }; + for (unsigned i = 0; i < 3; i++) { + VkResult result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, src[i], bytes[i], &bo[i]); + if (result != VK_SUCCESS) + return result; + data[2 * i] = (uint32_t)bo[i]->dev_addr.addr; + data[2 * i + 1] = (uint32_t)(bo[i]->dev_addr.addr >> 32); + data[6 + 2 * i] = g5p_doutd_control_at(up[i].common, up[i].dwords, i == 2); + } + struct pvr_pds_upload load_data; + VkResult result = pvr_arch_cmd_buffer_upload_pds(cmd_buffer, data, ARRAY_SIZE(data), 16, NULL, 0, 0, 16, + &load_data); + if (result != VK_SUCCESS) + return result; + pvr_csb_pack (&bgnd[0], CR_PDS_BGRND0_BASE, b0) { + b0.shader_addr = PVR_DEV_ADDR(p->a8_load_kick.data_offset); + b0.texunicode_addr = PVR_DEV_ADDR(p->a8_load_dma.code_offset); + } + pvr_csb_pack (&bgnd[2], CR_PDS_BGRND2_BASE, b2) { + b2.uniformdata_addr = PVR_DEV_ADDR(load_data.data_offset); + } + pvr_csb_pack (&bgnd[3], CR_PDS_BGRND3_SIZEINFO, b3) { + b3.pds_uniformsize = DIV_ROUND_UP(ARRAY_SIZE(data), ROGUE_CR_PDS_BGRND3_SIZEINFO_PDS_UNIFORMSIZE_UNIT_SIZE); + b3.usc_sharedsize = DIV_ROUND_UP(14, ROGUE_CR_PDS_BGRND3_SIZEINFO_USC_SHAREDSIZE_UNIT_SIZE); + } + return VK_SUCCESS; +} + VkResult PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd_gfx *sub_cmd, const struct pvr_renderpass_hwsetup_render *hw_render, @@ -708,6 +838,17 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd const bool byte_source = first_format == VK_FORMAT_R8_UINT || first_format == VK_FORMAT_R8_SINT; if (byte_source) raw = first_format == VK_FORMAT_R8_SINT ? 11 : 10; + /* The A8's RGBA8UI is a float-shaped pass (AGXMetalA8, a8-passcap rgba8u): the one-target end of tile with the + * integer conversion in its state word (below), the four clear uints DMA'd as the float colours are and packed + * by the A8 integer background program (g5p_mrt_clear_usc_g4p_rgba8ui()) */ + const bool a8_rgba8ui = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p) && n == 1 && + first_format == VK_FORMAT_R8G8B8A8_UINT; + if (a8_rgba8ui) + raw = 0; + /* and its R32F (a8-passcap r32f): the float pass with the f32 end-of-tile state word, one output word, the clear + * moving the red f32 (g5p_mrt_clear_usc_g4p_r32f()) */ + const bool a8_r32f = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p) && n == 1 && + first_format == VK_FORMAT_R32_SFLOAT; const bool narrow = raw >= 5 && (byte_source || first_format == VK_FORMAT_R16_UINT || first_format == VK_FORMAT_R16_SINT || first_format == VK_FORMAT_R8G8_UINT || first_format == VK_FORMAT_R8G8_SINT); @@ -746,6 +887,8 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd g->width = info->rstate->width; g->height = info->rstate->height; } + uint64_t target0_addr = 0; /* the A8's end of tile takes the target's address (below) */ + bool a8_load_twiddled = false; /* the A8's LOAD samples a twiddled target only (Apple's capture) */ for (unsigned t = 0; t < n && !null_target; t++) { const struct pvr_image_view *iview = info->attachments[hw_render->eot_surfaces[t].attachment_idx]; const struct pvr_image *image = pvr_image_view_get_image(iview); @@ -753,6 +896,8 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd const struct pvr_mip_level *level = &plane->mip_levels[iview->vk.base_mip_level]; const uint64_t addr = image->dev_addr.addr + plane->offset + level->offset + iview->vk.base_array_layer * plane->layer_size; + if (!t) + target0_addr = addr; uint32_t key; bool srgb; /* 4x (admitted above: one target): the samples stored as they are, or the PBE resolve into the @@ -760,6 +905,8 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd const bool ms_store = g->samples == 4 && !hw_render->eot_surfaces[t].need_resolve; const bool twiddled=image->memlayout!=PVR_MEMLAYOUT_LINEAR && !ms_store; + if (!t) + a8_load_twiddled = twiddled; /* A twiddled 2D target (a mip level, an array layer or a cube face, one * layer per render): q0 bit 3 and no row pitch, as the native A9 * captures store (g5p_pbe.h). Device-proven as the L11 probe (36 CTS @@ -776,6 +923,66 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd } if (!g5p_pbe_key(iview->vk.format, &key, &srgb)) return G5P_REFUSE(cmd_buffer, "colour target format %s", vk_Format_to_str(iview->vk.format)); + if (t == 1 && PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p)) { + /* The A8's second target (a8-passcap mrt2: BGRA8 + RGBA8): its own PBE pair, and the EOT format word + * with 0x120 (Apple's 0x122 for RGBA8; its meaning is not decoded, so only this shape is admitted) */ + const VkFormat f = iview->vk.format; + if (f != VK_FORMAT_B8G8R8A8_UNORM && f != VK_FORMAT_R8G8B8A8_UNORM) + return G5P_REFUSE(cmd_buffer, "A8 second colour target format %s (RGBA8/BGRA8 UNORM are captured)", + vk_Format_to_str(f)); + g->a8_eot_format_t1 = ROGUE_PBESTATE_PACKMODE_U8U8U8U8 << 16 | 0x120 | 2; + g->a8_t1_addr = addr; + g5p_pbe_a8_reg_words(g->a8_pbe_t1, iview->vk.extent.width, iview->vk.extent.height, + f == VK_FORMAT_B8G8R8A8_UNORM ? G5P_PBE_A8_SWIZZLE_BGRA : G5P_PBE_A8_SWIZZLE_RGBA, + twiddled, level->pitch / 4); + } + if (!t && PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p)) { + /* The A8's PBE words for target 0 (the kernel's 3D record W+0x118): the formats captured on the A8 */ + /* + * The formats whose A8 encoding the iPod captures prove: + * the end of tile's state word is the PBE packmode << 16 | the conversion (2 normalised, 0 float), the + * fragment program, clear and end-of-tile USC are BGRA8's for every 4-channel single target, and the + * PBE swizzle is BGRA8's or the identity + */ + const VkFormat f = iview->vk.format; + g->a8_narrow_halves = 0; + switch (f) { + case VK_FORMAT_B8G8R8A8_UNORM: + case VK_FORMAT_R8G8B8A8_UNORM: + g->a8_eot_format = ROGUE_PBESTATE_PACKMODE_U8U8U8U8 << 16 | 2; + break; + case VK_FORMAT_R16G16B16A16_SFLOAT: + g->a8_eot_format = ROGUE_PBESTATE_PACKMODE_F16F16F16F16 << 16 | 0; /* Apple: 0x000a0000 */ + break; + case VK_FORMAT_A2B10G10R10_UNORM_PACK32: + g->a8_eot_format = ROGUE_PBESTATE_PACKMODE_A2R10B10G10 << 16 | 2; /* Apple: 0x000e0002 */ + break; + case VK_FORMAT_R8_UNORM: + g->a8_eot_format = ROGUE_PBESTATE_PACKMODE_U8 << 16 | 2; /* Apple: 0x002f0002 */ + g->a8_narrow_halves = 1; + break; + case VK_FORMAT_R8G8_UNORM: + g->a8_eot_format = ROGUE_PBESTATE_PACKMODE_U8U8 << 16 | 2; /* Apple: 0x00230002 */ + g->a8_narrow_halves = 2; + break; + case VK_FORMAT_R8G8B8A8_UINT: + g->a8_eot_format = ROGUE_PBESTATE_PACKMODE_U8U8U8U8 << 16 | 8; /* Apple: 0x00000008, integer */ + break; + case VK_FORMAT_R32_SFLOAT: + g->a8_eot_format = ROGUE_PBESTATE_PACKMODE_F32 << 16 | 0; /* Apple: 0x00360000 */ + break; + default: + return G5P_REFUSE(cmd_buffer, "A8 colour target format %s (captured: BGRA8/RGBA8 UNORM, RGBA16F, RGB10A2)", + vk_Format_to_str(f)); + } + g5p_pbe_a8_reg_words(g->a8_pbe, iview->vk.extent.width, iview->vk.extent.height, + f == VK_FORMAT_B8G8R8A8_UNORM ? G5P_PBE_A8_SWIZZLE_BGRA + : f == VK_FORMAT_R8G8_UNORM ? 04510u /* Apple's RG8 swizzle 0x948 */ + : f == VK_FORMAT_R8_UNORM || f == VK_FORMAT_R32_SFLOAT ? 04550u /* Apple's R8 and R32F + * swizzle 0x968 */ + : G5P_PBE_A8_SWIZZLE_RGBA, + twiddled, level->pitch / 4); + } if ((level->pitch % 16 && !ms_store && !twiddled) || addr % 64) return G5P_REFUSE(cmd_buffer, "a colour target at %#" PRIx64 " with pitch %u (64-byte base, 16-byte rows)", addr, level->pitch); @@ -815,7 +1022,14 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd } } if (!t || compact) { - if(twiddled) { + if (!ms_store && PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p)) { + /* the A8 loads through Rogue's texture state: the view's own (pvr_arch_image.c keeps the Rogue words on + * the A8; Apple's load pass DMAs TEXSTATE_IMAGE_WORD0/1). A linear target (every image shared outside + * the driver: GBM/dmabuf, so the compositor's output buffers) has the stride form of the same two + * words, the ones the A8 already samples linear textures with. */ + load_state[t][0] = iview->image_state[PVR_TEXTURE_STATE_SAMPLE].words[0]; + load_state[t][1] = iview->image_state[PVR_TEXTURE_STATE_SAMPLE].words[1]; + } else if(twiddled) { struct pvr_image_descriptor desc; /* A one-layer array attachment is still a 2D store/restore * surface. Bake its base-layer offset into a 2D LOAD view; @@ -927,7 +1141,7 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd if (lr != VK_SUCCESS) return lr; } - if (load && !load_program->load_usc) + if (load && !load_program->load_usc && !PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p)) return G5P_REFUSE(cmd_buffer, "LOAD of a %scolour attachment%s: the load program: %s", g->samples == 4 ? "4x " : "", partial ? " (a partial render area)" : "", load_program->load_refusal); @@ -961,8 +1175,39 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd struct pvr_suballoc_bo *floats, *block; uint32_t pbe_words[G5P_MRT_PBE_STRIDE * G5P_MRT_MAX]; /* each target's source: output half 4t (8-bit RGBA, two dwords), 8t (raw32, four words) */ - const unsigned pbe_dwords = g5p_mrt_pbe_block_halves(pbe_words, nt, (const uint64_t(*)[3])pbe, - raw ? 2 * raw_words : compact ? 2 : 4); + unsigned pbe_dwords = g5p_mrt_pbe_block_halves(pbe_words, nt, (const uint64_t(*)[3])pbe, + raw ? 2 * raw_words : compact ? 2 : 4); + if (PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p)) { + /* + * The A8's end of tile DMAs four dwords: the target's address and 2, 0 + * (iPod touch 6, iOS 12.4, a8-shadercap: the EOT PDS data's DOUTD source + * for a one-target 64x64 BGRA8 pass; the program is g5p_mrt_eot_usc(), + * Apple's 1e010070). Its format, size and stride are not in these words: + * the A8 3D record's +0x118 block carries them (the kernel + * fills it). Only that pass shape is captured. + */ + if ((nt != 1 && nt != 2) || raw || compact || null_target || g->samples != 1 || (nt == 2 && g->a8_narrow_halves)) + return G5P_REFUSE(cmd_buffer, "A8 end of tile for %u targets, %u samples%s: one or two single-sample " + "colour targets are captured", nt, g->samples, raw ? ", integer" : ""); + pbe_words[0] = (uint32_t)target0_addr; + pbe_words[1] = (uint32_t)(target0_addr >> 32); + pbe_words[2] = g->a8_eot_format; /* packmode << 16 | conversion */ + pbe_words[3] = 0; + pbe_dwords = 4; + if (nt == 2) { + /* the second target's block (Apple's mrt2: {t1, 0x122, 0}), one 8-dword DMA */ + pbe_words[4] = (uint32_t)g->a8_t1_addr; + pbe_words[5] = (uint32_t)(g->a8_t1_addr >> 32); + pbe_words[6] = g->a8_eot_format_t1; + pbe_words[7] = 0; + pbe_dwords = 8; + /* its clear colours in reverse target order (g5p_mrt_clear_usc()) */ + float t0[4]; + memcpy(t0, clear[0], sizeof(t0)); + memcpy(clear[0], clear[1], sizeof(t0)); + memcpy(clear[1], t0, sizeof(t0)); + } + } VkResult result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, clear, 16 * nt, &floats); if (result != VK_SUCCESS) return result; @@ -994,7 +1239,22 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd /* PDS_BGRND0..3: the clear kick, the clear DMA, its data; sizes */ uint64_t bgnd[4] = { 0 }; - if (load) { + if (load && PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p)) { + /* Apple's capture: one single-sample twiddled BGRA8 target (8-bit UNORM targets share the program; the + * texture state carries the format) */ + const VkFormat f = first_format; + /* a linear target loads through its stride texture state (above): Apple's load program samples whatever the + * state describes. Without it, a compositor's damage-only redraw into its (linear, shared) output buffer was + * refused and the frame never drawn. */ + if (nt != 1 || raw || compact || g->samples != 1 || g->a8_narrow_halves || + (f != VK_FORMAT_B8G8R8A8_UNORM && f != VK_FORMAT_R8G8B8A8_UNORM)) + return G5P_REFUSE(cmd_buffer, "A8 LOAD of %s (%u targets, %u samples%s): one single-sample " + "BGRA8/RGBA8 target is captured", vk_Format_to_str(f), nt, g->samples, + a8_load_twiddled ? "" : ", linear"); + result = g5p_load_background_g4p(cmd_buffer, load_state[0], bgnd, p); + if (result != VK_SUCCESS) + return result; + } else if (load) { result = g5p_load_background(cmd_buffer, (const uint64_t(*)[4])load_state, g->width, g->height, bgnd, load_program); if (result != VK_SUCCESS) @@ -1002,6 +1262,9 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd } else { pvr_csb_pack (&bgnd[0], CR_PDS_BGRND0_BASE, b0) { b0.shader_addr = PVR_DEV_ADDR(raw ? p->raw_clear_kick[g5p_raw_variant(raw_total)].data_offset : g->samples == 4 ? p->clear_kick_ms.data_offset + : g->a8_narrow_halves ? p->clear_kick_narrow[g->a8_narrow_halves - 1].data_offset + : a8_rgba8ui ? p->clear_kick_rgba8ui.data_offset + : a8_r32f ? p->clear_kick_r32f.data_offset : p->clear_kick[v].data_offset); b0.texunicode_addr = PVR_DEV_ADDR(p->clear_dma.code_offset); } @@ -1014,10 +1277,30 @@ PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd ROGUE_CR_PDS_BGRND3_SIZEINFO_USC_SHAREDSIZE_UNIT_SIZE); } } + /* Apple's record +0x640 (D6ad) processes empty tiles only for a resolve: + * the captured clear/LOAD/MRT/4x-store passes have 0, resolves have 1. + * Skip them when no attachment is CLEARed: an untouched tile is neither + * loaded nor stored and keeps its memory, which is the LOAD result + * (don't-care: undefined). A CLEAR must still reach empty tiles. + */ + bool keep = g->msaa_resolve || getenv("G5P_ALL_TILES"); + for (unsigned i = 0; i < hw_render->color_init_count; i++) + keep |= hw_render->color_init[i].op == VK_ATTACHMENT_LOAD_OP_CLEAR; + for (unsigned i = 0; i < hw_render->eot_surface_count; i++) + keep |= hw_render->eot_surfaces[i].need_resolve; + if (hw_render->ds_attach_idx != VK_ATTACHMENT_UNUSED) { + keep |= hw_render->depth_init == VK_ATTACHMENT_LOAD_OP_CLEAR || + hw_render->stencil_init == VK_ATTACHMENT_LOAD_OP_CLEAR || + hw_render->ds_attach_resolve_idx != VK_ATTACHMENT_UNUSED || + hw_render->depth_resolve_mode != VK_RESOLVE_MODE_NONE || + hw_render->stencil_resolve_mode != VK_RESOLVE_MODE_NONE; + } + g->skip_empty_tiles = !keep; + memcpy(g->background, bgnd, sizeof(g->background)); g->eot[0] = eot_data.data_offset; g->eot[1] = p->eot.code_offset; - g->eot[2] = g5p_mrt_eot_info(nt); + g->eot[2] = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p) ? g5p_mrt_eot_info_g4p(nt) : g5p_mrt_eot_info(nt); g->enabled = true; return VK_SUCCESS; } @@ -1151,7 +1434,12 @@ PVR_PER_ARCH(g5p_frag_stream)(const struct pvr_device *device, const struct pvr_ /* the depth/stencil attachment's ZLS words and background object * (pvr_arch_g5p_ds.c; without one, depth 1.0f and tag | mask) */ struct pvr_g5p_zls_words zls; - pvr_arch_g5p_ds_pass(job, &zls); + pvr_arch_g5p_ds_pass(job, PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p), &zls); + /* bo_log: how often a depth/stencil attachment is loaded or stored (what lazily allocated memory would save) */ + if (unlikely(device->bo_log) && job->ds.iview && (job->has_depth_attachment || job->has_stencil_attachment)) + pvr_bo_log_zls((struct pvr_device *)device, job->ds.iview->vk.extent.width, job->ds.iview->vk.extent.height, + job->ds.load.d || job->ds.load.s, + job->ds.store.d || job->ds.store.s); g5p_put64(&s, scissor); g5p_put64(&s, dbias); @@ -1159,8 +1447,19 @@ PVR_PER_ARCH(g5p_frag_stream)(const struct pvr_device *device, const struct pvr_ g5p_put64(&s, 0); /* ZLSCTL and the load/store bases: in the extension */ g5p_put64(&s, 0); g5p_put64(&s, 0); - for (unsigned i = 0; i < 8 * 3; i++) - g5p_put64(&s, 0); /* PBE words: in the end-of-tile program's data */ + if (PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p)) { + /* The A8's PBE words, Rogue's pbe_word[8][3]: target 0's REG_WORD0/1, then the empty pair Apple writes for + * every unused target; word 2 is 0 (the kernel copies words 0-1 to W+0x118 + 16 i) */ + for (unsigned i = 0; i < 8; i++) { + const bool t1 = i == 1 && g->a8_t1_addr; + g5p_put64(&s, i ? (t1 ? g->a8_pbe_t1[0] : G5P_PBE_A8_EMPTY0) : g->a8_pbe[0]); + g5p_put64(&s, i ? (t1 ? g->a8_pbe_t1[1] : G5P_PBE_A8_EMPTY1) : g->a8_pbe[1]); + g5p_put64(&s, 0); + } + } else { + for (unsigned i = 0; i < 8 * 3; i++) + g5p_put64(&s, 0); /* PBE words: in the end-of-tile program's data */ + } g5p_put64(&s, 0); /* border colour table */ for (unsigned i = 0; i < 6; i++) g5p_put64(&s, 0); /* Rogue background words: in the extension */ @@ -1169,8 +1468,16 @@ PVR_PER_ARCH(g5p_frag_stream)(const struct pvr_device *device, const struct pvr_ g5p_put32(&s, 0); /* USC pixel output control */ g5p_put32(&s, zls.bgobj_depth); /* ISP_BGOBJDEPTH */ g5p_put32(&s, zls.bgobj_vals); /* ISP_BGOBJVALS */ - g5p_put32(&s, 0); /* ISP_AA, ISP_CTL: the kernel's, from the dataset's samples */ - g5p_put32(&s, 0); + g5p_put32(&s, 0); /* ISP_AA: the kernel's, from the dataset's samples */ + /* ISP_CTL: kernel K1 reads only bit 17, keeping its other bits and MSAA form. + * Older kernels ignore this field and continue processing every tile. + */ + /* The A8's kernel writes its own ISP_CTL (3D record +0x3c8, PROCESS_EMPTY_TILES set): this word stays 0 on the + * A8 until a kernel reads bit 17 from it for the A8 too. */ + if (PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p)) + g5p_put32(&s, 0); + else + g5p_put32(&s, 0x0021000f | (g->skip_empty_tiles ? 0 : 1u << 17)); g5p_put32(&s, 0); /* EVENT_PIXEL_PDS_INFO: in the extension's end-of-tile words */ g5p_put32(&s, 0); g5p_put32(&s, 0); diff --git a/src/imagination/vulkan/pvr_arch_g5p_compute.c b/src/imagination/vulkan/pvr_arch_g5p_compute.c index 5213860ead6..0b28c4f644e 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_compute.c +++ b/src/imagination/vulkan/pvr_arch_g5p_compute.c @@ -36,7 +36,35 @@ #include "pvr_winsys.h" #define G9P_KERNEL_DATA_DWORDS 8 static uint32_t usc_offset(const struct pvr_device *d, const struct pvr_suballoc_bo *bo) -{ return bo->dev_addr.addr - d->heaps.usc_heap->base_addr.addr; } +{ return bo->dev_addr.addr - PVR_G5P_USC_EXEC_BASE(d); } +/* The kernel does atomics on device memory: the A8 runs one workgroup per task for it (g5p_cdm_max_instances_g4p()) */ +static bool +g5p_nir_has_device_atomics(nir_shader *nir) +{ + nir_foreach_function_impl (impl, nir) { + nir_foreach_block (block, impl) { + nir_foreach_instr (instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + switch (nir_instr_as_intrinsic(instr)->intrinsic) { + case nir_intrinsic_ssbo_atomic: + case nir_intrinsic_ssbo_atomic_swap: + case nir_intrinsic_global_atomic: + case nir_intrinsic_global_atomic_swap: + case nir_intrinsic_image_atomic: + case nir_intrinsic_image_atomic_swap: + case nir_intrinsic_image_deref_atomic: + case nir_intrinsic_image_deref_atomic_swap: + return true; + default: + break; + } + } + } + } + return false; +} + VkResult PVR_PER_ARCH(g5p_compute_pipeline_compile)( struct pvr_device *device, @@ -97,6 +125,15 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( } else { pvr_g5p_shader_dump(&create_info->stage, create_info->layout, "comp", NULL, 0, "ADMIT", NULL); } + if (!PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_compute_dispatch)) { + /* A device entry without Apple's residency parameters */ + mesa_logw("G5P refuses: compute on %s, whose workgroup residency limits are not established", + device->pdevice->dev_info.ident.public_name); + result = vk_errorf(device, VK_ERROR_FEATURE_NOT_PRESENT, + "G5P compute: no workgroup residency limits for %s", + device->pdevice->dev_info.ident.public_name); + goto out; + } { const unsigned threads = shader.local_size[0] * shader.local_size[1] * shader.local_size[2]; unsigned max; @@ -134,6 +171,7 @@ PVR_PER_ARCH(g5p_compute_pipeline_compile)( state->a9_local_dwords = shader.a9_local_dwords; state->a9_has_barriers = shader.a9_has_barriers; state->a9_device_fence = shader.a9_device_fence; + state->a8_device_atomics = g5p_nir_has_device_atomics(nir); state->a9_num_images=shader.num_textures; for(unsigned i=0;ia9_images[i].set=shader.textures[i].set; @@ -282,6 +320,7 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, const uint32_t group_count[static 3]) { struct pvr_device *device = cmd_buffer->device; + const bool g4p = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p); const struct pvr_compute_pipeline *pipeline = cmd_buffer->state.compute_pipeline; const struct pvr_g9p_compute_state *state = &pipeline->g9p; const struct pvr_descriptor_state *desc = &cmd_buffer->state.compute_desc_state; @@ -307,6 +346,12 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, if (base_group[0] || base_group[1] || base_group[2]) return vk_errorf(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY, "G5P: base dispatch not implemented yet"); + /* Apple's A8 driver has no indirect dispatch: AGXMetalA8's + * dispatchThreadgroupsWithIndirectBuffer:indirectBufferOffset:threadsPerThreadgroup: is a bare return + * (0x1a14458d4; the sampled encoder's 0x1a1447638 only books statistics) */ + if (indirect_va && g4p) + return vk_errorf(cmd_buffer, VK_ERROR_FEATURE_NOT_PRESENT, + "G5P: an indirect dispatch on the A8 (Apple's A8 driver has none; not hardware-proven)"); if (indirect_va && ((indirect_va & 3) || indirect_va >> 40)) return vk_error(cmd_buffer, VK_ERROR_OUT_OF_HOST_MEMORY); if (!indirect_va && (!group_count[0] || !group_count[1] || !group_count[2])) @@ -349,8 +394,9 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, } /* bo_list is freed as pvr_suballoc_bo, despite its pvr_bo comment. * Give the dedicated generic BO a normal ownership wrapper. Reset - * may cache it in suballoc_general, so use that allocator's mapping - * flags too; a later ordinary upload must find a mapped BO. */ + * may cache it in the general command-buffer allocator, so use that + * allocator's mapping flags too; a later ordinary upload must find a + * mapped BO. */ struct pvr_suballoc_bo *owner = vk_zalloc(&device->vk.alloc, sizeof(*owner), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!owner) { @@ -365,7 +411,7 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, vk_command_buffer_set_error(&cmd_buffer->vk, result); return result; } - owner->allocator = &device->suballoc_general; + owner->allocator = pvr_cmd_suballocator(device, &device->suballoc_general); owner->bo = shared_spill; owner->dev_addr = shared_spill->vma->dev_addr; owner->size = groups * state->shared_spill_stride; @@ -490,6 +536,18 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, &uniforms_bo); if (result != VK_SUCCESS) return result; + /* The A8: the thread count in a block of its own, as AGXMetalA8's K0 reads it (iPod touch 6 capture: the + * address table at 0x1_00014100, the count at 0x1_00014140, each on a 64-byte line) rather than 8 bytes into + * the uniforms (8-byte aligned for an odd buffer count) */ + uint64_t count_va = 0; + if (g4p) { + const uint32_t count_block[4] = { count, 0, 0, 0 }; + struct pvr_suballoc_bo *count_bo; + result = pvr_arch_cmd_buffer_upload_general(cmd_buffer, count_block, sizeof(count_block), &count_bo); + if (result != VK_SUCCESS) + return result; + count_va = count_bo->dev_addr.addr; + } /* * Binding task data: the source addresses (64-bit each), then one DOUTD @@ -539,14 +597,17 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, dest=31;dwords=1; } else { src = uniforms_bo->dev_addr.addr + 8 * k; - if (k == n) + if (k == n) { dwords = 1; + if (g4p) + src = count_va; + } } bind_data[2 * k] = src; bind_data[2 * k + 1] = src >> 32; bind_data[ctrl0 + k] = (k+1 == transfers ? BITFIELD_BIT(31) : 0) | BITFIELD_BIT(28) | dest << 13 | dwords; - bind_code[nc++] = g9p_pds_dout(G9P_DOUTD, k, G9P_PDS_DATA, ctrl0 + k, + bind_code[nc++] = G5P_PDS_DOUT(g4p, G9P_DOUTD, k, G9P_PDS_DATA, ctrl0 + k, k+1 == transfers ? G9P_PDS_END : 0); } bind_code[nc++] = g9p_pds_halt(); @@ -575,11 +636,11 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, kernel_data[2] = 0x80000000; kernel_data[4] = 0x90000002; kernel_code[0] = g9p_pds_bra_not_if1(4); - kernel_code[1] = g9p_pds_dout(G9P_DOUTW, 0, G9P_PDS_TEMPS, 4, G9P_PDS_END); + kernel_code[1] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 0, G9P_PDS_TEMPS, 4, G9P_PDS_END); kernel_code[2] = g9p_pds_halt(); kernel_code[3] = g9p_pds_halt(); - kernel_code[4] = g9p_pds_dout(G9P_DOUTW, 0, G9P_PDS_TEMPS, 2, 0); - kernel_code[5] = g9p_pds_dout(G9P_DOUTU, 0, G9P_PDS_DATA, 0, + kernel_code[4] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 0, G9P_PDS_TEMPS, 2, 0); + kernel_code[5] = G5P_PDS_DOUT(g4p, G9P_DOUTU, 0, G9P_PDS_DATA, 0, G9P_PDS_END | G9P_PDS_CC); kernel_code[6] = g9p_pds_halt(); kernel_code[7] = g9p_pds_halt(); @@ -590,11 +651,11 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, kernel_dwords = 12; kernel_data[6] = BITFIELD_BIT(28) | 2u << 13 | 2; kernel_code[0] = g9p_pds_bra_not_if1(5); - kernel_code[1] = g9p_pds_dout(G9P_DOUTW, 4, G9P_PDS_DATA, 6, 0); - kernel_code[2] = g9p_pds_dout(G9P_DOUTW, 0, G9P_PDS_TEMPS, 4, G9P_PDS_END); + kernel_code[1] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 4, G9P_PDS_DATA, 6, 0); + kernel_code[2] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 0, G9P_PDS_TEMPS, 4, G9P_PDS_END); kernel_code[3] = kernel_code[4] = g9p_pds_halt(); - kernel_code[5] = g9p_pds_dout(G9P_DOUTW, 0, G9P_PDS_TEMPS, 2, 0); - kernel_code[6] = g9p_pds_dout(G9P_DOUTU, 0, G9P_PDS_DATA, 0, G9P_PDS_END | G9P_PDS_CC); + kernel_code[5] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 0, G9P_PDS_TEMPS, 2, 0); + kernel_code[6] = G5P_PDS_DOUT(g4p, G9P_DOUTU, 0, G9P_PDS_DATA, 0, G9P_PDS_END | G9P_PDS_CC); } if (state->a9_wgid_z) { /* Workgroup ID z, only for a kernel that reads it: AGXMetalA9's EncodeComputePDSProgram (0x207d1ea4c, flags @@ -609,13 +670,66 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_DEST_COMMON_STORE | G9P_A9_TG_WGID_Z << PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_AO_SHIFT | PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_BSIZE_UPPER; - kernel_code[at] = g9p_pds_dout(G9P_DOUTW, 0, G9P_PDS_TEMPS, 4, 0); - kernel_code[at + 1] = g9p_pds_dout(G9P_DOUTW, 1, G9P_PDS_TEMPS, 5, G9P_PDS_END); + kernel_code[at] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 0, G9P_PDS_TEMPS, 4, 0); + kernel_code[at + 1] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 1, G9P_PDS_TEMPS, 5, G9P_PDS_END); + } + unsigned kernel_code_dwords = ARRAY_SIZE(kernel_code); + if (g4p) { + /* + * The A8 kernel launch in AGXMetalA8's order (its compute PDS encoder, + * iOS 12.4, 0x1a1453d44): the branch past the coefficient (IF1) task's + * part, that part (the barrier counter's zero pair, the workgroup ID + * x/y and then z into the threadgroup words) with no END bit, its HALT; + * then the local ID into r0, DOUTU.END.CC and HALT. No padding HALTs. + * Without the IF1 writes this is Metal's A8 store kernel, c2c00002 + * d3000000 f0024001 fc000002 d3000000. The data words are the A9's. + */ + unsigned k = 1; + if (state->a9_has_barriers) + kernel_code[k++] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 4, G9P_PDS_DATA, 6, 0); + kernel_code[k++] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 0, G9P_PDS_TEMPS, 4, 0); + if (state->a9_wgid_z) + kernel_code[k++] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 1, G9P_PDS_TEMPS, 5, 0); + kernel_code[k++] = g9p_pds_halt(); + kernel_code[0] = g9p_pds_bra_not_if1(k); + kernel_code[k++] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 0, G9P_PDS_TEMPS, 2, 0); + kernel_code[k++] = G5P_PDS_DOUT(g4p, G9P_DOUTU, 0, G9P_PDS_DATA, 0, G9P_PDS_END | G9P_PDS_CC); + kernel_code[k++] = g9p_pds_halt(); + assert(k <= ARRAY_SIZE(kernel_code)); + kernel_code_dwords = k; + } + /* The A8's plain launches, AGXMetalA8's words (iPod touch 6 capture, all 25 kernels): + * one data unit, no fence. A kernel that reads neither the + * workgroup ID nor threadgroup memory gets no IF1 part and no common store (k_store_one: KERNEL0 0x2106, + * c2c00002 d3000000 f0024001 fc000002 d3000000, data USC, temps, 0x80000000). One that reads the ID gets + * Apple's layout, the ID's control word at c2 and the local ID's at c3 (c2c00003 f0024001 d3000000 f0034001 + * fc000002 d3000000, KERNEL0 0x42106); Apple writes 32 bits (0x90000000, its kernels read x only), we keep + * 64 (x and y, the form tested on the device, which completes). Barrier and z launches keep the form above. */ + /* a9_local_dwords is 4 threadgroup words + the shared memory (g9p_a9_compute.h) */ + const bool g4p_if1 = state->uses_workgroup_id || state->a9_local_dwords > 4; + const bool g4p_plain = g4p && !state->a9_has_barriers && !state->a9_wgid_z; + if (g4p_plain) { + unsigned k = 0; + memset(&kernel_data[2], 0, sizeof(kernel_data) - 2 * sizeof(kernel_data[0])); + kernel_code[k++] = g9p_pds_bra_not_if1(g4p_if1 ? 3 : 2); + if (g4p_if1) { + kernel_data[2] = PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_LAST_EN | + PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_DEST_COMMON_STORE | + PVR_ROGUE_PDSINST_DOUT_FIELDS_DOUTW_SRC1_BSIZE_ALL64; + kernel_code[k++] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 0, G9P_PDS_TEMPS, 2, 0); + } + kernel_code[k++] = g9p_pds_halt(); + kernel_data[g4p_if1 ? 3 : 2] = 0x80000000; + kernel_code[k++] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 0, G9P_PDS_TEMPS, g4p_if1 ? 3 : 2, 0); + kernel_code[k++] = G5P_PDS_DOUT(g4p, G9P_DOUTU, 0, G9P_PDS_DATA, 0, G9P_PDS_END | G9P_PDS_CC); + kernel_code[k++] = g9p_pds_halt(); + kernel_code_dwords = k; + kernel_dwords = 4; } result = pvr_arch_cmd_buffer_upload_pds(cmd_buffer, kernel_data, kernel_dwords, ROGUE_CDMCTRL_KERNEL1_DATA_ADDR_ALIGNMENT, - kernel_code, ARRAY_SIZE(kernel_code), + kernel_code, kernel_code_dwords, ROGUE_CDMCTRL_KERNEL2_CODE_ADDR_ALIGNMENT, ROGUE_CDMCTRL_KERNEL1_DATA_ADDR_ALIGNMENT, &kernel); @@ -625,14 +739,22 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, { struct pvr_csb *csb = &sub_cmd->control_stream; const struct g5p_cdm_record bind_rec = - g5p_cdm_dma_task(G9P_CDM_DATA_BLOCKS(bind_dwords / 4), bind.data_offset, + g5p_cdm_dma_task(G5P_CDM_DATA_UNITS(g4p, bind_dwords / 4), bind.data_offset, bind.code_offset, (state->a9_device_fence || images || nup) ? 4 : DIV_ROUND_UP(2 * n + 1, 16)); - const struct g5p_cdm_record kernel_rec = g5p_cdm_kernel( + struct g5p_cdm_record kernel_rec = g5p_cdm_kernel( G9P_CDM_KERNEL_REGS(bank1) | G9P_CDM_ANY_CLUSTER | - G9P_CDM_FENCE | - G9P_CDM_DATA_BLOCKS(kernel_dwords / 4), + /* the A8: no fence on a kernel (Apple's 25 records; the fence bit locks the job up) */ + (g4p ? 0 : G9P_CDM_FENCE) | + G5P_CDM_DATA_UNITS(g4p, kernel_dwords / 4), kernel.data_offset, kernel.code_offset, group_count[0], group_count[1], - group_count[2], hw_threads, DIV_ROUND_UP(state->a9_local_dwords, 16)); + group_count[2], hw_threads, + /* the A8's no-IF1 launch has no common store (k_store_one's 0x2106) */ + (g4p_plain && !g4p_if1) ? 0 : DIV_ROUND_UP(state->a9_local_dwords, 16)); + /* KERNEL8 max_instances: the A8 sets it the way AGXMetalA8 does (g5p_cdm_max_instances_g4p()); + * the A9 leaves it 0 (32 instances per task) */ + const uint32_t max_instances = + g4p ? g5p_cdm_max_instances_g4p(threads, state->a8_device_atomics, 4 * state->a9_local_dwords) << 27 : 0; + kernel_rec.w[6] |= max_instances; pvr_csb_set_relocation_mark(csb); emit_record(csb, &bind_rec); @@ -642,7 +764,11 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, kernel_rec.w[0], kernel.data_offset, kernel.code_offset, indirect_va, hw_threads, 0); uint32_t *w = pvr_arch_csb_alloc_dwords(csb, 6); - if (w) memcpy(w, rec.w, sizeof(rec.w)); + /* rec is const: KERNEL8 is its word 5 */ + if (w) { + memcpy(w, rec.w, sizeof(rec.w)); + w[5] |= max_instances; + } } else { emit_record(csb, &kernel_rec); } @@ -662,6 +788,7 @@ PVR_PER_ARCH(g5p_cmd_dispatch)(struct pvr_cmd_buffer *cmd_buffer, static VkResult upload_cdm_task(struct pvr_cmd_buffer *cmd, bool fenced, struct pvr_pds_upload *out) { + const bool g4p = PVR_HAS_FEATURE(&cmd->device->pdevice->dev_info, apple_g4p); uint8_t usc[128] = {0}; const uint16_t control = (0xc4u << 8) | 1u; unsigned at = 0; @@ -681,7 +808,7 @@ upload_cdm_task(struct pvr_cmd_buffer *cmd, bool fenced, struct pvr_pds_upload * /* Same unconditional DOUTU.END/HALT as the original ta_kick; only the * USC target changes, so probe2 does not also vary the IF1 launch. */ uint32_t code[2] = { - g9p_pds_dout(G9P_DOUTU, 0, G9P_PDS_DATA, 0, G9P_PDS_END), + G5P_PDS_DOUT(g4p, G9P_DOUTU, 0, G9P_PDS_DATA, 0, G9P_PDS_END), g9p_pds_halt(), }; return pvr_arch_cmd_buffer_upload_pds(cmd, data, 4, 16, code, 2, 32, 64, out); @@ -718,6 +845,7 @@ upload_cdm_task(struct pvr_cmd_buffer *cmd, bool fenced, struct pvr_pds_upload * static VkResult upload_end_barrier(struct pvr_cmd_buffer *cmd, struct pvr_pds_upload *out) { + const bool g4p = PVR_HAS_FEATURE(&cmd->device->pdevice->dev_info, apple_g4p); uint8_t usc[128] = {0}; const unsigned at = g5p_end_barrier_helper(usc); if (!at) @@ -746,19 +874,76 @@ upload_end_barrier(struct pvr_cmd_buffer *cmd, struct pvr_pds_upload *out) uint32_t data[8] = {(uint32_t)doutd_src0, (uint32_t)(doutd_src0 >> 32), doutd_src1, 0}; g9p_pds_kick_data(&data[4], usc_offset(cmd->device, bo)); /* q2: DOUTU source */ uint32_t code[3] = { - g9p_pds_dout(G9P_DOUTD, 0, G9P_PDS_DATA, 2, 0), - g9p_pds_dout(G9P_DOUTU, 2, G9P_PDS_DATA, 0, G9P_PDS_END), + G5P_PDS_DOUT(g4p, G9P_DOUTD, 0, G9P_PDS_DATA, 2, 0), + G5P_PDS_DOUT(g4p, G9P_DOUTU, 2, G9P_PDS_DATA, 0, G9P_PDS_END), g9p_pds_halt(), }; return pvr_arch_cmd_buffer_upload_pds(cmd, data, 8, 16, code, 3, 32, 64, out); } +/* + * The A8's end of a compute pass, AGXMetalA8's last two records after each encoder (iPod touch 6 capture, + * dispatch 0's K3 and K4): a task on all clusters that kicks an empty USC program (KERNEL0 0x4, one data unit; + * PDS c2c00002 d3000000 fc000002 d3000000, data {USC, 0x10}), then a fenced PDS-only task (KERNEL0 0x7, PDS + * c2c00002 d3000000 f0000005 d3000000: DOUTC, data zero). No type-3 fence blocks: Mesa's A9 tail (barrier task, + * fence blocks, the USC fence-ack task) locks the A8 up. Apple's K2 before them (a DMA of + * 12 dwords from a driver buffer at 0x1_00000000 and a USC program) is left out: K0 + K4 alone completes on + * the A8, and what K2's source holds on iOS is not known. + */ +static VkResult +g5p_compute_end_pass_g4p(struct pvr_cmd_buffer *cmd, struct pvr_sub_cmd_compute *sub) +{ + struct pvr_pds_upload kick, fence; + sub->g5p_kernel_since_end = false; + /* the empty USC program: a branch to END, padded to 64 bytes (upload_cdm_task) */ + uint8_t usc[128] = {0}; + unsigned at = g5p_usc_branch(usc, 64, false); + while (at < 64) + at += g5p_usc_special(usc + at, false); + at += g5p_usc_special(usc + at, true); + struct pvr_suballoc_bo *bo; + VkResult r = pvr_gpu_upload_usc(cmd->device, usc, ALIGN_POT(at, 64), 64, &bo); + if (r != VK_SUCCESS) return r; + list_add(&bo->link, &cmd->bo_list); + uint32_t kdata[4] = {usc_offset(cmd->device, bo), 0x10, 0, 0}; + uint32_t kcode[4] = { + g9p_pds_bra_not_if1(2), + g9p_pds_halt(), + G5P_PDS_DOUT(true, G9P_DOUTU, 0, G9P_PDS_DATA, 0, G9P_PDS_END | G9P_PDS_CC), + g9p_pds_halt(), + }; + r = pvr_arch_cmd_buffer_upload_pds(cmd, kdata, 4, 16, kcode, 4, 32, 64, &kick); + if (r != VK_SUCCESS) return r; + uint32_t fdata[4] = {0}; + uint32_t fcode[4] = { + g9p_pds_bra_not_if1(2), + g9p_pds_halt(), + G5P_PDS_DOUT(true, G9P_DOUTC, 0, G9P_PDS_DATA, 0, 0), + g9p_pds_halt(), + }; + r = pvr_arch_cmd_buffer_upload_pds(cmd, fdata, 4, 16, fcode, 4, 16, 16, &fence); + if (r != VK_SUCCESS) return r; + struct g5p_cdm_record rec; + pvr_csb_set_relocation_mark(&sub->control_stream); + /* K3: KERNEL1 0x8 | DMA (Apple's 0xa: usc_common_shared and the DMA kind) */ + rec = g5p_cdm_task(G5P_CDM_DATA_UNITS(true, 1), kick.data_offset, G9P_CDM_K1_DMA | 0x8u, kick.code_offset); + emit_record(&sub->control_stream, &rec); + /* K4: fence, any cluster, one unit = 0x7 */ + rec = g5p_cdm_task(G9P_CDM_FENCE | G9P_CDM_ANY_CLUSTER | G5P_CDM_DATA_UNITS(true, 1), fence.data_offset, + G9P_CDM_K1_FENCE, fence.code_offset); + emit_record(&sub->control_stream, &rec); + pvr_csb_clear_relocation_mark(&sub->control_stream); + return pvr_csb_get_status(&sub->control_stream); +} + VkResult PVR_PER_ARCH(g5p_compute_end_pass)(struct pvr_cmd_buffer *cmd, struct pvr_sub_cmd_compute *sub) { uint32_t data[4] = {0}, code[4]; struct pvr_pds_upload fence, fenced, kick, barrier; + if (PVR_HAS_FEATURE(&cmd->device->pdevice->dev_info, apple_g4p)) + return g5p_compute_end_pass_g4p(cmd, sub); /* after a dispatch only, as iOS: a pass whose dispatches were all * bodyless (omitted) keeps the plain tail */ const bool end_barrier = sub->g5p_kernel_since_end; diff --git a/src/imagination/vulkan/pvr_arch_g5p_ds.c b/src/imagination/vulkan/pvr_arch_g5p_ds.c index 909c689f8f9..32a372a99e9 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_ds.c +++ b/src/imagination/vulkan/pvr_arch_g5p_ds.c @@ -22,6 +22,8 @@ #include "pvr_cmd_buffer.h" #include "pvr_csb.h" +#include "pvr_device.h" +#include "pvr_physical_device.h" #include "pvr_g5p_ds.h" #include "pvr_g5p_shader_dump.h" #include "pvr_g5p_tex_encode.h" @@ -42,6 +44,13 @@ PVR_PER_ARCH(g5p_ds_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_r const struct pvr_g5p_ds_format *f = pvr_g5p_ds_format(iview->vk.format); struct pvr_g5p_ds_plane plane; + /* The A8: what a8-passcap captured (iPod touch 6, iOS 12.4): D32F + * and D32F_S8 at one sample, clear/load and store */ + if (PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g4p) && + (!pvr_g5p_ds_a8_attachment(iview->vk.format) || image->vk.samples != 1)) + return G5P_REFUSE(cmd_buffer, "A8 depth/stencil %s at %u samples (D32F and D32F_S8 at one sample are captured)", + vk_Format_to_str(iview->vk.format), image->vk.samples); + if (!f) return G5P_REFUSE(cmd_buffer, "depth/stencil format %s (the ZLS stores D16, D32F, S8, D32F_S8)", vk_Format_to_str(iview->vk.format)); @@ -76,7 +85,7 @@ PVR_PER_ARCH(g5p_ds_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_r } void -PVR_PER_ARCH(g5p_ds_pass)(const struct pvr_render_job *job, struct pvr_g5p_zls_words *out) +PVR_PER_ARCH(g5p_ds_pass)(const struct pvr_render_job *job, bool g4p, struct pvr_g5p_zls_words *out) { if (!job->ds.iview || !(job->has_depth_attachment || job->has_stencil_attachment)) { pvr_g5p_ds_zls_none(out); @@ -101,6 +110,7 @@ PVR_PER_ARCH(g5p_ds_pass)(const struct pvr_render_job *job, struct pvr_g5p_zls_w .sstore = job->ds.store.s, .depth_clear = job->ds_clear_value.depth, .stencil_clear = (uint8_t)job->ds_clear_value.stencil, + .g4p = g4p, }; /* g5p_ds_admit() refused what pvr_g5p_ds_zls() cannot build */ const char *why = pvr_g5p_ds_zls(&t, out); diff --git a/src/imagination/vulkan/pvr_arch_g5p_gfx.c b/src/imagination/vulkan/pvr_arch_g5p_gfx.c index cdbd14e4833..03797434349 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_gfx.c +++ b/src/imagination/vulkan/pvr_arch_g5p_gfx.c @@ -45,6 +45,7 @@ #include "pvr_descriptor_set.h" #include "pvr_cmd_buffer.h" #include "pvr_csb.h" +#include "pvr_g5p_a8_fetch.h" #include "pvr_device.h" #include "pvr_physical_device.h" #include "pvr_framebuffer.h" @@ -96,11 +97,11 @@ pds_pack(pdsc_program *p, uint32_t *dest) static uint64_t doutu_src0(uint32_t exe_off, unsigned temps) { - pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-doutu"); + /* packing a field reads only the feature set: no program allocation per draw */ + static const pdsc_program pack = { .name = "g5p-doutu", .features = PDSC_FEATURE_DOUT_EXT }; + const pdsc_program *p = &pack; struct pdsc_doutu_src0 s = { .exe_off = exe_off, .temps = temps }; const uint64_t v = pdsc_doutu_src0_pack(p, &s); - ralloc_free(p); return v; } @@ -108,23 +109,23 @@ doutu_src0(uint32_t exe_off, unsigned temps) static uint32_t doutw_control(unsigned reg, bool upper, bool last) { - pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-doutw"); + /* packing a field reads only the feature set: no program allocation per draw */ + static const pdsc_program pack = { .name = "g5p-doutw", .features = PDSC_FEATURE_DOUT_EXT }; + const pdsc_program *p = &pack; struct pdsc_doutw_src1 s = { .last_issue = last, .store_dest = PDSC_STORE_DEST_UNIFIED, .ao = reg, .bsize = upper ? PDSC_DOUTW_BSIZE_UPPER : PDSC_DOUTW_BSIZE_LOWER }; const uint32_t v = pdsc_doutw_src1_pack(p, &s); - ralloc_free(p); return v; } static uint64_t ddmad_src3(unsigned dwords, unsigned ao, bool last) { - pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-ddmad"); + /* packing a field reads only the feature set: no program allocation per draw */ + static const pdsc_program pack = { .name = "g5p-ddmad", .features = PDSC_FEATURE_DOUT_EXT }; + const pdsc_program *p = &pack; struct pdsc_ddmad_src3 s = { .last_issue = last, .bsize = dwords, .ao = ao }; const uint64_t v = pdsc_ddmad_src3_pack(p, &s); - ralloc_free(p); return v; } @@ -133,12 +134,12 @@ ddmad_src3(unsigned dwords, unsigned ao, bool last) static uint32_t doutd_control(unsigned common, unsigned dwords, bool last) { - pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-doutd"); + /* packing a field reads only the feature set: no program allocation per draw */ + static const pdsc_program pack = { .name = "g5p-doutd", .features = PDSC_FEATURE_DOUT_EXT }; + const pdsc_program *p = &pack; struct pdsc_doutd_src1 s = { .last_issue = last, .store_dest = PDSC_STORE_DEST_COMMON, .ao = common, .bsize = dwords }; const uint32_t v = pdsc_doutd_src1_pack(p, &s); - ralloc_free(p); return v; } @@ -149,10 +150,10 @@ doutd_control(unsigned common, unsigned dwords, bool last) * the UNIFORMDATABASE pointer. */ static unsigned -loader_program(uint32_t *dest, unsigned n) +loader_program(uint32_t *dest, unsigned n, enum pdsc_feature features) { pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-fs-loader"); + pdsc_program *p = pdsc_program_create(NULL, features, "g5p-fs-loader"); for (unsigned i = 0; i < n; i++) pdsc_dout(p, pdsc_ref_const64(2 * i), pdsc_ref_const32(2 * n + 2 * i), pdsc_ref_null(), .dstdout = PDSC_DSTDOUT_D, .end = i == n - 1); @@ -252,7 +253,8 @@ fetch_reads_instance(const struct pvr_g5p_pipeline *g) } static unsigned -fetch_program(uint32_t *dest, const struct pvr_g5p_pipeline *g, bool base_instance, bool indirect) +fetch_program(uint32_t *dest, const struct pvr_g5p_pipeline *g, bool base_instance, bool indirect, + enum pdsc_feature features) { const struct pvr_g5p_fetch *list = g->fetch_list; const unsigned fetches = g->fetch_count, id_mask = g->vs_id_mask; @@ -260,7 +262,7 @@ fetch_program(uint32_t *dest, const struct pvr_g5p_pipeline *g, bool base_instan unsigned slot[PVR_G5P_MAX_FETCH]; const unsigned ndiv = fetch_divisors(g, div, slot); pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-vertex-fetch"); + pdsc_program *p = pdsc_program_create(NULL, features, "g5p-vertex-fetch"); /* A9's indirect base-instance variant reads persistent temporary 1 * (encoded selector 0xc1), not a CPU-specialized data constant. */ const pdsc_ref base = indirect ? pdsc_ref_ptemp32(1) : pdsc_ref_const32(fetch_base_instance(fetches)); @@ -313,10 +315,10 @@ fetch_program(uint32_t *dest, const struct pvr_g5p_pipeline *g, bool base_instan /* The fragment launch: one DOUTU.end from c0:1, no CC and no HALT (the * captured form) */ static unsigned -launch_program(uint32_t *dest) +launch_program(uint32_t *dest, enum pdsc_feature features) { pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-fragment"); + pdsc_program *p = pdsc_program_create(NULL, features, "g5p-fragment"); pdsc_dout(p, pdsc_ref_const64(0), pdsc_ref_null(), pdsc_ref_null(), .dstdout = PDSC_DSTDOUT_U, .end = true); const unsigned n = pds_pack(p, dest); ralloc_free(p); @@ -343,10 +345,10 @@ static_assert(4 * PCO_G5P_MAX_FS_VARYINGS == 112, "G5P varying components"); static unsigned varying_program(const struct pco_g5p_program *fs, const struct pco_g5p_program *vs, uint32_t *data, - uint32_t *code, unsigned *code_dwords, unsigned pv) + uint32_t *code, unsigned *code_dwords, unsigned pv, enum pdsc_feature features) { pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-varying"); + pdsc_program *p = pdsc_program_create(NULL, features, "g5p-varying"); struct pdsc_douti_src0 item[2 + PCO_G5P_MAX_FS_VARYINGS]; unsigned items = 0; /* Apple's item order: position Z, then W, then the varyings (the A9 @@ -397,7 +399,7 @@ varying_program(const struct pco_g5p_program *fs, const struct pco_g5p_program * static uint32_t usc_off(const struct pvr_device *device, const struct pvr_suballoc_bo *bo) { - return (uint32_t)(bo->dev_addr.addr - device->heaps.usc_heap->base_addr.addr); + return (uint32_t)(bo->dev_addr.addr - PVR_G5P_USC_EXEC_BASE(device)); } /* ---- pipelines ---- */ @@ -579,6 +581,13 @@ g5p_check_state(struct pvr_device *device, const VkGraphicsPipelineCreateInfo *c } if (raw32) key->color_output = raw32 == 2 ? PCO_G5P_COLOR_SINT32 : PCO_G5P_COLOR_UINT32; + if (PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p) && colors == 1 && + (cf == VK_FORMAT_R8_UNORM || cf == VK_FORMAT_R8G8_UNORM)) + key->a8_narrow_halves = cf == VK_FORMAT_R8_UNORM ? 1 : 2; + if (PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p) && colors == 1 && cf == VK_FORMAT_R8G8B8A8_UINT) + key->a8_packed_word = true; + if (PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p) && colors == 1 && cf == VK_FORMAT_R32_SFLOAT) + key->color_output = PCO_G5P_COLOR_F32; /* no colour attachment: the program writes no colour (the no-colour ISP form: depth/stencil only, a * discarding program still runs for its feedback) */ if (!colors) @@ -969,18 +978,18 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo /* PDS: vertex fetch code */ uint32_t code[96 + 4 * PVR_G5P_MAX_FS_UPLOADS], data[2 * (1 + PCO_G5P_MAX_FS_VARYINGS) + 4]; - unsigned n = fetch_program(code, g, false, false); + unsigned n = fetch_program(code, g, false, false, PVR_G5P_PDS_FEATURES(&device->pdevice->dev_info)); result = pvr_gpu_upload_pds(device, NULL, 0, 0, code, n, 16, 64, &g->fetch); if (result != VK_SUCCESS) return result; if (fetch_reads_instance(g)) { - n = fetch_program(code, g, true, false); + n = fetch_program(code, g, true, false, PVR_G5P_PDS_FEATURES(&device->pdevice->dev_info)); result = pvr_gpu_upload_pds(device, NULL, 0, 0, code, n, 16, 64, &g->fetch_base); if (result != VK_SUCCESS) return result; } - n = fetch_program(code, g, fetch_reads_instance(g), true); + n = fetch_program(code, g, fetch_reads_instance(g), true, PVR_G5P_PDS_FEATURES(&device->pdevice->dev_info)); result = pvr_gpu_upload_pds(device, NULL, 0, 0, code, n, 16, 64, &g->fetch_indirect); if (result != VK_SUCCESS) return result; @@ -990,7 +999,7 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo /* at 4x the fragment program runs at sample rate (pvr_g5p_msaa_doutu()) */ const uint64_t doutu = pvr_g5p_msaa_doutu(doutu_src0(usc_off(device, g->fs_usc), g->fs_temps), g->samples); const uint32_t launch_data[4] = { (uint32_t)doutu, (uint32_t)(doutu >> 32), 0, 0 }; - n = launch_program(code); + n = launch_program(code, PVR_G5P_PDS_FEATURES(&device->pdevice->dev_info)); result = pvr_gpu_upload_pds(device, launch_data, ARRAY_SIZE(launch_data), 16, code, n, 0x20, 64, &g->fs_launch); if (result != VK_SUCCESS) @@ -1004,12 +1013,12 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo unsigned varying_data_dwords = 0; if (fs->num_varyings || fs->w_set || fs->z_set) { unsigned code_dwords; - varying_data_dwords = varying_program(fs, vs, data, code, &code_dwords, 0); + varying_data_dwords = varying_program(fs, vs, data, code, &code_dwords, 0, PVR_G5P_PDS_FEATURES(&device->pdevice->dev_info)); result = pvr_gpu_upload_pds(device, data, varying_data_dwords, 16, code, code_dwords, 16, 64, &g->varying); if (result != VK_SUCCESS) return result; for (unsigned pv = 1; g->fs_flat && pv <= 2; pv++) { - varying_program(fs, vs, data, code, &code_dwords, pv); + varying_program(fs, vs, data, code, &code_dwords, pv, PVR_G5P_PDS_FEATURES(&device->pdevice->dev_info)); result = pvr_gpu_upload_pds(device, data, varying_data_dwords, 16, code, code_dwords, 16, 64, &g->varying_pv[pv - 1]); if (result != VK_SUCCESS) @@ -1022,7 +1031,7 @@ g5p_pipeline_build(struct pvr_device *device, const VkGraphicsPipelineCreateInfo /* PDS: the uniform/texture loader (the device HALT without uploads) */ if (g->fs_num_uploads) { - n = loader_program(code, g->fs_num_dmas); + n = loader_program(code, g->fs_num_dmas, PVR_G5P_PDS_FEATURES(&device->pdevice->dev_info)); result = pvr_gpu_upload_pds(device, NULL, 0, 0, code, n, 16, 64, &g->fs_loader); if (result != VK_SUCCESS) return result; @@ -2042,6 +2051,25 @@ fs_uploads(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_pipeline *g, u->set, u->binding, u->element, lb->descriptor_count); const uint32_t *desc = (const uint32_t *)((const uint8_t *)set->mapping + lb->offset + (size_t)u->element * lb->stride); + if (lb->type == VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER && + PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g4p)) { + /* the A8's texel buffer (pvr_arch_image.c CreateBufferView): Rogue's four state dwords, the element count + * for the size, the fetch sampler for its read (the fold's 2D fetch, pco_g5p_general.c a8_fetch_as_sample) */ + bool ok = desc[7] == PVR_G5P_TBO_MAGIC && desc[6]; + if (ok && u->kind == PCO_G5P_UPLOAD_IMAGE && u->dwords == 4) + memcpy(out, desc, 16); + else if (ok && u->kind == PCO_G5P_UPLOAD_IMAGE_SIZE && u->dwords == 4 && !u->offset) { + out[0] = desc[6]; + out[1] = 0; + out[2] = out[3] = 1; + } else if (ok && u->kind == PCO_G5P_UPLOAD_SAMPLER && u->dwords == 4 && (u->offset & PCO_G5P_UPLOAD_SAMPLER_FETCH)) + pvr_g5p_a8_fetch_sampler(out, NULL, 0); + else + return G5P_REFUSE(cmd_buffer, "A8 texel-buffer upload (kind %d, %u dwords, offset %#x) at set %u binding %u " + "element %u", (int)u->kind, u->dwords, u->offset, u->set, u->binding, u->element); + copied[i] = true; + continue; + } if (lb->type == VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER) { if (!pvr_g5p_tbo_upload(u->kind, u->offset, u->dwords, desc, out)) return G5P_REFUSE(cmd_buffer, "unqualified texel-buffer upload at set %u binding %u element %u", @@ -2064,10 +2092,17 @@ fs_uploads(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_pipeline *g, /* a combined descriptor's sampler is at +32 (the upload's offset), * a standalone sampler's at +0 */ if (lb->type != VK_DESCRIPTOR_TYPE_SAMPLER) - desc += u->offset / 4; + desc += (u->offset & 0xffu) / 4; if (!pvr_g5p_sampler_valid(desc)) return G5P_REFUSE(cmd_buffer, "set %u binding %u: a sampler without A9 words", u->set, u->binding); memcpy(out, desc, 4 * u->dwords); + /* the A8's texelFetch drawn as a sample: the binding's sampler as a fetch sampler */ + if ((u->offset & PCO_G5P_UPLOAD_SAMPLER_FETCH) && u->dwords == 4) { + if (u->offset & PCO_G5P_UPLOAD_SAMPLER_READ) /* Apple's read form: its constant read sampler */ + pvr_g5p_a8_read_sampler(out); + else + pvr_g5p_a8_fetch_sampler(out, desc, (u->offset >> PCO_G5P_UPLOAD_SAMPLER_FETCH_LEVEL_SHIFT) & 0xf); + } break; case PCO_G5P_UPLOAD_IMAGE_SIZE: if (u->dwords != 4 || !pvr_g5p_tex_size(desc, u->offset, out)) @@ -2087,6 +2122,22 @@ fs_uploads(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g5p_pipeline *g, default: return G5P_REFUSE(cmd_buffer, "fragment upload kind %u", u->kind); } + /* read once: this runs for every texture upload of every draw */ + static int texdiag = -1; + if (texdiag < 0) + texdiag = getenv("PVR_G5P_TEXDIAG") != NULL; + if (texdiag && (u->kind == PCO_G5P_UPLOAD_IMAGE || u->kind == PCO_G5P_UPLOAD_SAMPLER)) { + /* the texture and sampler words a fragment program's DOUTDs take into the common store */ + fprintf(stderr, "G5P texdiag: %s set %u binding %u -> common %u, %u dwords:", u->kind == PCO_G5P_UPLOAD_IMAGE ? + "image" : "sampler", u->set, u->binding, u->common, u->dwords); + for (unsigned k = 0; k < u->dwords; k++) + fprintf(stderr, " %08x", out[k]); + if (u->kind == PCO_G5P_UPLOAD_IMAGE && u->dwords >= 4) { + const uint64_t w1 = out[2] | (uint64_t)out[3] << 32; + fprintf(stderr, " (Rogue texaddr %#" PRIx64 ")", (uint64_t)(((w1 >> 16) & ((UINT64_C(1) << 38) - 1)) << 2)); + } + fprintf(stderr, "\n"); + } copied[i] = true; } bool any_copied = false; @@ -2271,6 +2322,11 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g if (!g) return G5P_REFUSE(cmd_buffer, "a draw of a pipeline without G5P programs (refused at creation, named there)"); + /* Apple's A8 driver has no indirect draw: AGXMetalA8's render encoders raise APINotSupported for both + * indirect draws (0x1a144b89c, 0x1a144b8dc; the sampled encoder 0x1a145176c, 0x1a14517ac), so there is no + * A8 form to follow and the VDM's indirect index list is untested on the A8. */ + if (draw->indirect_addr && PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p)) + return G5P_REFUSE(cmd_buffer, "an indirect draw on the A8 (Apple's A8 driver has none; not hardware-proven)"); /* transform feedback: while capturing, the pipeline's capturing variant * (pvr_g5p_xfb.h); a pipeline that captures nothing draws as it is */ const bool xfb = g->xfb && pvr_arch_g5p_xfb_capturing(cmd_buffer); @@ -2709,6 +2765,11 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g c.clip_mode = (enum ROGUE_TA_CLIP_MODE)clip_mode; c.flatshade_vtx = ROGUE_TA_FLATSHADE_VTX_VERTEX_0; c.wclampen = true; + /* The A8 draws clipped edges: AGXMetalA8's render encoder init writes + * PPP_CTRL 0x6a0 (0x1a1450cb8) and no setter clears bit 9 (cull + * 0x1a1449a90, clip mode 0x1a1449b00, 0x1a144ac2c, 0x1a144ac90). The + * A9 captures carry 0x4a0. */ + c.drawclippededges = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p); if (dyn->rs.cull_mode == VK_CULL_MODE_BACK_BIT || dyn->rs.cull_mode == VK_CULL_MODE_FRONT_BIT) { /* Rogue's rule (pvr_setup_ppp_control) */ const bool ccw = dyn->rs.front_face == VK_FRONT_FACE_COUNTER_CLOCKWISE; @@ -2739,7 +2800,10 @@ PVR_PER_ARCH(g5p_cmd_draw)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_g pvr_csb_pack (&w[1], TA_STATE_STREAM_OUT0, s0) { s0.stream0_ta_output = true; } - w[2] = G5P_STREAM_OUT1; + /* The A8: PDS data 1, temps 1, no bit 14. AGXMetalA8 0x1a144f654 sets + * the low 10 bits to 0x11 over a default of 0 (0x1a1459500), for the + * same 16 zero bytes + HALT program (0x1a144f61c). */ + w[2] = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p) ? 0x11u : G5P_STREAM_OUT1; pvr_csb_pack (&w[3], TA_STATE_STREAM_OUT2, s2) { s2.pds_data_addr = PVR_DEV_ADDR(dp->halt.data_offset); } diff --git a/src/imagination/vulkan/pvr_arch_g5p_layout.c b/src/imagination/vulkan/pvr_arch_g5p_layout.c index d333074bc16..dc191cf76a8 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_layout.c +++ b/src/imagination/vulkan/pvr_arch_g5p_layout.c @@ -211,7 +211,7 @@ g5p_ds_plane(const struct g5p_surface *s, VkImageAspectFlagBits aspect, p->available -= off; p->bpp = p->cpp; p->twiddled = true; - /* The allocation pads to a square, but ZLS stores rectangular + /* The A9's allocation pads to a square (the A8's to the rectangle), but ZLS stores rectangular * Morton order over independently 32-pixel-aligned sides (R17's * device-proven unequal-side sampling). Match that order in CPU * clears/copies too; the unused allocation tail remains padding. */ diff --git a/src/imagination/vulkan/pvr_arch_g5p_meta.c b/src/imagination/vulkan/pvr_arch_g5p_meta.c index ea09c845e3a..063e968c03c 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_meta.c +++ b/src/imagination/vulkan/pvr_arch_g5p_meta.c @@ -29,6 +29,7 @@ #include "pvr_device.h" #include "pvr_g5p.h" #include "pvr_image.h" +#include "pvr_physical_device.h" #include "pvr_g5p_meta.h" @@ -147,6 +148,11 @@ g5p_meta_blit_eligible(const struct pvr_cmd_buffer *cmd_buffer, const VkBlitImag return false; if (!g5p_meta_image(src, VK_IMAGE_USAGE_SAMPLED_BIT) || !g5p_meta_image(dst, VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT)) return false; + /* The A8 keeps every blit a CPU transfer: vk_meta's blit shader samples with an explicit LOD, a form the A8 + * compiler refuses (the LOD operand is not decoded), so the + * command buffer failed (Zink's glGenerateMipmap lost the GL context). The A8 renders no sRGB target either. */ + if (PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g4p)) + return false; for (uint32_t i = 0; i < info->regionCount; i++) { const VkImageBlit2 *r = &info->pRegions[i]; @@ -161,6 +167,77 @@ g5p_meta_blit_eligible(const struct pvr_cmd_buffer *cmd_buffer, const VkBlitImag return true; } +/* + * vkCmdClearColorImage on the GPU, or false to keep it a CPU transfer: a render pass per level and range whose + * colour attachment is cleared by its background object and stored (vk_meta_clear_color_image(); Metal clears the + * same way, with a load action). The CPU path writes every texel through the GPU's mapping after the GPU drains: + * vkmark's clear scene at 750x1334 took 12.8 ms a frame on the A8. A8 only for now (the A9 keeps the CPU path it + * was qualified with); the image must be one the meta blits render (g5p_meta_image(), g5p_meta_dst_level()). + */ +bool +PVR_PER_ARCH(g5p_meta_clear_color)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_image *image, + VkImageLayout layout, const VkClearColorValue *color, uint32_t range_count, + const VkImageSubresourceRange *ranges) +{ + struct pvr_device *device = cmd_buffer->device; + + if (!PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p) || !device->g5p || !device->g5p->meta_ready || + cmd_buffer->state.vis_test_enabled || cmd_buffer->vk.level != VK_COMMAND_BUFFER_LEVEL_PRIMARY || + !g5p_meta_image(image, VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) || getenv("PVR_G5P_CPU_CLEAR")) + return false; + /* the A8 renders no sRGB target yet (pvr_arch_g5p.c refuses it by name): an sRGB image keeps the CPU clear */ + if (vk_format_is_srgb(image->vk.format)) + return false; + for (uint32_t i = 0; i < range_count; i++) { + if (ranges[i].aspectMask != VK_IMAGE_ASPECT_COLOR_BIT || + vk_image_subresource_layer_count(&image->vk, &ranges[i]) != 1) + return false; + const uint32_t levels = vk_image_subresource_level_count(&image->vk, &ranges[i]); + for (uint32_t l = 0; l < levels; l++) { + const VkImageSubresourceLayers sub = { VK_IMAGE_ASPECT_COLOR_BIT, ranges[i].baseMipLevel + l, + ranges[i].baseArrayLayer, 1 }; + if (!g5p_meta_dst_level(image, &sub)) + return false; + } + } + + struct g5p_meta_save *save = malloc(sizeof(*save)); + if (!save) { + vk_command_buffer_set_error(&cmd_buffer->vk, VK_ERROR_OUT_OF_HOST_MEMORY); + return true; + } + VkResult result = pvr_arch_cmd_buffer_end_sub_cmd(cmd_buffer); + if (result != VK_SUCCESS) { + free(save); + return true; + } + struct list_head *mark = cmd_buffer->sub_cmds.prev; + + g5p_meta_begin(cmd_buffer, save); + pvr_g5p_internal_pipeline++; + vk_meta_clear_color_image(&cmd_buffer->vk, &device->g5p->meta, &image->vk, layout, image->vk.format, color, + range_count, ranges); + pvr_g5p_internal_pipeline--; + g5p_meta_end(cmd_buffer, save); + free(save); + + bool rendered = false; + for (struct list_head *l = mark->next; l != &cmd_buffer->sub_cmds; l = l->next) { + struct pvr_sub_cmd *sub_cmd = list_entry(l, struct pvr_sub_cmd, link); + + if (sub_cmd->type == PVR_SUB_CMD_TYPE_GRAPHICS) { + sub_cmd->gfx.g5p_transfer = true; + rendered = true; + } + } + /* later transfer-stage barriers wait for these renders as well */ + if (rendered) { + for (uint32_t i = 0; i < ARRAY_SIZE(cmd_buffer->state.barriers_needed); i++) + cmd_buffer->state.barriers_needed[i] |= PVR_PIPELINE_STAGE_TRANSFER_BIT; + } + return true; +} + /* vkCmdBlitImage2 on the GPU, or false to keep it a CPU transfer */ bool PVR_PER_ARCH(g5p_meta_blit)(struct pvr_cmd_buffer *cmd_buffer, const VkBlitImageInfo2 *info) diff --git a/src/imagination/vulkan/pvr_arch_g5p_query.c b/src/imagination/vulkan/pvr_arch_g5p_query.c index b8968d100fc..87855c91d0a 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_query.c +++ b/src/imagination/vulkan/pvr_arch_g5p_query.c @@ -317,7 +317,7 @@ void PVR_PER_ARCH(g5p_query_programs_destroy)(struct pvr_device *device) static uint32_t usc_offset(const struct pvr_device *device, const struct pvr_suballoc_bo *bo) { - return bo->dev_addr.addr - device->heaps.usc_heap->base_addr.addr; + return bo->dev_addr.addr - PVR_G5P_USC_EXEC_BASE(device); } static void emit_record(struct pvr_csb *csb, @@ -338,6 +338,7 @@ emit_query_dispatch(struct pvr_cmd_buffer *cmd_buffer, const struct g5p_query_params *params) { struct pvr_device *device = cmd_buffer->device; + const bool g4p = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p); const unsigned num_buffers = program->g5p_num_buffers; const unsigned transfers = num_buffers + 1; uint32_t uniforms[2 * 4 + 1] = { 0 }; @@ -401,7 +402,7 @@ emit_query_dispatch(struct pvr_cmd_buffer *cmd_buffer, (i + 1 == transfers ? BITFIELD_BIT(31) : 0) | BITFIELD_BIT(28) | (2 * i) << 13 | dwords; bind_code[code_count++] = - g9p_pds_dout(G9P_DOUTD, + G5P_PDS_DOUT(g4p, G9P_DOUTD, i, G9P_PDS_DATA, control_base + i, @@ -428,14 +429,25 @@ emit_query_dispatch(struct pvr_cmd_buffer *cmd_buffer, kernel_data[2] = 0x80000000; kernel_data[4] = 0x90000002; kernel_code[0] = g9p_pds_bra_not_if1(4); - kernel_code[1] = g9p_pds_dout(G9P_DOUTW, 0, G9P_PDS_TEMPS, 4, G9P_PDS_END); + kernel_code[1] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 0, G9P_PDS_TEMPS, 4, G9P_PDS_END); kernel_code[2] = g9p_pds_halt(); kernel_code[3] = g9p_pds_halt(); - kernel_code[4] = g9p_pds_dout(G9P_DOUTW, 0, G9P_PDS_TEMPS, 2, 0); + kernel_code[4] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 0, G9P_PDS_TEMPS, 2, 0); kernel_code[5] = - g9p_pds_dout(G9P_DOUTU, 0, G9P_PDS_DATA, 0, G9P_PDS_END | G9P_PDS_CC); + G5P_PDS_DOUT(g4p, G9P_DOUTU, 0, G9P_PDS_DATA, 0, G9P_PDS_END | G9P_PDS_CC); kernel_code[6] = g9p_pds_halt(); kernel_code[7] = g9p_pds_halt(); + unsigned kernel_code_dwords = ARRAY_SIZE(kernel_code); + if (g4p) { + /* The A8 launch, as g5p_cmd_dispatch() builds it (AGXMetalA8's order, no END in the IF1 part, no padding) */ + kernel_code[0] = g9p_pds_bra_not_if1(3); + kernel_code[1] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 0, G9P_PDS_TEMPS, 4, 0); + kernel_code[2] = g9p_pds_halt(); + kernel_code[3] = G5P_PDS_DOUT(g4p, G9P_DOUTW, 0, G9P_PDS_TEMPS, 2, 0); + kernel_code[4] = G5P_PDS_DOUT(g4p, G9P_DOUTU, 0, G9P_PDS_DATA, 0, G9P_PDS_END | G9P_PDS_CC); + kernel_code[5] = g9p_pds_halt(); + kernel_code_dwords = 6; + } result = pvr_arch_cmd_buffer_upload_pds(cmd_buffer, @@ -443,7 +455,7 @@ emit_query_dispatch(struct pvr_cmd_buffer *cmd_buffer, ARRAY_SIZE(kernel_data), ROGUE_CDMCTRL_KERNEL1_DATA_ADDR_ALIGNMENT, kernel_code, - ARRAY_SIZE(kernel_code), + kernel_code_dwords, ROGUE_CDMCTRL_KERNEL2_CODE_ADDR_ALIGNMENT, ROGUE_CDMCTRL_KERNEL1_DATA_ADDR_ALIGNMENT, &kernel); @@ -451,14 +463,15 @@ emit_query_dispatch(struct pvr_cmd_buffer *cmd_buffer, return result; const struct g5p_cdm_record bind_record = - g5p_cdm_dma_task(G9P_CDM_DATA_BLOCKS(bind_dwords / 4), + g5p_cdm_dma_task(G5P_CDM_DATA_UNITS(g4p, bind_dwords / 4), bind.data_offset, bind.code_offset, DIV_ROUND_UP(2 * num_buffers + 1, 16)); const uint32_t groups = DIV_ROUND_UP(params->count, G5P_QUERY_LOCAL_SIZE); - const struct g5p_cdm_record kernel_record = g5p_cdm_kernel( + struct g5p_cdm_record kernel_record = g5p_cdm_kernel( G9P_CDM_KERNEL_REGS(program->g5p_num_regs) | G9P_CDM_ANY_CLUSTER | - G9P_CDM_FENCE | G9P_CDM_DATA_BLOCKS(G5P_QUERY_KERNEL_DATA_DWORDS / 4), + /* the A8: no fence on a kernel, data units counted directly (pvr_arch_g5p_compute.c) */ + (g4p ? 0 : G9P_CDM_FENCE) | G5P_CDM_DATA_UNITS(g4p, G5P_QUERY_KERNEL_DATA_DWORDS / 4), kernel.data_offset, kernel.code_offset, groups, @@ -466,6 +479,8 @@ emit_query_dispatch(struct pvr_cmd_buffer *cmd_buffer, 1, g5p_cdm_workgroup_threads(G5P_QUERY_LOCAL_SIZE), DIV_ROUND_UP(program->g5p_local_dwords, 16)); + if (g4p) + kernel_record.w[6] |= g5p_cdm_max_instances_g4p(G5P_QUERY_LOCAL_SIZE, false, 4 * program->g5p_local_dwords) << 27; pvr_csb_set_relocation_mark(&sub_cmd->control_stream); emit_record(&sub_cmd->control_stream, &bind_record); diff --git a/src/imagination/vulkan/pvr_arch_g5p_upload.c b/src/imagination/vulkan/pvr_arch_g5p_upload.c index 3cf3c9dbd70..051dc48ff18 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_upload.c +++ b/src/imagination/vulkan/pvr_arch_g5p_upload.c @@ -32,6 +32,7 @@ #include "pvr_csb.h" #include "pvr_descriptor_set.h" #include "pvr_device.h" +#include "pvr_physical_device.h" #include "pvr_g5p_upload.h" #include "pvr_g5p_vtex.h" @@ -39,10 +40,10 @@ /* The loader: DOUTD c(2i):c(2i+1) -> common store with the control word in * c(2n+2i), the last one ends, then HALT (the fragment loader's shape) */ static unsigned -g5p_up_loader_program(uint32_t *dest, unsigned n) +g5p_up_loader_program(uint32_t *dest, unsigned n, enum pdsc_feature features) { pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-vs-loader"); + pdsc_program *p = pdsc_program_create(NULL, features, "g5p-vs-loader"); for (unsigned i = 0; i < n; i++) pdsc_dout(p, pdsc_ref_const64(2 * i), pdsc_ref_const32(2 * n + 2 * i), pdsc_ref_null(), .dstdout = PDSC_DSTDOUT_D, .end = i == n - 1); @@ -58,12 +59,12 @@ g5p_up_loader_program(uint32_t *dest, unsigned n) static uint32_t g5p_up_doutd_control(unsigned common, unsigned dwords, bool last) { - pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-doutd"); + /* packing a field reads only the feature set: no program allocation per draw */ + static const pdsc_program pack = { .name = "g5p-doutd", .features = PDSC_FEATURE_DOUT_EXT }; + const pdsc_program *p = &pack; struct pdsc_doutd_src1 s = { .last_issue = last, .store_dest = PDSC_STORE_DEST_COMMON, .ao = common, .bsize = dwords }; const uint32_t v = pdsc_doutd_src1_pack(p, &s); - ralloc_free(p); return v; } @@ -134,7 +135,7 @@ PVR_PER_ARCH(g5p_vs_uploads_create)(struct pvr_device *device, const struct pco_ } uint32_t code[4 * PVR_G5P_MAX_STAGE_UPLOADS + 4]; - const unsigned n = g5p_up_loader_program(code, up->count); + const unsigned n = g5p_up_loader_program(code, up->count, PVR_G5P_PDS_FEATURES(&device->pdevice->dev_info)); VkResult result = pvr_gpu_upload_pds(device, NULL, 0, 0, code, n, 16, 64, &up->loader); if (result != VK_SUCCESS) { vk_free(&device->vk.alloc, up); diff --git a/src/imagination/vulkan/pvr_arch_g5p_vertex.c b/src/imagination/vulkan/pvr_arch_g5p_vertex.c index 3806bd02911..cc44bd237ed 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_vertex.c +++ b/src/imagination/vulkan/pvr_arch_g5p_vertex.c @@ -22,6 +22,7 @@ #include "pvr_cmd_buffer.h" #include "pvr_csb.h" #include "pvr_device.h" +#include "pvr_physical_device.h" #include "pvr_g5p.h" #include "pvr_pipeline.h" #include "pvr_g5p_vertex.h" @@ -85,14 +86,20 @@ PVR_PER_ARCH(g5p_vertex_index_list)(struct pvr_cmd_buffer *cmd_buffer, const str v1.cut_index = isize == 1 ? 0xff : isize == 2 ? 0xffff : 0xffffffff; } } + /* The A8 sends no split count, and index_size 0 when not indexed: + * AGXMetalA8 emits 0x6e000000 | topology, count, instances - 1, start + * (0x1a144b3c8, 0x1a144b484) and, indexed, 0x7e000000 | topology | index + * size (0x20000 / 0x40000) | address (0x1a144b684, 0x1a144b830). */ + const bool g4p = PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g4p); pvr_csb_pack (&w[(*n)++], VDMCTRL_INDEX_LIST0, l0) { l0.index_count_present = true; l0.index_instance_count_present = true; l0.index_offset_present = true; - l0.split_count_present = true; - l0.index_size = isize == 1 ? ROGUE_VDMCTRL_INDEX_SIZE_B8 - : isize == 2 ? ROGUE_VDMCTRL_INDEX_SIZE_B16 - : ROGUE_VDMCTRL_INDEX_SIZE_B32; + l0.split_count_present = !g4p; + l0.index_size = (g4p && !draw->indexed) ? ROGUE_VDMCTRL_INDEX_SIZE_B8 + : isize == 1 ? ROGUE_VDMCTRL_INDEX_SIZE_B8 + : isize == 2 ? ROGUE_VDMCTRL_INDEX_SIZE_B16 + : ROGUE_VDMCTRL_INDEX_SIZE_B32; l0.primitive_topology = hw_topology; if (draw->indexed) { l0.index_addr_present = true; @@ -109,7 +116,8 @@ PVR_PER_ARCH(g5p_vertex_index_list)(struct pvr_cmd_buffer *cmd_buffer, const str /* indexed: the vertex offset (added to every index, so the PDS's vertex * index is gl_VertexIndex); else the first vertex */ w[(*n)++] = draw->indexed ? (uint32_t)draw->vertex_offset : draw->first; - w[(*n)++] = split; + if (!g4p) + w[(*n)++] = split; return VK_SUCCESS; } diff --git a/src/imagination/vulkan/pvr_arch_g5p_vtex.c b/src/imagination/vulkan/pvr_arch_g5p_vtex.c index 2cf6c436306..737c7f83d6c 100644 --- a/src/imagination/vulkan/pvr_arch_g5p_vtex.c +++ b/src/imagination/vulkan/pvr_arch_g5p_vtex.c @@ -18,6 +18,7 @@ #include "util/u_dynarray.h" #include "util/u_math.h" #include "pco_g5p_general.h" +#include "g5p_usc.h" #include "pdsc.h" #include "pdsc_internal.h" #include "pdsc_isa_pack.h" @@ -25,8 +26,10 @@ #include "pvr_bo.h" #include "pvr_cmd_buffer.h" #include "pvr_csb.h" +#include "pvr_g5p_a8_fetch.h" #include "pvr_descriptor_set.h" #include "pvr_device.h" +#include "pvr_physical_device.h" #include "pvr_g5p_tex.h" #include "pvr_g5p_tex_encode.h" #include "pvr_g5p_vtex.h" @@ -40,12 +43,13 @@ static_assert(VTEX_COMMON >= PCO_G5P_COMMON_DWORDS, "vertex common-store plan bo static uint32_t vtex_control(unsigned common, unsigned dwords, bool last) { - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-vtex-control"); + /* packing a field reads only the feature set: no program allocation per draw */ + static const pdsc_program pack = { .name = "g5p-vtex-control", .features = PDSC_FEATURE_DOUT_EXT }; + const pdsc_program *p = &pack; struct pdsc_doutd_src1 s = { .last_issue = last, .store_dest = PDSC_STORE_DEST_COMMON, .ao = common, .bsize = dwords, }; uint32_t word = pdsc_doutd_src1_pack(p, &s); - ralloc_free(p); return word; } @@ -69,7 +73,7 @@ PVR_PER_ARCH(g5p_vtex_create)(struct pvr_device *device, const struct pco_g5p_pr up->count = plan.dmas; up->common_dwords = vs->common_dwords; pdsc_init(); - pdsc_program *p = pdsc_program_create(NULL, PDSC_FEATURE_DOUT_EXT, "g5p-vtex-loader"); + pdsc_program *p = pdsc_program_create(NULL, PVR_G5P_PDS_FEATURES(&device->pdevice->dev_info), "g5p-vtex-loader"); for (unsigned i = 0; i < plan.dmas; i++) pdsc_dout(p, pdsc_ref_const64(2*i), pdsc_ref_const32(2*plan.dmas + 2*i), pdsc_ref_null(), .dstdout = PDSC_DSTDOUT_D, .end = i == plan.dmas - 1); @@ -118,6 +122,26 @@ vtex_descriptor(struct pvr_cmd_buffer *cmd, const struct pco_g5p_upload *u, uint return VK_SUCCESS; } bool image = lb->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER || lb->type == VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE; + /* the A8: its Rogue words as the fragment stage takes them, four dwords of image state, four of + * sampler state at +32 in a combined descriptor */ + if (PVR_HAS_FEATURE(&cmd->device->pdevice->dev_info, apple_g4p)) { + if (u->kind == PCO_G5P_UPLOAD_IMAGE) { + if (!image || u->dwords != 4 || !pvr_g5p_tex_state_valid(desc)) + return VTEX_REFUSE(cmd, "set %u binding %u has no A8 image state", u->set, u->binding); + memcpy(out, desc, 16); + return VK_SUCCESS; + } + if (u->kind == PCO_G5P_UPLOAD_SAMPLER) { + if (lb->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER) desc += 8; + if (u->dwords != 4 || !pvr_g5p_sampler_valid(desc)) + return VTEX_REFUSE(cmd, "set %u binding %u has no A8 sampler state", u->set, u->binding); + memcpy(out, desc, 16); + /* a vertex texelFetch: the fetch sampler, Apple's read sampler words (an iOS capture) */ + if (u->offset & PCO_G5P_UPLOAD_SAMPLER_FETCH) + pvr_g5p_a8_fetch_sampler(out, desc, (u->offset >> PCO_G5P_UPLOAD_SAMPLER_FETCH_LEVEL_SHIFT) & 0xf); + return VK_SUCCESS; + } + } if (u->kind == PCO_G5P_UPLOAD_IMAGE) { if (!image || u->dwords != 8 || !pvr_g5p_tex_state_valid(desc)) return VTEX_REFUSE(cmd, "set %u binding %u has no A9 image state", u->set, u->binding); @@ -160,6 +184,12 @@ PVR_PER_ARCH(g5p_vtex_draw)(struct pvr_cmd_buffer *cmd, const struct pvr_g5p_sta if (result != VK_SUCCESS) return result; break; case PCO_G5P_UPLOAD_FETCH_SAMPLER: { + if (u->dwords == 4 && PVR_HAS_FEATURE(&cmd->device->pdevice->dev_info, apple_g4p)) { + /* the A8: Rogue sampler words, point, non-normalized, clamped, level 0: Apple's read sampler, the constant + * 0x0202240000000fff of its vertex and fragment reads (from an iOS capture) */ + pvr_g5p_a8_fetch_sampler(dst, NULL, 0); + break; + } if (u->dwords != 2) return VTEX_REFUSE(cmd, "invalid samplerless read policy size"); uint64_t policy = pvr_g5p_fetch_sampler_word(); memcpy(dst, &policy, sizeof(policy)); diff --git a/src/imagination/vulkan/pvr_arch_image.c b/src/imagination/vulkan/pvr_arch_image.c index fb4be87c5a8..5aea1705e63 100644 --- a/src/imagination/vulkan/pvr_arch_image.c +++ b/src/imagination/vulkan/pvr_arch_image.c @@ -21,6 +21,9 @@ #include "pvr_physical_device.h" #include "pvr_tex_state.h" #include "pvr_ycbcr.h" +#include "g5p_texel_defs.h" +#include "util/log.h" +#include "vk_enum_to_str.h" static void pvr_adjust_non_compressed_view(const struct pvr_image *image, const struct pvr_image_plane *plane, @@ -203,7 +206,10 @@ VkResult PVR_PER_ARCH(CreateImageView)(VkDevice _device, util_format_compose_swizzles(format_swizzle, input_swizzle, info.swizzle); - info.skip_rogue_words = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g5p) || + /* The A8's TPU takes Rogue's texture state (AGXMetalA8's TEXSTATE_IMAGE_WORD0/1 in the a8-passcap load pass: + * 60008861 003f00fc 40000001 0000403d), so its views keep the Rogue words; the A9 and A10 have their own */ + info.skip_rogue_words = (PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g5p) && + !PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p)) || PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g9p); result = pvr_arch_pack_tex_state(device, &info, @@ -222,7 +228,8 @@ VkResult PVR_PER_ARCH(CreateImageView)(VkDevice _device, */ if (info.is_cube && image->vk.usage & VK_IMAGE_USAGE_STORAGE_BIT) { info.tex_state_type = PVR_TEXTURE_STATE_STORAGE; - info.skip_rogue_words = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g5p); + info.skip_rogue_words = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g5p) && + !PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p); result = pvr_arch_pack_tex_state(device, @@ -344,9 +351,31 @@ PVR_PER_ARCH(CreateBufferView)(VkDevice _device, format_swizzle = pvr_get_format_swizzle(info.format); memcpy(info.swizzle, format_swizzle, sizeof(info.swizzle)); + /* the A8: the A9's texel-buffer contract (g5p_texel_defs.h) over Rogue's state, a plain linear 2D image + * PVR_G5P_TBO_WIDTH texels wide (no index lookup), which the program reads at the folded coordinate as a 2D + * fetch; the element count and the marker in the descriptor's software words (pvr_g5p_texel_buffer.c) */ + const bool a8 = PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p); + if (a8) { + info.extent.width = PVR_G5P_TBO_WIDTH; + info.extent.height = DIV_ROUND_UP(bview->vk.elements, PVR_G5P_TBO_WIDTH); + info.stride = PVR_G5P_TBO_WIDTH; + info.flags = 0; + if (info.addr.addr % PVR_G5P_TBO_ALIGN || !bview->vk.elements || bview->vk.elements > PVR_G5P_TBO_LIMIT) { + mesa_loge("G5P refuses: %s texel buffer: base %% 64 = %u, %u elements (64-byte base, 1..65536)", + vk_Format_to_str(info.format), (unsigned)(info.addr.addr % PVR_G5P_TBO_ALIGN), + (unsigned)bview->vk.elements); + result = vk_error(device, VK_ERROR_FORMAT_NOT_SUPPORTED); + goto err_vk_buffer_view_destroy; + } + } + result = pvr_arch_pack_tex_state(device, &info, &bview->image_state); if (result != VK_SUCCESS) goto err_vk_buffer_view_destroy; + if (a8) { + bview->image_state.meta[2] = bview->vk.elements; + bview->image_state.meta[3] = PVR_G5P_TBO_MAGIC; + } *pView = pvr_buffer_view_to_handle(bview); diff --git a/src/imagination/vulkan/pvr_arch_job_render.c b/src/imagination/vulkan/pvr_arch_job_render.c index 26b622787f1..c425d257522 100644 --- a/src/imagination/vulkan/pvr_arch_job_render.c +++ b/src/imagination/vulkan/pvr_arch_job_render.c @@ -500,6 +500,50 @@ pvr_rt_rgn_headers_data_init(struct pvr_device *device, return VK_SUCCESS; } +/* + * The A8's region headers (AGXG4P): one ring per render target, not a buffer + * per frame. Its size is the power of two >= 16 x the per-frame bytes (the region headers rounded up to 0x1000; the + * kext's second per-frame part, RTM +0x30, has no counterpart here and is 0), it lives in the general (0x1_) heap, + * and it is mapped twice back to back (kext 0x847d738 maps a flag-0x200000 resource at VA and VA + length), so a + * frame's slice may run past the ring's end. The kernel picks each frame's slice (RTM +0x1a0: the first frame at + * offset 0, then + per-frame bytes): both render-target datas name the ring's base. Allocated before the TPC, which + * Apple's allocation order puts after the ring (P5b: ring 0x1_00460000, TPC 0x1_00484000). + */ +static VkResult pvr_rt_a8_rgn_ring_init(struct pvr_device *device, + struct pvr_rt_dataset *rt_dataset, + const struct pvr_rt_mtile_info *mtile_info, + uint32_t width, + uint32_t height, + uint32_t samples, + uint32_t layers) +{ + uint64_t rgn_headers_size; + + pvr_rt_get_region_headers_stride_size(device, mtile_info, layers, &rt_dataset->rgn_headers_stride, + &rgn_headers_size); + /* The region headers of Apple's record (the A8/A9 kernel's s8003_rt_derive(), which refuses a smaller ring): + * five bytes per tile of the padded tile grid, 32x32 tiles in macrotiles of 4, the grid in multiples of 4 + * macrotiles, twice as wide and high at 4 samples */ + const uint32_t mx = ALIGN_POT((DIV_ROUND_UP(width, 32) + 3) >> 2, 4); + const uint32_t my = ALIGN_POT((DIV_ROUND_UP(height, 32) + 3) >> 2, 4); + const uint32_t s = samples == 4 ? 2 : 1; + const uint64_t apple_headers = (uint64_t)(mx * s * 4) * (my * s * 4) * 5 * layers; + const uint64_t frame = ALIGN_POT(MAX2(rgn_headers_size, apple_headers), 0x1000); + const uint64_t ring = util_next_power_of_two64(16 * frame); + + VkResult result = pvr_bo_alloc_mirrored(device, device->heaps.general_heap, ring, + MAX2(ROGUE_CR_TE_PSGREGION_ADDR_BASE_ALIGNMENT, 0x1000), + PVR_BO_ALLOC_FLAG_GPU_UNCACHED, &rt_dataset->rgn_headers_bo); + if (result != VK_SUCCESS) + return result; + + rt_dataset->a8_rgn_frame_size = frame; + rt_dataset->a8_rgn_ring_size = ring; + for (uint32_t i = 0; i < ARRAY_SIZE(rt_dataset->rt_datas); i++) + rt_dataset->rt_datas[i].rgn_headers_dev_addr = rt_dataset->rgn_headers_bo->vma->dev_addr; + return VK_SUCCESS; +} + static VkResult pvr_rt_datas_init(struct pvr_device *device, struct pvr_rt_dataset *rt_dataset, const struct pvr_free_list *global_free_list, @@ -520,8 +564,9 @@ static VkResult pvr_rt_datas_init(struct pvr_device *device, if (result != VK_SUCCESS) goto err_pvr_rt_mta_data_fini; - result = - pvr_rt_rgn_headers_data_init(device, rt_dataset, mtile_info, layers); + /* the A8's ring is allocated already (pvr_rt_a8_rgn_ring_init()) */ + result = rt_dataset->rgn_headers_bo ? VK_SUCCESS + : pvr_rt_rgn_headers_data_init(device, rt_dataset, mtile_info, layers); if (result != VK_SUCCESS) goto err_pvr_rt_mlist_data_fini; @@ -699,6 +744,12 @@ VkResult pvr_arch_render_target_dataset_create( if (result != VK_SUCCESS) goto err_pvr_free_list_destroy; + if (PVR_HAS_FEATURE(dev_info, apple_g4p)) { + result = pvr_rt_a8_rgn_ring_init(device, rt_dataset, &mtile_info, width, height, samples, layers); + if (result != VK_SUCCESS) + goto err_pvr_rt_vheap_rtc_data_fini; + } + result = pvr_rt_tpc_data_init(device, rt_dataset, &mtile_info, layers); if (result != VK_SUCCESS) goto err_pvr_rt_vheap_rtc_data_fini; @@ -727,6 +778,24 @@ VkResult pvr_arch_render_target_dataset_create( if (result != VK_SUCCESS) goto err_pvr_rt_datas_fini; + struct pvr_bo *cache_bos[] = { + rt_dataset->vheap_rtc_bo, + rt_dataset->tpc_bo, + rt_dataset->rgn_headers_bo, + rt_dataset->mlist_bo, + rt_dataset->mta_bo, + }; + + for (uint32_t i = 0; i < ARRAY_SIZE(cache_bos); i++) { + if (cache_bos[i]) + rt_dataset->cache_bytes += cache_bos[i]->bo->size; + } + + if (rt_dataset->local_free_list->bo && + rt_dataset->local_free_list != device->g9p.free_list && + !(device->g5p && rt_dataset->local_free_list == device->g5p->free_list)) + rt_dataset->cache_bytes += rt_dataset->local_free_list->bo->bo->size; + *rt_dataset_out = rt_dataset; if (PVR_HAS_FEATURE(dev_info, apple_g9p) && getenv("G9P_SYNC")) { static unsigned n; @@ -744,6 +813,9 @@ VkResult pvr_arch_render_target_dataset_create( pvr_rt_tpc_data_fini(rt_dataset); err_pvr_rt_vheap_rtc_data_fini: + /* the A8's ring when the TPC or the render-target datas failed (pvr_rt_datas_fini() clears it otherwise) */ + pvr_bo_free(device, rt_dataset->rgn_headers_bo); + rt_dataset->rgn_headers_bo = NULL; pvr_rt_vheap_rtc_data_fini(rt_dataset); err_pvr_free_list_destroy: diff --git a/src/imagination/vulkan/pvr_arch_queue.c b/src/imagination/vulkan/pvr_arch_queue.c index 4b230c0b9a6..0253cf20a18 100644 --- a/src/imagination/vulkan/pvr_arch_queue.c +++ b/src/imagination/vulkan/pvr_arch_queue.c @@ -30,6 +30,7 @@ */ #include "util/os_time.h" +#include "pvr_bo_log.h" #include "pvr_g9p.h" #include "pvr_queue.h" #include "pvr_rt_dataset.h" @@ -1723,6 +1724,9 @@ static VkResult pvr_driver_queue_submit(struct vk_queue *queue, struct pvr_queue *driver_queue = container_of(queue, struct pvr_queue, vk); const bool is_g5p = PVR_HAS_FEATURE(&driver_queue->device->pdevice->dev_info, apple_g5p); + /* bo_log's periodic summary also for a client whose BO set is steady (a desktop shell) */ + if (unlikely(driver_queue->device->bo_log)) + pvr_bo_log_tick(driver_queue->device); if (is_g5p && vk_queue_is_lost(queue)) { for (uint32_t i = 0; i < submit->signal_count; i++) diff --git a/src/imagination/vulkan/pvr_arch_tex_state.c b/src/imagination/vulkan/pvr_arch_tex_state.c index 662aa274d90..2db9f66e003 100644 --- a/src/imagination/vulkan/pvr_arch_tex_state.c +++ b/src/imagination/vulkan/pvr_arch_tex_state.c @@ -111,6 +111,15 @@ static uint32_t setup_pck_info(VkFormat vk_format) return pck_info; } +/* The A8 (AGXMetalA8, every texture state of the a8-passcap tex_* and load passes: BGRA8 and RGBA8, one level and + * mipmapped) leaves alpha_msb clear for 8-bit-per-channel formats, where upstream sets it for any alpha-in-W format */ +static bool pvr_tex_state_a8_no_alpha_msb(const struct pvr_device_info *dev_info, VkFormat format) +{ + const struct util_format_description *desc = vk_format_description(format); + return PVR_HAS_FEATURE(dev_info, apple_g4p) && desc->nr_channels == 4 && desc->block.bits == 32 && + desc->channel[0].size == 8 && desc->channel[3].size == 8; +} + VkResult pvr_arch_pack_tex_state(struct pvr_device *device, const struct pvr_texture_state_info *info, struct pvr_image_descriptor *state) @@ -254,7 +263,7 @@ VkResult pvr_arch_pack_tex_state(struct pvr_device *device, word1.texaddr = PVR_DEV_ADDR_OFFSET(info->addr, info->offset); - if (vk_format_is_alpha_on_msb(info->format)) + if (vk_format_is_alpha_on_msb(info->format) && !pvr_tex_state_a8_no_alpha_msb(dev_info, info->format)) word1.alpha_msb = true; if (!PVR_HAS_FEATURE(dev_info, tpu_extended_integer_lookup) && @@ -275,7 +284,10 @@ VkResult pvr_arch_pack_tex_state(struct pvr_device *device, } else { pvr_csb_pack (&state->words[1], TEXSTATE_IMAGE_WORD1, word1) { word1.num_mip_levels = info->mip_levels; - word1.mipmaps_present = info->mipmaps_present; + /* the A8 sets mipmaps_present only for a state of several levels (AGXMetalA8: 0 for the one-level load + * target, 1 with seven levels in tex_plain); a one-level view of a mipmapped image is one level to it */ + word1.mipmaps_present = info->mipmaps_present && + (!PVR_HAS_FEATURE(dev_info, apple_g4p) || info->mip_levels > 1); word1.baselevel = info->base_level; if (iview_type == VK_IMAGE_VIEW_TYPE_3D || @@ -287,7 +299,10 @@ VkResult pvr_arch_pack_tex_state(struct pvr_device *device, } else { uint32_t array_layers = info->array_size; - if (iview_type == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY) + /* the A8 counts cubes, not faces, for a cube view as well (AGXMetalA8's tex_cube state: word1 depth 0 + * for one 6-face cube, passcap round 1: 60008863 003f00fc 30008007 000040b0) */ + if (iview_type == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY || + (iview_type == VK_IMAGE_VIEW_TYPE_CUBE && PVR_HAS_FEATURE(dev_info, apple_g4p))) array_layers /= 6; if (array_layers > 0) @@ -309,7 +324,7 @@ VkResult pvr_arch_pack_tex_state(struct pvr_device *device, if (info->flags & PVR_TEXFLAGS_BORDER) word1.border = true; - if (vk_format_is_alpha_on_msb(info->format)) + if (vk_format_is_alpha_on_msb(info->format) && !pvr_tex_state_a8_no_alpha_msb(dev_info, info->format)) word1.alpha_msb = true; if (PVR_HAS_FEATURE(dev_info, tpu_image_state_v2) && diff --git a/src/imagination/vulkan/pvr_bo.c b/src/imagination/vulkan/pvr_bo.c index e4bf38f2f6a..72fe492da42 100644 --- a/src/imagination/vulkan/pvr_bo.c +++ b/src/imagination/vulkan/pvr_bo.c @@ -322,6 +322,50 @@ static inline void pvr_bo_free_bo(const struct pvr_device *const device, vk_free(&device->vk.alloc, ptr); } +VkResult pvr_bo_alloc_mirrored(struct pvr_device *device, + struct pvr_winsys_heap *heap, + uint64_t size, + uint64_t alignment, + uint64_t flags, + struct pvr_bo **const pvr_bo_out) +{ + struct pvr_bo *pvr_bo; + VkResult result; + + if (!device->ws->ops->vma_map_mirror || (flags & PVR_BO_ALLOC_FLAG_CPU_MAPPED)) + return vk_error(device, VK_ERROR_FEATURE_NOT_PRESENT); + + pvr_bo = pvr_bo_alloc_bo(device); + if (!pvr_bo) + return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + pvr_bo->ref_count = 1; + + result = device->ws->ops->buffer_create(device->ws, size, alignment, PVR_WINSYS_BO_TYPE_GPU, + pvr_bo_alloc_to_winsys_flags(flags), &pvr_bo->bo); + if (result != VK_SUCCESS) + goto err_free_bo; + + result = device->ws->ops->heap_alloc(heap, 2 * pvr_bo->bo->size, alignment, &pvr_bo->vma); + if (result != VK_SUCCESS) + goto err_buffer_destroy; + + result = device->ws->ops->vma_map_mirror(pvr_bo->vma, pvr_bo->bo); + if (result != VK_SUCCESS) + goto err_heap_free; + + pvr_bo_store_insert(device->bo_store, pvr_bo); + *pvr_bo_out = pvr_bo; + return VK_SUCCESS; + +err_heap_free: + device->ws->ops->heap_free(pvr_bo->vma); +err_buffer_destroy: + device->ws->ops->buffer_destroy(pvr_bo->bo); +err_free_bo: + pvr_bo_free_bo(device, pvr_bo); + return result; +} + /** * \brief Helper interface to allocate a GPU buffer and map it to both host and * device virtual memory. Host mapping is conditional and is controlled by @@ -558,6 +602,32 @@ static inline uint64_t pvr_suballoc_cache_max(const struct pvr_suballocator *a) return 2 * (uint64_t)a->default_size; } +/* The caller holds allocator->mtx. Logging can run from pvr_bo_alloc/free + * under that same mutex, so publish counters atomically for its snapshots. + */ +static void pvr_suballoc_log_bo_add(struct pvr_suballocator *allocator, + const struct pvr_bo *bo) +{ + if (likely(!allocator->device->bo_log)) + return; + + p_atomic_inc(&allocator->live_bos); + p_atomic_add(&allocator->live_bo_bytes, bo->bo->size); + p_atomic_set(&allocator->peak_bo_bytes, + MAX2(p_atomic_read(&allocator->peak_bo_bytes), + p_atomic_read(&allocator->live_bo_bytes))); +} + +static void pvr_suballoc_log_bo_remove(struct pvr_suballocator *allocator, + const struct pvr_bo *bo) +{ + if (likely(!allocator->device->bo_log)) + return; + + p_atomic_dec(&allocator->live_bos); + p_atomic_add(&allocator->live_bo_bytes, -bo->bo->size); +} + /** * \brief Interface to sub-allocate buffer objects. * @@ -618,6 +688,10 @@ VkResult pvr_bo_suballoc(struct pvr_suballocator *allocator, return result; } + pvr_suballoc_log_bo_add(allocator, bo); + if (unlikely(allocator->device->bo_log)) + p_atomic_add(&allocator->live_bytes, aligned_size); + suballoc_bo->allocator = allocator; suballoc_bo->bo = bo; suballoc_bo->dev_addr = bo->vma->dev_addr; @@ -636,6 +710,14 @@ VkResult pvr_bo_suballoc(struct pvr_suballocator *allocator, if (allocator->bo) { uint32_t aligned_offset = ALIGN_POT(allocator->next_offset, align); + /* Every sub-allocation of the current buffer has been freed: start it + * over instead of freeing it and creating another one. + */ + if (aligned_offset + aligned_size > allocator->bo->bo->size && + aligned_size <= allocator->bo->bo->size && + p_atomic_read(&allocator->bo->ref_count) == 1) + aligned_offset = 0; + if (aligned_offset + aligned_size <= allocator->bo->bo->size) { suballoc_bo->allocator = allocator; suballoc_bo->bo = pvr_bo_get_ref(allocator->bo); @@ -645,6 +727,8 @@ VkResult pvr_bo_suballoc(struct pvr_suballocator *allocator, suballoc_bo->size = aligned_size; allocator->next_offset = aligned_offset + aligned_size; + if (unlikely(allocator->device->bo_log)) + p_atomic_add(&allocator->live_bytes, aligned_size); if (zero_on_alloc) memset(pvr_bo_suballoc_get_map_addr(suballoc_bo), 0, aligned_size); @@ -654,6 +738,8 @@ VkResult pvr_bo_suballoc(struct pvr_suballocator *allocator, return VK_SUCCESS; } else { + if (p_atomic_read(&allocator->bo->ref_count) == 1) + pvr_suballoc_log_bo_remove(allocator, allocator->bo); pvr_bo_free(allocator->device, allocator->bo); allocator->bo = NULL; } @@ -664,10 +750,16 @@ VkResult pvr_bo_suballoc(struct pvr_suballocator *allocator, if (allocator->bo_cached) { struct pvr_winsys_bo *bo_cached = allocator->bo_cached->bo; - if (alloc_size <= bo_cached->size) + bool retained = true; + result = VK_SUCCESS; + if (allocator->device->ws->ops->buffer_set_purgeable) + result = allocator->device->ws->ops->buffer_set_purgeable(bo_cached, false, &retained); + if (result == VK_SUCCESS && retained && alloc_size <= bo_cached->size) { allocator->bo = allocator->bo_cached; - else + pvr_suballoc_log_bo_add(allocator, allocator->bo); + } else { pvr_bo_free(allocator->device, allocator->bo_cached); + } allocator->bo_cached = NULL; } @@ -684,8 +776,12 @@ VkResult pvr_bo_suballoc(struct pvr_suballocator *allocator, simple_mtx_unlock(&allocator->mtx); return result; } + pvr_suballoc_log_bo_add(allocator, allocator->bo); } + if (unlikely(allocator->device->bo_log)) + p_atomic_add(&allocator->live_bytes, aligned_size); + suballoc_bo->allocator = allocator; suballoc_bo->bo = pvr_bo_get_ref(allocator->bo); suballoc_bo->dev_addr = allocator->bo->vma->dev_addr; @@ -717,10 +813,21 @@ void pvr_bo_suballoc_free(struct pvr_suballoc_bo *suballoc_bo) simple_mtx_lock(&suballoc_bo->allocator->mtx); + if (unlikely(suballoc_bo->allocator->device->bo_log)) + p_atomic_add(&suballoc_bo->allocator->live_bytes, -(uint64_t)suballoc_bo->size); + if (p_atomic_read(&suballoc_bo->bo->ref_count) == 1) + pvr_suballoc_log_bo_remove(suballoc_bo->allocator, suballoc_bo->bo); + if (p_atomic_read(&suballoc_bo->bo->ref_count) == 1 && !suballoc_bo->allocator->bo_cached && suballoc_bo->bo->bo->size <= pvr_suballoc_cache_max(suballoc_bo->allocator)) { suballoc_bo->allocator->bo_cached = suballoc_bo->bo; + if (suballoc_bo->allocator->device->ws->ops->buffer_set_purgeable) { + bool retained; + VkResult ignored = suballoc_bo->allocator->device->ws->ops->buffer_set_purgeable( + suballoc_bo->bo->bo, true, &retained); + (void)ignored; /* The winsys warns once if DONTNEED fails. */ + } } else { pvr_bo_free(suballoc_bo->allocator->device, suballoc_bo->bo); } diff --git a/src/imagination/vulkan/pvr_bo.h b/src/imagination/vulkan/pvr_bo.h index 941f98b9e2b..eba862620f7 100644 --- a/src/imagination/vulkan/pvr_bo.h +++ b/src/imagination/vulkan/pvr_bo.h @@ -70,6 +70,15 @@ struct pvr_suballocator { /* Mutex to protect access to all of the members below this point */ simple_mtx_t mtx; + /* Updated under mtx; atomic accesses let BO logging sample these without + * taking mtx while the log mutex is held (alloc/free can already hold mtx). + * Empty cached BOs are excluded until reused. + */ + uint32_t live_bos; + uint64_t live_bo_bytes; + uint64_t live_bytes; + uint64_t peak_bo_bytes; + /* Current buffer object where sub-allocations are made from */ struct pvr_bo *bo; /* Previous buffer that can be used when a new buffer object is needed */ @@ -125,6 +134,14 @@ VkResult pvr_bo_alloc_tagged(const char *tag, uint64_t flags, struct pvr_bo **const bo_out); #define pvr_bo_alloc(...) pvr_bo_alloc_tagged(__func__, __VA_ARGS__) +/* A GPU-only buffer of size bytes mapped twice back to back (at vma->dev_addr and vma->dev_addr + size, the vma + * 2 * size long): winsys vma_map_mirror(). pvr_bo_free() releases it. */ +VkResult pvr_bo_alloc_mirrored(struct pvr_device *device, + struct pvr_winsys_heap *heap, + uint64_t size, + uint64_t alignment, + uint64_t flags, + struct pvr_bo **const pvr_bo_out); VkResult pvr_bo_cpu_map(struct pvr_device *device, struct pvr_bo *bo); void pvr_bo_cpu_unmap(struct pvr_device *device, struct pvr_bo *bo); diff --git a/src/imagination/vulkan/pvr_bo_log.c b/src/imagination/vulkan/pvr_bo_log.c index 3875cbb2f4f..d8da16ddd70 100644 --- a/src/imagination/vulkan/pvr_bo_log.c +++ b/src/imagination/vulkan/pvr_bo_log.c @@ -17,6 +17,7 @@ #include "pvr_image.h" #include "util/hash_table.h" #include "util/list.h" +#include "util/os_time.h" #include "util/simple_mtx.h" #include "util/u_math.h" #include "util/log.h" @@ -51,6 +52,8 @@ struct pvr_bo_log_binding { struct pvr_bo_log { struct list_head link; simple_mtx_t mutex; + struct pvr_device *device; + uint64_t last_summary_ns; struct hash_table *entries; struct hash_table *bindings; uint64_t min_size; @@ -63,11 +66,14 @@ struct pvr_bo_log { uint64_t peak; uint64_t failed; uint64_t tracking_failed; + uint64_t zls_passes, zls_load, zls_store, zls_neither; + struct { uint32_t w, h; uint64_t passes, load, store; } zls_size[8]; /* the first eight attachment sizes */ }; static struct list_head logs = { &logs, &logs }; static simple_mtx_t logs_mutex = SIMPLE_MTX_INITIALIZER; DEBUG_GET_ONCE_NUM_OPTION(pvr_bo_log_min_kib, "PVR_BO_LOG_MIN_KIB", 64) +DEBUG_GET_ONCE_NUM_OPTION(pvr_bo_log_period_s, "PVR_BO_LOG_PERIOD_S", 0) static const char *thread_name(char buf[17]) { @@ -165,6 +171,30 @@ static void summary_locked(struct pvr_bo_log *log, const char *why, bool tables) return; } + const struct { + const char *name; + struct pvr_suballocator *allocator; + } suballocators[] = { + { "general", &log->device->suballoc_general }, + { "pds", &log->device->suballoc_pds }, + { "transfer", &log->device->suballoc_transfer }, + { "usc", &log->device->suballoc_usc }, + { "vis_test", &log->device->suballoc_vis_test }, + { "cmd_general", &log->device->suballoc_cmd_general }, + { "cmd_pds", &log->device->suballoc_cmd_pds }, + { "cmd_usc", &log->device->suballoc_cmd_usc }, + }; + for (unsigned i = 0; i < ARRAY_SIZE(suballocators); i++) { + const struct pvr_suballocator *a = suballocators[i].allocator; + mesa_logi("pvr bo summary-suballoc name=%s bos=%u bo_kib=%" PRIu64 + " live_kib=%" PRIu64 " peak_bo_kib=%" PRIu64, + suballocators[i].name, + p_atomic_read(&a->live_bos), + p_atomic_read(&a->live_bo_bytes) >> 10, + p_atomic_read(&a->live_bytes) >> 10, + p_atomic_read(&a->peak_bo_bytes) >> 10); + } + uint64_t images = 0, total = 0, tight = 0; hash_table_foreach (log->bindings, h) { const struct pvr_bo_log_binding *b = h->data; @@ -174,6 +204,13 @@ static void summary_locked(struct pvr_bo_log *log, const char *why, bool tables) tight += b->tight; } } + mesa_logi("pvr bo summary-zls passes=%" PRIu64 " load=%" PRIu64 " store=%" PRIu64 " neither=%" PRIu64 + " (renders with a depth/stencil attachment; neither: memory a lazy allocation would never commit)\n", + log->zls_passes, log->zls_load, log->zls_store, log->zls_neither); + for (unsigned i = 0; i < ARRAY_SIZE(log->zls_size) && log->zls_size[i].passes; i++) + mesa_logi("pvr bo summary-zls-size %ux%u passes=%" PRIu64 " load=%" PRIu64 " store=%" PRIu64 "\n", + log->zls_size[i].w, log->zls_size[i].h, log->zls_size[i].passes, log->zls_size[i].load, + log->zls_size[i].store); mesa_logi("pvr bo summary-pad images=%" PRIu64 " total_kib=%" PRIu64 " tight_kib=%" PRIu64 " pad_kib=%" PRId64 "\n", images, @@ -265,6 +302,51 @@ static void summary_locked(struct pvr_bo_log *log, const char *why, bool tables) } +static void summary_period_locked(struct pvr_bo_log *log) +{ + if (!PVR_IS_DEBUG_SET(BO_LOG)) + return; + + const int64_t period_s = debug_get_option_pvr_bo_log_period_s(); + if (period_s <= 0) + return; + + const uint64_t now = os_time_get_nano(); + if ((now - log->last_summary_ns) / 1000000000ULL >= (uint64_t)period_s) { + summary_locked(log, "period", true); + log->last_summary_ns = now; + } +} + +void pvr_bo_log_zls(struct pvr_device *device, uint32_t w, uint32_t h, bool load, bool store) +{ + struct pvr_bo_log *log = device->bo_log; + simple_mtx_lock(&log->mutex); + for (unsigned i = 0; i < ARRAY_SIZE(log->zls_size); i++) { + if (log->zls_size[i].passes && (log->zls_size[i].w != w || log->zls_size[i].h != h)) + continue; + log->zls_size[i].w = w; + log->zls_size[i].h = h; + log->zls_size[i].passes++; + log->zls_size[i].load += load; + log->zls_size[i].store += store; + break; + } + log->zls_passes++; + log->zls_load += load; + log->zls_store += store; + log->zls_neither += !load && !store; + simple_mtx_unlock(&log->mutex); +} + +void pvr_bo_log_tick(struct pvr_device *device) +{ + struct pvr_bo_log *log = device->bo_log; + simple_mtx_lock(&log->mutex); + summary_period_locked(log); + simple_mtx_unlock(&log->mutex); +} + void pvr_bo_log_summary(struct pvr_device *device, const char *why) { struct pvr_bo_log *log = device->bo_log; @@ -289,6 +371,8 @@ void pvr_bo_log_device_init(struct pvr_device *device) free(log); return; } + log->device = device; + log->last_summary_ns = os_time_get_nano(); log->min_size = MAX2(0, debug_get_option_pvr_bo_log_min_kib()); log->min_size = MIN2(log->min_size, UINT64_MAX / 1024) * 1024; simple_mtx_init(&log->mutex, mtx_plain); @@ -372,6 +456,7 @@ static void record_alloc(struct pvr_bo_log *log, summary_locked(log, "peak", false); if (result != VK_SUCCESS && log->failed <= 3) summary_locked(log, "alloc-failed", true); + summary_period_locked(log); simple_mtx_unlock(&log->mutex); } @@ -479,6 +564,7 @@ void pvr_bo_log_free(struct pvr_device *device, const void *key) free(e); _mesa_hash_table_remove(log->entries, h); } + summary_period_locked(log); simple_mtx_unlock(&log->mutex); } diff --git a/src/imagination/vulkan/pvr_bo_log.h b/src/imagination/vulkan/pvr_bo_log.h index 28606747031..759fc13188d 100644 --- a/src/imagination/vulkan/pvr_bo_log.h +++ b/src/imagination/vulkan/pvr_bo_log.h @@ -2,6 +2,7 @@ #ifndef PVR_BO_LOG_H #define PVR_BO_LOG_H +#include #include #include @@ -15,6 +16,10 @@ struct pvr_winsys_heap; void pvr_bo_log_device_init(struct pvr_device *device); void pvr_bo_log_device_finish(struct pvr_device *device); void pvr_bo_log_summary(struct pvr_device *device, const char *why); +/* PVR_BO_LOG_PERIOD_S's summary when the period has passed (called per queue submission) */ +void pvr_bo_log_tick(struct pvr_device *device); +/* a render with a depth/stencil attachment, and whether its ZLS loads or stores it */ +void pvr_bo_log_zls(struct pvr_device *device, uint32_t w, uint32_t h, bool load, bool store); void pvr_bo_log_driver_alloc(struct pvr_device *device, const char *tag, struct pvr_winsys_heap *heap, diff --git a/src/imagination/vulkan/pvr_device.h b/src/imagination/vulkan/pvr_device.h index 022b999dc3a..d28e7d32198 100644 --- a/src/imagination/vulkan/pvr_device.h +++ b/src/imagination/vulkan/pvr_device.h @@ -168,6 +168,17 @@ struct pvr_device { struct pvr_suballocator suballoc_usc; struct pvr_suballocator suballoc_vis_test; + /* Command buffer uploads (general, PDS and USC heaps). They live until + * their command buffer is reset, so they are kept apart from pipelines, + * descriptor pools and queries, which live much longer: mixing the two + * lets one long-lived allocation keep a whole buffer of freed uploads. + * PVR_CMD_SUBALLOC=0 shares the allocators above instead. + */ + bool cmd_suballoc; + struct pvr_suballocator suballoc_cmd_general; + struct pvr_suballocator suballoc_cmd_pds; + struct pvr_suballocator suballoc_cmd_usc; + struct { struct pvr_pds_upload pds; struct pvr_suballoc_bo *usc; @@ -258,6 +269,7 @@ struct pvr_device { simple_mtx_t rt_cache_mtx; struct list_head rt_dataset_cache; uint32_t rt_dataset_cache_count; + uint64_t rt_dataset_cache_bytes; /* Data Source for GPU tracing */ struct pvr_ds_device ds; @@ -291,6 +303,25 @@ VkResult pvr_sync_wait_many(struct vk_device *vk, uint32_t count, VkResult pvr_device_tile_buffer_ensure_cap(struct pvr_device *device, uint32_t capacity); +/* The allocator a command buffer upload takes in place of a device-lifetime + * one: general, PDS and USC uploads go to the command buffer allocators while + * they are enabled; every other heap keeps its allocator. + */ +static inline struct pvr_suballocator * +pvr_cmd_suballocator(struct pvr_device *device, + struct pvr_suballocator *shared) +{ + if (!device->cmd_suballoc) + return shared; + if (shared == &device->suballoc_general) + return &device->suballoc_cmd_general; + if (shared == &device->suballoc_pds) + return &device->suballoc_cmd_pds; + if (shared == &device->suballoc_usc) + return &device->suballoc_cmd_usc; + return shared; +} + static inline void pvr_device_free_tile_buffer_state(struct pvr_device *device) { for (uint32_t i = 0; i < device->tile_buffer_state.buffer_count; i++) diff --git a/src/imagination/vulkan/pvr_g5p.h b/src/imagination/vulkan/pvr_g5p.h index d0b6151cd6d..5a58ce35576 100644 --- a/src/imagination/vulkan/pvr_g5p.h +++ b/src/imagination/vulkan/pvr_g5p.h @@ -27,26 +27,69 @@ static inline VkResult pvr_refusal_to_api(VkResult result) return result == VK_ERROR_FEATURE_NOT_PRESENT ? VK_ERROR_OUT_OF_HOST_MEMORY : result; } +#include #include "util/log.h" #include "util/u_atomic.h" #include "vk_log.h" +/* Whether a refusal's text is new at its call site: a small table of text hashes (racing threads can at worst + * print one twice); a full table leaves only the count rule below. */ +static inline bool pvr_g5p_refusal_new(uint32_t *seen, unsigned n, const char *text) +{ + uint32_t h = 2166136261u; + for (const char *p = text; *p; p++) + h = (h ^ (uint8_t)*p) * 16777619u; + h |= !h; + for (unsigned i = 0; i < n; i++) { + const uint32_t v = p_atomic_read(&seen[i]); + if (v == h) + return false; + if (!v && p_atomic_cmpxchg(&seen[i], 0, h) == 0) + return true; + } + return false; +} + /* Refusal kinds are keyed by their call site (pipeline validation or pass/draw - * validation). Count atomically across recording threads; warn on occurrences - * 1, 2, 4, ... so tolerant workloads cannot emit one warning per draw. Keep - * vk_errorf for debug callbacks and preserve the internal refusal result. */ + * validation). Every distinct reason is printed once; a repeated one on + * occurrences 1, 2, 4, ... of its call site, so tolerant workloads cannot emit + * one warning per draw (a pipeline refused again for the same program is such + * a repeat). Keep vk_errorf for debug callbacks and preserve the internal + * refusal result. */ #define G5P_REFUSE(obj, ...) \ ({ \ - static uint32_t refusal_count; \ + static uint32_t refusal_count, refusal_seen[32]; \ + char refusal_text[768]; \ + snprintf(refusal_text, sizeof(refusal_text), __VA_ARGS__); \ uint32_t occurrence = p_atomic_inc_return(&refusal_count); \ - if (occurrence && !(occurrence & (occurrence - 1))) \ - mesa_logw("G5P refuses: " __VA_ARGS__); \ - vk_errorf((obj), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: " __VA_ARGS__); \ + if (pvr_g5p_refusal_new(refusal_seen, 32, refusal_text) || \ + (occurrence && !(occurrence & (occurrence - 1)))) \ + mesa_logw("G5P refuses: %s", refusal_text); \ + vk_errorf((obj), VK_ERROR_FEATURE_NOT_PRESENT, "G5P: %s", refusal_text); \ }) #include "pvr_common.h" #include "pvr_limits.h" +/* The pdsc features of a PDS program: the A9 encodes DOUT in Apple's extended + * form (src0 at bits 3-9, src1 at 10-17), as the A10 does; the A8 in Rogue's + * (src0 at 8-14, src1 at 16-23). The A8's form is decoded from Metal's PDS + * programs on an A8 (OrchardLinux/gpu a8/M2-IOS-CAPTURE.md): DOUTW of temps q0 + * with the control in data dword 2 is 0xf0024001 there, 0xf0000a01 on the A10. + * Data words (DOUTU/DOUTD/DOUTW sources and controls) are the same in both. + * For files that include pdsc.h. */ +/* The base DOUTU's USC offsets count from. The A9's firmware has a PDS and a USC execution base (PLATFORM + * +0x10/+0x18, powervr's two heap bases). The A8's has one: Apple writes 0x3_00000000 to both of its pair + * (PLATFORM +0x20/+0x28; Apple's code runs there and faults at 0x1_ on the device), so its USC code + * counts from the PDS heap's base: the kernel puts the PDS heap at that base and the USC heap above it in the + * same 4 GiB window. For files that include pvr_device.h. */ +#define PVR_G5P_USC_EXEC_BASE(device) \ + (PVR_HAS_FEATURE(&(device)->pdevice->dev_info, apple_g4p) ? (device)->heaps.pds_heap->base_addr.addr \ + : (device)->heaps.usc_heap->base_addr.addr) + +#define PVR_G5P_PDS_FEATURES(dev_info) \ + (PVR_HAS_FEATURE(dev_info, apple_g4p) ? (enum pdsc_feature)0 : PDSC_FEATURE_DOUT_EXT) + /* A sampler's mipLodBias, clamped to +-maxSamplerLodBias, as an f32 in its descriptor's metadata * (pvr_sampler_descriptor::meta): the A9 sampler word has no bias field, so the LOD-bias program variant adds * it in the shader (pvr_g5p_nir.c lod_bias_lower()). */ @@ -95,6 +138,15 @@ struct pvr_g5p_programs { struct pvr_suballoc_bo *eot_usc[PVR_G5P_MRT_VARIANTS]; /* PDS kicks of the clear programs (PDS_BGRND0's shader) */ struct pvr_pds_upload clear_kick[PVR_G5P_MRT_VARIANTS]; + /* the A8's background programs for one R8 [0] or RG8 [1] target (g5p_mrt_clear_usc_g4p_narrow()) */ + struct pvr_suballoc_bo *clear_usc_narrow[2]; + struct pvr_pds_upload clear_kick_narrow[2]; + /* the A8's background program for one RGBA8UI target (g5p_mrt_clear_usc_g4p_rgba8ui()) */ + struct pvr_suballoc_bo *clear_usc_rgba8ui; + struct pvr_pds_upload clear_kick_rgba8ui; + /* the A8's background program for one R32F target (g5p_mrt_clear_usc_g4p_r32f()) */ + struct pvr_suballoc_bo *clear_usc_r32f; + struct pvr_pds_upload clear_kick_r32f; /* PDS code: the clear colour's DOUTD (PDS_BGRND0's texunicode; its data, * {floats, control}, is per render) and the end of tile, DOUTD of the PBE * words then DOUTU of the end-of-tile program (data per render) */ @@ -133,6 +185,9 @@ struct pvr_g5p_programs { struct pvr_g5p_programs *mrt_load[8]; /* Lazy 4x LOAD (one UNORM/float target, pvr_g5p_compile_load_ms()), kicked at sample rate */ struct pvr_g5p_programs *ms_load; + /* the A8's colour LOAD background (g5p_mrt_load_usc_g4p()): its USC, kick and the three-DOUTD loader */ + struct pvr_suballoc_bo *a8_load_usc; + struct pvr_pds_upload a8_load_kick, a8_load_dma; /* raw clear programs by total output words: 2, 4, 8 (two 32-bit integer targets) */ struct pvr_suballoc_bo *raw_clear_usc[4]; /*2/4/8 words, then scalar1 */ /* raw end-of-tile programs by target count: 1, 2 */ @@ -143,12 +198,18 @@ struct pvr_g5p_programs { /* * The parameter buffer of a render (bytes): what the device's free list asks - * the kernel for. On the A9 the list holds no pages; its size is the size of - * the parameter manager's pool, which the kernel raises to what the render - * target's tiles need and which does not grow during a render. 4 MiB (32 - * blocks of 128 KiB) is the pool the device's renders ran with. + * the kernel for. On the A9 the list holds no pages; its size is the smallest + * parameter manager pool the kernel builds for the render, which it raises to + * what the render target's tiles need, at most 128 blocks of 128 KiB. + * + * The pool does not grow during a scene, and the kernel has no partial + * render: a scene whose geometry needs more parameter memory than the pool + * stops the geometry job without a fault, and the job times out. A pool sized + * from a small target's tiles (4 MiB, 32 blocks, below 974 tiles) can be too + * small for a 3D scene drawn into it. Ask for the largest pool the kernel + * builds, 16 MiB. PVR_G5P_FREE_LIST_MB overrides it. */ -#define PVR_G5P_FREE_LIST_SIZE (4U * 1024U * 1024U) +#define PVR_G5P_FREE_LIST_SIZE (16U * 1024U * 1024U) /* G5P's per-device state (pvr_device::g5p), allocated by g5p_device_init() */ struct pvr_g5p_device { @@ -177,6 +238,13 @@ struct pvr_g5p_job { uint64_t vdm_va; /* the control stream, one block */ uint32_t background[8]; /* PDS_BGRND0..3 (64-bit registers: 8-byte aligned) */ uint32_t eot[4]; /* EVENT_PIXEL_PDS data, code, info */ + uint64_t a8_pbe[2]; /* the A8: target 0's PBESTATE_REG_WORD0/1 (g5p_pbe_a8_reg_words()) */ + uint32_t a8_eot_format; /* the A8: the EOT state's format word, PBE packmode << 16 | conversion */ + uint8_t a8_narrow_halves; /* the A8: one R8 (1) or RG8 (2) target: one-word output (Apple's 111c) */ + /* the A8's second colour target (MRT2): its PBE words, address and EOT format word */ + uint64_t a8_pbe_t1[2]; + uint64_t a8_t1_addr; + uint32_t a8_eot_format_t1; uint32_t vdm_bytes; /* through STREAM_TERMINATE */ uint32_t width, height; unsigned targets; /* colour targets, 1, 2 or 4 */ @@ -186,6 +254,7 @@ struct pvr_g5p_job { uint32_t scissor_count; /* entries of upstream's scissor table, 0 without one */ uint32_t samples; /* 1 or 4 (hw_render->sample_count) */ bool msaa_resolve; /* 4x: the PBE resolves target 0 into a one-sample view */ + bool skip_empty_tiles; bool enabled; }; @@ -486,6 +555,9 @@ VkResult PVR_PER_ARCH(g5p_device_init)(struct pvr_device *device); VkResult PVR_PER_ARCH(g5p_meta_init)(struct pvr_device *device); void PVR_PER_ARCH(g5p_meta_finish)(struct pvr_device *device); bool PVR_PER_ARCH(g5p_meta_blit)(struct pvr_cmd_buffer *cmd_buffer, const VkBlitImageInfo2 *info); +bool PVR_PER_ARCH(g5p_meta_clear_color)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_image *image, + VkImageLayout layout, const VkClearColorValue *color, uint32_t range_count, + const VkImageSubresourceRange *ranges); void PVR_PER_ARCH(g5p_device_finish)(struct pvr_device *device); VkResult PVR_PER_ARCH(g5p_job_init)(struct pvr_cmd_buffer *cmd_buffer, struct pvr_sub_cmd_gfx *sub_cmd, @@ -503,6 +575,7 @@ uint32_t PVR_PER_ARCH(g5p_compute_stream)(const struct pvr_device *device, uint6 # define pvr_arch_g5p_device_init PVR_PER_ARCH(g5p_device_init) # define pvr_arch_g5p_meta_blit PVR_PER_ARCH(g5p_meta_blit) +# define pvr_arch_g5p_meta_clear_color PVR_PER_ARCH(g5p_meta_clear_color) # define pvr_arch_g5p_device_finish PVR_PER_ARCH(g5p_device_finish) # define pvr_arch_g5p_job_init PVR_PER_ARCH(g5p_job_init) # define pvr_arch_g5p_geom_stream PVR_PER_ARCH(g5p_geom_stream) diff --git a/src/imagination/vulkan/pvr_g5p_a8_fetch.h b/src/imagination/vulkan/pvr_g5p_a8_fetch.h new file mode 100644 index 00000000000..eefb0b7eb29 --- /dev/null +++ b/src/imagination/vulkan/pvr_g5p_a8_fetch.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: MIT */ +/* + * The A8's fetch sampler: the Rogue sampler words a texelFetch drawn as a sample + * reads, point filtered, non-normalized, clamped to the edge and to one level, so the texture's own sampler state + * cannot change the texel. From the binding's sampler words (its border index, its comparison off) or from zero. + * Include after pvr_csb.h. + */ +#ifndef PVR_G5P_A8_FETCH_H +#define PVR_G5P_A8_FETCH_H + +#include +#include + +static inline void +pvr_g5p_a8_fetch_sampler(uint32_t out[4], const uint32_t *app, unsigned level) +{ + uint64_t w0 = 0, w1 = 0; + struct ROGUE_TEXSTATE_SAMPLER_WORD0 s; + + if (app) { + memcpy(&w0, app, sizeof(w0)); + memcpy(&w1, app + 2, sizeof(w1)); + } + ROGUE_TEXSTATE_SAMPLER_WORD0_unpack(&w0, &s); + s.minfilter = ROGUE_TEXSTATE_FILTER_POINT; + s.magfilter = ROGUE_TEXSTATE_FILTER_POINT; + s.mipfilter = false; + s.anisoctl = ROGUE_TEXSTATE_ANISOCTL_DISABLED; + s.addrmode_u = ROGUE_TEXSTATE_ADDRMODE_CLAMP_TO_EDGE; + s.addrmode_v = ROGUE_TEXSTATE_ADDRMODE_CLAMP_TO_EDGE; + s.addrmode_w = ROGUE_TEXSTATE_ADDRMODE_CLAMP_TO_EDGE; + s.non_normalized_coords = true; + /* the level the fetch reads (0, or a constant level with PVR_A8_TEXFETCH_LEVEL) */ + s.minlod = level << ROGUE_TEXSTATE_CLAMP_FRACTIONAL_BITS; + s.maxlod = level << ROGUE_TEXSTATE_CLAMP_FRACTIONAL_BITS; + s.dadjust = ROGUE_TEXSTATE_DADJUST_ZERO_UINT; + ROGUE_TEXSTATE_SAMPLER_WORD0_pack(&w0, &s); + memcpy(out, &w0, sizeof(w0)); + memcpy(out + 2, &w1, sizeof(w1)); +} + +/* The sampler of PVR_A8_TEXFETCH_LEVEL=apple (PCO_G5P_UPLOAD_SAMPLER_READ): point, mip nearest, CLAMP_TO_EDGE, + * normalized, LOD 0..MAX, no LOD bias (0x02002401df800fff, word 1 0), as read from Apple's compiler binary; with byte 6 bit 2 and the level + * in a register it read levels 1 and 2 right on the iPod. Apple's own read sampler, in an iOS capture, is + * the fetch sampler's words instead (0x0202240000000fff: non-normalized, LOD 0..0): pvr_g5p_a8_fetch_sampler(). */ +static inline void +pvr_g5p_a8_read_sampler(uint32_t out[4]) +{ + uint64_t w0 = 0; + struct ROGUE_TEXSTATE_SAMPLER_WORD0 s; + + memset(&s, 0, sizeof(s)); + s.minfilter = ROGUE_TEXSTATE_FILTER_POINT; + s.magfilter = ROGUE_TEXSTATE_FILTER_POINT; + s.mipfilter = false; + s.anisoctl = ROGUE_TEXSTATE_ANISOCTL_DISABLED; + s.addrmode_u = ROGUE_TEXSTATE_ADDRMODE_CLAMP_TO_EDGE; + s.addrmode_v = ROGUE_TEXSTATE_ADDRMODE_CLAMP_TO_EDGE; + s.addrmode_w = ROGUE_TEXSTATE_ADDRMODE_CLAMP_TO_EDGE; + s.non_normalized_coords = false; + s.minlod = ROGUE_TEXSTATE_CLAMP_MIN; + s.maxlod = ROGUE_TEXSTATE_CLAMP_MAX; + s.dadjust = ROGUE_TEXSTATE_DADJUST_ZERO_UINT; + ROGUE_TEXSTATE_SAMPLER_WORD0_pack(&w0, &s); + memcpy(out, &w0, sizeof(w0)); + out[2] = out[3] = 0; +} + +#endif diff --git a/src/imagination/vulkan/pvr_g5p_compute_state.h b/src/imagination/vulkan/pvr_g5p_compute_state.h index 727502925bb..e7f2597c8bf 100644 --- a/src/imagination/vulkan/pvr_g5p_compute_state.h +++ b/src/imagination/vulkan/pvr_g5p_compute_state.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: MIT */ #ifndef PVR_G5P_COMPUTE_STATE_H #define PVR_G5P_COMPUTE_STATE_H -#define PVR_G5P_COMPUTE_STATE_FIELDS unsigned shared_spill_stride; int shared_spill_logged; char a9_pipeline_id[13]; bool a9_capture_stub; bool a9_compute_log; unsigned a9_log_local[3], a9_log_shared; unsigned a9_bank1_regs; bool a9_bodyless; bool a9_wgid_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; unsigned a9_num_images; struct { unsigned set,binding; } a9_images[1]; \ +#define PVR_G5P_COMPUTE_STATE_FIELDS unsigned shared_spill_stride; int shared_spill_logged; char a9_pipeline_id[13]; bool a9_capture_stub; bool a9_compute_log; unsigned a9_log_local[3], a9_log_shared; unsigned a9_bank1_regs; bool a9_bodyless; bool a9_wgid_z; unsigned a9_local_dwords; bool a9_has_barriers; bool a9_device_fence; bool a8_device_atomics; unsigned a9_num_images; struct { unsigned set,binding; } a9_images[1]; \ unsigned a9_num_uploads; struct { unsigned char kind, set, binding, element, combined, level, common, dwords; uint32_t value; } a9_uploads[10]; #define PVR_G5P_NATIVE_STORAGE(d) PVR_HAS_FEATURE((d), apple_g5p) #endif diff --git a/src/imagination/vulkan/pvr_g5p_ds.c b/src/imagination/vulkan/pvr_g5p_ds.c index 5b05cb80f5d..0c16202fd37 100644 --- a/src/imagination/vulkan/pvr_g5p_ds.c +++ b/src/imagination/vulkan/pvr_g5p_ds.c @@ -13,6 +13,8 @@ #include #include +#include "util/u_math.h" + /* addresses are plain integers here: this file never includes pvr_csb.h */ #define __pvr_address_type uint64_t #define __pvr_get_address(addr) (addr) @@ -21,6 +23,7 @@ #include "csbgen/rogue/ppp.h" #include "pvr_g5p_ds.h" +#include "g5p/g5p_usc.h" #include "pvr_g5p_shader_dump.h" static uint32_t @@ -77,6 +80,9 @@ pvr_g5p_ds_format(VkFormat format) return NULL; } +bool pvr_g5p_ds_rect_planes; +bool pvr_g5p_ds_pot_extents; + bool pvr_g5p_ds_layout(VkFormat format, uint32_t width, uint32_t height, uint32_t samples, struct pvr_g5p_ds_plane *out) @@ -87,7 +93,13 @@ pvr_g5p_ds_layout(VkFormat format, uint32_t width, uint32_t height, uint32_t sam if (!f || !width || !height || (samples != 1 && samples != 4)) return false; out->side = g5p_pot(g5p_align(width > height ? width : height, 32)); - const uint64_t texels = (uint64_t)out->side * out->side * samples; + /* With pvr_g5p_ds_rect_planes, each plane takes the rectangle the ZLS addresses: its Morton order runs over + * each side 32-aligned and padded to its own power of two (proven on the A9 by sampling and CPU copies of + * unequal-side planes, pvr_g5p_ds_texture_plane() and the CPU transfers' rectangular order), so the square + * of the larger side holds that rectangle and the rest. */ + const uint64_t texels = pvr_g5p_ds_rect_planes + ? (uint64_t)g5p_pot(g5p_align(width, 32)) * g5p_pot(g5p_align(height, 32)) * samples + : (uint64_t)out->side * out->side * samples; out->depth_bytes = f->depth ? texels * f->depth_bytes : 0; if (f->stencil) out->stencil_offset = (out->depth_bytes + PVR_G5P_ZLS_PLANE_ALIGN - 1) & ~(uint64_t)(PVR_G5P_ZLS_PLANE_ALIGN - 1); @@ -136,7 +148,7 @@ pvr_g5p_ds_zls(const struct pvr_g5p_ds_target *t, struct pvr_g5p_zls_words *out) * and stencil: ZLSCTL 0xe2004 / 0xaa006 / 0x26002 and ZLS_PIXELS 0x1f803f at 64x64, as at one sample; the * stencil plane at depth + 0x10000, four times the one-sample plane) */ - const struct ROGUE_CR_ISP_ZLSCTL ctl = { + struct ROGUE_CR_ISP_ZLSCTL ctl = { .loadtwiddled = true, .storetwiddled = true, .zloadformat = f->zls_type, @@ -149,6 +161,18 @@ pvr_g5p_ds_zls(const struct pvr_g5p_ds_target *t, struct pvr_g5p_zls_words *out) .forcezstore = zstore || sstore, }; uint32_t dw[2]; + if (t->g4p) { + /* The A8 (Rogue's form, no ZLS_PIXELS): the extent in 32-pixel tiles minus 1 in ZLSCTL. iPod touch 6, + * iOS 12.4, a8-passcap at 64x64: 0x080e200c depth+stencil clear/store, 0x080ee00e loaded, 0x080a200c + * depth only (x_z = y_z = 1). A stored depth or stencil read back wrong on the iPod at every size whose + * tile count is not a power of two on both sides (640x1136, 528x80, 640x52, 528x464, 100x75 failed; + * 37x29 and 64x64 passed) while the extent was the plain tile count. Each extent is now its tile count + * rounded up to a power of two, as Rogue's ZLSCTL documents for twiddled mode ("zlsextent uses above + * result and rounds the value up to the nearest value which is power of 2"); every size then passed. */ + const uint32_t tx = DIV_ROUND_UP(t->width, 32), ty = DIV_ROUND_UP(t->height, 32); + ctl.zlsextent_x_z = (pvr_g5p_ds_pot_extents ? g5p_pot(tx) : tx) - 1; + ctl.zlsextent_y_z = (pvr_g5p_ds_pot_extents ? g5p_pot(ty) : ty) - 1; + } ROGUE_CR_ISP_ZLSCTL_pack(dw, &ctl); out->zls[0] = g5p_dw64(dw); @@ -174,7 +198,8 @@ pvr_g5p_ds_zls(const struct pvr_g5p_ds_target *t, struct pvr_g5p_zls_words *out) const struct ROGUE_CR_ISP_ZLS_PIXELS px = { .x = w - 1, .y = h - 1 }; uint32_t pixels; ROGUE_CR_ISP_ZLS_PIXELS_pack(&pixels, &px); - out->zls[5] = pixels; + /* the A8's 3D record has no ZLS_PIXELS word (a8-passcap: none set in any depth pass) */ + out->zls[5] = t->g4p ? 0 : pixels; for (unsigned i = 1; i <= 4; i++) if (out->zls[i]) @@ -330,6 +355,16 @@ pvr_g5p_ds_texture_plane(VkFormat format, VkImageAspectFlags aspect, uint32_t wi } } +/* The depth/stencil attachments the A8 renders: what a8-passcap captured (iPod touch 6, iOS 12.4), + * D32F and D32F_S8. The render path refuses any other by name + * (pvr_arch_g5p_ds.c), and the format features offer no other, so Zink renders GL's DEPTH_COMPONENT16 with + * one of these instead of being refused at the draw. */ +bool +pvr_g5p_ds_a8_attachment(VkFormat format) +{ + return format == VK_FORMAT_D32_SFLOAT || format == VK_FORMAT_D32_SFLOAT_S8_UINT; +} + VkFormatFeatureFlags2 pvr_g5p_ds_format_features(VkFormat format) { @@ -337,7 +372,7 @@ pvr_g5p_ds_format_features(VkFormat format) if (!f) return 0; - return VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT | + return (g5p_usc_g4p && !pvr_g5p_ds_a8_attachment(format) ? 0 : VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT) | VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT | VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT | (f->sampled ? VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT : 0) | (f->sampled && f->depth ? VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_DEPTH_COMPARISON_BIT : 0); diff --git a/src/imagination/vulkan/pvr_g5p_ds.h b/src/imagination/vulkan/pvr_g5p_ds.h index 44021577de3..3a7dc5ced8a 100644 --- a/src/imagination/vulkan/pvr_g5p_ds.h +++ b/src/imagination/vulkan/pvr_g5p_ds.h @@ -52,6 +52,11 @@ struct pvr_g5p_ds_plane { bool pvr_g5p_ds_layout(VkFormat format, uint32_t width, uint32_t height, uint32_t samples, struct pvr_g5p_ds_plane *out); +/* The planes take the ZLS's rectangle (each side 32-aligned, then a power of two) instead of the larger side's + * square. Set by the driver on the A8. */ +extern bool pvr_g5p_ds_rect_planes; +/* The A8's ZLSCTL extents in tiles rounded up to powers of two (pvr_g5p_ds_zls()); set by the driver on the A8 */ +extern bool pvr_g5p_ds_pot_extents; /* The depth/stencil attachment of one render, as the ZLS sees it */ struct pvr_g5p_ds_target { @@ -65,6 +70,7 @@ struct pvr_g5p_ds_target { bool zload, zstore, sload, sstore; float depth_clear; /* ISP_BGOBJDEPTH */ uint8_t stencil_clear; /* ISP_BGOBJVALS.stencil */ + bool g4p; /* the A8: ZLS extent in ZLSCTL, no ZLS_PIXELS */ }; /* The ZLS words of a render's fragment stream (pvr_arch_g5p.c): @@ -130,6 +136,7 @@ const char *pvr_g5p_ds_texture_plane(VkFormat format, VkImageAspectFlags aspect, * 32-bit float texture, and ES 3.0 asks none of DEPTH_COMPONENT32F), and no * transfers (the planes are in the ZLS's Morton order). */ VkFormatFeatureFlags2 pvr_g5p_ds_format_features(VkFormat format); +bool pvr_g5p_ds_a8_attachment(VkFormat format); /* pvr_image_init(): a depth/stencil image of a ZLS format takes exactly the * ZLS planes' size (pvr_g5p_ds_layout()), not upstream's larger D32F_S8 @@ -157,7 +164,7 @@ struct vk_dynamic_graphics_state; VkResult PVR_PER_ARCH(g5p_ds_admit)(struct pvr_cmd_buffer *cmd_buffer, const struct pvr_renderpass_hwsetup_render *hw_render); /* frag_stream: the job's ZLS and background words from pvr_render_job::ds */ -void PVR_PER_ARCH(g5p_ds_pass)(const struct pvr_render_job *job, struct pvr_g5p_zls_words *out); +void PVR_PER_ARCH(g5p_ds_pass)(const struct pvr_render_job *job, bool g4p, struct pvr_g5p_zls_words *out); /* the draw: w[0..4] as pvr_g5p_ds_isp(), from the dynamic state */ void PVR_PER_ARCH(g5p_ds_isp)(const struct vk_dynamic_graphics_state *dyn, VkImageAspectFlags aspects, uint32_t samples, bool lines, uint32_t passtype, bool scissor, bool dbias, diff --git a/src/imagination/vulkan/pvr_g5p_formats.c b/src/imagination/vulkan/pvr_g5p_formats.c index 09f1d2919dd..6564e89f9ec 100644 --- a/src/imagination/vulkan/pvr_g5p_formats.c +++ b/src/imagination/vulkan/pvr_g5p_formats.c @@ -24,6 +24,7 @@ #include "pvr_g5p_format_table.h" #include "pvr_g5p_formats.h" +#include "g5p/g5p_usc.h" /* Formats that have not run on the device are not advertised */ bool @@ -46,6 +47,16 @@ row_probe(VkFormat format) return pvr_g5p_formats_probe() || tex16_row(format); } +/* Colour targets the A8 renders beyond the A9 table's: RGBA8UI and R32F. AGXMetalA8's passes for them + * (a8-passcap rgba8u, r32f) are the float pass shape with their own end-of-tile state word, and our fragment, + * clear and end-of-tile programs for them match Apple's byte for byte (pvr_arch_g5p.c, g5p_mrt_programs.h). + * On the A9, RGBA8UI stays a probe row and R32F is not rendered. */ +static bool +a8_render_row(VkFormat format) +{ + return g5p_usc_g4p && (format == VK_FORMAT_R8G8B8A8_UINT || format == VK_FORMAT_R32_SFLOAT); +} + bool pvr_g5p_format_tex(VkFormat format, uint32_t *key, bool *srgb) { @@ -69,6 +80,12 @@ pvr_g5p_packed_word_pbe_key(void) bool pvr_g5p_format_pbe(VkFormat format, uint32_t *key, bool *srgb) { + if (a8_render_row(format)) { + const struct pvr_g5p_format *f = pvr_g5p_format_find(format); + *key = f->key; + *srgb = f->srgb; + return true; + } return pvr_g5p_format_pbe_key(format, row_probe(format), key, srgb); } @@ -79,7 +96,11 @@ pvr_g5p_color_format_features(VkFormat format, VkImageTiling tiling, VkFormatFea /* the A9's own answer; upstream's flags only say the format exists */ (void)flags; - return pvr_g5p_format_features(f, tiling == VK_IMAGE_TILING_LINEAR, row_probe(format)); + VkFormatFeatureFlags2 r = pvr_g5p_format_features(f, tiling == VK_IMAGE_TILING_LINEAR, row_probe(format)); + if (a8_render_row(format)) + r |= VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT | VK_FORMAT_FEATURE_2_BLIT_DST_BIT | + VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT | VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT; + return r; } VkResult diff --git a/src/imagination/vulkan/pvr_g5p_hooks.h b/src/imagination/vulkan/pvr_g5p_hooks.h index 97682e7cd79..ca09f93c78b 100644 --- a/src/imagination/vulkan/pvr_g5p_hooks.h +++ b/src/imagination/vulkan/pvr_g5p_hooks.h @@ -25,8 +25,11 @@ #include "pvr_g5p_compute_hooks.h" /* Uniform buffer textures: independent R5 probe; no storage exposure. */ +/* The A8 (apple_g4p) keeps upstream's Rogue texture, sampler and texel-buffer state: its TPU is Rogue's (the + * a8-passcap load pass's TEXSTATE words); the A9's own forms are below. */ #define PVR_G5P_HOOK_texel_state(device, info, state) \ - if (PVR_HAS_FEATURE(&(device)->pdevice->dev_info, apple_g5p) && (info)->buffer_elems) \ + if (PVR_HAS_FEATURE(&(device)->pdevice->dev_info, apple_g5p) && \ + !PVR_HAS_FEATURE(&(device)->pdevice->dev_info, apple_g4p) && (info)->buffer_elems) \ return pvr_g5p_tbo_pack(device, info, state); #define PVR_G5P_HOOK_texel_features(pdevice, format, flags) \ if (PVR_HAS_FEATURE(&(pdevice)->dev_info, apple_g5p)) \ @@ -43,13 +46,14 @@ * the A9's three-qword image state (pvr_arch_g5p_pack_image_view()). */ #define PVR_G5P_HOOK_image_view(device, iview) \ - if (PVR_HAS_FEATURE(&(device)->pdevice->dev_info, apple_g5p)) \ + if (PVR_HAS_FEATURE(&(device)->pdevice->dev_info, apple_g5p) && \ + !PVR_HAS_FEATURE(&(device)->pdevice->dev_info, apple_g4p)) \ pvr_arch_g5p_pack_image_view( \ iview, &(iview)->image_state[PVR_TEXTURE_STATE_SAMPLE]); /* Only storage descriptors reinterpret cube faces as a 2D array. */ #define PVR_G5P_HOOK_storage_image(dev_info, iview, desc) \ - if (PVR_HAS_FEATURE(dev_info, apple_g5p) && \ + if (PVR_HAS_FEATURE(dev_info, apple_g5p) && !PVR_HAS_FEATURE(dev_info, apple_g4p) && \ ((iview)->vk.view_type == VK_IMAGE_VIEW_TYPE_CUBE || \ (iview)->vk.view_type == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY)) \ pvr_arch_g5p_pack_storage_image_view(iview, desc); @@ -75,7 +79,8 @@ * forms (pvr_arch_g5p_pack_sampler()). */ #define PVR_G5P_HOOK_sampler(device, create_info, sampler) \ - if (PVR_HAS_FEATURE(&(device)->pdevice->dev_info, apple_g5p)) \ + if (PVR_HAS_FEATURE(&(device)->pdevice->dev_info, apple_g5p) && \ + !PVR_HAS_FEATURE(&(device)->pdevice->dev_info, apple_g4p)) \ pvr_arch_g5p_pack_sampler(create_info, &(sampler)->descriptor); /* pvr_arch_descriptor_set.c, write_image_sampler(): a combined descriptor @@ -84,7 +89,10 @@ #define PVR_G5P_HOOK_image_sampler(set, image_info, desc) \ if (PVR_HAS_FEATURE(&container_of((set)->base.device, struct pvr_device, \ vk)->pdevice->dev_info, \ - apple_g5p)) \ + apple_g5p) && \ + !PVR_HAS_FEATURE(&container_of((set)->base.device, struct pvr_device, \ + vk)->pdevice->dev_info, \ + apple_g4p)) \ pvr_arch_g5p_combine_sampler( \ desc, (image_info)->imageView \ ? pvr_image_view_from_handle((image_info)->imageView) \ diff --git a/src/imagination/vulkan/pvr_g5p_image.c b/src/imagination/vulkan/pvr_g5p_image.c index 78a04852c15..6f5d3ea96c9 100644 --- a/src/imagination/vulkan/pvr_g5p_image.c +++ b/src/imagination/vulkan/pvr_g5p_image.c @@ -47,6 +47,9 @@ #include "pvr_g5p_tex.h" #include "pvr_image.h" #include "pvr_physical_device.h" +#include "g5p/g5p_usc.h" + +bool pvr_g5p_image_a8_linear; void pvr_g5p_image_layout(struct pvr_image *image, unsigned stride_align) @@ -65,7 +68,18 @@ pvr_g5p_image_layout(struct pvr_image *image, unsigned stride_align) (image->vk.usage & VK_IMAGE_USAGE_STORAGE_BIT) || (image->vk.create_flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) || (image->vk.array_layers > 1 && (image->vk.usage & VK_IMAGE_USAGE_SAMPLED_BIT)) || - vk_format_is_compressed(image->vk.format); + vk_format_is_compressed(image->vk.format) || + /* The A8 twiddles single-level colour images as Metal does: every A8 colour target + * Apple's captures show is twiddled (a8-passcap base, load, mrt2: PBE memlayout 1), and + * the A8's LOAD background is captured for a twiddled target only, so a linear one-level + * target (Zink's GL textures and renderbuffers) could not be drawn twice. Images shared + * outside the driver stay linear. */ + (g5p_usc_g4p && !image->vk.external_handle_types && + /* a single-level image whose sides are not both powers of two + * stays linear, as on the A9, instead of padding each side to a power of two (the A8 + * renders, LOADs and samples linear colour images: shared buffers take that path) */ + !(pvr_g5p_image_a8_linear && (!util_is_power_of_two_nonzero(image->vk.extent.width) || + !util_is_power_of_two_nonzero(image->vk.extent.height)))); if (!twiddled) { const unsigned cpp = vk_format_get_blocksize(image->vk.format); @@ -132,6 +146,11 @@ pvr_g5p_image_layout(struct pvr_image *image, unsigned stride_align) plane->layer_size = image->vk.array_layers > 1 || is3d ? align64(size, size >= PVR_G5P_IMAGE_BIG_LAYER ? PVR_G5P_IMAGE_BIG_LAYER : 64) : own; + /* The A8 (a Rogue TPU) steps a 2D array's layers and a cube's faces by the chain's size rounded up to 4 KiB: + * AGXMetalA8's texture size (iOS 12.4, sub_1a13cfb68) rounds one face's or layer's chain to 0x1000 and + * multiplies by 6 for a cube (by the layer count for an array of two or more), Rogue's CEM 4K face packing */ + if (g5p_usc_g4p && image->vk.array_layers > 1 && !is3d) + plane->layer_size = align64(size, 4096); } plane->size = plane->layer_size * (is3d ? image->vk.extent.depth : image->vk.array_layers); plane->offset = 0; diff --git a/src/imagination/vulkan/pvr_g5p_tex.h b/src/imagination/vulkan/pvr_g5p_tex.h index 4a130bfdcfd..5c0c1d3d672 100644 --- a/src/imagination/vulkan/pvr_g5p_tex.h +++ b/src/imagination/vulkan/pvr_g5p_tex.h @@ -72,5 +72,8 @@ void PVR_PER_ARCH(g5p_combine_sampler)(struct pvr_combined_image_sampler_descrip */ #define PVR_G5P_IMAGE_BIG_LAYER 16384u void pvr_g5p_image_layout(struct pvr_image *image, unsigned stride_align); +/* On the A8, single-level colour images whose sides are not both powers of two stay linear + * (pvr_g5p_image_layout()) */ +extern bool pvr_g5p_image_a8_linear; #endif /* PVR_G5P_TEX_H */ diff --git a/src/imagination/vulkan/pvr_physical_device.c b/src/imagination/vulkan/pvr_physical_device.c index d3c25d5fd1c..56c2b63bf34 100644 --- a/src/imagination/vulkan/pvr_physical_device.c +++ b/src/imagination/vulkan/pvr_physical_device.c @@ -10,6 +10,7 @@ * SPDX-License-Identifier: MIT */ +#include "g5p_usc.h" #include "pvr_physical_device.h" #include @@ -19,6 +20,7 @@ #include "util/disk_cache.h" #include "util/ralloc.h" #include "util/os_misc.h" +#include "util/u_debug.h" #include "vk_util.h" #include "vk_log.h" @@ -29,6 +31,8 @@ #include "pco/pco.h" #include "pco_uscgen_programs.h" #include "pvr_g5p_ds_hooks.h" +#include "pvr_g5p_ds.h" +#include "pvr_g5p_tex.h" #include "pvr_device.h" #include "pvr_dump_info.h" #include "pvr_entrypoints.h" @@ -256,8 +260,9 @@ static void pvr_physical_device_get_supported_features( .sampleRateShading = true, .dualSrcBlend = false, .logicOp = true, - .multiDrawIndirect = true, - .drawIndirectFirstInstance = true, + /* the A8: no indirect draw in Apple's driver; vkCmdDraw*Indirect is refused by name (pvr_arch_g5p_gfx.c) */ + .multiDrawIndirect = !PVR_HAS_FEATURE(dev_info, apple_g4p), + .drawIndirectFirstInstance = !PVR_HAS_FEATURE(dev_info, apple_g4p), .depthClamp = false, .depthBiasClamp = true, .fillModeNonSolid = false, @@ -1252,7 +1257,7 @@ VkResult pvr_physical_device_init(struct pvr_physical_device *pdevice, pdevice->render_devid = render_stat.st_rdev; result = - pvr_winsys_create(render_path, display_path, true, + pvr_winsys_create(render_path, display_path, true, true, &instance->vk.alloc, &ws); if (result != VK_SUCCESS) goto err_vk_free_display_path; @@ -1389,8 +1394,39 @@ VkResult pvr_physical_device_init(struct pvr_physical_device *pdevice, goto err_vk_physical_device_finish; } + /* The A8's USC forms (g5p_usc.h); one process drives one Apple GPU */ + if (PVR_HAS_FEATURE(&pdevice->dev_info, apple_g4p)) { + g5p_usc_g4p = true; + /* the A9's instructions where the A8's captured forms cannot express the operands (g5p_usc.h) */ + g5p_usc_g4p_a9_forms = debug_get_bool_option("PVR_A8_A9_FORMS", true); + /* The A8's depth/stencil layout (pvr_g5p_ds.h): ZLS extents rounded up to powers of two and planes over the + * ZLS's rectangle. On the iPod every depth and stencil buffer stored by one render and loaded by the next + * read back right with them, at seven sizes from 640x1136 to 37x29. */ + pvr_g5p_ds_pot_extents = true; + pvr_g5p_ds_rect_planes = true; + /* single-level non-power-of-two colour images stay linear (pvr_g5p_tex.h) */ + pvr_g5p_image_a8_linear = true; + /* texelFetch (g5p_usc.h): Apple's read form, which reads every texel right on the iPod touch 6 at level 0 + * (RGBA8, RGBA32F, RGBA32I); a level above 0 with PVR_A8_TEXFETCH_LEVEL=level|scale|apple */ + g5p_usc_g4p_texfetch = 2; + const char *level = getenv("PVR_A8_TEXFETCH_LEVEL"); + g5p_usc_g4p_texfetch_level = !level ? 0 : !strcmp(level, "level") ? 1 : !strcmp(level, "scale") ? 2 : + !strcmp(level, "apple") ? 3 : 0; + /* vertex textures: refused by default; PVR_A8_VTEX=1 takes the A9's recipe on the A8's words, which does not + * work yet, PVR_A8_VTEX_APPLE=1 Apple's form (g5p_usc.h) */ + g5p_usc_g4p_vtex_apple = debug_get_bool_option("PVR_A8_VTEX_APPLE", false); + g5p_usc_g4p_vtex = debug_get_bool_option("PVR_A8_VTEX", false) || g5p_usc_g4p_vtex_apple; + /* fragment texel-buffer reads (g5p_usc.h) */ + g5p_usc_g4p_tbo = debug_get_bool_option("PVR_A8_TBO", true); + } + + /* The A8 skips refused work by default: a refused program or pass loses its draws (named on stderr) instead of + * failing the command buffer, which Zink turns into a lost context for the whole application. PVR_A8_STRICT=1 + * restores the failure. */ pdevice->skip_refused_work = PVR_HAS_FEATURE(&pdevice->dev_info, apple_g5p) && - instance->drirc.misc.pvr_skip_refused_work; + (instance->drirc.misc.pvr_skip_refused_work || + (PVR_HAS_FEATURE(&pdevice->dev_info, apple_g4p) && + !debug_get_bool_option("PVR_A8_STRICT", false))); pdevice->sync_type = ws->syncobj_type; if (ws->job_sync_size) { diff --git a/src/imagination/vulkan/pvr_rt_dataset.c b/src/imagination/vulkan/pvr_rt_dataset.c index e14682cf0f4..d853906be1e 100644 --- a/src/imagination/vulkan/pvr_rt_dataset.c +++ b/src/imagination/vulkan/pvr_rt_dataset.c @@ -9,6 +9,7 @@ #include "pvr_device.h" #include "pvr_free_list.h" #include "pvr_g5p.h" +#include "pvr_physical_device.h" static void pvr_rt_rgn_headers_data_fini(struct pvr_rt_dataset *rt_dataset) { @@ -79,10 +80,28 @@ void pvr_render_target_dataset_destroy(struct pvr_rt_dataset *rt_dataset) vk_free(&device->vk.alloc, rt_dataset); } -/* Unused datasets kept per device: enough for a compositor's outputs and - * the offscreen sizes it renders at, bounded so that memory stays small. +/* Cache demand is passes per command buffer, not distinct sizes: each pass + * takes its own dataset. For example, glmark2 terrain has four 256x256 passes + * and three 2048x1536 passes, among about 24-29 passes including mip renders. + * Zink returns a batch's datasets together when it resets the finished command + * buffer (pvr_cmd_buffer_free_sub_cmd -> pvr_rstate_entry_remove), before + * recording the next batch. Keep enough for a batch, with room for a second + * burst. Matching remains exact in width, height, samples and layers. + * + * A9 datasets share the device's local free list and own five small BOs, + * rounded to at least 16 KiB each. Only region headers scale with size + * (3072 tiles * 5 bytes at 2048x1536), so roughly 0.1-0.2 MiB per dataset + * makes the count limit bind first (about 13 MiB for 64). Rogue datasets own + * a local free list too, so bound their total BO bytes as well. Keep a single + * oversized dataset to avoid repeated allocation for a large target. + * Only released datasets are cached; rendering one or two sizes does not + * itself fill the cache. */ -#define PVR_RT_DATASET_CACHE_MAX 4 +#define PVR_RT_DATASET_CACHE_MAX 64 +#define PVR_RT_DATASET_CACHE_BYTES (32ull << 20) +/* The A8 (1 GB shared by the CPU and GPU): an A8 dataset owns the region-header ring (16 frames of the target's + * headers, 128 KiB at 640x1136) besides the small BOs, so 8 MiB holds some 30 full-screen datasets. */ +#define PVR_RT_DATASET_CACHE_BYTES_A8 (8ull << 20) struct pvr_rt_dataset * pvr_render_target_dataset_cache_take(struct pvr_device *device, @@ -102,6 +121,7 @@ pvr_render_target_dataset_cache_take(struct pvr_device *device, rt_dataset->samples == samples && rt_dataset->layers == layers) { list_del(&rt_dataset->cache_link); device->rt_dataset_cache_count--; + device->rt_dataset_cache_bytes -= rt_dataset->cache_bytes; found = rt_dataset; break; } @@ -117,21 +137,34 @@ pvr_render_target_dataset_cache_take(struct pvr_device *device, void pvr_render_target_dataset_release(struct pvr_rt_dataset *rt_dataset) { struct pvr_device *device = rt_dataset->device; - struct pvr_rt_dataset *evict = NULL; + struct list_head evicted; + + list_inithead(&evicted); simple_mtx_lock(&device->rt_cache_mtx); list_add(&rt_dataset->cache_link, &device->rt_dataset_cache); - if (++device->rt_dataset_cache_count > PVR_RT_DATASET_CACHE_MAX) { - evict = list_last_entry(&device->rt_dataset_cache, - struct pvr_rt_dataset, - cache_link); + device->rt_dataset_cache_count++; + device->rt_dataset_cache_bytes += rt_dataset->cache_bytes; + while (device->rt_dataset_cache_count > PVR_RT_DATASET_CACHE_MAX || + (device->rt_dataset_cache_count > 1 && + device->rt_dataset_cache_bytes > (PVR_HAS_FEATURE(&device->pdevice->dev_info, apple_g4p) + ? PVR_RT_DATASET_CACHE_BYTES_A8 + : PVR_RT_DATASET_CACHE_BYTES))) { + struct pvr_rt_dataset *evict = + list_last_entry(&device->rt_dataset_cache, + struct pvr_rt_dataset, + cache_link); list_del(&evict->cache_link); device->rt_dataset_cache_count--; + device->rt_dataset_cache_bytes -= evict->cache_bytes; + list_addtail(&evict->cache_link, &evicted); } simple_mtx_unlock(&device->rt_cache_mtx); - if (evict) + list_for_each_entry_safe (struct pvr_rt_dataset, evict, &evicted, cache_link) { + list_del(&evict->cache_link); pvr_render_target_dataset_destroy(evict); + } } void pvr_render_target_dataset_cache_finish(struct pvr_device *device) @@ -144,5 +177,6 @@ void pvr_render_target_dataset_cache_finish(struct pvr_device *device) pvr_render_target_dataset_destroy(rt_dataset); } device->rt_dataset_cache_count = 0; + device->rt_dataset_cache_bytes = 0; simple_mtx_destroy(&device->rt_cache_mtx); } diff --git a/src/imagination/vulkan/pvr_rt_dataset.h b/src/imagination/vulkan/pvr_rt_dataset.h index 0537b52e583..2956df608e0 100644 --- a/src/imagination/vulkan/pvr_rt_dataset.h +++ b/src/imagination/vulkan/pvr_rt_dataset.h @@ -27,6 +27,7 @@ struct pvr_rt_dataset { /* In pvr_device::rt_dataset_cache while unused. */ struct list_head cache_link; + uint64_t cache_bytes; /* RT dataset information */ uint32_t width; @@ -53,6 +54,8 @@ struct pvr_rt_dataset { struct pvr_bo *rgn_headers_bo; uint64_t rgn_headers_stride; + /* The A8's region-header ring (pvr_rt_a8_rgn_ring_init()): its size and each frame's slice; 0 elsewhere */ + uint64_t a8_rgn_ring_size, a8_rgn_frame_size; bool need_frag; diff --git a/src/imagination/vulkan/rogue/pvr_blit.c b/src/imagination/vulkan/rogue/pvr_blit.c index 7e2cc331726..f026f35da15 100644 --- a/src/imagination/vulkan/rogue/pvr_blit.c +++ b/src/imagination/vulkan/rogue/pvr_blit.c @@ -1437,6 +1437,10 @@ void pvr_rogue_CmdClearColorImage(VkCommandBuffer commandBuffer, if (PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g9p) && pvr_arch_g9p_meta_clear_color(cmd_buffer, image, imageLayout, pColor, rangeCount, pRanges)) return; + /* G5P (the A8 for now): a render pass whose background object clears (pvr_arch_g5p_meta.c) */ + if (PVR_HAS_FEATURE(&cmd_buffer->device->pdevice->dev_info, apple_g5p) && + pvr_arch_g5p_meta_clear_color(cmd_buffer, image, imageLayout, pColor, rangeCount, pRanges)) + return; for (uint32_t i = 0; i < rangeCount; i++) { const VkResult result = diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm.c b/src/imagination/vulkan/winsys/powervr/pvr_drm.c index 9231b94c7de..e66bcb47076 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm.c @@ -63,11 +63,51 @@ void pvr_drm_report_loss(struct pvr_drm_winsys *ws, const char *job_type, } } +/* A capable kernel restores a VM before accepting a submit or bind. Reclaim may need time + * to catch up, but retrying any other error could repeat accepted work. + */ +int pvr_drm_ioctl_retry(struct pvr_drm_winsys *ws, unsigned long request, + void *args) +{ + int ret = drmIoctl(ws->base.render_fd, request, args); + if (!ret || errno != ENOMEM || !(ws->caps & PVR_DRM_CAP_RETRY_ENOMEM)) + return ret; + + const int64_t start = os_time_get_nano(); + const int64_t deadline = start + 2000000000LL; + unsigned delay_us = 500; + int err = ENOMEM; + int64_t now; + + while ((now = os_time_get_nano()) < deadline) { + os_time_sleep(MIN2(delay_us, (deadline - now + 999) / 1000)); + ret = drmIoctl(ws->base.render_fd, request, args); + err = ret ? errno : 0; + if (err != ENOMEM) + break; + delay_us = MIN2(delay_us * 2, 50000); + } + + now = os_time_get_nano(); + if (now - start >= 100000000 || err == ENOMEM) { + const int64_t last = p_atomic_read(&ws->retry_log_time); + if (now - last >= 1000000000LL && + p_atomic_cmpxchg(&ws->retry_log_time, last, now) == last) { + mesa_logw("pvr: %s residency retry took %.1f ms (errno %d)", + request == DRM_IOCTL_PVR_SUBMIT_JOBS ? "SUBMIT_JOBS" : "VM_MAP", + (now - start) / 1000000.0, err); + } + } + + errno = err; + return ret; +} + VkResult pvr_drm_submit_jobs(struct pvr_drm_winsys *ws, struct drm_pvr_ioctl_submit_jobs_args *args, const char *job_type) { - if (drmIoctl(ws->base.render_fd, DRM_IOCTL_PVR_SUBMIT_JOBS, args) == 0) + if (pvr_drm_ioctl_retry(ws, DRM_IOCTL_PVR_SUBMIT_JOBS, args) == 0) return VK_SUCCESS; const int err = errno; if (err == EIO) @@ -154,6 +194,9 @@ static void pvr_drm_winsys_destroy(struct pvr_winsys *ws) .handle = drm_ws->vm_context, }; + if (drm_ws->query_only) + goto out; + pvr_winsys_helper_free_static_memory(drm_ws->general_vma, drm_ws->pds_vma, drm_ws->usc_vma); @@ -165,6 +208,7 @@ static void pvr_drm_winsys_destroy(struct pvr_winsys *ws) &destroy_vm_context_args, VK_ERROR_UNKNOWN); +out: util_sparse_array_finish(&drm_ws->bo_map); u_rwlock_destroy(&drm_ws->dmabuf_bo_lock); @@ -469,6 +513,7 @@ static const struct pvr_winsys_ops drm_winsys_ops = { .buffer_create = pvr_drm_winsys_buffer_create, .buffer_create_from_fd = pvr_drm_winsys_buffer_create_from_fd, .buffer_destroy = pvr_drm_winsys_buffer_destroy, + .buffer_set_purgeable = pvr_drm_winsys_buffer_set_purgeable, .buffer_get_fd = pvr_drm_winsys_buffer_get_fd, .buffer_map = pvr_drm_winsys_buffer_map, .buffer_unmap = pvr_drm_winsys_buffer_unmap, @@ -476,6 +521,7 @@ static const struct pvr_winsys_ops drm_winsys_ops = { .heap_free = pvr_drm_winsys_heap_free, .vma_map = pvr_drm_winsys_vma_map, .vma_unmap = pvr_drm_winsys_vma_unmap, + .vma_map_mirror = pvr_drm_winsys_vma_map_mirror, .free_list_create = pvr_drm_winsys_free_list_create, .free_list_destroy = pvr_drm_winsys_free_list_destroy, .render_target_dataset_create = pvr_drm_render_target_dataset_create, @@ -750,6 +796,7 @@ static VkResult pvr_drm_setup_heaps(struct pvr_drm_winsys *const drm_ws) VkResult pvr_drm_winsys_create(const int render_fd, const int display_fd, + const bool query_only, const VkAllocationCallbacks *alloc, struct pvr_winsys **const ws_out) { @@ -773,6 +820,16 @@ VkResult pvr_drm_winsys_create(const int render_fd, drm_ws->base.display_fd = display_fd; drm_ws->base.alloc = alloc; + struct drm_pvr_dev_query_caps caps = { 0 }; + struct drm_pvr_ioctl_dev_query_args caps_query = { + .type = DRM_PVR_DEV_QUERY_CAPS_GET, + .size = sizeof(caps), + .pointer = (uintptr_t)&caps, + }; + if (!drmIoctl(render_fd, DRM_IOCTL_PVR_DEV_QUERY, &caps_query)) + drm_ws->caps = caps.caps; + /* An unknown query on an older kernel leaves capabilities disabled. */ + os_get_page_size(&drm_ws->base.page_size); drm_ws->base.log2_page_size = util_logbase2(drm_ws->base.page_size); @@ -803,6 +860,14 @@ VkResult pvr_drm_winsys_create(const int render_fd, PVR_HAS_FEATURE(&info, apple_g9p); drm_ws->base.job_sync_size = drm_ws->job_fence_errors ? sizeof(struct pvr_drm_job_sync) : 0; + /* The A8's physical-device winsys answers device queries only: no VM context, heaps or static memory, so + * a process holds one VM (its VkDevice's) instead of two. powervr's A8 variant gives every VM the hardware + * object window, the zero and data pages and a user slot, and a client that never submits needs none. */ + drm_ws->query_only = query_only && PVR_HAS_FEATURE(&info, apple_g4p); + } + if (drm_ws->query_only) { + *ws_out = &drm_ws->base; + return VK_SUCCESS; } result = pvr_ioctl(render_fd, diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm.h b/src/imagination/vulkan/winsys/powervr/pvr_drm.h index 2e5379f26fb..956c4232fcd 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm.h +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm.h @@ -51,6 +51,11 @@ struct pvr_drm_winsys { uint64_t bvnc; bool is_g5p; + /* pvr_drm_winsys_create(query_only) on the A8: device queries only, no VM context */ + bool query_only; + uint64_t caps; + int madvise_warning; + int64_t retry_log_time; /* The kernel sets a job fence's error when it resets (pvr_apple.c); * upstream pvr_queue.c does not. */ @@ -101,6 +106,10 @@ struct pvr_drm_winsys { uint32_t vm_context; }; +#define PVR_DRM_CAP_RETRY_ENOMEM DRM_PVR_CAP_RETRY_ENOMEM +#define PVR_DRM_CAP_BO_MADVISE DRM_PVR_CAP_BO_MADVISE +int pvr_drm_ioctl_retry(struct pvr_drm_winsys *ws, unsigned long request, void *args); + struct drm_pvr_ioctl_submit_jobs_args; VkResult pvr_drm_submit_jobs(struct pvr_drm_winsys *ws, struct drm_pvr_ioctl_submit_jobs_args *args, diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c index 75ad551a814..5cb57455220 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.c @@ -333,6 +333,29 @@ void pvr_drm_winsys_buffer_destroy(struct pvr_winsys_bo *bo) pvr_drm_buffer_release(drm_bo); } +VkResult pvr_drm_winsys_buffer_set_purgeable(struct pvr_winsys_bo *bo, + bool purgeable, bool *retained) +{ + struct pvr_drm_winsys *ws = to_pvr_drm_winsys(bo->ws); + struct pvr_drm_winsys_bo *drm_bo = to_pvr_drm_winsys_bo(bo); + *retained = true; + if (!(ws->caps & PVR_DRM_CAP_BO_MADVISE) || bo->is_imported || + (drm_bo->flags & DRM_PVR_BO_PM_FW_PROTECT)) + return VK_SUCCESS; + struct drm_pvr_ioctl_bo_madvise_args args = { + .handle = drm_bo->handle, + .madv = purgeable ? DRM_PVR_MADV_DONTNEED : DRM_PVR_MADV_WILLNEED, + }; + if (drmIoctl(ws->base.render_fd, DRM_IOCTL_PVR_BO_MADVISE, &args)) { + const int err = errno; + if (purgeable && p_atomic_cmpxchg(&ws->madvise_warning, 0, 1) == 0) + mesa_logw("pvr: DONTNEED failed (errno %d)", err); + return vk_error(NULL, VK_ERROR_OUT_OF_DEVICE_MEMORY); + } + *retained = args.retained; + return VK_SUCCESS; +} + VkResult pvr_drm_winsys_buffer_get_fd(struct pvr_winsys_bo *bo, int *const fd_out) { @@ -538,7 +561,7 @@ VkResult pvr_drm_winsys_vma_map(struct pvr_winsys_vma *vma, return vk_error(NULL, VK_ERROR_MEMORY_MAP_FAILED); } - if (drmIoctl(drm_ws->base.render_fd, DRM_IOCTL_PVR_VM_MAP, &args)) { + if (pvr_drm_ioctl_retry(drm_ws, DRM_IOCTL_PVR_VM_MAP, &args)) { const int err = errno; const char *message = strerror(err); return vk_errorf(NULL, @@ -559,8 +582,60 @@ VkResult pvr_drm_winsys_vma_map(struct pvr_winsys_vma *vma, return VK_SUCCESS; } +VkResult pvr_drm_winsys_vma_map_mirror(struct pvr_winsys_vma *vma, struct pvr_winsys_bo *bo) +{ + struct pvr_drm_winsys_bo *const drm_bo = to_pvr_drm_winsys_bo(bo); + struct pvr_drm_winsys *const drm_ws = to_pvr_drm_winsys(bo->ws); + const uint64_t size = bo->size; + + assert(!vma->bo); + if (size % vma->heap->page_size || 2 * size > vma->size) + return vk_error(NULL, VK_ERROR_MEMORY_MAP_FAILED); + + for (unsigned i = 0; i < 2; i++) { + struct drm_pvr_ioctl_vm_map_args args = { .device_addr = vma->dev_addr.addr + i * size, + .flags = 0U, + .handle = drm_bo->handle, + .offset = 0, + .size = size, + .vm_context_handle = drm_ws->vm_context }; + if (drmIoctl(drm_ws->base.render_fd, DRM_IOCTL_PVR_VM_MAP, &args)) { + const int err = errno; + if (i) { + struct drm_pvr_ioctl_vm_unmap_args un = { .vm_context_handle = drm_ws->vm_context, + .device_addr = vma->dev_addr.addr, + .size = size }; + drmIoctl(drm_ws->base.render_fd, DRM_IOCTL_PVR_VM_UNMAP, &un); + } + return vk_errorf(NULL, err == EIO ? VK_ERROR_OUT_OF_HOST_MEMORY : VK_ERROR_OUT_OF_DEVICE_MEMORY, + "DRM_IOCTL_PVR_VM_MAP (mirror %u) failed (errno %d: %s)", i, err, strerror(err)); + } + } + + pvr_drm_buffer_acquire(drm_bo); + vma->bo = &drm_bo->base; + vma->bo_offset = 0; + vma->mapped_size = 2 * size; + vma->mirrored = true; + return VK_SUCCESS; +} + void pvr_drm_winsys_vma_unmap(struct pvr_winsys_vma *vma) { + if (vma->mirrored) { + /* two mappings of the buffer (pvr_drm_winsys_vma_map_mirror()): the second one first, then the first + * through the ordinary path */ + struct pvr_drm_winsys *const ws = to_pvr_drm_winsys(vma->bo->ws); + const uint64_t half = vma->mapped_size / 2; + struct drm_pvr_ioctl_vm_unmap_args un = { .vm_context_handle = ws->vm_context, + .device_addr = vma->dev_addr.addr + half, + .size = half }; + if (drmIoctl(ws->base.render_fd, DRM_IOCTL_PVR_VM_UNMAP, &un) && !p_atomic_read(&ws->lost)) + mesa_logw("G5P: VM_UNMAP of the mirror half failed (errno %d)", errno); + vma->mapped_size = half; + vma->mirrored = false; + } + struct pvr_drm_winsys_bo *const drm_bo = to_pvr_drm_winsys_bo(vma->bo); struct pvr_drm_winsys *const drm_ws = to_pvr_drm_winsys(vma->bo->ws); diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.h b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.h index f102325df8a..323ce6e4d04 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.h +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_bo.h @@ -65,6 +65,8 @@ pvr_drm_winsys_buffer_create_from_fd(struct pvr_winsys *ws, int fd, struct pvr_winsys_bo **const bo_out); void pvr_drm_winsys_buffer_destroy(struct pvr_winsys_bo *bo); +VkResult pvr_drm_winsys_buffer_set_purgeable(struct pvr_winsys_bo *bo, + bool purgeable, bool *retained); VkResult pvr_drm_winsys_buffer_get_fd(struct pvr_winsys_bo *bo, int *const fd_out); @@ -89,6 +91,7 @@ VkResult pvr_drm_winsys_vma_map(struct pvr_winsys_vma *vma, uint64_t size, pvr_dev_addr_t *dev_addr_out); void pvr_drm_winsys_vma_unmap(struct pvr_winsys_vma *vma); +VkResult pvr_drm_winsys_vma_map_mirror(struct pvr_winsys_vma *vma, struct pvr_winsys_bo *bo); /******************************************* helper macros diff --git a/src/imagination/vulkan/winsys/powervr/pvr_drm_public.h b/src/imagination/vulkan/winsys/powervr/pvr_drm_public.h index a3c719f28db..62b15439af5 100644 --- a/src/imagination/vulkan/winsys/powervr/pvr_drm_public.h +++ b/src/imagination/vulkan/winsys/powervr/pvr_drm_public.h @@ -30,6 +30,7 @@ VkResult pvr_drm_winsys_create(int render_fd, int display_fd, + bool query_only, const VkAllocationCallbacks *alloc, struct pvr_winsys **ws_out); diff --git a/src/imagination/vulkan/winsys/pvr_winsys.c b/src/imagination/vulkan/winsys/pvr_winsys.c index fb4387576e7..c99f04d468b 100644 --- a/src/imagination/vulkan/winsys/pvr_winsys.c +++ b/src/imagination/vulkan/winsys/pvr_winsys.c @@ -67,6 +67,7 @@ void pvr_winsys_destroy(struct pvr_winsys *ws) VkResult pvr_winsys_create(const char *render_path, const char *display_path, bool keep_display_master, + bool query_only, const VkAllocationCallbacks *alloc, struct pvr_winsys **const ws_out) { @@ -112,7 +113,7 @@ VkResult pvr_winsys_create(const char *render_path, } if (strcmp(version->name, PVR_DRM_DRIVER_NAME) == 0) { - result = pvr_drm_winsys_create(render_fd, display_fd, alloc, ws_out); + result = pvr_drm_winsys_create(render_fd, display_fd, query_only, alloc, ws_out); #if defined(PVR_SUPPORT_SERVICES_DRIVER) } else if (strcmp(version->name, PVR_SRV_DRIVER_NAME) == 0) { result = pvr_srv_winsys_create(render_fd, display_fd, alloc, ws_out); diff --git a/src/imagination/vulkan/winsys/pvr_winsys.h b/src/imagination/vulkan/winsys/pvr_winsys.h index 5202e8d4630..50335a567cf 100644 --- a/src/imagination/vulkan/winsys/pvr_winsys.h +++ b/src/imagination/vulkan/winsys/pvr_winsys.h @@ -137,6 +137,8 @@ struct pvr_winsys_vma { pvr_dev_addr_t dev_addr; uint64_t size; uint64_t mapped_size; + /* vma_map_mirror(): the whole buffer is mapped twice, at dev_addr and at dev_addr + bo->size */ + bool mirrored; }; struct pvr_winsys_free_list { @@ -387,6 +389,9 @@ struct pvr_winsys_ops { int fd, struct pvr_winsys_bo **const bo_out); void (*buffer_destroy)(struct pvr_winsys_bo *bo); + /* Only idle cached chunks may be advised. Replace a non-retained BO. */ + VkResult (*buffer_set_purgeable)(struct pvr_winsys_bo *bo, bool purgeable, + bool *retained); VkResult (*buffer_get_fd)(struct pvr_winsys_bo *bo, int *const fd_out); @@ -405,6 +410,11 @@ struct pvr_winsys_ops { uint64_t size, pvr_dev_addr_t *dev_addr_out); void (*vma_unmap)(struct pvr_winsys_vma *vma); + /* The whole of bo mapped twice back to back, at vma->dev_addr and vma->dev_addr + bo->size (the vma holds + * at least 2 * bo->size): the A8's region-header ring, which AGXG4P maps that way (kext 0x847d738, resources + * created with flag 0x200000) so a frame's slice can run past the ring's end. Two VM_MAPs of one buffer; + * vma_unmap() removes both. NULL where the winsys has no such mapping. */ + VkResult (*vma_map_mirror)(struct pvr_winsys_vma *vma, struct pvr_winsys_bo *bo); VkResult (*free_list_create)( struct pvr_winsys *ws, @@ -493,9 +503,12 @@ void pvr_winsys_count_bo(int kind, int64_t delta); uint64_t pvr_winsys_counted_bytes(int kind); void pvr_winsys_destroy(struct pvr_winsys *ws); +/* query_only: the physical device's winsys, used for device queries; where the winsys supports it (the A8's powervr + * winsys), it then creates no VM context */ VkResult pvr_winsys_create(const char *render_path, const char *display_path, bool keep_display_master, + bool query_only, const VkAllocationCallbacks *alloc, struct pvr_winsys **ws_out);