diff --git a/ps2xRuntime/include/runtime/ps2_vu1.h b/ps2xRuntime/include/runtime/ps2_vu1.h index 67c8184fd..c19926aa9 100644 --- a/ps2xRuntime/include/runtime/ps2_vu1.h +++ b/ps2xRuntime/include/runtime/ps2_vu1.h @@ -3,6 +3,8 @@ #include #include +#include +#include class GS; class PS2Memory; @@ -115,8 +117,24 @@ class VU1Interpreter bool tBit = false; uint8_t upperVfShadowReg = 0; uint8_t suppressedLowerVf = 0; + // Precomputed from the usages: ready-time slots (see m_ready) the + // pair reads, and the slots it writes with their latencies, in the + // order markPairWrites applies them. + uint8_t readSlotCount = 0; + uint8_t writeSlotCount = 0; + uint8_t firstViWrite = 0; + uint8_t maxWriteLatency = 0; + std::array readSlots{}; + std::array writeSlots{}; + std::array writeLatencies{}; }; + // m_ready slots: VF register r lane c, then VI registers, then ACC lanes. + static constexpr uint32_t kReadyVfBase = 0u; + static constexpr uint32_t kReadyViBase = 128u; + static constexpr uint32_t kReadyAccBase = 144u; + static constexpr uint32_t kReadySlotCount = 148u; + struct FlagPipelineEntry { uint64_t readyCycle = 0; @@ -149,34 +167,6 @@ class VU1Interpreter bool valid = false; }; - struct PendingVfWrite - { - uint64_t readyCycle = 0; - uint64_t sequence = 0; - std::array value{}; - uint8_t reg = 0; - uint8_t laneMask = 0; - bool valid = false; - }; - - struct PendingViWrite - { - uint64_t readyCycle = 0; - uint64_t sequence = 0; - int32_t value = 0; - uint8_t reg = 0; - bool valid = false; - }; - - struct PendingAccWrite - { - uint64_t readyCycle = 0; - uint64_t sequence = 0; - std::array value{}; - uint8_t laneMask = 0; - bool valid = false; - }; - struct XgkickPipeline { static constexpr uint32_t kBufferSize = 0x10000u; @@ -195,9 +185,6 @@ class VU1Interpreter static constexpr uint32_t kAccForwardLatency = 1u; static constexpr uint32_t kMaxFlagEntries = 8u; static constexpr uint32_t kMaxPendingStores = 8u; - static constexpr uint32_t kMaxPendingVfWrites = 16u; - static constexpr uint32_t kMaxPendingViWrites = 8u; - static constexpr uint32_t kMaxPendingAccWrites = 8u; static constexpr uint32_t kMaxDecodedPairs = 0x4000u / 8u; Unit m_unit; @@ -213,22 +200,35 @@ class VU1Interpreter ScalarPipelineEntry m_fdiv{}; std::array m_efu{}; std::array m_storePipeline{}; - std::array m_vfWritePipeline{}; - std::array m_viWritePipeline{}; - std::array m_accWritePipeline{}; XgkickPipeline m_xgkick{}; - std::array, 32> m_vfReady{}; - std::array m_viReady{}; - std::array m_accReady{}; - std::array, 32> m_vfLatestWrite{}; - std::array m_viLatestWrite{}; - std::array m_accLatestWrite{}; + // Cycle at which each VF lane / VI register / ACC lane write becomes + // readable by a following instruction. + std::array m_ready{}; uint64_t m_cycle = 0; - uint64_t m_nextWriteSequence = 0; uint64_t m_efuResourceReady = 0; + // Earliest readyCycle of any valid pipeline entry (max when none), so + // the per-cycle commit and pending checks don't scan every pipeline. + uint64_t m_nextCommitCycle = std::numeric_limits::max(); + // Latest cycle at which an issued VF/VI/ACC write lands. The values are + // written back at issue (see run), but a program only ends once its + // writes would have committed. + uint64_t m_writebackHorizon = 0; + // Occupied slots per pipeline (bit i = entry i valid); commits walk set + // bits in ascending slot order, the same order as a full array scan. + uint32_t m_flagMask = 0; + uint32_t m_efuMask = 0; + uint32_t m_storeMask = 0; + DecodedInstructionPair m_uncachedDecoded{}; uint32_t m_workingClip = 0; uint32_t m_currentUpperInstruction = 0; + // Normalized operands of the executing upper instruction (set by + // execUpper, read by normalizeFmacResult). + const float *m_fmacVs = nullptr; + const float *m_fmacVt = nullptr; + const float *m_fmacAcc = nullptr; + float m_fmacQ = 0.0f; + float m_fmacI = 0.0f; int32_t m_viBranchBackupValue = 0; uint8_t m_viBranchBackupReg = 0; bool m_viBranchBackupValid = false; @@ -250,7 +250,7 @@ class VU1Interpreter static void addVfWrite(InstructionUsage &usage, uint8_t reg, uint8_t lanes); static uint8_t vfReadLanes(const InstructionUsage &usage, uint8_t reg); DecodedInstructionPair decodeInstructionPair(const uint8_t *vuCode, uint32_t pc) const; - DecodedInstructionPair getDecodedInstructionPairForPc(const uint8_t *vuCode, uint32_t codeSize, PS2Memory *memory, uint32_t pc); + const DecodedInstructionPair &getDecodedInstructionPairForPc(const uint8_t *vuCode, uint32_t codeSize, PS2Memory *memory, uint32_t pc); void rebuildDecodedCodeCache(const uint8_t *vuCode, uint32_t codeSize, const PS2Memory *memory, uint64_t generation); void execUpper(uint32_t instr); @@ -260,10 +260,11 @@ class VU1Interpreter void applyDestAcc(const float *result, uint8_t dest); void applyFmacDest(float *dst, float *result, uint8_t dest); void applyFmacDestAcc(float *result, uint8_t dest); - void normalizeFmacResult(float *result, uint8_t dest, uint8_t laneFlags[4]); - bool calculateFmacExactResult(uint32_t component, long double &result) const; - uint8_t normalizeFmacExactResult(float &value, long double exactResult) const; - uint32_t calculateFmacProductSticky(uint8_t dest) const; + void normalizeFmacResult(float *result, uint8_t dest, uint8_t laneFlags[4], uint32_t &productSticky); + template + bool calculateFmacExactResult(uint32_t component, Real &result) const; + template + uint8_t normalizeFmacExactResult(float &value, Real exactResult) const; void updateFmacFlags(const uint8_t laneFlags[4], uint8_t dest, uint32_t extraSticky); void queueFsset(uint16_t immediate); void queueClip(uint32_t clip); @@ -271,13 +272,16 @@ class VU1Interpreter void queueQ(float value, uint32_t latency, uint32_t statusDi); void queueP(float value, uint32_t latency); void queueStore(uint32_t address, const uint32_t words[4], uint8_t laneMask); - void queueVfWrite(uint8_t reg, uint8_t laneMask, const float value[4], uint32_t latency); - void queueViWrite(uint8_t reg, int32_t value, uint32_t latency); - void queueAccWrite(uint8_t laneMask, const float value[4], uint32_t latency); void startXgkick(uint32_t qwordAddress); void resetScheduler(); void commitReadyPipelines(); + void commitDuePipelines(); + void noteReadyCycle(uint64_t readyCycle) + { + if (readyCycle < m_nextCommitCycle) + m_nextCommitCycle = readyCycle; + } void advanceOneCycle(); void advanceTo(uint64_t targetCycle); void flushPipelines(); @@ -287,7 +291,19 @@ class VU1Interpreter void markPairWrites(const DecodedInstructionPair &decoded); bool pipelinesPending() const; - float normalizeOperand(float value) const; + // Denormals flush to signed zero; Inf/NaN clamp to the signed max. + float normalizeOperand(float value) const + { + uint32_t bits = 0; + std::memcpy(&bits, &value, sizeof(bits)); + const uint32_t exponent = (bits >> 23) & 0xFFu; + if (exponent == 0u) + bits &= 0x80000000u; + else if (exponent == 0xFFu) + bits = (bits & 0x80000000u) | 0x7F7FFFFFu; + std::memcpy(&value, &bits, sizeof(value)); + return value; + } float normalizeResult(float value, uint32_t &laneFlags) const; uint32_t microAddressMask() const; int32_t readBranchVi(uint8_t reg) const; diff --git a/ps2xRuntime/src/lib/vu/ps2_vu1_core.cpp b/ps2xRuntime/src/lib/vu/ps2_vu1_core.cpp index bd75bba31..80d6a0cc6 100644 --- a/ps2xRuntime/src/lib/vu/ps2_vu1_core.cpp +++ b/ps2xRuntime/src/lib/vu/ps2_vu1_core.cpp @@ -5,6 +5,7 @@ #include "ps2_vu1_detail.h" #include +#include #include #include #include @@ -20,6 +21,99 @@ namespace { return static_cast(1u << (3u - component)); } + + // Claims the lowest free slot of an N-entry pipeline; -1 when full. + template + int claimSlot(uint32_t &mask) + { + const uint32_t freeSlots = ~mask & ((1u << N) - 1u); + if (freeSlots == 0u) + return -1; + const int slot = std::countr_zero(freeSlots); + mask |= 1u << slot; + return slot; + } + + // What an FMAC upper instruction computes per lane, from its normalized + // operands: A is VS (VS crossed for OPMULA/OPMSUB), B the broadcast lane, + // Q, I, VT, or VT crossed. Mirrors calculateFmacExactResult. + enum class FmacOp : uint8_t + { + None, + Add, // A + B + Sub, // A - B + Mul, // A * B + MAdd, // ACC + A * B + MSub, // ACC - A * B + OuterMul, // A * B, w = 0 (OPMULA) + OuterMSub // ACC - A * B, w = 0 (OPMSUB) + }; + + enum class FmacSource : uint8_t + { + Broadcast, + Q, + I, + Full, + Cross, + }; + + struct FmacDecode + { + FmacOp op = FmacOp::None; + FmacSource source = FmacSource::Full; + uint8_t broadcast = 0; + }; + + FmacDecode decodeFmac(uint32_t upper) + { + const uint32_t op = upper & 0x3Fu; + const bool special = op >= 0x3Cu; + const uint32_t code = special ? ((upper & 3u) | ((upper >> 4) & 0x7Cu)) : op; + const uint8_t bc = static_cast(code & 3u); + if (code <= 0x03u) + return {FmacOp::Add, FmacSource::Broadcast, bc}; + if (code <= 0x07u) + return {FmacOp::Sub, FmacSource::Broadcast, bc}; + if (code <= 0x0Bu) + return {FmacOp::MAdd, FmacSource::Broadcast, bc}; + if (code <= 0x0Fu) + return {FmacOp::MSub, FmacSource::Broadcast, bc}; + if (code >= 0x18u && code <= 0x1Bu) + return {FmacOp::Mul, FmacSource::Broadcast, bc}; + switch (code) + { + case 0x1Cu: return {FmacOp::Mul, FmacSource::Q}; + case 0x1Eu: return {FmacOp::Mul, FmacSource::I}; + case 0x20u: return {FmacOp::Add, FmacSource::Q}; + case 0x21u: return {FmacOp::MAdd, FmacSource::Q}; + case 0x22u: return {FmacOp::Add, FmacSource::I}; + case 0x23u: return {FmacOp::MAdd, FmacSource::I}; + case 0x24u: return {FmacOp::Sub, FmacSource::Q}; + case 0x25u: return {FmacOp::MSub, FmacSource::Q}; + case 0x26u: return {FmacOp::Sub, FmacSource::I}; + case 0x27u: return {FmacOp::MSub, FmacSource::I}; + case 0x28u: return {FmacOp::Add, FmacSource::Full}; + case 0x29u: return {FmacOp::MAdd, FmacSource::Full}; + case 0x2Au: return {FmacOp::Mul, FmacSource::Full}; + case 0x2Cu: return {FmacOp::Sub, FmacSource::Full}; + case 0x2Du: return {FmacOp::MSub, FmacSource::Full}; + case 0x2Eu: return {special ? FmacOp::OuterMul : FmacOp::OuterMSub, FmacSource::Cross}; + default: return {}; + } + } + + bool isProductSum(FmacOp op) + { + return op == FmacOp::MAdd || op == FmacOp::MSub || op == FmacOp::OuterMSub; + } + + inline int nextSlot(uint32_t &bits) + { + const int slot = std::countr_zero(bits); + bits &= bits - 1u; + return slot; + } } void VU1Interpreter::addVfRead(InstructionUsage &usage, uint8_t reg, uint8_t lanes) @@ -70,18 +164,14 @@ void VU1Interpreter::resetScheduler() m_fdiv = {}; m_efu = {}; m_storePipeline = {}; - m_vfWritePipeline = {}; - m_viWritePipeline = {}; - m_accWritePipeline = {}; m_xgkick = {}; - m_vfReady = {}; - m_viReady = {}; - m_accReady = {}; - m_vfLatestWrite = {}; - m_viLatestWrite = {}; - m_accLatestWrite = {}; - m_nextWriteSequence = 0; + m_ready = {}; m_efuResourceReady = 0; + m_nextCommitCycle = std::numeric_limits::max(); + m_writebackHorizon = 0; + m_flagMask = 0; + m_efuMask = 0; + m_storeMask = 0; m_workingClip = m_state.clip; m_viBranchBackupValue = 0; m_viBranchBackupReg = 0; @@ -106,23 +196,6 @@ float VU1Interpreter::broadcast(const float *vf, uint8_t bc) return normalizeOperand(vf[bc & 3u]); } -float VU1Interpreter::normalizeOperand(float value) const -{ - uint32_t bits = 0; - std::memcpy(&bits, &value, sizeof(bits)); - const uint32_t exponent = (bits >> 23) & 0xFFu; - if (exponent == 0u) - { - bits &= 0x80000000u; - } - else if (exponent == 0xFFu) - { - bits = (bits & 0x80000000u) | 0x7F7FFFFFu; - } - std::memcpy(&value, &bits, sizeof(value)); - return value; -} - float VU1Interpreter::normalizeResult(float value, uint32_t &laneFlags) const { uint32_t bits = 0; @@ -195,29 +268,137 @@ void VU1Interpreter::applyDestAcc(const float *result, uint8_t dest) } void VU1Interpreter::normalizeFmacResult(float *result, uint8_t dest, - uint8_t laneFlags[4]) + uint8_t laneFlags[4], uint32_t &productSticky) { + productSticky = 0u; + const FmacDecode fmac = decodeFmac(m_currentUpperInstruction); + const bool single = fmac.op == FmacOp::Add || fmac.op == FmacOp::Sub || + fmac.op == FmacOp::Mul || fmac.op == FmacOp::OuterMul; + const bool productSum = isProductSum(fmac.op); + + // Operands as calculateFmacExactResult derives them, from the values + // execUpper already normalized. + bool haveOperands = false; + double a[4]{}, b[4]{}, acc[4]{}; + auto gatherOperands = [&]() + { + if (haveOperands) + return; + haveOperands = true; + static constexpr uint8_t crossLeft[4] = {1u, 2u, 0u, 3u}; + static constexpr uint8_t crossRight[4] = {2u, 0u, 1u, 3u}; + const bool cross = fmac.source == FmacSource::Cross; + for (uint32_t lane = 0; lane < 4u; ++lane) + { + a[lane] = m_fmacVs[cross ? crossLeft[lane] : lane]; + acc[lane] = m_fmacAcc[lane]; + } + switch (fmac.source) + { + case FmacSource::Broadcast: + for (uint32_t lane = 0; lane < 4u; ++lane) + b[lane] = m_fmacVt[fmac.broadcast]; + break; + case FmacSource::Q: + for (uint32_t lane = 0; lane < 4u; ++lane) + b[lane] = m_fmacQ; + break; + case FmacSource::I: + for (uint32_t lane = 0; lane < 4u; ++lane) + b[lane] = m_fmacI; + break; + case FmacSource::Full: + for (uint32_t lane = 0; lane < 4u; ++lane) + b[lane] = m_fmacVt[lane]; + break; + case FmacSource::Cross: + for (uint32_t lane = 0; lane < 4u; ++lane) + b[lane] = m_fmacVt[crossRight[lane]]; + break; + } + }; + for (uint32_t component = 0; component < 4u; ++component) { laneFlags[component] = 0u; if ((dest & laneForComponent(component)) == 0u) continue; - long double exactResult = 0.0L; - if (calculateFmacExactResult(component, exactResult)) + if (productSum) { - laneFlags[component] = normalizeFmacExactResult(result[component], exactResult); + // Every product condition accumulates into the sticky flags (the + // current flags come from the sum). A float product is exact in + // double. + gatherOperands(); + float ignored = 0.0f; + productSticky |= normalizeFmacExactResult(ignored, a[component] * b[component]) & 0xFu; + } + + // With a single rounding toward zero, a normal result below FLT_MAX + // means the exact result is nonzero, same-signed and in range: the + // exact check below would only report the sign and keep the value. + if (single) + { + uint32_t bits = 0; + std::memcpy(&bits, &result[component], sizeof(bits)); + const uint32_t exponent = (bits >> 23) & 0xFFu; + if (exponent != 0u && exponent != 0xFFu && (bits & 0x7FFFFFFFu) != 0x7F7FFFFFu) + { + laneFlags[component] = (bits & 0x80000000u) != 0u ? 0x2u : 0u; + continue; + } + // Operands are zero or normal, so a sum or difference is a + // multiple of the smallest denormal: one that rounds to zero is + // exactly zero, signed by the same IEEE rule as the exact sum. + if ((bits & 0x7FFFFFFFu) == 0u && (fmac.op == FmacOp::Add || fmac.op == FmacOp::Sub)) + { + laneFlags[component] = (bits & 0x80000000u) != 0u ? 0x3u : 0x1u; + continue; + } + } + + if (fmac.op == FmacOp::None) + { + uint32_t flags = 0u; + result[component] = normalizeResult(result[component], flags); + laneFlags[component] = static_cast(flags); continue; } - uint32_t flags = 0u; - result[component] = normalizeResult(result[component], flags); - laneFlags[component] = static_cast(flags); + // The exact result in double classifies like the long double one: a + // float product is exact in double, and zero, sign and FLT_MIN are + // preserved by rounding toward zero. Only a sum that rounds to exactly + // FLT_MAX may hide an overflow, so that case takes the long double. + gatherOperands(); + const double product = a[component] * b[component]; + double exactResult = 0.0; + switch (fmac.op) + { + case FmacOp::Add: exactResult = a[component] + b[component]; break; + case FmacOp::Sub: exactResult = a[component] - b[component]; break; + case FmacOp::Mul: exactResult = product; break; + case FmacOp::MAdd: exactResult = acc[component] + product; break; + case FmacOp::MSub: exactResult = acc[component] - product; break; + case FmacOp::OuterMul: exactResult = component == 3u ? 0.0 : product; break; + case FmacOp::OuterMSub: exactResult = component == 3u ? 0.0 : acc[component] - product; break; + case FmacOp::None: break; + } + if (std::fabs(exactResult) == static_cast(std::numeric_limits::max())) + { + long double preciseResult = 0.0L; + calculateFmacExactResult(component, preciseResult); + laneFlags[component] = normalizeFmacExactResult(result[component], preciseResult); + } + else + { + laneFlags[component] = normalizeFmacExactResult(result[component], exactResult); + } } } +template bool VU1Interpreter::calculateFmacExactResult(uint32_t component, - long double &result) const + Real &result) const { const uint32_t upper = m_currentUpperInstruction; const uint8_t op = static_cast(upper & 0x3Fu); @@ -229,7 +410,7 @@ bool VU1Interpreter::calculateFmacExactResult(uint32_t component, const auto operand = [this](float value) { - return static_cast(normalizeOperand(value)); + return static_cast(normalizeOperand(value)); }; const auto vs = [&](uint32_t lane) { @@ -244,8 +425,8 @@ bool VU1Interpreter::calculateFmacExactResult(uint32_t component, return operand(m_state.acc[lane]); }; - const long double q = operand(m_state.q); - const long double i = operand(m_state.i); + const Real q = operand(m_state.q); + const Real i = operand(m_state.i); if (op < 0x3Cu) { @@ -313,7 +494,7 @@ bool VU1Interpreter::calculateFmacExactResult(uint32_t component, static constexpr uint8_t left[4] = {1u, 2u, 0u, 3u}; static constexpr uint8_t right[4] = {2u, 0u, 1u, 3u}; result = component == 3u - ? 0.0L + ? Real(0) : acc(component) - vs(left[component]) * vt(right[component]); break; } @@ -388,7 +569,7 @@ bool VU1Interpreter::calculateFmacExactResult(uint32_t component, static constexpr uint8_t left[4] = {1u, 2u, 0u, 3u}; static constexpr uint8_t right[4] = {2u, 0u, 1u, 3u}; result = component == 3u - ? 0.0L + ? Real(0) : vs(left[component]) * vt(right[component]); break; } @@ -399,17 +580,18 @@ bool VU1Interpreter::calculateFmacExactResult(uint32_t component, return true; } +template uint8_t VU1Interpreter::normalizeFmacExactResult(float &value, - long double exactResult) const + Real exactResult) const { const bool negative = std::signbit(exactResult); - const long double magnitude = std::fabs(exactResult); - const long double maximum = static_cast(std::numeric_limits::max()); - const long double minimum = static_cast(std::numeric_limits::min()); + const Real magnitude = std::fabs(exactResult); + const Real maximum = static_cast(std::numeric_limits::max()); + const Real minimum = static_cast(std::numeric_limits::min()); uint8_t flags = negative ? 0x2u : 0u; uint32_t bits = negative ? 0x80000000u : 0u; - if (magnitude == 0.0L) + if (magnitude == Real(0)) { flags |= 0x1u; std::memcpy(&value, &bits, sizeof(value)); @@ -429,65 +611,6 @@ uint8_t VU1Interpreter::normalizeFmacExactResult(float &value, return flags; } -uint32_t VU1Interpreter::calculateFmacProductSticky(uint8_t dest) const -{ - uint32_t extraSticky = 0u; - const uint32_t upper = m_currentUpperInstruction; - const uint8_t op = static_cast(upper & 0x3Fu); - const uint8_t special = op >= 0x3Cu ? static_cast((upper & 3u) | ((upper >> 4) & 0x7Cu)) : 0xFFu; - const bool productSum = - (op >= 0x08u && op <= 0x0Fu) || - op == 0x21u || op == 0x23u || op == 0x25u || op == 0x27u || - op == 0x29u || op == 0x2Du || op == 0x2Eu || - (special >= 0x08u && special <= 0x0Fu) || - special == 0x21u || special == 0x23u || special == 0x25u || - special == 0x27u || special == 0x29u || special == 0x2Du; - if (!productSum) - return 0u; - - const uint8_t fs = FS(upper); - const uint8_t ft = FT(upper); - for (uint32_t component = 0; component < 4u; ++component) - { - if ((dest & laneForComponent(component)) == 0u) - continue; - static constexpr uint8_t crossLeft[4] = {1u, 2u, 0u, 3u}; - static constexpr uint8_t crossRight[4] = {2u, 0u, 1u, 3u}; - const uint8_t leftComponent = op == 0x2Eu ? crossLeft[component] : static_cast(component); - const float left = normalizeOperand(m_state.vf[fs][leftComponent]); - float right = 0.0f; - if ((op >= 0x08u && op <= 0x0Fu) || (special >= 0x08u && special <= 0x0Fu)) - { - right = normalizeOperand(m_state.vf[ft][(op >= 0x08u && op <= 0x0Fu ? op : special) & 3u]); - } - else if (op == 0x21u || op == 0x25u || special == 0x21u || special == 0x25u) - { - right = normalizeOperand(m_state.q); - } - else if (op == 0x23u || op == 0x27u || special == 0x23u || special == 0x27u) - { - right = normalizeOperand(m_state.i); - } - else if (op == 0x2Eu) - { - right = normalizeOperand(m_state.vf[ft][crossRight[component]]); - } - else - { - right = normalizeOperand(m_state.vf[ft][component]); - } - - float product = left * right; - const long double exactProduct = static_cast(left) * static_cast(right); - const uint8_t productFlags = normalizeFmacExactResult(product, exactProduct); - // Product-sum instructions report Z/S/U/O from the add/subtract result - // as current flags, while every product condition accumulates into the - // corresponding sticky flag. - extraSticky |= productFlags & 0xFu; - } - return extraSticky; -} - void VU1Interpreter::updateFmacFlags(const uint8_t laneFlags[4], uint8_t dest, uint32_t extraSticky) { @@ -514,25 +637,19 @@ void VU1Interpreter::updateFmacFlags(const uint8_t laneFlags[4], uint8_t dest, status |= flags; } - FlagPipelineEntry *entry = nullptr; - for (FlagPipelineEntry &candidate : m_flagPipeline) - { - if (!candidate.valid) - { - entry = &candidate; - break; - } - } - if (!entry) + const int slot = claimSlot(m_flagMask); + if (slot < 0) { reportReservedInstruction(true, 0xFFFFFFFFu); return; } + FlagPipelineEntry *entry = &m_flagPipeline[static_cast(slot)]; *entry = {}; entry->valid = true; entry->issueCycle = m_cycle; entry->readyCycle = m_cycle + kFmacLatency; + noteReadyCycle(entry->readyCycle); entry->mac = mac; entry->status = status; entry->extraSticky = extraSticky; @@ -543,35 +660,39 @@ void VU1Interpreter::updateFmacFlags(const uint8_t laneFlags[4], uint8_t dest, void VU1Interpreter::applyFmacDest(float *dst, float *result, uint8_t dest) { uint8_t laneFlags[4]{}; - normalizeFmacResult(result, dest, laneFlags); - updateFmacFlags(laneFlags, dest, calculateFmacProductSticky(dest)); + uint32_t productSticky = 0u; + normalizeFmacResult(result, dest, laneFlags, productSticky); + updateFmacFlags(laneFlags, dest, productSticky); applyDest(dst, result, dest); } void VU1Interpreter::applyFmacDestAcc(float *result, uint8_t dest) { uint8_t laneFlags[4]{}; - normalizeFmacResult(result, dest, laneFlags); - updateFmacFlags(laneFlags, dest, calculateFmacProductSticky(dest)); + uint32_t productSticky = 0u; + normalizeFmacResult(result, dest, laneFlags, productSticky); + updateFmacFlags(laneFlags, dest, productSticky); applyDestAcc(result, dest); } void VU1Interpreter::queueFsset(uint16_t immediate) { - for (FlagPipelineEntry &entry : m_flagPipeline) + for (uint32_t bits = m_flagMask; bits != 0u;) { - if (entry.valid && entry.issueCycle == m_cycle) + FlagPipelineEntry &entry = m_flagPipeline[static_cast(nextSlot(bits))]; + if (entry.issueCycle == m_cycle) entry.writesStatus = false; } - for (FlagPipelineEntry &entry : m_flagPipeline) + if (const int slot = claimSlot(m_flagMask); slot >= 0) { - if (!entry.valid) + FlagPipelineEntry &entry = m_flagPipeline[static_cast(slot)]; { entry = {}; entry.valid = true; entry.issueCycle = m_cycle; entry.readyCycle = m_cycle + kFmacLatency; + noteReadyCycle(entry.readyCycle); entry.status = static_cast(immediate) & 0xFC0u; entry.writesSticky = true; return; @@ -583,14 +704,15 @@ void VU1Interpreter::queueFsset(uint16_t immediate) void VU1Interpreter::queueClip(uint32_t clip) { m_workingClip = ((m_workingClip << 6) | (clip & 0x3Fu)) & 0xFFFFFFu; - for (FlagPipelineEntry &entry : m_flagPipeline) + if (const int slot = claimSlot(m_flagMask); slot >= 0) { - if (!entry.valid) + FlagPipelineEntry &entry = m_flagPipeline[static_cast(slot)]; { entry = {}; entry.valid = true; entry.issueCycle = m_cycle; entry.readyCycle = m_cycle + kFmacLatency; + noteReadyCycle(entry.readyCycle); entry.clip = m_workingClip; entry.writesClip = true; return; @@ -602,19 +724,21 @@ void VU1Interpreter::queueClip(uint32_t clip) void VU1Interpreter::queueFcset(uint32_t clip) { m_workingClip = clip & 0xFFFFFFu; - for (FlagPipelineEntry &entry : m_flagPipeline) + for (uint32_t bits = m_flagMask; bits != 0u;) { - if (entry.valid && entry.issueCycle == m_cycle) + FlagPipelineEntry &entry = m_flagPipeline[static_cast(nextSlot(bits))]; + if (entry.issueCycle == m_cycle) entry.writesClip = false; } - for (FlagPipelineEntry &entry : m_flagPipeline) + if (const int slot = claimSlot(m_flagMask); slot >= 0) { - if (!entry.valid) + FlagPipelineEntry &entry = m_flagPipeline[static_cast(slot)]; { entry = {}; entry.valid = true; entry.issueCycle = m_cycle; entry.readyCycle = m_cycle + kFmacLatency; + noteReadyCycle(entry.readyCycle); entry.clip = m_workingClip; entry.writesClip = true; return; @@ -629,6 +753,7 @@ void VU1Interpreter::queueQ(float value, uint32_t latency, uint32_t statusDi) value = normalizeResult(value, ignoredFlags); m_fdiv.valid = true; m_fdiv.readyCycle = m_cycle + latency; + noteReadyCycle(m_fdiv.readyCycle); m_fdiv.value = value; m_fdiv.statusDi = statusDi & 0x30u; } @@ -637,12 +762,13 @@ void VU1Interpreter::queueP(float value, uint32_t latency) { uint32_t ignoredFlags = 0u; value = normalizeResult(value, ignoredFlags); - for (ScalarPipelineEntry &entry : m_efu) + if (const int slot = claimSlot<2u>(m_efuMask); slot >= 0) { - if (!entry.valid) + ScalarPipelineEntry &entry = m_efu[static_cast(slot)]; { entry.valid = true; entry.readyCycle = m_cycle + latency; + noteReadyCycle(entry.readyCycle); entry.value = value; // EFU throughput is one cycle shorter than result visibility. m_efuResourceReady = m_cycle + (latency > 0u ? latency - 1u : 0u); @@ -654,12 +780,13 @@ void VU1Interpreter::queueP(float value, uint32_t latency) void VU1Interpreter::queueStore(uint32_t address, const uint32_t words[4], uint8_t laneMask) { - for (PendingStore &store : m_storePipeline) + if (const int slot = claimSlot(m_storeMask); slot >= 0) { - if (!store.valid) + PendingStore &store = m_storePipeline[static_cast(slot)]; { store.valid = true; store.readyCycle = m_cycle + 1u; + noteReadyCycle(store.readyCycle); store.address = address; store.laneMask = laneMask; std::copy(words, words + 4, store.words.begin()); @@ -669,85 +796,25 @@ void VU1Interpreter::queueStore(uint32_t address, const uint32_t words[4], uint8 reportReservedInstruction(false, 0xFFFFFFFCu); } -void VU1Interpreter::queueVfWrite(uint8_t reg, uint8_t laneMask, - const float value[4], uint32_t latency) +void VU1Interpreter::commitReadyPipelines() { - if (reg == 0u || laneMask == 0u) - return; - for (PendingVfWrite &write : m_vfWritePipeline) - { - if (!write.valid) - { - write = {}; - write.valid = true; - write.readyCycle = m_cycle + latency; - write.sequence = ++m_nextWriteSequence; - write.reg = reg; - write.laneMask = laneMask; - std::copy(value, value + 4, write.value.begin()); - for (uint32_t component = 0; component < 4u; ++component) - { - if ((laneMask & laneForComponent(component)) != 0u) - m_vfLatestWrite[reg][component] = write.sequence; - } - return; - } - } - reportReservedInstruction(false, 0xFFFFFFF7u); + if (m_cycle >= m_nextCommitCycle) + commitDuePipelines(); } -void VU1Interpreter::queueViWrite(uint8_t reg, int32_t value, uint32_t latency) +void VU1Interpreter::commitDuePipelines() { - if (reg == 0u) - return; - for (PendingViWrite &write : m_viWritePipeline) - { - if (!write.valid) - { - write = {}; - write.valid = true; - write.readyCycle = m_cycle + latency; - write.sequence = ++m_nextWriteSequence; - write.reg = reg; - write.value = value; - m_viLatestWrite[reg] = write.sequence; - return; - } - } - reportReservedInstruction(false, 0xFFFFFFF6u); -} + uint64_t next = std::numeric_limits::max(); -void VU1Interpreter::queueAccWrite(uint8_t laneMask, const float value[4], uint32_t latency) -{ - if (laneMask == 0u) - return; - for (PendingAccWrite &write : m_accWritePipeline) + for (uint32_t bits = m_flagMask; bits != 0u;) { - if (!write.valid) + const int slot = nextSlot(bits); + FlagPipelineEntry &entry = m_flagPipeline[static_cast(slot)]; + if (entry.readyCycle > m_cycle) { - write = {}; - write.valid = true; - write.readyCycle = m_cycle + latency; - write.sequence = ++m_nextWriteSequence; - write.laneMask = laneMask; - std::copy(value, value + 4, write.value.begin()); - for (uint32_t component = 0; component < 4u; ++component) - { - if ((laneMask & laneForComponent(component)) != 0u) - m_accLatestWrite[component] = write.sequence; - } - return; - } - } - reportReservedInstruction(true, 0xFFFFFFF5u); -} - -void VU1Interpreter::commitReadyPipelines() -{ - for (FlagPipelineEntry &entry : m_flagPipeline) - { - if (!entry.valid || entry.readyCycle > m_cycle) + next = std::min(next, entry.readyCycle); continue; + } if (entry.writesMac) m_state.mac = entry.mac; @@ -763,29 +830,47 @@ void VU1Interpreter::commitReadyPipelines() if (entry.writesClip) m_state.clip = entry.clip; entry = {}; + m_flagMask &= ~(1u << slot); } - if (m_fdiv.valid && m_fdiv.readyCycle <= m_cycle) + if (m_fdiv.valid) { - m_state.q = m_fdiv.value; - const uint32_t currentDi = m_fdiv.statusDi & 0x30u; - m_state.status = (m_state.status & 0xFCFu) | currentDi | (currentDi << 6); - m_fdiv = {}; + if (m_fdiv.readyCycle <= m_cycle) + { + m_state.q = m_fdiv.value; + const uint32_t currentDi = m_fdiv.statusDi & 0x30u; + m_state.status = (m_state.status & 0xFCFu) | currentDi | (currentDi << 6); + m_fdiv = {}; + } + else + { + next = std::min(next, m_fdiv.readyCycle); + } } - for (ScalarPipelineEntry &entry : m_efu) + for (uint32_t bits = m_efuMask; bits != 0u;) { - if (entry.valid && entry.readyCycle <= m_cycle) + const int slot = nextSlot(bits); + ScalarPipelineEntry &entry = m_efu[static_cast(slot)]; + if (entry.readyCycle > m_cycle) { - m_state.p = entry.value; - entry = {}; + next = std::min(next, entry.readyCycle); + continue; } + m_state.p = entry.value; + entry = {}; + m_efuMask &= ~(1u << slot); } - for (PendingStore &store : m_storePipeline) + for (uint32_t bits = m_storeMask; bits != 0u;) { - if (!store.valid || store.readyCycle > m_cycle) + const int slot = nextSlot(bits); + PendingStore &store = m_storePipeline[static_cast(slot)]; + if (store.readyCycle > m_cycle) + { + next = std::min(next, store.readyCycle); continue; + } if (m_activeVuData && store.address + 16u <= m_activeVuDataSize) { uint32_t oldWords[4]{}; @@ -798,46 +883,10 @@ void VU1Interpreter::commitReadyPipelines() std::memcpy(m_activeVuData + store.address, oldWords, sizeof(oldWords)); } store = {}; + m_storeMask &= ~(1u << slot); } - for (PendingVfWrite &write : m_vfWritePipeline) - { - if (!write.valid || write.readyCycle > m_cycle) - continue; - for (uint32_t component = 0; component < 4u; ++component) - { - if ((write.laneMask & laneForComponent(component)) != 0u && - m_vfLatestWrite[write.reg][component] == write.sequence) - { - m_state.vf[write.reg][component] = write.value[component]; - } - } - write = {}; - } - - for (PendingViWrite &write : m_viWritePipeline) - { - if (!write.valid || write.readyCycle > m_cycle) - continue; - if (m_viLatestWrite[write.reg] == write.sequence) - m_state.vi[write.reg] = static_cast(write.value); - write = {}; - } - - for (PendingAccWrite &write : m_accWritePipeline) - { - if (!write.valid || write.readyCycle > m_cycle) - continue; - for (uint32_t component = 0; component < 4u; ++component) - { - if ((write.laneMask & laneForComponent(component)) != 0u && - m_accLatestWrite[component] == write.sequence) - { - m_state.acc[component] = write.value[component]; - } - } - write = {}; - } + m_nextCommitCycle = next; } void VU1Interpreter::progressXgkick() @@ -857,10 +906,18 @@ void VU1Interpreter::progressXgkick() } const uint32_t qwordOffset = m_xgkick.copiedBytes; - for (uint32_t i = 0; i < 16u; ++i) + const uint32_t qwordSource = (m_xgkick.sourceAddress + m_xgkick.copiedBytes) % m_activeVuDataSize; + if (qwordSource + 16u <= m_activeVuDataSize) { - const uint32_t source = (m_xgkick.sourceAddress + m_xgkick.copiedBytes + i) % m_activeVuDataSize; - m_xgkick.packet[m_xgkick.copiedBytes + i] = m_activeVuData[source]; + std::memcpy(m_xgkick.packet.data() + qwordOffset, m_activeVuData + qwordSource, 16u); + } + else + { + for (uint32_t i = 0; i < 16u; ++i) + { + const uint32_t source = (m_xgkick.sourceAddress + m_xgkick.copiedBytes + i) % m_activeVuDataSize; + m_xgkick.packet[m_xgkick.copiedBytes + i] = m_activeVuData[source]; + } } m_xgkick.copiedBytes += 16u; @@ -952,70 +1009,46 @@ void VU1Interpreter::advanceOneCycle() void VU1Interpreter::advanceTo(uint64_t targetCycle) { while (m_cycle < targetCycle) + { + // With no XGKICK in flight a cycle does nothing until the next + // pipeline commit, so skip straight to the cycle before it. + if (!m_xgkick.active) + { + const uint64_t idleEnd = std::min(targetCycle, m_nextCommitCycle) - 1u; + if (idleEnd > m_cycle) + m_cycle = idleEnd; + } advanceOneCycle(); + } } bool VU1Interpreter::pipelinesPending() const { - if (m_fdiv.valid || m_xgkick.active) - return true; - for (const ScalarPipelineEntry &entry : m_efu) - if (entry.valid) - return true; - for (const FlagPipelineEntry &entry : m_flagPipeline) - if (entry.valid) - return true; - for (const PendingStore &store : m_storePipeline) - if (store.valid) - return true; - for (const PendingVfWrite &write : m_vfWritePipeline) - if (write.valid) - return true; - for (const PendingViWrite &write : m_viWritePipeline) - if (write.valid) - return true; - for (const PendingAccWrite &write : m_accWritePipeline) - if (write.valid) - return true; - return false; + return m_xgkick.active || m_nextCommitCycle != std::numeric_limits::max() || + m_writebackHorizon > m_cycle; } void VU1Interpreter::flushPipelines() { while (pipelinesPending()) + { + if (!m_xgkick.active) + { + uint64_t next = m_nextCommitCycle; + if (m_writebackHorizon > m_cycle) + next = std::min(next, m_writebackHorizon); + if (next > m_cycle + 1u) + m_cycle = next - 1u; + } advanceOneCycle(); + } } uint64_t VU1Interpreter::calculatePairReadyCycle(const DecodedInstructionPair &decoded) const { uint64_t ready = m_cycle; - const InstructionUsage *usages[2] = { - &decoded.upperUsage, - &decoded.lowerUsage}; - for (const InstructionUsage *usage : usages) - { - if (!usage) - continue; - for (uint32_t index = 0; index < usage->vfReadCount; ++index) - { - const VfAccess &access = usage->vfRead[index]; - for (uint32_t component = 0; component < 4u; ++component) - { - if ((access.lanes & laneForComponent(component)) != 0u) - ready = std::max(ready, m_vfReady[access.reg][component]); - } - } - for (uint32_t reg = 1; reg < m_viReady.size(); ++reg) - { - if ((usage->viRead & (1u << reg)) != 0u) - ready = std::max(ready, m_viReady[reg]); - } - for (uint32_t component = 0; component < 4u; ++component) - { - if ((usage->accRead & laneForComponent(component)) != 0u) - ready = std::max(ready, m_accReady[component]); - } - } + for (uint32_t index = 0; index < decoded.readSlotCount; ++index) + ready = std::max(ready, m_ready[decoded.readSlots[index]]); if (decoded.lowerUsage.pipeline == PipelineFdiv && m_fdiv.valid) ready = std::max(ready, m_fdiv.readyCycle); @@ -1036,43 +1069,10 @@ uint64_t VU1Interpreter::calculatePairReadyCycle(const DecodedInstructionPair &d void VU1Interpreter::markPairWrites(const DecodedInstructionPair &decoded) { - const VfAccess lowerWrite = decoded.lowerUsage.vfWrite; - if (lowerWrite.reg != 0u && - decoded.suppressedLowerVf != lowerWrite.reg) - { - const uint32_t latency = decoded.lowerUsage.vfLatency != 0u - ? decoded.lowerUsage.vfLatency - : decoded.lowerUsage.latency; - for (uint32_t component = 0; component < 4u; ++component) - { - if ((lowerWrite.lanes & laneForComponent(component)) != 0u) - m_vfReady[lowerWrite.reg][component] = m_cycle + latency; - } - } - - const VfAccess upperWrite = decoded.upperUsage.vfWrite; - if (upperWrite.reg != 0u) - { - const uint32_t latency = decoded.upperUsage.vfLatency != 0u - ? decoded.upperUsage.vfLatency - : decoded.upperUsage.latency; - for (uint32_t component = 0; component < 4u; ++component) - { - if ((upperWrite.lanes & laneForComponent(component)) != 0u) - m_vfReady[upperWrite.reg][component] = m_cycle + latency; - } - } - - for (uint32_t reg = 1; reg < m_viReady.size(); ++reg) - { - if ((decoded.lowerUsage.viWrite & (1u << reg)) != 0u) - m_viReady[reg] = m_cycle + (decoded.lowerUsage.viLatency != 0u ? decoded.lowerUsage.viLatency : decoded.lowerUsage.latency); - } - for (uint32_t component = 0; component < 4u; ++component) - { - if ((decoded.upperUsage.accWrite & laneForComponent(component)) != 0u) - m_accReady[component] = m_cycle + kAccForwardLatency; - } + for (uint32_t index = 0; index < decoded.writeSlotCount; ++index) + m_ready[decoded.writeSlots[index]] = m_cycle + decoded.writeLatencies[index]; + if (decoded.writeSlotCount != 0u) + m_writebackHorizon = std::max(m_writebackHorizon, m_cycle + decoded.maxWriteLatency); } VU1Interpreter::InstructionUsage VU1Interpreter::decodeUpperUsage(uint32_t upper) const @@ -1528,6 +1528,66 @@ VU1Interpreter::DecodedInstructionPair VU1Interpreter::decodeInstructionPair(con if (decoded.lowerUsage.vfWrite.reg == upperWriteReg) decoded.suppressedLowerVf = upperWriteReg; } + + // Hazard slots, matching the checks calculatePairReadyCycle and + // markPairWrites used to derive from the usages on every issue. + auto addRead = [&decoded](uint32_t slot) + { + decoded.readSlots[decoded.readSlotCount++] = static_cast(slot); + }; + for (const InstructionUsage *usage : {&decoded.upperUsage, &decoded.lowerUsage}) + { + for (uint32_t index = 0; index < usage->vfReadCount; ++index) + { + const VfAccess &access = usage->vfRead[index]; + if (access.reg == 0u) + continue; // VF0 is never written, so never pending. + for (uint32_t component = 0; component < 4u; ++component) + if ((access.lanes & laneForComponent(component)) != 0u) + addRead(kReadyVfBase + access.reg * 4u + component); + } + for (uint32_t bits = usage->viRead & 0xFFFEu; bits != 0u; bits &= bits - 1u) + addRead(kReadyViBase + static_cast(std::countr_zero(bits))); + for (uint32_t component = 0; component < 4u; ++component) + if ((usage->accRead & laneForComponent(component)) != 0u) + addRead(kReadyAccBase + component); + } + + auto addWrite = [&decoded](uint32_t slot, uint32_t latency) + { + decoded.writeSlots[decoded.writeSlotCount] = static_cast(slot); + decoded.writeLatencies[decoded.writeSlotCount] = static_cast(latency); + decoded.maxWriteLatency = std::max(decoded.maxWriteLatency, static_cast(latency)); + ++decoded.writeSlotCount; + }; + const VfAccess lowerWrite = decoded.lowerUsage.vfWrite; + if (lowerWrite.reg != 0u && decoded.suppressedLowerVf != lowerWrite.reg) + { + const uint32_t latency = decoded.lowerUsage.vfLatency != 0u ? decoded.lowerUsage.vfLatency + : decoded.lowerUsage.latency; + for (uint32_t component = 0; component < 4u; ++component) + if ((lowerWrite.lanes & laneForComponent(component)) != 0u) + addWrite(kReadyVfBase + lowerWrite.reg * 4u + component, latency); + } + const VfAccess upperWrite = decoded.upperUsage.vfWrite; + if (upperWrite.reg != 0u) + { + const uint32_t latency = decoded.upperUsage.vfLatency != 0u ? decoded.upperUsage.vfLatency + : decoded.upperUsage.latency; + for (uint32_t component = 0; component < 4u; ++component) + if ((upperWrite.lanes & laneForComponent(component)) != 0u) + addWrite(kReadyVfBase + upperWrite.reg * 4u + component, latency); + } + const uint32_t viLatency = decoded.lowerUsage.viLatency != 0u ? decoded.lowerUsage.viLatency + : decoded.lowerUsage.latency; + for (uint32_t bits = decoded.lowerUsage.viWrite & 0xFFFEu; bits != 0u; bits &= bits - 1u) + addWrite(kReadyViBase + static_cast(std::countr_zero(bits)), viLatency); + for (uint32_t component = 0; component < 4u; ++component) + if ((decoded.upperUsage.accWrite & laneForComponent(component)) != 0u) + addWrite(kReadyAccBase + component, kAccForwardLatency); + + if ((decoded.lowerUsage.viWrite & 0xFFFEu) != 0u) + decoded.firstViWrite = static_cast(std::countr_zero(decoded.lowerUsage.viWrite & 0xFFFEu)); return decoded; } @@ -1545,17 +1605,17 @@ void VU1Interpreter::rebuildDecodedCodeCache(const uint8_t *vuCode, uint32_t cod m_decodedCodeCacheValid = true; } -VU1Interpreter::DecodedInstructionPair VU1Interpreter::getDecodedInstructionPairForPc( +const VU1Interpreter::DecodedInstructionPair &VU1Interpreter::getDecodedInstructionPairForPc( const uint8_t *vuCode, uint32_t codeSize, PS2Memory *memory, uint32_t pc) { if ((pc & 7u) != 0u) - return decodeInstructionPair(vuCode, pc); + return m_uncachedDecoded = decodeInstructionPair(vuCode, pc); const bool trackedVu1Code = memory != nullptr && ((m_unit == Unit::VU1 && vuCode == memory->getVU1Code()) || (m_unit == Unit::VU0 && vuCode == memory->getVU0Code())); if (!trackedVu1Code) - return decodeInstructionPair(vuCode, pc); + return m_uncachedDecoded = decodeInstructionPair(vuCode, pc); const uint64_t generation = m_unit == Unit::VU1 ? memory->getVU1CodeGeneration() : memory->getVU0CodeGeneration(); if (!m_decodedCodeCacheValid || @@ -1568,7 +1628,7 @@ VU1Interpreter::DecodedInstructionPair VU1Interpreter::getDecodedInstructionPair } const uint32_t pairIndex = pc / 8u; if (pairIndex >= kMaxDecodedPairs) - return decodeInstructionPair(vuCode, pc); + return m_uncachedDecoded = decodeInstructionPair(vuCode, pc); return m_decodedCodeCache[pairIndex]; } @@ -1639,7 +1699,7 @@ void VU1Interpreter::run(uint8_t *vuCode, uint32_t codeSize, if (m_state.pc + 8u > codeSize) break; - const DecodedInstructionPair decoded = getDecodedInstructionPairForPc(vuCode, codeSize, memory, m_state.pc); + const DecodedInstructionPair &decoded = getDecodedInstructionPairForPc(vuCode, codeSize, memory, m_state.pc); if (decoded.upperUsage.reserved || decoded.lowerUsage.reserved) { reportReservedInstruction(decoded.upperUsage.reserved, decoded.upperUsage.reserved ? decoded.upper : decoded.lower); @@ -1660,35 +1720,8 @@ void VU1Interpreter::run(uint8_t *vuCode, uint32_t codeSize, if (m_cycle >= budgetEnd) break; - uint8_t writtenVi = 0u; - int32_t oldVi = 0; - for (uint32_t reg = 1; reg < 16u; ++reg) - { - if ((decoded.lowerUsage.viWrite & (1u << reg)) != 0u) - { - writtenVi = static_cast(reg); - oldVi = m_state.vi[reg]; - break; - } - } - - const VfAccess upperWrite = decoded.upperUsage.vfWrite; - const VfAccess lowerWrite = decoded.lowerUsage.vfWrite; - const bool hasUpperWrite = upperWrite.reg != 0u; - const bool hasLowerWrite = lowerWrite.reg != 0u && decoded.suppressedLowerVf != lowerWrite.reg; - const bool hasDistinctLowerWrite = hasLowerWrite && (!hasUpperWrite || lowerWrite.reg != upperWrite.reg); - float oldUpperVf[4]{}; - float newUpperVf[4]{}; - float oldLowerVf[4]{}; - float newLowerVf[4]{}; - float oldAcc[4]{}; - float newAcc[4]{}; - if (hasUpperWrite) - std::memcpy(oldUpperVf, m_state.vf[upperWrite.reg], sizeof(oldUpperVf)); - if (hasDistinctLowerWrite) - std::memcpy(oldLowerVf, m_state.vf[lowerWrite.reg], sizeof(oldLowerVf)); - if (decoded.upperUsage.accWrite != 0u) - std::memcpy(oldAcc, m_state.acc, sizeof(oldAcc)); + const uint8_t writtenVi = decoded.firstViWrite; + const int32_t oldVi = writtenVi != 0u ? m_state.vi[writtenVi] : 0; if (decoded.iBit) { @@ -1724,45 +1757,11 @@ void VU1Interpreter::run(uint8_t *vuCode, uint32_t codeSize, m_viBranchBackupValid = false; - if (hasUpperWrite) - { - std::memcpy(newUpperVf, m_state.vf[upperWrite.reg], sizeof(newUpperVf)); - std::memcpy(m_state.vf[upperWrite.reg], oldUpperVf, sizeof(oldUpperVf)); - const uint32_t latency = - decoded.upperUsage.vfLatency != 0u - ? decoded.upperUsage.vfLatency - : decoded.upperUsage.latency; - queueVfWrite(upperWrite.reg, upperWrite.lanes, newUpperVf, latency); - } - if (hasDistinctLowerWrite) - { - std::memcpy(newLowerVf, m_state.vf[lowerWrite.reg], sizeof(newLowerVf)); - std::memcpy(m_state.vf[lowerWrite.reg], oldLowerVf, sizeof(oldLowerVf)); - const uint32_t latency = decoded.lowerUsage.vfLatency != 0u - ? decoded.lowerUsage.vfLatency - : decoded.lowerUsage.latency; - queueVfWrite(lowerWrite.reg, lowerWrite.lanes, newLowerVf, latency); - } - if (decoded.upperUsage.accWrite != 0u) - { - std::memcpy(newAcc, m_state.acc, sizeof(newAcc)); - std::memcpy(m_state.acc, oldAcc, sizeof(oldAcc)); - // ACC is forwarded to the next upper instruction. Its arithmetic - // flags still use the normal four-cycle FMAC timeline. - queueAccWrite(decoded.upperUsage.accWrite, newAcc, - kAccForwardLatency); - } - if (writtenVi != 0u) - { - const int32_t newVi = m_state.vi[writtenVi]; - m_state.vi[writtenVi] = oldVi; - const uint32_t latency = - decoded.lowerUsage.viLatency != 0u - ? decoded.lowerUsage.viLatency - : decoded.lowerUsage.latency; - queueViWrite(writtenVi, newVi, latency); - } - + // VF, VI and ACC results are written back immediately: these + // registers are interlocked, so calculatePairReadyCycle stalls every + // reader until the result's latency has elapsed and an early value + // can never be observed. (Flags, Q, P and stores are not interlocked + // and keep their pipelines.) markPairWrites(decoded); if (writtenVi != 0u && decoded.lowerUsage.delaysNextBranchRead) recordViWriteForBranch(writtenVi, oldVi); diff --git a/ps2xRuntime/src/lib/vu/ps2_vu1_upper.cpp b/ps2xRuntime/src/lib/vu/ps2_vu1_upper.cpp index 69742546f..79d200085 100644 --- a/ps2xRuntime/src/lib/vu/ps2_vu1_upper.cpp +++ b/ps2xRuntime/src/lib/vu/ps2_vu1_upper.cpp @@ -4,9 +4,38 @@ #include #include #include +#if defined(__SSE4_1__) +#include +#endif namespace { + // normalizeOperand on four lanes. + void normalizeOperands(const float *in, float *out) + { +#if defined(__SSE4_1__) + const __m128i bits = _mm_loadu_si128(reinterpret_cast(in)); + const __m128i exponent = _mm_and_si128(bits, _mm_set1_epi32(0x7F800000)); + const __m128i sign = _mm_and_si128(bits, _mm_set1_epi32(static_cast(0x80000000u))); + __m128i normalized = _mm_blendv_epi8(bits, sign, _mm_cmpeq_epi32(exponent, _mm_setzero_si128())); + normalized = _mm_blendv_epi8(normalized, _mm_or_si128(sign, _mm_set1_epi32(0x7F7FFFFF)), + _mm_cmpeq_epi32(exponent, _mm_set1_epi32(0x7F800000))); + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), normalized); +#else + for (uint32_t lane = 0; lane < 4u; ++lane) + { + uint32_t bits = 0; + std::memcpy(&bits, &in[lane], sizeof(bits)); + const uint32_t exponent = (bits >> 23) & 0xFFu; + if (exponent == 0u) + bits &= 0x80000000u; + else if (exponent == 0xFFu) + bits = (bits & 0x80000000u) | 0x7F7FFFFFu; + std::memcpy(&out[lane], &bits, sizeof(bits)); + } +#endif + } + int32_t vuFloatToInt(float value, float scale) { const double scaled = static_cast(value) * static_cast(scale); @@ -34,17 +63,19 @@ void VU1Interpreter::execUpper(uint32_t instr) float normalizedVs[4]; float normalizedVt[4]; float normalizedAcc[4]; - for (uint32_t component = 0; component < 4u; ++component) - { - normalizedVs[component] = normalizeOperand(m_state.vf[fs][component]); - normalizedVt[component] = normalizeOperand(m_state.vf[ft][component]); - normalizedAcc[component] = normalizeOperand(m_state.acc[component]); - } + normalizeOperands(m_state.vf[fs], normalizedVs); + normalizeOperands(m_state.vf[ft], normalizedVt); + normalizeOperands(m_state.acc, normalizedAcc); const float *vs = normalizedVs; const float *vt = normalizedVt; const float *acc = normalizedAcc; const float q = normalizeOperand(m_state.q); const float i = normalizeOperand(m_state.i); + m_fmacVs = vs; + m_fmacVt = vt; + m_fmacAcc = acc; + m_fmacQ = q; + m_fmacI = i; float result[4]; // Upper opcode decoding (bits 5:0 of upper word)