diff --git a/src/dxbc/dxbc_compiler.cpp b/src/dxbc/dxbc_compiler.cpp index cdee97bbb..61bf18301 100644 --- a/src/dxbc/dxbc_compiler.cpp +++ b/src/dxbc/dxbc_compiler.cpp @@ -1571,7 +1571,7 @@ namespace dxvk { DxbcRegisterValue cos; cos.type = cosInput.type; - cos.id = m_module.opSin( + cos.id = m_module.opCos( getVectorTypeId(cos.type), cosInput.id); @@ -1600,12 +1600,110 @@ namespace dxvk { void DxbcCompiler::emitAtomic(const DxbcShaderInstruction& ins) { + // atomic_* operations have the following operands: + // (dst0) Destination u# or g# register + // (src0) Index into the texture or buffer + // (src1) The source value for the operation + // (src2) Second source operand (optional) + // imm_atomic_* operations have the following operands: + // (dst0) Register that receives the result + // (dst1) Destination u# or g# register + // (srcX) As above + const bool isImm = ins.dstCount == 2; + const bool isUav = ins.dst[ins.dstCount - 1].type == DxbcOperandType::UnorderedAccessView; + + // Retrieve destination pointer for the atomic operation + const DxbcRegisterPointer pointer = emitGetAtomicPointer( + ins.dst[ins.dstCount - 1], ins.src[0]); + + // Load source values + std::array src; + + for (uint32_t i = 1; i < ins.srcCount; i++) { + src[i - 1] = emitRegisterLoad(ins.src[i], + DxbcRegMask(true, false, false, false)); + } + + // Define memory scope and semantics based on the operands + uint32_t semantics = isUav + ? spv::MemorySemanticsUniformMemoryMask + : spv::MemorySemanticsWorkgroupMemoryMask; + + // TODO verify whether this is even remotely correct + semantics |= spv::MemorySemanticsAcquireReleaseMask; + + // TODO for UAVs, respect globally coherent flag + const uint32_t scopeId = m_module.constu32(spv::ScopeWorkgroup); + const uint32_t semanticsId = m_module.constu32(semantics); + + // Perform the atomic operation on the given pointer + DxbcRegisterValue value; + value.type.ctype = ins.dst[0].dataType; + value.type.ccount = 1; + value.id = 0; + + // The result type, which is a scalar integer + const uint32_t typeId = getVectorTypeId(value.type); + + // TODO add signed min/max switch (ins.op) { + case DxbcOpcode::ImmAtomicExch: + value.id = m_module.opAtomicExchange(typeId, + pointer.id, scopeId, semanticsId, + src[0].id); + break; + + case DxbcOpcode::AtomicIAdd: + case DxbcOpcode::ImmAtomicIAdd: + value.id = m_module.opAtomicIAdd(typeId, + pointer.id, scopeId, semanticsId, + src[0].id); + break; + + case DxbcOpcode::AtomicAnd: + case DxbcOpcode::ImmAtomicAnd: + value.id = m_module.opAtomicAnd(typeId, + pointer.id, scopeId, semanticsId, + src[0].id); + break; + + case DxbcOpcode::AtomicOr: + case DxbcOpcode::ImmAtomicOr: + value.id = m_module.opAtomicOr(typeId, + pointer.id, scopeId, semanticsId, + src[0].id); + break; + + case DxbcOpcode::AtomicXor: + case DxbcOpcode::ImmAtomicXor: + value.id = m_module.opAtomicXor(typeId, + pointer.id, scopeId, semanticsId, + src[0].id); + break; + + case DxbcOpcode::AtomicUMin: + value.id = m_module.opAtomicUMin(typeId, + pointer.id, scopeId, semanticsId, + src[0].id); + break; + + case DxbcOpcode::AtomicUMax: + value.id = m_module.opAtomicUMin(typeId, + pointer.id, scopeId, semanticsId, + src[0].id); + break; + default: Logger::warn(str::format( "DxbcCompiler: Unhandled instruction: ", ins.op)); + return; } + + // Write back the result to the destination + // register if this is an imm_atomic_* opcode. + if (isImm) + emitRegisterStore(ins.dst[0], value); } @@ -3111,6 +3209,62 @@ namespace dxvk { } + DxbcRegisterPointer DxbcCompiler::emitGetAtomicPointer( + const DxbcRegister& operand, + const DxbcRegister& address) { + // Query information about the resource itself + const uint32_t registerId = operand.idx[0].offset; + const DxbcBufferInfo resourceInfo = getBufferInfo(operand); + + // For UAVs and shared memory, different methods + // of obtaining the final pointer are used. + const bool isUav = operand.type == DxbcOperandType::UnorderedAccessView; + + // Compute the actual address into the resource + const DxbcRegisterValue addressValue = [&] { + switch (resourceInfo.type) { + case DxbcResourceType::Raw: + return emitCalcBufferIndexRaw(emitRegisterLoad( + address, DxbcRegMask(true, false, false, false))); + + case DxbcResourceType::Structured: { + const DxbcRegisterValue addressComponents = emitRegisterLoad( + address, DxbcRegMask(true, true, false, false)); + + return emitCalcBufferIndexStructured( + emitRegisterExtract(addressComponents, DxbcRegMask(true, false, false, false)), + emitRegisterExtract(addressComponents, DxbcRegMask(false, true, false, false)), + resourceInfo.stride); + }; + + case DxbcResourceType::Typed: { + if (!isUav) + throw DxvkError("DxbcCompiler: TGSM cannot be typed"); + + return emitRegisterLoad(address, getTexCoordMask( + m_uavs.at(registerId).imageInfo)); + } + + default: + throw DxvkError("DxbcCompiler: Unhandled resource type"); + } + }(); + + // Compute the actual pointer + DxbcRegisterPointer result; + result.type.ctype = operand.dataType; + result.type.ccount = 1; + result.id = isUav + ? m_module.opImageTexelPointer( + m_module.defPointerType(getVectorTypeId(result.type), spv::StorageClassImage), + m_uavs.at(registerId).varId, addressValue.id, m_module.constu32(0)) + : m_module.opAccessChain( + m_module.defPointerType(getVectorTypeId(result.type), spv::StorageClassWorkgroup), + m_gRegs.at(registerId).varId, 1, &addressValue.id); + return result; + } + + DxbcRegisterValue DxbcCompiler::emitRawBufferLoad( const DxbcRegister& operand, DxbcRegisterValue elementIndex, diff --git a/src/dxbc/dxbc_compiler.h b/src/dxbc/dxbc_compiler.h index bab7ef3f7..fc0edd83c 100644 --- a/src/dxbc/dxbc_compiler.h +++ b/src/dxbc/dxbc_compiler.h @@ -562,6 +562,10 @@ namespace dxvk { DxbcRegisterPointer emitGetOperandPtr( const DxbcRegister& operand); + DxbcRegisterPointer emitGetAtomicPointer( + const DxbcRegister& operand, + const DxbcRegister& address); + /////////////////////////////// // Resource load/store methods DxbcRegisterValue emitRawBufferLoad( diff --git a/src/spirv/spirv_module.cpp b/src/spirv/spirv_module.cpp index f2fbc6f04..68d9d4f1e 100644 --- a/src/spirv/spirv_module.cpp +++ b/src/spirv/spirv_module.cpp @@ -591,6 +591,283 @@ namespace dxvk { } + uint32_t SpirvModule::opAtomicLoad( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics) { + uint32_t resultId = this->allocateId(); + + m_code.putIns (spv::OpAtomicLoad, 6); + m_code.putWord(resultType); + m_code.putWord(resultId); + m_code.putWord(pointer); + m_code.putWord(scope); + m_code.putWord(semantics); + return resultId; + } + + + void SpirvModule::opAtomicStore( + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value) { + m_code.putIns (spv::OpAtomicStore, 5); + m_code.putWord(pointer); + m_code.putWord(scope); + m_code.putWord(semantics); + m_code.putWord(value); + } + + + uint32_t SpirvModule::opAtomicExchange( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value) { + uint32_t resultId = this->allocateId(); + + m_code.putIns (spv::OpAtomicExchange, 7); + m_code.putWord(resultType); + m_code.putWord(resultId); + m_code.putWord(pointer); + m_code.putWord(scope); + m_code.putWord(semantics); + m_code.putWord(value); + return resultId; + } + + + uint32_t SpirvModule::opAtomicCompareExchange( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t equal, + uint32_t unequal, + uint32_t value, + uint32_t comparator) { + uint32_t resultId = this->allocateId(); + + m_code.putIns (spv::OpAtomicCompareExchange, 9); + m_code.putWord(resultType); + m_code.putWord(resultId); + m_code.putWord(pointer); + m_code.putWord(scope); + m_code.putWord(equal); + m_code.putWord(unequal); + m_code.putWord(value); + m_code.putWord(comparator); + return resultId; + } + + + uint32_t SpirvModule::opAtomicIIncrement( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics) { + uint32_t resultId = this->allocateId(); + + m_code.putIns (spv::OpAtomicIIncrement, 6); + m_code.putWord(resultType); + m_code.putWord(resultId); + m_code.putWord(pointer); + m_code.putWord(scope); + m_code.putWord(semantics); + return resultId; + } + + + uint32_t SpirvModule::opAtomicIDecrement( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics) { + uint32_t resultId = this->allocateId(); + + m_code.putIns (spv::OpAtomicIDecrement, 6); + m_code.putWord(resultType); + m_code.putWord(resultId); + m_code.putWord(pointer); + m_code.putWord(scope); + m_code.putWord(semantics); + return resultId; + } + + + uint32_t SpirvModule::opAtomicIAdd( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value) { + uint32_t resultId = this->allocateId(); + + m_code.putIns (spv::OpAtomicIAdd, 7); + m_code.putWord(resultType); + m_code.putWord(resultId); + m_code.putWord(pointer); + m_code.putWord(scope); + m_code.putWord(semantics); + m_code.putWord(value); + return resultId; + } + + + uint32_t SpirvModule::opAtomicISub( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value) { + uint32_t resultId = this->allocateId(); + + m_code.putIns (spv::OpAtomicISub, 7); + m_code.putWord(resultType); + m_code.putWord(resultId); + m_code.putWord(pointer); + m_code.putWord(scope); + m_code.putWord(semantics); + m_code.putWord(value); + return resultId; + } + + + uint32_t SpirvModule::opAtomicSMin( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value) { + uint32_t resultId = this->allocateId(); + + m_code.putIns (spv::OpAtomicSMin, 7); + m_code.putWord(resultType); + m_code.putWord(resultId); + m_code.putWord(pointer); + m_code.putWord(scope); + m_code.putWord(semantics); + m_code.putWord(value); + return resultId; + } + + + uint32_t SpirvModule::opAtomicSMax( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value) { + uint32_t resultId = this->allocateId(); + + m_code.putIns (spv::OpAtomicSMax, 7); + m_code.putWord(resultType); + m_code.putWord(resultId); + m_code.putWord(pointer); + m_code.putWord(scope); + m_code.putWord(semantics); + m_code.putWord(value); + return resultId; + } + + + uint32_t SpirvModule::opAtomicUMin( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value) { + uint32_t resultId = this->allocateId(); + + m_code.putIns (spv::OpAtomicUMin, 7); + m_code.putWord(resultType); + m_code.putWord(resultId); + m_code.putWord(pointer); + m_code.putWord(scope); + m_code.putWord(semantics); + m_code.putWord(value); + return resultId; + } + + + uint32_t SpirvModule::opAtomicUMax( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value) { + uint32_t resultId = this->allocateId(); + + m_code.putIns (spv::OpAtomicUMax, 7); + m_code.putWord(resultType); + m_code.putWord(resultId); + m_code.putWord(pointer); + m_code.putWord(scope); + m_code.putWord(semantics); + m_code.putWord(value); + return resultId; + } + + + uint32_t SpirvModule::opAtomicAnd( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value) { + uint32_t resultId = this->allocateId(); + + m_code.putIns (spv::OpAtomicAnd, 7); + m_code.putWord(resultType); + m_code.putWord(resultId); + m_code.putWord(pointer); + m_code.putWord(scope); + m_code.putWord(semantics); + m_code.putWord(value); + return resultId; + } + + + uint32_t SpirvModule::opAtomicOr( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value) { + uint32_t resultId = this->allocateId(); + + m_code.putIns (spv::OpAtomicOr, 7); + m_code.putWord(resultType); + m_code.putWord(resultId); + m_code.putWord(pointer); + m_code.putWord(scope); + m_code.putWord(semantics); + m_code.putWord(value); + return resultId; + } + + + uint32_t SpirvModule::opAtomicXor( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value) { + uint32_t resultId = this->allocateId(); + + m_code.putIns (spv::OpAtomicXor, 7); + m_code.putWord(resultType); + m_code.putWord(resultId); + m_code.putWord(pointer); + m_code.putWord(scope); + m_code.putWord(semantics); + m_code.putWord(value); + return resultId; + } + + uint32_t SpirvModule::opBitcast( uint32_t resultType, uint32_t operand) { @@ -1902,6 +2179,23 @@ namespace dxvk { } + uint32_t SpirvModule::opImageTexelPointer( + uint32_t resultType, + uint32_t image, + uint32_t coordinates, + uint32_t sample) { + uint32_t resultId = this->allocateId(); + + m_code.putIns (spv::OpImageTexelPointer, 6); + m_code.putWord(resultType); + m_code.putWord(resultId); + m_code.putWord(image); + m_code.putWord(coordinates); + m_code.putWord(sample); + return resultId; + } + + uint32_t SpirvModule::opImageQuerySizeLod( uint32_t resultType, uint32_t image, diff --git a/src/spirv/spirv_module.h b/src/spirv/spirv_module.h index e094f111f..b4610a7b1 100644 --- a/src/spirv/spirv_module.h +++ b/src/spirv/spirv_module.h @@ -250,10 +250,113 @@ namespace dxvk { uint32_t indexCount, const uint32_t* indexArray); + uint32_t opAtomicLoad( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics); + + void opAtomicStore( + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value); + + uint32_t opAtomicExchange( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value); + + uint32_t opAtomicCompareExchange( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t equal, + uint32_t unequal, + uint32_t value, + uint32_t comparator); + + uint32_t opAtomicIIncrement( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics); + + uint32_t opAtomicIDecrement( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics); + + uint32_t opAtomicIAdd( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value); + + uint32_t opAtomicISub( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value); + + uint32_t opAtomicSMin( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value); + + uint32_t opAtomicSMax( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value); + + uint32_t opAtomicUMin( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value); + + uint32_t opAtomicUMax( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value); + + uint32_t opAtomicAnd( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value); + + uint32_t opAtomicOr( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value); + + uint32_t opAtomicXor( + uint32_t resultType, + uint32_t pointer, + uint32_t scope, + uint32_t semantics, + uint32_t value); + uint32_t opBitcast( uint32_t resultType, uint32_t operand); - + uint32_t opBitFieldInsert( uint32_t resultType, uint32_t base, @@ -661,6 +764,12 @@ namespace dxvk { uint32_t texel, const SpirvImageOperands& operands); + uint32_t opImageTexelPointer( + uint32_t resultType, + uint32_t image, + uint32_t coordinates, + uint32_t sample); + uint32_t opSampledImage( uint32_t resultType, uint32_t image,