submission 923166

from __future__ import annotations

import triton
import triton.language as tl
from triton.language.extra import cuda as tl_cuda
from triton.experimental import gluon
from triton.experimental.gluon import language as gl
_IA1 = gl.constexpr('{.reg .pred first_warp,first_thread,producer,producer_first;.reg .pred consumer_first,complete,more,reuse,accumulate;.reg .b32 tid,warp_id,local,cta_id,smem_addr,taddr;.reg .b32 warp_bits,warp_taddr,scale_value,bar_addr;.reg .b32 iter,stage,cycle,parity,kbytes,row_index;.reg .b32 row_smem,swizzle,stage_smem,chunk,column,physical;.reg .b32 a_smem,b_smem,ready_addr,empty_addr,idesc0,idesc1;.reg .b32 lane,tensor_dim;.reg .b64 a_global,b_global,a_chunk,b_chunk,desc_base;.reg .b64 a_desc0,b_desc0,a_desc1,b_desc1;.reg .b64 a_desc2,b_desc2,a_desc3,b_desc3,state;griddepcontrol.wait;mov.u32 tid,%tid.x;shr.u32 warp_id,tid,5;mov.u32 cta_id,%ctaid.x;setp.lt.u32 first_warp,tid,32;setp.eq.u32 first_thread,tid,0;setp.ge.u32 producer,tid,128;setp.eq.u32 producer_first,tid,128;setp.eq.u32 consumer_first,tid,0;mov.u32 smem_addr,global_smem;ld.shared.b32 taddr,[smem_addr];and.b32 lane,tid,31;shl.b32 physical,lane,2;add.u32 physical,physical,smem_addr;@first_warp st.shared.b32[physical],0;bar.sync 0,256;cvt.u64.u32 desc_base,smem_addr;@first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64[desc_base],$1;@first_thread tensormap.replace.tile.rank.shared::cta.b1024.b32[desc_base],0x1;mov.u32 tensor_dim,128;@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32[desc_base],0x0,tensor_dim;@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;mov.u32 tensor_dim,$7;@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32[desc_base],0x0,tensor_dim;mov.u32 tensor_dim,$6;@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;mov.u32 tensor_dim,$7;cvt.u64.u32 a_global,tensor_dim;@first_thread tensormap.replace.tile.global_stride.shared::cta.b1024.b64[desc_base],0x0,a_global;mov.u32 tensor_dim,1;@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32[desc_base],0x0,tensor_dim;@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;@first_thread tensormap.replace.tile.elemtype.shared::cta.b1024.b32[desc_base],0x0;@first_thread tensormap.replace.tile.interleave_layout.shared::cta.b1024.b32[desc_base],0x0;@first_thread tensormap.replace.tile.swizzle_mode.shared::cta.b1024.b32[desc_base],0x3;@first_thread tensormap.replace.tile.fill_mode.shared::cta.b1024.b32[desc_base],0x0;mad.wide.u32 state,$8,128,$2;add.u64 state,state,128;@first_warp tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned[state],[desc_base],0x80;@first_warp fence.proxy.tensormap::generic.acquire.gpu[state],0x80;@first_warp cp.async.bulk.commit_group;@first_warp cp.async.bulk.wait_group.read 0;bar.sync 0,256;and.b32 warp_bits,warp_id,3;shl.b32 warp_bits,warp_bits,21;add.u32 warp_taddr,taddr,warp_bits;ld.global.u8 scale_value,[$2];mul.lo.u32 scale_value,scale_value,0x01010101;@!producer tcgen05.st.sync.aligned.32x32b.x8.b32[warp_taddr + 128],{scale_value,scale_value,scale_value,scale_value,scale_value,scale_value,scale_value,scale_value};@!producer tcgen05.st.sync.aligned.32x32b.x8.b32[warp_taddr + 136],{scale_value,scale_value,scale_value,scale_value,scale_value,scale_value,scale_value,scale_value};@!producer tcgen05.wait::st.sync.aligned;bar.sync 0,256;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98304],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98312],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98320],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98328],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98336],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98344],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98352],1;bar.sync 0,256;@producer bra.uni producer_path;consumer_path:\nmov.u32 iter,0;mov.u32 idesc0,0x08a00480;mov.u32 idesc1,0x48a00480;consumer_loop:\nrem.u32 stage,iter,3;div.u32 cycle,iter,3;and.b32 parity,cycle,1;shl.b32 ready_addr,stage,3;add.u32 ready_addr,ready_addr,smem_addr;add.u32 ready_addr,ready_addr,98304;consumer_ready_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[ready_addr],parity;@!complete bra.uni consumer_ready_wait;bar.sync 3,128;shl.b32 stage_smem,stage,15;add.u32 a_smem,smem_addr,stage_smem;add.u32 b_smem,a_smem,16384;bfe.u32 a_smem,a_smem,4,14;bfe.u32 b_smem,b_smem,4,14;cvt.u64.u32 desc_base,a_smem;or.b64 a_desc0,desc_base,0x4000404000000000;cvt.u64.u32 desc_base,b_smem;or.b64 b_desc0,desc_base,0x4000404000000000;add.u64 a_desc1,a_desc0,2;add.u64 b_desc1,b_desc0,2;add.u64 a_desc2,a_desc0,4;add.u64 b_desc2,b_desc0,4;add.u64 a_desc3,a_desc0,6;add.u64 b_desc3,b_desc0,6;setp.ne.u32 accumulate,iter,0;@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32[taddr],a_desc0,b_desc0,idesc0,[taddr + 128],[taddr + 136],accumulate;mov.pred accumulate,1;@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32[taddr],a_desc1,b_desc1,idesc1,[taddr + 128],[taddr + 136],accumulate;@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32[taddr],a_desc2,b_desc2,idesc0,[taddr + 132],[taddr + 140],accumulate;@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32[taddr],a_desc3,b_desc3,idesc1,[taddr + 132],[taddr + 140],accumulate;shl.b32 empty_addr,stage,3;add.u32 empty_addr,empty_addr,smem_addr;add.u32 empty_addr,empty_addr,98328;@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64[empty_addr];add.u32 iter,iter,1;setp.lt.u32 more,iter,$3;@more bra.uni consumer_loop;@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64[smem_addr + 98352];consumer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[smem_addr + 98352],0;@!complete bra.uni consumer_final_wait;bra.uni final_barrier;producer_path:\nmov.u32 row_index,$4;mov.u32 local,$5;mov.u32 iter,0;producer_loop:\nrem.u32 stage,iter,3;setp.ge.u32 reuse,iter,3;@!reuse bra.uni producer_load;div.u32 cycle,iter,3;sub.u32 cycle,cycle,1;and.b32 parity,cycle,1;shl.b32 empty_addr,stage,3;add.u32 empty_addr,empty_addr,smem_addr;add.u32 empty_addr,empty_addr,98328;producer_empty_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[empty_addr],parity;@!complete bra.uni producer_empty_wait;producer_load:\nshl.b32 stage_smem,stage,15;add.u32 a_smem,stage_smem,smem_addr;add.u32 b_smem,a_smem,16384;shl.b32 ready_addr,stage,3;add.u32 ready_addr,ready_addr,smem_addr;add.u32 ready_addr,ready_addr,98304;shl.b32 column,iter,7;@producer_first mbarrier.arrive.expect_tx.shared::cta.b64 _,[ready_addr],32768;fence.proxy.async.shared::cta;@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes[a_smem],[state,{column,row_index}],[ready_addr];@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes[b_smem],[state,{column,local}],[ready_addr];add.u32 iter,iter,1;setp.lt.u32 more,iter,$3;@more bra.uni producer_loop;producer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[smem_addr + 98352],0;@!complete bra.uni producer_final_wait;bra.uni final_barrier;final_barrier:\nbar.sync 0,256;mov.u32 $0,0;}')
_IA2 = gl.constexpr('{.reg .pred first_warp,first_thread,more;.reg .b32 tid,lane,physical,smem_addr,tensor_dim,iter;.reg .b64 desc_base,state,stride,history_base;griddepcontrol.launch_dependents;mov.u32 tid,%tid.x;and.b32 lane,tid,31;setp.lt.u32 first_warp,tid,32;setp.eq.u32 first_thread,tid,0;mov.u32 smem_addr,global_smem;shl.b32 physical,lane,2;add.u32 physical,physical,smem_addr;@first_warp st.shared.b32[physical],0;bar.sync 0,32;cvt.u64.u32 desc_base,smem_addr;@first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64[desc_base],$1;@first_thread tensormap.replace.tile.rank.shared::cta.b1024.b32[desc_base],0x1;mov.u32 tensor_dim,128;@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32[desc_base],0x0,tensor_dim;@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;mov.u32 tensor_dim,$3;@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32[desc_base],0x0,tensor_dim;mov.u32 tensor_dim,$4;@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;cvt.u64.u32 stride,$3;@first_thread tensormap.replace.tile.global_stride.shared::cta.b1024.b64[desc_base],0x0,stride;mov.u32 tensor_dim,1;@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32[desc_base],0x0,tensor_dim;@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;@first_thread tensormap.replace.tile.elemtype.shared::cta.b1024.b32[desc_base],0x0;@first_thread tensormap.replace.tile.interleave_layout.shared::cta.b1024.b32[desc_base],0x0;@first_thread tensormap.replace.tile.swizzle_mode.shared::cta.b1024.b32[desc_base],0x3;@first_thread tensormap.replace.tile.fill_mode.shared::cta.b1024.b32[desc_base],0x0;mov.u64 state,$2;add.u64 state,state,128;mov.u64 history_base,$1;mov.u32 iter,0;publish_pair:\n@first_warp tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned[state],[desc_base],0x80;mov.u32 tensor_dim,256;@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;add.u64 state,state,128;@first_warp tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned[state],[desc_base],0x80;add.u32 iter,iter,1;setp.lt.u32 more,iter,4;@!more bra.uni publish_done;add.u64 state,state,128;add.u64 history_base,history_base,4096;@first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64[desc_base],history_base;mov.u32 tensor_dim,128;@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;bra.uni publish_pair;publish_done:\nbar.sync 0,32;griddepcontrol.launch_dependents;mov.u32 $0,0;}')
_IA3 = gl.constexpr('{.reg .pred first_warp,first_thread,producer,producer_first;.reg .pred consumer_first,complete,more,reuse,accumulate;.reg .b32 tid,warp_id,local,cta_id,smem_addr,taddr;.reg .b32 warp_bits,warp_taddr,scale_value,bar_addr;.reg .b32 iter,stage,cycle,parity,kbytes,row_index;.reg .b32 row_smem,swizzle,stage_smem,chunk,column,physical;.reg .b32 a_smem,b_smem,ready_addr,empty_addr,idesc0,idesc1;.reg .b32 lane,tensor_dim;.reg .b64 a_global,b_global,a_chunk,b_chunk,desc_base;.reg .b64 a_desc0,b_desc0,a_desc1,b_desc1;.reg .b64 a_desc2,b_desc2,a_desc3,b_desc3,state;griddepcontrol.wait;griddepcontrol.launch_dependents;mov.u32 tid,%tid.x;shr.u32 warp_id,tid,5;mov.u32 cta_id,%ctaid.x;setp.lt.u32 first_warp,tid,32;setp.eq.u32 first_thread,tid,0;setp.ge.u32 producer,tid,128;setp.eq.u32 producer_first,tid,128;setp.eq.u32 consumer_first,tid,0;mov.u32 smem_addr,global_smem;ld.shared.b32 taddr,[smem_addr];mov.u64 state,$2;add.u64 state,state,128;@first_warp fence.proxy.tensormap::generic.acquire.gpu[state],0x80;bar.sync 0,256;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98304],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98312],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98320],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98328],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98336],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98344],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98352],1;bar.sync 0,256;@producer bra.uni producer_path;consumer_path:\nmov.u32 iter,0;mov.u32 idesc0,0x08200010;mov.u32 idesc1,0x08200010;consumer_loop:\nrem.u32 stage,iter,3;div.u32 cycle,iter,3;and.b32 parity,cycle,1;shl.b32 ready_addr,stage,3;add.u32 ready_addr,ready_addr,smem_addr;add.u32 ready_addr,ready_addr,98304;consumer_ready_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[ready_addr],parity;@!complete bra.uni consumer_ready_wait;bar.sync 3,128;mul.lo.u32 stage_smem,stage,32768;add.u32 a_smem,smem_addr,stage_smem;add.u32 b_smem,a_smem,16384;bfe.u32 a_smem,a_smem,4,14;bfe.u32 b_smem,b_smem,4,14;cvt.u64.u32 desc_base,a_smem;or.b64 a_desc0,desc_base,0x4000404000000000;cvt.u64.u32 desc_base,b_smem;or.b64 b_desc0,desc_base,0x4000404000000000;add.u64 a_desc1,a_desc0,2;add.u64 b_desc1,b_desc0,2;add.u64 a_desc2,a_desc0,4;add.u64 b_desc2,b_desc0,4;add.u64 a_desc3,a_desc0,6;add.u64 b_desc3,b_desc0,6;setp.ne.u32 accumulate,iter,0;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr],a_desc0,b_desc0,idesc0,accumulate;mov.pred accumulate,1;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr],a_desc1,b_desc1,idesc0,accumulate;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr],a_desc2,b_desc2,idesc0,accumulate;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr],a_desc3,b_desc3,idesc0,accumulate;shl.b32 empty_addr,stage,3;add.u32 empty_addr,empty_addr,smem_addr;add.u32 empty_addr,empty_addr,98328;@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64[empty_addr];add.u32 iter,iter,1;setp.lt.u32 more,iter,$3;@more bra.uni consumer_loop;@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64[smem_addr + 98352];consumer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[smem_addr + 98352],0;@!complete bra.uni consumer_final_wait;bra.uni final_barrier;producer_path:\nmov.u32 row_index,$4;mov.u32 local,$5;mov.u32 iter,0;producer_loop:\nrem.u32 stage,iter,3;setp.ge.u32 reuse,iter,3;@!reuse bra.uni producer_load;div.u32 cycle,iter,3;sub.u32 cycle,cycle,1;and.b32 parity,cycle,1;shl.b32 empty_addr,stage,3;add.u32 empty_addr,empty_addr,smem_addr;add.u32 empty_addr,empty_addr,98328;producer_empty_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[empty_addr],parity;@!complete bra.uni producer_empty_wait;producer_load:\nmul.lo.u32 stage_smem,stage,32768;add.u32 a_smem,stage_smem,smem_addr;add.u32 b_smem,a_smem,16384;shl.b32 ready_addr,stage,3;add.u32 ready_addr,ready_addr,smem_addr;add.u32 ready_addr,ready_addr,98304;shl.b32 column,iter,7;@producer_first mbarrier.arrive.expect_tx.shared::cta.b64 _,[ready_addr],32768;fence.proxy.async.shared::cta;@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes[a_smem],[state,{column,row_index}],[ready_addr];@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes[b_smem],[state,{column,local}],[ready_addr];add.u32 iter,iter,1;setp.lt.u32 more,iter,$3;@more bra.uni producer_loop;producer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[smem_addr + 98352],0;@!complete bra.uni producer_final_wait;bra.uni final_barrier;final_barrier:\nbar.sync 0,256;mov.u32 $0,0;}')
_IA4 = gl.constexpr('{.reg .pred first_warp,first_thread,producer,producer_first;.reg .pred consumer_first,complete,more,reuse,accumulate,last;.reg .b32 tid,warp_id,local,cta_id,smem_addr,taddr;.reg .b32 warp_bits,warp_taddr,scale_value,bar_addr;.reg .b32 iter,stage,cycle,parity,kbytes,row_index;.reg .b32 row_smem,swizzle,stage_smem,chunk,column,physical;.reg .b32 a_smem,b_smem,ready_addr,empty_addr,idesc0,idesc1;.reg .b32 lane,tensor_dim;.reg .b64 a_global,b_global,a_chunk,b_chunk,desc_base;.reg .b64 a_desc0,b_desc0,a_desc1,b_desc1;.reg .b64 a_desc2,b_desc2,a_desc3,b_desc3,state,b_state;.reg .b64 b_second0,b_second1,b_second2,b_second3;mov.u32 tid,%tid.x;shr.u32 warp_id,tid,5;mov.u32 cta_id,%ctaid.x;setp.lt.u32 first_warp,tid,32;setp.eq.u32 first_thread,tid,0;setp.ge.u32 producer,tid,128;setp.eq.u32 producer_first,tid,128;setp.eq.u32 consumer_first,tid,0;mov.u32 smem_addr,global_smem;ld.shared.b32 taddr,[smem_addr];mov.u64 state,$2;add.u64 state,state,128;add.u64 b_state,state,128;@first_warp fence.proxy.tensormap::generic.acquire.gpu[state],0x80;@first_warp fence.proxy.tensormap::generic.acquire.gpu[b_state],0x80;bar.sync 0,256;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98304],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98312],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98320],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98328],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98336],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98344],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98352],1;bar.sync 0,256;@producer bra.uni producer_path;consumer_path:\nmov.u32 iter,0;mov.u32 idesc0,0x08400010;mov.u32 idesc1,0x08200010;consumer_loop:\nrem.u32 stage,iter,2;div.u32 cycle,iter,2;and.b32 parity,cycle,1;shl.b32 ready_addr,stage,3;add.u32 ready_addr,ready_addr,smem_addr;add.u32 ready_addr,ready_addr,98304;consumer_ready_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[ready_addr],parity;@!complete bra.uni consumer_ready_wait;bar.sync 3,128;mul.lo.u32 stage_smem,stage,49152;add.u32 a_smem,smem_addr,stage_smem;add.u32 b_smem,a_smem,16384;bfe.u32 a_smem,a_smem,4,14;bfe.u32 b_smem,b_smem,4,14;cvt.u64.u32 desc_base,a_smem;or.b64 a_desc0,desc_base,0x4000404000000000;cvt.u64.u32 desc_base,b_smem;or.b64 b_desc0,desc_base,0x4000404000000000;add.u64 a_desc1,a_desc0,2;add.u64 b_desc1,b_desc0,2;add.u64 a_desc2,a_desc0,4;add.u64 b_desc2,b_desc0,4;add.u64 a_desc3,a_desc0,6;add.u64 b_desc3,b_desc0,6;sub.u32 tensor_dim,$3,1;setp.eq.u32 last,iter,tensor_dim;@last bra.uni consumer_tail_issue;setp.ne.u32 accumulate,iter,0;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr],a_desc0,b_desc0,idesc0,accumulate;mov.pred accumulate,1;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr],a_desc1,b_desc1,idesc0,accumulate;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr],a_desc2,b_desc2,idesc0,accumulate;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr],a_desc3,b_desc3,idesc0,accumulate;bra.uni consumer_issue_done;consumer_tail_issue:\nadd.u64 b_second0,b_desc0,1024;add.u64 b_second1,b_desc1,1024;add.u64 b_second2,b_desc2,1024;add.u64 b_second3,b_desc3,1024;mov.pred accumulate,1;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr + 128],a_desc0,b_second0,idesc1,accumulate;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr + 128],a_desc1,b_second1,idesc1,accumulate;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr + 128],a_desc2,b_second2,idesc1,accumulate;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr + 128],a_desc3,b_second3,idesc1,accumulate;consumer_issue_done:\nshl.b32 empty_addr,stage,3;add.u32 empty_addr,empty_addr,smem_addr;add.u32 empty_addr,empty_addr,98328;@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64[empty_addr];add.u32 iter,iter,1;setp.lt.u32 more,iter,$3;@more bra.uni consumer_loop;@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64[smem_addr + 98352];consumer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[smem_addr + 98352],0;@!complete bra.uni consumer_final_wait;bra.uni final_barrier;producer_path:\nmov.u32 row_index,$4;mov.u32 local,$5;mov.u32 iter,0;producer_loop:\nrem.u32 stage,iter,2;setp.ge.u32 reuse,iter,2;@!reuse bra.uni producer_load;div.u32 cycle,iter,2;sub.u32 cycle,cycle,1;and.b32 parity,cycle,1;shl.b32 empty_addr,stage,3;add.u32 empty_addr,empty_addr,smem_addr;add.u32 empty_addr,empty_addr,98328;producer_empty_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[empty_addr],parity;@!complete bra.uni producer_empty_wait;producer_load:\nmul.lo.u32 stage_smem,stage,49152;add.u32 a_smem,stage_smem,smem_addr;add.u32 b_smem,a_smem,16384;shl.b32 ready_addr,stage,3;add.u32 ready_addr,ready_addr,smem_addr;add.u32 ready_addr,ready_addr,98304;shl.b32 column,iter,7;@producer_first mbarrier.arrive.expect_tx.shared::cta.b64 _,[ready_addr],49152;fence.proxy.async.shared::cta;@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes[a_smem],[state,{column,row_index}],[ready_addr];@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes[b_smem],[b_state,{column,local}],[ready_addr];add.u32 iter,iter,1;setp.lt.u32 more,iter,$3;@more bra.uni producer_loop;producer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[smem_addr + 98352],0;@!complete bra.uni producer_final_wait;bra.uni final_barrier;final_barrier:\nbar.sync 0,256;mov.u32 $0,0;}')
_IA8 = gl.constexpr('{.reg .pred first_thread;.reg .b32 tid,smem_addr,lane_addr,tensor_dim;.reg .b64 desc_base,state,global_stride;mov.u32 tid,%tid.x;setp.eq.u32 first_thread,tid,0;mov.u32 smem_addr,global_smem;shl.b32 lane_addr,tid,2;add.u32 lane_addr,lane_addr,smem_addr;st.shared.b32[lane_addr],0;bar.sync 0,32;cvt.u64.u32 desc_base,smem_addr;@first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64[desc_base],$1;@first_thread tensormap.replace.tile.rank.shared::cta.b1024.b32[desc_base],0x1;mov.u32 tensor_dim,128;@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32[desc_base],0x0,tensor_dim;@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;mov.u32 tensor_dim,$3;@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32[desc_base],0x0,tensor_dim;mov.u32 tensor_dim,$4;@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;cvt.u64.u32 global_stride,$5;@first_thread tensormap.replace.tile.global_stride.shared::cta.b1024.b64[desc_base],0x0,global_stride;mov.u32 tensor_dim,1;@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32[desc_base],0x0,tensor_dim;@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;@first_thread tensormap.replace.tile.elemtype.shared::cta.b1024.b32[desc_base],0x0;@first_thread tensormap.replace.tile.interleave_layout.shared::cta.b1024.b32[desc_base],0x0;@first_thread tensormap.replace.tile.swizzle_mode.shared::cta.b1024.b32[desc_base],0x3;@first_thread tensormap.replace.tile.fill_mode.shared::cta.b1024.b32[desc_base],0x0;mov.u64 state,$2;tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned[state],[desc_base],0x80;fence.proxy.tensormap::generic.acquire.gpu[state],0x80;bar.sync 0,32;@first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64[desc_base],$6;add.u64 state,state,128;tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned[state],[desc_base],0x80;fence.proxy.tensormap::generic.acquire.gpu[state],0x80;bar.sync 0,32;mov.u32 $0,0;}')
_IA9 = gl.constexpr('{.reg .pred first_warp,first_thread,producer;.reg .pred producer_first,consumer_first,complete;.reg .pred more,reuse,accumulate;.reg .b32 tid,smem_addr,taddr,result;.reg .b32 iter,stage,cycle,parity,stage_smem;.reg .b32 column,a_smem,b_smem,ready_addr,empty_addr;.reg .b32 a_compact,b_compact,idesc;.reg .b64 desc_base,state;.reg .b64 a_desc0,b_desc0,a_desc1,b_desc1;.reg .b64 a_desc2,b_desc2,a_desc3,b_desc3;griddepcontrol.wait;mov.u32 tid,%tid.x;setp.lt.u32 first_warp,tid,32;setp.eq.u32 first_thread,tid,0;setp.ge.u32 producer,tid,128;setp.eq.u32 producer_first,tid,128;setp.eq.u32 consumer_first,tid,0;mov.u32 smem_addr,global_smem;ld.shared.b32 taddr,[smem_addr];mad.wide.u32 state,$5,128,$1;@first_warp fence.proxy.tensormap::generic.acquire.gpu[state],0x80;bar.sync 0,256;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98304],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98312],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98320],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98328],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98336],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98344],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98352],1;bar.sync 0,256;@producer bra.uni producer_path;consumer_path:\nmov.u32 iter,0;mov.u32 idesc,0x08200010;consumer_loop:\nrem.u32 stage,iter,3;div.u32 cycle,iter,3;and.b32 parity,cycle,1;shl.b32 ready_addr,stage,3;add.u32 ready_addr,ready_addr,smem_addr;add.u32 ready_addr,ready_addr,98304;consumer_ready_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[ready_addr],parity;@!complete bra.uni consumer_ready_wait;bar.sync 3,128;fence.proxy.async.shared::cta;shl.b32 stage_smem,stage,15;add.u32 a_smem,smem_addr,stage_smem;add.u32 b_smem,a_smem,16384;bfe.u32 a_compact,a_smem,4,14;bfe.u32 b_compact,b_smem,4,14;cvt.u64.u32 desc_base,a_compact;or.b64 a_desc0,desc_base,0x4000404000000000;cvt.u64.u32 desc_base,b_compact;or.b64 b_desc0,desc_base,0x4000404000000000;add.u64 a_desc1,a_desc0,2;add.u64 b_desc1,b_desc0,2;add.u64 a_desc2,a_desc0,4;add.u64 b_desc2,b_desc0,4;add.u64 a_desc3,a_desc0,6;add.u64 b_desc3,b_desc0,6;setp.ne.u32 accumulate,iter,0;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr],a_desc0,b_desc0,idesc,accumulate;mov.pred accumulate,1;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr],a_desc1,b_desc1,idesc,accumulate;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr],a_desc2,b_desc2,idesc,accumulate;@consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4[taddr],a_desc3,b_desc3,idesc,accumulate;shl.b32 empty_addr,stage,3;add.u32 empty_addr,empty_addr,smem_addr;add.u32 empty_addr,empty_addr,98328;@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64[empty_addr];add.u32 iter,iter,1;setp.lt.u32 more,iter,$2;@more bra.uni consumer_loop;@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64[smem_addr + 98352];consumer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[smem_addr + 98352],0;@!complete bra.uni consumer_final_wait;bra.uni final_barrier;producer_path:\nmov.u32 iter,0;producer_loop:\nrem.u32 stage,iter,3;setp.ge.u32 reuse,iter,3;@!reuse bra.uni producer_load;div.u32 cycle,iter,3;sub.u32 cycle,cycle,1;and.b32 parity,cycle,1;shl.b32 empty_addr,stage,3;add.u32 empty_addr,empty_addr,smem_addr;add.u32 empty_addr,empty_addr,98328;producer_empty_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[empty_addr],parity;@!complete bra.uni producer_empty_wait;producer_load:\nshl.b32 stage_smem,stage,15;add.u32 a_smem,stage_smem,smem_addr;add.u32 b_smem,a_smem,16384;shl.b32 ready_addr,stage,3;add.u32 ready_addr,ready_addr,smem_addr;add.u32 ready_addr,ready_addr,98304;shl.b32 column,iter,7;@producer_first mbarrier.arrive.expect_tx.shared::cta.b64 _,[ready_addr],32768;fence.proxy.async.shared::cta;@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes[a_smem],[state,{column,$3}],[ready_addr];@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes[b_smem],[state,{column,$4}],[ready_addr];add.u32 iter,iter,1;setp.lt.u32 more,iter,$2;@more bra.uni producer_loop;producer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[smem_addr + 98352],0;@!complete bra.uni producer_final_wait;final_barrier:\nbar.sync 0,256;mov.u32 result,0;mov.u32 $0,result;}')
_IA10 = gl.constexpr('{.reg .pred first_thread;.reg .b32 tid,smem_addr,lane_addr,tensor_dim;.reg .b64 desc_base,state,global_stride;mov.u32 tid,%tid.x;setp.eq.u32 first_thread,tid,0;mov.u32 smem_addr,global_smem;shl.b32 lane_addr,tid,2;add.u32 lane_addr,lane_addr,smem_addr;st.shared.b32[lane_addr],0;bar.sync 0,32;cvt.u64.u32 desc_base,smem_addr;@first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64[desc_base],$1;@first_thread tensormap.replace.tile.rank.shared::cta.b1024.b32[desc_base],0x1;mov.u32 tensor_dim,64;@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32[desc_base],0x0,tensor_dim;mov.u32 tensor_dim,128;@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;mov.u32 tensor_dim,$3;@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32[desc_base],0x0,tensor_dim;mov.u32 tensor_dim,$4;@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;cvt.u64.u32 global_stride,$5;@first_thread tensormap.replace.tile.global_stride.shared::cta.b1024.b64[desc_base],0x0,global_stride;mov.u32 tensor_dim,1;@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32[desc_base],0x0,tensor_dim;@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;@first_thread tensormap.replace.tile.elemtype.shared::cta.b1024.b32[desc_base],0x6;@first_thread tensormap.replace.tile.interleave_layout.shared::cta.b1024.b32[desc_base],0x0;@first_thread tensormap.replace.tile.swizzle_mode.shared::cta.b1024.b32[desc_base],0x3;@first_thread tensormap.replace.tile.fill_mode.shared::cta.b1024.b32[desc_base],0x0;mov.u64 state,$2;tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned[state],[desc_base],0x80;fence.proxy.tensormap::generic.acquire.gpu[state],0x80;bar.sync 0,32;@first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64[desc_base],$6;add.u64 state,state,128;tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned[state],[desc_base],0x80;fence.proxy.tensormap::generic.acquire.gpu[state],0x80;bar.sync 0,32;mov.u32 $0,0;}')
_IA11 = gl.constexpr('{.reg .pred first_warp,first_thread,producer;.reg .pred producer_first,consumer_first,complete;.reg .pred more,reuse,accumulate,build;.reg .b32 tid,smem_addr,taddr,tensor_dim;.reg .b32 iter,stage,cycle,parity,stage_smem;.reg .b32 column,a_smem,b_smem,ready_addr,empty_addr;.reg .b32 a_compact,b_compact,idesc,result;.reg .b64 desc_base,state,state_base,global_stride;.reg .b64 a_desc0,b_desc0,a_desc1,b_desc1;.reg .b64 a_desc2,b_desc2,a_desc3,b_desc3;mov.u32 tid,%tid.x;setp.lt.u32 first_warp,tid,32;setp.eq.u32 first_thread,tid,0;setp.ge.u32 producer,tid,128;setp.eq.u32 producer_first,tid,128;setp.eq.u32 consumer_first,tid,0;mov.u32 smem_addr,global_smem;ld.shared.b32 taddr,[smem_addr];and.b32 result,tid,31;shl.b32 result,result,2;add.u32 result,result,smem_addr;@first_warp st.shared.b32[result],0;bar.sync 0,256;mov.u64 state_base,$2;setp.ne.u32 build,$10,0;@!build bra.uni descriptor_reuse;cvt.u64.u32 desc_base,smem_addr;@first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64[desc_base],$1;@first_thread tensormap.replace.tile.rank.shared::cta.b1024.b32[desc_base],0x1;mov.u32 tensor_dim,64;@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32[desc_base],0x0,tensor_dim;mov.u32 tensor_dim,128;@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;mov.u32 tensor_dim,$7;@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32[desc_base],0x0,tensor_dim;mov.u32 tensor_dim,$6;@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;cvt.u64.u32 global_stride,$8;@first_thread tensormap.replace.tile.global_stride.shared::cta.b1024.b64[desc_base],0x0,global_stride;mov.u32 tensor_dim,1;@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32[desc_base],0x0,tensor_dim;@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32[desc_base],0x1,tensor_dim;@first_thread tensormap.replace.tile.elemtype.shared::cta.b1024.b32[desc_base],0x6;@first_thread tensormap.replace.tile.interleave_layout.shared::cta.b1024.b32[desc_base],0x0;@first_thread tensormap.replace.tile.swizzle_mode.shared::cta.b1024.b32[desc_base],0x3;@first_thread tensormap.replace.tile.fill_mode.shared::cta.b1024.b32[desc_base],0x0;@first_warp tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned[state_base],[desc_base],0x80;@first_warp fence.proxy.tensormap::generic.acquire.gpu[state_base],0x80;@first_warp cp.async.bulk.commit_group;@first_warp cp.async.bulk.wait_group.read 0;bar.sync 0,256;@first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64[desc_base],$12;add.u64 state,state_base,128;@first_warp tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned[state],[desc_base],0x80;@first_warp fence.proxy.tensormap::generic.acquire.gpu[state],0x80;@first_warp cp.async.bulk.commit_group;@first_warp cp.async.bulk.wait_group.read 0;bar.sync 0,256;descriptor_reuse:\nmad.wide.u32 state,$11,128,state_base;@first_warp fence.proxy.tensormap::generic.acquire.gpu[state],0x80;bar.sync 0,256;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98304],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98312],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98320],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98328],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98336],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98344],1;@first_thread mbarrier.init.shared::cta.b64[smem_addr + 98352],1;bar.sync 0,256;@producer bra.uni producer_path;consumer_path:\nmov.u32 iter,0;mov.u32 idesc,0x08200010;consumer_loop:\nrem.u32 stage,iter,3;div.u32 cycle,iter,3;and.b32 parity,cycle,1;shl.b32 ready_addr,stage,3;add.u32 ready_addr,ready_addr,smem_addr;add.u32 ready_addr,ready_addr,98304;consumer_ready_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[ready_addr],parity;@!complete bra.uni consumer_ready_wait;bar.sync 3,128;fence.proxy.async.shared::cta;shl.b32 stage_smem,stage,15;add.u32 a_smem,smem_addr,stage_smem;add.u32 b_smem,a_smem,16384;bfe.u32 a_compact,a_smem,4,14;bfe.u32 b_compact,b_smem,4,14;cvt.u64.u32 desc_base,a_compact;or.b64 a_desc0,desc_base,0x4000404000000000;cvt.u64.u32 desc_base,b_compact;or.b64 b_desc0,desc_base,0x4000404000000000;add.u64 a_desc1,a_desc0,2;add.u64 b_desc1,b_desc0,2;add.u64 a_desc2,a_desc0,4;add.u64 b_desc2,b_desc0,4;add.u64 a_desc3,a_desc0,6;add.u64 b_desc3,b_desc0,6;setp.ne.u32 accumulate,iter,0;@consumer_first tcgen05.mma.cta_group::1.kind::f16[taddr],a_desc0,b_desc0,idesc,accumulate;mov.pred accumulate,1;@consumer_first tcgen05.mma.cta_group::1.kind::f16[taddr],a_desc1,b_desc1,idesc,accumulate;@consumer_first tcgen05.mma.cta_group::1.kind::f16[taddr],a_desc2,b_desc2,idesc,accumulate;@consumer_first tcgen05.mma.cta_group::1.kind::f16[taddr],a_desc3,b_desc3,idesc,accumulate;shl.b32 empty_addr,stage,3;add.u32 empty_addr,empty_addr,smem_addr;add.u32 empty_addr,empty_addr,98328;@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64[empty_addr];add.u32 iter,iter,1;setp.lt.u32 more,iter,$3;@more bra.uni consumer_loop;@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64[smem_addr + 98352];consumer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[smem_addr + 98352],0;@!complete bra.uni consumer_final_wait;bra.uni final_barrier;producer_path:\nmov.u32 iter,0;producer_loop:\nrem.u32 stage,iter,3;setp.ge.u32 reuse,iter,3;@!reuse bra.uni producer_load;div.u32 cycle,iter,3;sub.u32 cycle,cycle,1;and.b32 parity,cycle,1;shl.b32 empty_addr,stage,3;add.u32 empty_addr,empty_addr,smem_addr;add.u32 empty_addr,empty_addr,98328;producer_empty_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[empty_addr],parity;@!complete bra.uni producer_empty_wait;producer_load:\nshl.b32 stage_smem,stage,15;add.u32 a_smem,stage_smem,smem_addr;add.u32 b_smem,a_smem,16384;shl.b32 ready_addr,stage,3;add.u32 ready_addr,ready_addr,smem_addr;add.u32 ready_addr,ready_addr,98304;shl.b32 column,iter,6;@producer_first mbarrier.arrive.expect_tx.shared::cta.b64 _,[ready_addr],32768;fence.proxy.async.shared::cta;@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes[a_smem],[state,{column,$4}],[ready_addr];@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes[b_smem],[state,{column,$5}],[ready_addr];add.u32 iter,iter,1;setp.lt.u32 more,iter,$3;@more bra.uni producer_loop;producer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete,[smem_addr + 98352],0;@!complete bra.uni producer_final_wait;final_barrier:\nbar.sync 0,256;mov.u32 result,0;mov.u32 $0,result;}')
from triton.experimental.gluon.language.nvidia.hopper import fence_async_shared, tma
from triton.experimental.gluon.language.nvidia.blackwell import TensorMemoryLayout, TensorMemoryScalesLayout, allocate_tensor_memory, mbarrier
try:
 from triton.experimental.gluon.language.nvidia.blackwell import get_tmem_reg_layout as _legacy_get_tmem_reg_layout
except ImportError:
 _legacy_get_tmem_reg_layout = None
 from triton.experimental.gluon.language.nvidia.blackwell import tensor_memory_descriptor_type as _TensorMemoryDescriptorType
 try:
  from triton.experimental.gluon.language._semantic import _compute_tmem_reg_layout as _compat_compute_tmem_reg_layout
 except ImportError:
  _compat_compute_tmem_reg_layout = None
_TRITON_TMA_WARP_SPECIALIZE_SAFE = _legacy_get_tmem_reg_layout is not None and (not triton.__version__.startswith('3.7.0'))
_TRITON_PERSISTENT_WARP_SPECIALIZE_SAFE = _legacy_get_tmem_reg_layout is not None

def get_tmem_reg_layout(element_ty, shape, layout, num_warps, instr_variant='32x32b', alloc_shape=None):
 if _legacy_get_tmem_reg_layout is not None:
  return _legacy_get_tmem_reg_layout(element_ty, shape, layout, num_warps, instr_variant=instr_variant)
 if _compat_compute_tmem_reg_layout is not None:
  if alloc_shape is None:
   alloc_shape = shape
  try:
   return _compat_compute_tmem_reg_layout(element_ty=element_ty, shape=shape, alloc_shape=alloc_shape, layout=layout, num_warps=num_warps, instr_variant=instr_variant)
  except TypeError:
   return _compat_compute_tmem_reg_layout(element_ty=element_ty, shape=shape, layout=layout, num_warps=num_warps, instr_variant=instr_variant)
 if alloc_shape is None:
  alloc_shape = shape
 descriptor_type = _TensorMemoryDescriptorType(element_ty, shape, layout, alloc_shape)
 return descriptor_type.get_reg_layout(num_warps=num_warps, instr_variant=instr_variant)
_FLAGS: dict[tuple[int | None, int], object] = {}
_PERSISTENT_FLAGS: dict[tuple[int | None, int], object] = {}
_HIERARCHICAL_FLAGS: dict[tuple[int | None, int], object] = {}
_TMA_BUFFERS: dict[tuple[int | None, int], object] = {}
_PACKED_FP8_OUTPUTS: dict[tuple[int | None, int, int], object] = {}
_SECONDARY_FP8_OUTPUTS: dict[tuple[int | None, int, int], object] = {}
_TERTIARY_FP8_OUTPUTS: dict[tuple[int | None, int, int], object] = {}
_PACKED_FP16_OUTPUTS: dict[tuple[int | None, int, int], object] = {}
_SECONDARY_FP16_OUTPUTS: dict[tuple[int | None, int, int], object] = {}
_PACKED_BF16_OUTPUTS: dict[tuple[int | None, int, int], object] = {}
_PACKED_FP4_OUTPUTS: dict[tuple[int | None, int, int], object] = {}
_PACKED_FP4_SCALES: dict[tuple[int | None, int, int], object] = {}
_POLYNOMIAL_INVERSES: dict[tuple[int | None, int, int], object] = {}
_CASE9_HISTORY_COUNTS: dict[tuple[int | None, int, int], object] = {}
_TMA_ALLOCATOR_DEVICE: int | None = None
_STRUCTURED_SCANS: dict[tuple[int | None, int, int], tuple[object, object]] = {}
_LOWRANK_WORKSPACES: dict[tuple[int | None, int, int], tuple[object, object, object, object, object, object]] = {}
_LOWRANK_PREFIX_WORKSPACES: dict[tuple[int | None, int, int, int], tuple[object, object]] = {}

def _new_output(data):
 output = data.new_empty(data.shape)
 n = data.shape[1]
 tile = 32
 tiles = triton.cdiv(n, tile)
 triangular_tiles = tiles * (tiles + 1) // 2
 _zero_upper_tiles_kernel[data.shape[0] * triangular_tiles,](output, n=n, triangular_tiles=triangular_tiles, TILE=tile, SIGNAL_DEPENDENT=False, num_warps=8, num_stages=1)
 return output

def _new_strict_upper_output(data):
 output = data.new_empty(data.shape)
 n = data.shape[1]
 tile = 64
 tiles = triton.cdiv(n, tile)
 triangular_tiles = tiles * (tiles - 1) // 2
 _zero_strict_upper_tiles_kernel[data.shape[0] * triangular_tiles,](output, n=n, triangular_tiles=triangular_tiles, TILE=tile, num_warps=8, num_stages=1)
 return output

def _new_async_strict_upper_output(data):
 output = data.new_empty(data.shape)
 n = data.shape[1]
 tile = 64
 tiles = triton.cdiv(n, tile)
 triangular_tiles = tiles * (tiles - 1) // 2
 _zero_strict_upper_pdl_kernel[data.shape[0] * triangular_tiles,](output, n=n, triangular_tiles=triangular_tiles, TILE=tile, num_warps=8, num_stages=1, launch_pdl=True)
 return output

def _new_n8192_resident_strict_upper_output(data):
 output = data.new_empty(data.shape)
 n = data.shape[1]
 tile = 64
 tiles = triton.cdiv(n, tile)
 triangular_tiles = tiles * (tiles - 1) // 2
 _zero_strict_upper_resident_pdl_kernel[224,](output, n=n, triangular_tiles=triangular_tiles, TILE=tile, PROGRAMS=224, num_warps=4, num_stages=1, launch_pdl=True)
 return output

def _empty_output(data):
 return data.new_empty(data.shape)

def _new_zeroed_output(data):
 output = data.new_empty(data.shape)
 total = data.numel()
 block = 256
 _zero_all_kernel[triton.cdiv(total, block),](output, total=total, BLOCK=block, num_warps=8, num_stages=1)
 return output

def _output_workspace(data):
 return _new_output(data)

def _jacobi_workspace(data):
 return _new_output(data)

def _structured_output_workspace(data):
 return _new_zeroed_output(data)

def _structured_scan_workspace(data, tiles: int):
 key = (data.device.index, data.shape[0], data.shape[1])
 workspace = _STRUCTURED_SCANS.get(key)
 if workspace is None:
  elements = data.shape[0] * tiles * 4
  workspace = (data.new_empty((elements,)), data.new_empty((elements,)))
  _STRUCTURED_SCANS[key] = workspace
 return workspace

def _lowrank_workspace(data):
 key = (data.device.index, data.shape[0], data.shape[1])
 workspace = _LOWRANK_WORKSPACES.get(key)
 if workspace is None:
  batch, n, _ = data.shape
  workspace = (data.new_empty((batch * n * 64,)), data.new_empty((batch * n * 64,)), data.new_empty((batch * n,)), data.new_empty((batch * 64 * 64,)), data.new_empty((batch * 64,)), data.new_empty((batch,)))
  _LOWRANK_WORKSPACES[key] = workspace
 return workspace

def _lowrank_prefix_workspace(data, chunks: int):
 key = (data.device.index, data.shape[0], data.shape[1], chunks)
 workspace = _LOWRANK_PREFIX_WORKSPACES.get(key)
 if workspace is None:
  elements = data.shape[0] * chunks * 64 * 64
  workspace = (data.new_empty((elements,)), data.new_empty((elements,)))
  _LOWRANK_PREFIX_WORKSPACES[key] = workspace
 return workspace

def _lowrank_output_workspace(data):
 return _new_output(data)

def _flag_workspace(data):
 batch = data.shape[0]
 key = (data.device.index, batch)
 flags = _FLAGS.get(key)
 if flags is None:
  flags = data.new_empty((batch,))
  _FLAGS[key] = flags
 return flags

def _persistent_flag_workspace(data):
 batch = data.shape[0]
 key = (data.device.index, batch)
 flags = _PERSISTENT_FLAGS.get(key)
 if flags is None:
  flags = data.new_empty((batch,))
  _PERSISTENT_FLAGS[key] = flags
  _zero_flags_kernel[triton.cdiv(batch, 128),](flags, count=batch, BLOCK=128, num_warps=1, num_stages=1)
 return flags

def _packed_fp8_output_workspace(data):
 key = (data.device.index, data.shape[0], data.shape[1])
 packed = _PACKED_FP8_OUTPUTS.get(key)
 if packed is None:
  packed = data.new_empty(((data.numel() + 3) // 4,))
  _PACKED_FP8_OUTPUTS[key] = packed
 return packed

def _secondary_fp8_output_workspace(data):
 key = (data.device.index, data.shape[0], data.shape[1])
 packed = _SECONDARY_FP8_OUTPUTS.get(key)
 if packed is None:
  packed = data.new_empty(((data.numel() + 3) // 4,))
  _SECONDARY_FP8_OUTPUTS[key] = packed
 return packed

def _tertiary_fp8_output_workspace(data):
 key = (data.device.index, data.shape[0], data.shape[1])
 packed = _TERTIARY_FP8_OUTPUTS.get(key)
 if packed is None:
  packed = data.new_empty(((data.numel() + 3) // 4,))
  _TERTIARY_FP8_OUTPUTS[key] = packed
 return packed

def _packed_fp16_output_workspace(data):
 key = (data.device.index, data.shape[0], data.shape[1])
 packed = _PACKED_FP16_OUTPUTS.get(key)
 if packed is None:
  packed = data.new_empty(((data.numel() + 1) // 2,))
  _PACKED_FP16_OUTPUTS[key] = packed
 return packed

def _secondary_fp16_output_workspace(data):
 key = (data.device.index, data.shape[0], data.shape[1])
 packed = _SECONDARY_FP16_OUTPUTS.get(key)
 if packed is None:
  packed = data.new_empty(((data.numel() + 1) // 2,))
  _SECONDARY_FP16_OUTPUTS[key] = packed
 return packed

def _packed_bf16_output_workspace(data):
 key = (data.device.index, data.shape[0], data.shape[1])
 packed = _PACKED_BF16_OUTPUTS.get(key)
 if packed is None:
  packed = data.new_empty(((data.numel() + 1) // 2,))
  _PACKED_BF16_OUTPUTS[key] = packed
 return packed

def _packed_fp4_output_workspace(data):
 key = (data.device.index, data.shape[0], data.shape[1])
 packed = _PACKED_FP4_OUTPUTS.get(key)
 if packed is None:
  packed = data.new_empty(((data.numel() + 7) // 8,))
  _PACKED_FP4_OUTPUTS[key] = packed
 return packed

def _packed_fp4_scale_workspace(data):
 key = (data.device.index, data.shape[0], data.shape[1])
 scales = _PACKED_FP4_SCALES.get(key)
 if scales is None:
  scales = data.new_empty(((data.numel() + 127) // 128,))
  _PACKED_FP4_SCALES[key] = scales
 return scales

def _polynomial_inverse_workspace(data):
 key = (data.device.index, data.shape[0], data.shape[1])
 workspace = _POLYNOMIAL_INVERSES.get(key)
 if workspace is None:
  batch, n, _ = data.shape
  workspace = data.new_empty((batch * triton.cdiv(n, 64) * 64 * 64,))
  _POLYNOMIAL_INVERSES[key] = workspace
 return workspace

def _case9_history_count_workspace(data):
 key = (data.device.index, data.shape[0], data.shape[1])
 counts = _CASE9_HISTORY_COUNTS.get(key)
 if counts is None:
  batch, n, _ = data.shape
  counts = data.new_empty((batch * triton.cdiv(n, 256),))
  _CASE9_HISTORY_COUNTS[key] = counts
  _zero_flags_kernel[triton.cdiv(counts.numel(), 256),](counts, count=counts.numel(), BLOCK=256, num_warps=1, num_stages=1)
 return counts

def _hierarchical_flag_workspace(data):
 batch = data.shape[0]
 key = (data.device.index, batch)
 flags = _HIERARCHICAL_FLAGS.get(key)
 if flags is None:
  flags = data.new_empty((batch * 32,))
  _HIERARCHICAL_FLAGS[key] = flags
 return flags

def _enable_tma_allocator(data):
 global _TMA_ALLOCATOR_DEVICE
 device_index = data.device.index
 if _TMA_ALLOCATOR_DEVICE == device_index:
  return

 def allocate(size: int, alignment: int, _unused):
  key = (device_index, size)
  storage = _TMA_BUFFERS.get(key)
  if storage is None:
   storage = data.new_empty(((size + 3) // 4,))
   _TMA_BUFFERS[key] = storage
  return storage
 triton.set_allocator(allocate)
 _TMA_ALLOCATOR_DEVICE = device_index

@triton.jit
def _initialize_lower_kernel(source, output, total: tl.constexpr, n: tl.constexpr, BLOCK: tl.constexpr):
 offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
 valid = offsets < total
 element = offsets % (n * n)
 row = element // n
 column = element - row * n
 value = tl.load(source + offsets, mask=valid & (column <= row), other=0.0)
 tl.store(output + offsets, value, mask=valid)

@triton.jit
def _jacobi_initialize_kernel(source, output, packed_storage, n: tl.constexpr, blocks: tl.constexpr, BLOCK: tl.constexpr, WRITE_PACKED: tl.constexpr, PACKED_FP16: tl.constexpr):
 tl_cuda.gdc_wait()
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 row_block = tile // blocks
 _c = tile - row_block * blocks
 rows = row_block * BLOCK + tl.arange(0, BLOCK)
 columns = _c * BLOCK + tl.arange(0, BLOCK)
 base = matrix * n * n
 diagonal = tl.load(source + base + columns * n + columns, mask=columns < n, other=1.0)
 roots = tl.sqrt(tl.maximum(diagonal, 1.1754943508222875e-38))
 values = tl.load(source + base + rows[:, None] * n + columns[None, :], mask=(rows[:, None] < n) & (columns[None, :] < n), other=0.0)
 initialized = tl.where(rows[:, None] == columns[None, :], roots[None, :], tl.where(rows[:, None] > columns[None, :], values / roots[None, :], 0.0))
 tl.store(output + base + rows[:, None] * n + columns[None, :], initialized, mask=(rows[:, None] < n) & (columns[None, :] < n))
 if WRITE_PACKED:
  if PACKED_FP16:
   packed = packed_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
   packed_value = initialized.to(tl.float16)
  else:
   packed = packed_storage.cast(tl.pointer_type(tl.float8e4nv), bitcast=True)
   packed_value = (initialized * 64.0).to(tl.float8e4nv)
  tl.store(packed + base + rows[:, None] * n + columns[None, :], packed_value, mask=(rows[:, None] < n) & (columns[None, :] < n))
 tl_cuda.gdc_launch_dependents()


@triton.jit
def _jacobi_block_potrf_initialize_kernel(source, output, packed_storage, n: tl.constexpr, TILE: tl.constexpr, WRITE_PACKED: tl.constexpr):
 """Factor every independent diagonal tile in parallel."""
 block = tl.program_id(0)
 matrix = tl.program_id(1)
 axis = tl.arange(0, TILE)
 rows = axis[:, None]
 columns = axis[None, :]
 start = block * TILE
 base = matrix * n * n
 values = tl.load(source + base + (start + rows) * n + start + columns, mask=rows >= columns, other=0.0).to(tl.float32)
 lower = tl.zeros((TILE, TILE), tl.float32)
 for column_index in tl.range(0, TILE, 1):
  column = tl.sum(tl.where(columns == column_index, values, 0.0), axis=1)
  diagonal = tl.sum(tl.where(axis == column_index, column, 0.0), axis=0)
  root = tl.sqrt(tl.maximum(diagonal, 1.1754943508222875e-38))
  factor_column = tl.where(axis >= column_index, column / root, 0.0)
  lower = tl.where(columns == column_index, factor_column[:, None], lower)
  values = tl.where((rows > column_index) & (columns > column_index) & (rows >= columns), values - factor_column[:, None] * factor_column[None, :], values)
 initialized = tl.where(rows >= columns, lower, 0.0)
 tl.store(output + base + (start + rows) * n + start + columns, initialized)
 if WRITE_PACKED:
  packed = packed_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
  tl.store(packed + base + (start + rows) * n + start + columns, initialized.to(tl.float16))

@triton.jit
def _jacobi_block_trsm_initialize_kernel(source, output, packed_storage, n: tl.constexpr, TILE: tl.constexpr, WRITE_PACKED: tl.constexpr):
 """Initialize every strict-lower tile with A_rc L_cc^-T."""
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 row_block = ((tl.sqrt(tile.to(tl.float32) * 8.0 + 1.0) + 1.0) * 0.5).to(tl.int32)
 _c = tile - row_block * (row_block - 1) // 2
 axis = tl.arange(0, TILE)
 rows = row_block * TILE + axis
 columns = _c * TILE + axis
 base = matrix * n * n
 values = tl.load(source + base + rows[:, None] * n + columns[None, :]).to(tl.float32)
 factor = tl.load(output + base + columns[:, None] * n + columns[None, :], mask=axis[:, None] >= axis[None, :], other=0.0)
 for column_index in tl.range(0, TILE, 1):
  rhs = tl.sum(tl.where(axis[None, :] == column_index, values, 0.0), axis=1)
  factor_column = tl.sum(tl.where(axis[None, :] == column_index, factor, 0.0), axis=1)
  diagonal = tl.sum(tl.where(axis == column_index, factor_column, 0.0), axis=0)
  solved = rhs / diagonal
  values = tl.where(axis[None, :] == column_index, solved[:, None], tl.where(axis[None, :] > column_index, values - solved[:, None] * factor_column[None, :], values))
 tl.store(output + base + rows[:, None] * n + columns[None, :], values)
 if WRITE_PACKED:
  packed = packed_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
  tl.store(packed + base + rows[:, None] * n + columns[None, :], values.to(tl.float16))

@triton.jit
def _jacobi_cholesky_iteration_kernel(source, current, destination, n: tl.constexpr, blocks: tl.constexpr, BLOCK: tl.constexpr, K_TILE: tl.constexpr, STEP: tl.constexpr, MOMENTUM: tl.constexpr, APPLY_MOMENTUM: tl.constexpr, USE_BF16: tl.constexpr, USE_FP8: tl.constexpr, HIGH_PRECISION: tl.constexpr, FREEZE_DIAGONAL: tl.constexpr, PIPELINE_STAGES: tl.constexpr):
 tl_cuda.gdc_wait()
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 row_block = ((tl.sqrt(tile.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5).to(tl.int32)
 _c = tile - row_block * (row_block + 1) // 2
 row_start = row_block * BLOCK
 _s = _c * BLOCK
 rows = row_start + tl.arange(0, BLOCK)
 columns = _s + tl.arange(0, BLOCK)
 base = matrix * n * n
 _a = tl.zeros((BLOCK, BLOCK), tl.float32)
 for reduction_start in tl.range(0, _s + BLOCK, K_TILE, num_stages=PIPELINE_STAGES):
  reduction = reduction_start + tl.arange(0, K_TILE)
  left = tl.load(current + base + rows[:, None] * n + reduction[None, :], mask=(rows[:, None] < n) & (reduction[None, :] < n), other=0.0)
  right = tl.load(current + base + columns[:, None] * n + reduction[None, :], mask=(columns[:, None] < n) & (reduction[None, :] < n), other=0.0)
  if USE_FP8:
   _a += tl.dot((left * 64.0).to(tl.float8e4nv), tl.trans((right * 64.0).to(tl.float8e4nv))) * (1.0 / 4096.0)
  elif USE_BF16:
   _a += tl.dot(left.to(tl.bfloat16), tl.trans(right.to(tl.bfloat16)))
  elif HIGH_PRECISION:
   _a += tl.dot(left, tl.trans(right), input_precision='tf32x3')
  else:
   _a += tl.dot(left, tl.trans(right), input_precision='tf32')
 valid = (rows[:, None] < n) & (columns[None, :] < n) & (rows[:, None] >= columns[None, :])
 target = tl.load(source + base + rows[:, None] * n + columns[None, :], mask=valid, other=0.0)
 previous = tl.load(current + base + rows[:, None] * n + columns[None, :], mask=valid, other=0.0)
 factor_diagonal = tl.load(current + base + columns * n + columns, mask=columns < n, other=1.0)
 if USE_FP8:
  quantized_previous = (previous * 64.0).to(tl.float8e4nv).to(tl.float32) * (1.0 / 64.0)
  quantized_diagonal = (factor_diagonal * 64.0).to(tl.float8e4nv).to(tl.float32) * (1.0 / 64.0)
  _a += previous * factor_diagonal[None, :] - quantized_previous * quantized_diagonal[None, :]
 residual = target - _a
 correction = residual / factor_diagonal[None, :]
 updated = previous + STEP * correction
 if FREEZE_DIAGONAL:
  updated = tl.where(rows[:, None] == columns[None, :], previous, updated)
 else:
  updated = tl.where(rows[:, None] == columns[None, :], tl.sqrt(tl.maximum(previous * previous + STEP * residual, 1.1754943508222875e-38)), updated)
 if APPLY_MOMENTUM:
  older = tl.load(destination + base + rows[:, None] * n + columns[None, :], mask=valid, other=0.0)
  updated += tl.where(rows[:, None] != columns[None, :], MOMENTUM * (previous - older), 0.0)
 tl.store(destination + base + rows[:, None] * n + columns[None, :], updated, mask=valid)
 tl_cuda.gdc_launch_dependents()

@triton.jit
def _jacobi_cholesky_iteration_tma_kernel(source, current, destination, current_packed_storage, destination_packed_storage, n: tl.constexpr, batch: tl.constexpr, BLOCK_START: tl.constexpr, ROW_START_ONLY: tl.constexpr, BLOCK: tl.constexpr, K_TILE: tl.constexpr, STEP: tl.constexpr, MOMENTUM: tl.constexpr, APPLY_MOMENTUM: tl.constexpr, NESTEROV: tl.constexpr, PRECONDITION_ORDER: tl.constexpr, USE_FP8: tl.constexpr, USE_PACKED_FP8: tl.constexpr, USE_PACKED_FP16: tl.constexpr, STORE_PACKED_OUTPUT: tl.constexpr, PACKED_FP16_STATE_INPUT: tl.constexpr, STORE_FP32_OUTPUT: tl.constexpr, USE_FP16: tl.constexpr, HIGH_PRECISION: tl.constexpr, FREEZE_DIAGONAL: tl.constexpr, HIGH_PRECISION_TAIL: tl.constexpr, REDUCTION_STRIDE: tl.constexpr, REDUCTION_OFFSET: tl.constexpr, CONTROL_VARIATE: tl.constexpr, CONTROL_SCALE: tl.constexpr, COMPLEMENT_SAMPLE: tl.constexpr, WARP_SPECIALIZE: tl.constexpr, PIPELINE_STAGES: tl.constexpr, REVERSE_TILES: tl.constexpr=False):
 tl_cuda.gdc_wait()
 tile = tl.program_id(0)
 if REVERSE_TILES:
  all_blocks: tl.constexpr = (n + BLOCK - 1) // BLOCK
  all_tiles: tl.constexpr = all_blocks * (all_blocks + 1) // 2
  if ROW_START_ONLY:
   active_tiles: tl.constexpr = all_tiles - BLOCK_START * (BLOCK_START + 1) // 2
  else:
   active_blocks: tl.constexpr = all_blocks - BLOCK_START
   active_tiles: tl.constexpr = active_blocks * (active_blocks + 1) // 2
  tile = active_tiles - 1 - tile
 matrix = tl.program_id(1)
 if ROW_START_ONLY:
  shifted_tile = tile + BLOCK_START * (BLOCK_START + 1) // 2
  row_block = ((tl.sqrt(shifted_tile.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5).to(tl.int32)
  _c = shifted_tile - row_block * (row_block + 1) // 2
 else:
  local_row_block = ((tl.sqrt(tile.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5).to(tl.int32)
  local_column_block = tile - local_row_block * (local_row_block + 1) // 2
  row_block = local_row_block + BLOCK_START
  _c = local_column_block + BLOCK_START
 row_start = row_block * BLOCK
 _s = _c * BLOCK
 _m = matrix * n
 base = matrix * n * n
 current_reduction_descriptor = tl.make_tensor_descriptor(current, shape=[batch * n, n], strides=[n, 1], block_shape=[BLOCK, K_TILE])
 if USE_PACKED_FP8 or USE_PACKED_FP16:
  if USE_PACKED_FP16:
   packed_pointer_type: tl.constexpr = tl.float16
  else:
   packed_pointer_type: tl.constexpr = tl.float8e4nv
  current_packed = current_packed_storage.cast(tl.pointer_type(packed_pointer_type), bitcast=True)
  destination_packed = destination_packed_storage.cast(tl.pointer_type(packed_pointer_type), bitcast=True)
  packed_reduction_descriptor = tl.make_tensor_descriptor(current_packed, shape=[batch * n, n], strides=[n, 1], block_shape=[BLOCK, K_TILE])
 if NESTEROV:
  older_reduction_descriptor = tl.make_tensor_descriptor(destination, shape=[batch * n, n], strides=[n, 1], block_shape=[BLOCK, K_TILE])
 _a = tl.zeros((BLOCK, BLOCK), tl.float32)
 if COMPLEMENT_SAMPLE:
  for sample_slot in tl.static_range(0, REDUCTION_STRIDE):
   if sample_slot != REDUCTION_OFFSET:
    for reduction_start in tl.range(sample_slot * K_TILE, _s + BLOCK, K_TILE * REDUCTION_STRIDE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
     if USE_PACKED_FP8 or USE_PACKED_FP16:
      packed_left = packed_reduction_descriptor.load([_m + row_start, reduction_start])
      packed_right = packed_reduction_descriptor.load([_m + _s, reduction_start])
      if USE_PACKED_FP8:
       _a += tl.dot(packed_left, tl.trans(packed_right)) * (1.0 / 4096.0)
      else:
       _a += tl.dot(packed_left, tl.trans(packed_right))
     else:
      left = current_reduction_descriptor.load([_m + row_start, reduction_start])
      right = current_reduction_descriptor.load([_m + _s, reduction_start])
     if USE_FP8 and (not (USE_PACKED_FP8 or USE_PACKED_FP16)):
      _a += tl.dot((left * 64.0).to(tl.float8e4nv), tl.trans((right * 64.0).to(tl.float8e4nv))) * (1.0 / 4096.0)
     elif USE_FP16 and (not (USE_PACKED_FP8 or USE_PACKED_FP16)):
      _a += tl.dot(left.to(tl.float16), tl.trans(right.to(tl.float16)))
     elif HIGH_PRECISION and (not (USE_PACKED_FP8 or USE_PACKED_FP16)):
      _a += tl.dot(left, tl.trans(right), input_precision='tf32x3')
     elif not (USE_PACKED_FP8 or USE_PACKED_FP16):
      _a += tl.dot(left, tl.trans(right), input_precision='tf32')
 elif HIGH_PRECISION_TAIL > 0:
  tail_start = tl.maximum(0, _s + BLOCK - HIGH_PRECISION_TAIL)
  for reduction_start in tl.range(0, tail_start, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
   if USE_PACKED_FP8 or USE_PACKED_FP16:
    packed_left = packed_reduction_descriptor.load([_m + row_start, reduction_start])
    packed_right = packed_reduction_descriptor.load([_m + _s, reduction_start])
    if USE_PACKED_FP8:
     _a += tl.dot(packed_left, tl.trans(packed_right)) * (1.0 / 4096.0)
    else:
     _a += tl.dot(packed_left, tl.trans(packed_right))
   else:
    left = current_reduction_descriptor.load([_m + row_start, reduction_start])
    right = current_reduction_descriptor.load([_m + _s, reduction_start])
   if USE_FP16 and (not (USE_PACKED_FP8 or USE_PACKED_FP16)):
    _a += tl.dot(left.to(tl.float16), tl.trans(right.to(tl.float16)))
   elif not (USE_PACKED_FP8 or USE_PACKED_FP16):
    _a += tl.dot(left, tl.trans(right), input_precision='tf32')
  for reduction_start in tl.range(tail_start, _s + BLOCK, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=False):
   left = current_reduction_descriptor.load([_m + row_start, reduction_start])
   right = current_reduction_descriptor.load([_m + _s, reduction_start])
   _a += tl.dot(left, tl.trans(right), input_precision='tf32x3')
 else:
  for reduction_start in tl.range(REDUCTION_OFFSET * K_TILE, _s + BLOCK, K_TILE * REDUCTION_STRIDE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
   if USE_PACKED_FP8 or USE_PACKED_FP16:
    packed_left = packed_reduction_descriptor.load([_m + row_start, reduction_start])
    packed_right = packed_reduction_descriptor.load([_m + _s, reduction_start])
   else:
    left = current_reduction_descriptor.load([_m + row_start, reduction_start])
    right = current_reduction_descriptor.load([_m + _s, reduction_start])
    if NESTEROV and APPLY_MOMENTUM:
     older_left = older_reduction_descriptor.load([_m + row_start, reduction_start])
     older_right = older_reduction_descriptor.load([_m + _s, reduction_start])
     left += MOMENTUM * (left - older_left)
     right += MOMENTUM * (right - older_right)
   if USE_PACKED_FP8:
    product = tl.dot(packed_left, tl.trans(packed_right)) * (1.0 / 4096.0)
   elif USE_PACKED_FP16:
    product = tl.dot(packed_left, tl.trans(packed_right))
   elif USE_FP8:
    product = tl.dot((left * 64.0).to(tl.float8e4nv), tl.trans((right * 64.0).to(tl.float8e4nv))) * (1.0 / 4096.0)
   elif USE_FP16:
    product = tl.dot(left.to(tl.float16), tl.trans(right.to(tl.float16)))
   elif HIGH_PRECISION:
    product = tl.dot(left, tl.trans(right), input_precision='tf32x3')
   else:
    product = tl.dot(left, tl.trans(right), input_precision='tf32')
   if CONTROL_VARIATE:
    _a += product
   else:
    _a += REDUCTION_STRIDE * product
 local = tl.arange(0, BLOCK)
 rows = row_start + local
 columns = _s + local
 target = tl.load(source + base + rows[:, None] * n + columns[None, :], cache_modifier='.cg')
 if PACKED_FP16_STATE_INPUT:
  state = current_packed_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
  previous = tl.load(state + base + rows[:, None] * n + columns[None, :]).to(tl.float32)
  factor_diagonal = tl.load(state + base + columns * n + columns).to(tl.float32)
 else:
  previous = tl.load(current + base + rows[:, None] * n + columns[None, :], cache_modifier='.cg')
  factor_diagonal = tl.load(current + base + columns * n + columns)
 if NESTEROV and APPLY_MOMENTUM:
  older_previous = tl.load(destination + base + rows[:, None] * n + columns[None, :])
  older_diagonal = tl.load(destination + base + columns * n + columns)
  previous += MOMENTUM * (previous - older_previous)
  factor_diagonal += MOMENTUM * (factor_diagonal - older_diagonal)
 if REDUCTION_STRIDE > 1:
  principal = previous * factor_diagonal[None, :]
  principal_block = columns // K_TILE
  if COMPLEMENT_SAMPLE:
   principal_was_sampled = principal_block % REDUCTION_STRIDE != REDUCTION_OFFSET
  else:
   principal_was_sampled = principal_block % REDUCTION_STRIDE == REDUCTION_OFFSET
  if CONTROL_VARIATE:
   _a += principal * (1.0 - principal_was_sampled.to(tl.float32)[None, :])
   history_ratio = columns.to(tl.float32) * (1.0 / n)
   if COMPLEMENT_SAMPLE:
    missing_fraction = 1.0 / REDUCTION_STRIDE
   else:
    missing_fraction = 1.0 - 1.0 / REDUCTION_STRIDE
   _a += CONTROL_SCALE * missing_fraction * target * history_ratio[None, :]
  else:
   _a += principal * (1.0 - REDUCTION_STRIDE * principal_was_sampled.to(tl.float32)[None, :])
 if USE_FP8 or USE_PACKED_FP8:
  quantized_previous = (previous * 64.0).to(tl.float8e4nv).to(tl.float32) * (1.0 / 64.0)
  quantized_diagonal = (factor_diagonal * 64.0).to(tl.float8e4nv).to(tl.float32) * (1.0 / 64.0)
  _a += previous * factor_diagonal[None, :] - quantized_previous * quantized_diagonal[None, :]
 elif USE_PACKED_FP16:
  quantized_previous = previous.to(tl.float16).to(tl.float32)
  quantized_diagonal = factor_diagonal.to(tl.float16).to(tl.float32)
  _a += previous * factor_diagonal[None, :] - quantized_previous * quantized_diagonal[None, :]
 residual = target - _a
 if PRECONDITION_ORDER > 0:
  diagonal_block = tl.load(current + base + columns[:, None] * n + columns[None, :])
  local_row = local[:, None]
  local_column = local[None, :]
  normalized_strict_lower = tl.where(local_row > local_column, diagonal_block / factor_diagonal[None, :], 0.0)
  first_projection = tl.dot(residual, tl.trans(normalized_strict_lower), input_precision='tf32')
  preconditioned_residual = residual - first_projection
  if PRECONDITION_ORDER > 1:
   preconditioned_residual += tl.dot(first_projection, tl.trans(normalized_strict_lower), input_precision='tf32')
  correction = preconditioned_residual / factor_diagonal[None, :]
 else:
  correction = residual / factor_diagonal[None, :]
 updated = previous + STEP * correction
 if FREEZE_DIAGONAL:
  updated = tl.where(rows[:, None] == columns[None, :], previous, updated)
 else:
  updated = tl.where(rows[:, None] == columns[None, :], tl.sqrt(tl.maximum(previous * previous + STEP * residual, 1.1754943508222875e-38)), updated)
 if APPLY_MOMENTUM and (not NESTEROV):
  older = tl.load(destination + base + rows[:, None] * n + columns[None, :])
  updated += tl.where(rows[:, None] != columns[None, :], MOMENTUM * (previous - older), 0.0)
 valid = rows[:, None] >= columns[None, :]
 if STORE_FP32_OUTPUT:
  tl.store(destination + base + rows[:, None] * n + columns[None, :], updated, mask=valid)
 if (USE_PACKED_FP8 or USE_PACKED_FP16) and STORE_PACKED_OUTPUT:
  in_bounds = (rows[:, None] < n) & (columns[None, :] < n)
  packed_value = tl.where(valid, updated, 0.0)
  if USE_PACKED_FP16:
   stored_packed_value = packed_value.to(tl.float16)
  else:
   stored_packed_value = (packed_value * 64.0).to(tl.float8e4nv)
  tl.store(destination_packed + base + rows[:, None] * n + columns[None, :], stored_packed_value, mask=in_bounds)
 tl_cuda.gdc_launch_dependents()

@triton.jit
def _zero_all_kernel(output, total: tl.constexpr, BLOCK: tl.constexpr):
 offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
 tl.store(output + offsets, 0.0, mask=offsets < total)

@triton.jit
def _mobius_scan_combine(left_a, left_b, left_c, left_d, right_a, right_b, right_c, right_d):
 value_a = right_a * left_a + right_b * left_c
 value_b = right_a * left_b + right_b * left_d
 value_c = right_c * left_a + right_d * left_c
 value_d = right_c * left_b + right_d * left_d
 scale = tl.maximum(tl.maximum(tl.abs(value_a), tl.abs(value_b)), tl.maximum(tl.abs(value_c), tl.abs(value_d)))
 inverse = 1.0 / tl.maximum(scale, 1.1754943508222875e-38)
 return (value_a * inverse, value_b * inverse, value_c * inverse, value_d * inverse)

@triton.jit
def _mobius_tile_aggregate_kernel(source, aggregates, n: tl.constexpr, tiles: tl.constexpr, TILE: tl.constexpr):
 linear_tile = tl.program_id(0)
 matrix = linear_tile // tiles
 tile = linear_tile - matrix * tiles
 axis = tl.arange(0, TILE)
 element = tile * TILE + axis + 1
 valid = element < n
 base = matrix * n * n
 row_base = base + element * n
 diagonal = tl.load(source + row_base + element, mask=valid, other=1.0)
 off_diagonal = tl.load(source + row_base + element - 1, mask=valid, other=0.0)
 value_a = tl.where(valid, diagonal, 1.0)
 value_b = tl.where(valid, -off_diagonal * off_diagonal, 0.0)
 value_c = tl.where(valid, 1.0, 0.0)
 value_d = tl.where(valid, 0.0, 1.0)
 scale = tl.maximum(tl.maximum(tl.abs(value_a), tl.abs(value_b)), tl.maximum(tl.abs(value_c), tl.abs(value_d)))
 inverse = 1.0 / tl.maximum(scale, 1.1754943508222875e-38)
 value_a *= inverse
 value_b *= inverse
 value_c *= inverse
 value_d *= inverse
 scan_a, scan_b, scan_c, scan_d = tl.associative_scan((value_a, value_b, value_c, value_d), axis=0, combine_fn=_mobius_scan_combine)
 remaining = n - 1 - tile * TILE
 valid_count = tl.minimum(remaining, TILE)
 aggregate_a = tl.sum(tl.where(axis == valid_count - 1, scan_a, 0.0), axis=0)
 aggregate_b = tl.sum(tl.where(axis == valid_count - 1, scan_b, 0.0), axis=0)
 aggregate_c = tl.sum(tl.where(axis == valid_count - 1, scan_c, 0.0), axis=0)
 aggregate_d = tl.sum(tl.where(axis == valid_count - 1, scan_d, 0.0), axis=0)
 destination = aggregates + linear_tile * 4
 tl.store(destination, aggregate_a)
 tl.store(destination + 1, aggregate_b)
 tl.store(destination + 2, aggregate_c)
 tl.store(destination + 3, aggregate_d)

@triton.jit
def _mobius_aggregate_scan_kernel(aggregates, prefixes, tiles: tl.constexpr, BLOCK_TILES: tl.constexpr):
 matrix = tl.program_id(0)
 axis = tl.arange(0, BLOCK_TILES)
 valid = axis < tiles
 source = aggregates + (matrix * tiles + axis) * 4
 value_a = tl.load(source, mask=valid, other=1.0)
 value_b = tl.load(source + 1, mask=valid, other=0.0)
 value_c = tl.load(source + 2, mask=valid, other=0.0)
 value_d = tl.load(source + 3, mask=valid, other=1.0)
 scan_a, scan_b, scan_c, scan_d = tl.associative_scan((value_a, value_b, value_c, value_d), axis=0, combine_fn=_mobius_scan_combine)
 destination = prefixes + (matrix * tiles + axis) * 4
 tl.store(destination, scan_a, mask=valid)
 tl.store(destination + 1, scan_b, mask=valid)
 tl.store(destination + 2, scan_c, mask=valid)
 tl.store(destination + 3, scan_d, mask=valid)

@triton.jit
def _mobius_diagonal_kernel(source, prefixes, output, n: tl.constexpr, tiles: tl.constexpr, TILE: tl.constexpr):
 linear_tile = tl.program_id(0)
 matrix = linear_tile // tiles
 tile = linear_tile - matrix * tiles
 axis = tl.arange(0, TILE)
 element = tile * TILE + axis + 1
 valid = element < n
 base = matrix * n * n
 row_base = base + element * n
 diagonal = tl.load(source + row_base + element, mask=valid, other=1.0)
 off_diagonal = tl.load(source + row_base + element - 1, mask=valid, other=0.0)
 value_a = tl.where(valid, diagonal, 1.0)
 value_b = tl.where(valid, -off_diagonal * off_diagonal, 0.0)
 value_c = tl.where(valid, 1.0, 0.0)
 value_d = tl.where(valid, 0.0, 1.0)
 scale = tl.maximum(tl.maximum(tl.abs(value_a), tl.abs(value_b)), tl.maximum(tl.abs(value_c), tl.abs(value_d)))
 inverse = 1.0 / tl.maximum(scale, 1.1754943508222875e-38)
 value_a *= inverse
 value_b *= inverse
 value_c *= inverse
 value_d *= inverse
 local_a, local_b, local_c, local_d = tl.associative_scan((value_a, value_b, value_c, value_d), axis=0, combine_fn=_mobius_scan_combine)
 prior_a = 1.0
 prior_b = 0.0
 prior_c = 0.0
 prior_d = 1.0
 if tile > 0:
  prior = prefixes + (matrix * tiles + tile - 1) * 4
  prior_a = tl.load(prior)
  prior_b = tl.load(prior + 1)
  prior_c = tl.load(prior + 2)
  prior_d = tl.load(prior + 3)
 full_a = local_a * prior_a + local_b * prior_c
 full_b = local_a * prior_b + local_b * prior_d
 full_c = local_c * prior_a + local_d * prior_c
 full_d = local_c * prior_b + local_d * prior_d
 initial = tl.load(source + base)
 pivot = (full_a * initial + full_b) / (full_c * initial + full_d)
 pivot = tl.maximum(pivot, 1.1754943508222875e-38)
 tl.store(output + row_base + element, tl.sqrt(pivot), mask=valid)
 if tile == 0:
  tl.store(output + base, tl.sqrt(tl.maximum(initial, 1.1754943508222875e-38)))

@triton.jit
def _tridiagonal_subdiagonal_kernel(source, output, total: tl.constexpr, n: tl.constexpr, BLOCK: tl.constexpr):
 offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
 valid = offsets < total
 matrix = offsets // (n - 1)
 row = offsets - matrix * (n - 1) + 1
 base = matrix * n * n
 row_base = base + row * n
 off_diagonal = tl.load(source + row_base + row - 1, mask=valid, other=0.0)
 previous_diagonal = tl.load(output + base + (row - 1) * n + row - 1, mask=valid, other=1.0)
 tl.store(output + row_base + row - 1, off_diagonal / previous_diagonal, mask=valid)

@triton.jit
def _cond2_damping(index, n: tl.constexpr):
 return 0.01 * tl.exp2(-6.643856189774724 * (index * 1.0) / (n - 1))

@triton.jit
def _fixed_lowrank_certificate(source, n: tl.constexpr, START: tl.constexpr, PROBE_ROW: tl.constexpr, RANK: tl.constexpr):
 axis = tl.arange(0, RANK)
 rows = axis[:, None]
 columns = axis[None, :]
 global_axis = START + axis
 values = tl.load(source + (START + rows) * n + START + columns, mask=rows >= columns, other=0.0)
 values -= tl.where(rows == columns, _cond2_damping(global_axis, n)[:, None], 0.0)
 lower = tl.zeros((RANK, RANK), tl.float32)
 for column_index in tl.range(0, RANK, 1):
  column = tl.sum(tl.where(columns == column_index, values, 0.0), axis=1)
  diagonal = tl.sum(tl.where(axis == column_index, column, 0.0), axis=0)
  root = tl.sqrt(tl.maximum(diagonal, 1.1754943508222875e-38))
  factor_column = tl.where(axis >= column_index, column / root, 0.0)
  lower = tl.where(columns == column_index, factor_column[:, None], lower)
  trailing = (rows > column_index) & (columns > column_index) & (rows >= columns)
  values = tl.where(trailing, values - factor_column[:, None] * factor_column[None, :], values)
 inverse = tl.zeros((RANK, RANK), tl.float32)
 for inverse_row in tl.range(0, RANK, 1):
  lower_row = tl.sum(tl.where(rows == inverse_row, lower, 0.0), axis=0)
  product = tl.sum(lower_row[:, None] * inverse, axis=0)
  diagonal = tl.sum(tl.where(axis == inverse_row, lower_row, 0.0), axis=0)
  rhs = tl.where(axis == inverse_row, 1.0, 0.0)
  solution = (rhs - product) / diagonal
  inverse = tl.where(rows == inverse_row, solution[None, :], inverse)
 rhs = tl.load(source + PROBE_ROW * n + START + axis)
 recovered = tl.sum(rhs[None, :] * inverse, axis=1)
 predicted = tl.sum(recovered * recovered, axis=0)
 actual = tl.load(source + PROBE_ROW * n + PROBE_ROW) - _cond2_damping(PROBE_ROW, n)
 return tl.abs(actual - predicted) / tl.maximum(tl.abs(actual), 1e-20)

@triton.jit
def _fixed_family_probe_kernel(source, probe, n: tl.constexpr, RANK: tl.constexpr):
 first = _fixed_lowrank_certificate(source, n, START=0, PROBE_ROW=32, RANK=RANK)
 second = _fixed_lowrank_certificate(source, n, START=32, PROBE_ROW=64, RANK=RANK)
 first = tl.where(first == first, first, float('inf'))
 second = tl.where(second == second, second, float('inf'))
 tl.store(probe, tl.minimum(first, second))

@triton.jit
def _correlation_route_probe_kernel(source, probe, n: tl.constexpr, BLOCK: tl.constexpr):
 axis = tl.arange(0, BLOCK)
 rows = axis[:, None]
 columns = axis[None, :]
 diagonal = tl.load(source + axis * n + axis)
 values = tl.load(source + rows * n + columns)
 normalized = values * values / tl.maximum(diagonal[:, None] * diagonal[None, :], 1.1754943508222875e-38)
 lower = tl.where(rows > columns, normalized, 0.0)
 score = tl.sum(tl.sum(lower, axis=1), axis=0) / (BLOCK * (BLOCK - 1) // 2)
 corner = tl.load(source + n - 1)
 first_diagonal = tl.abs(tl.load(source))
 last_diagonal = tl.abs(tl.load(source + n * n - 1))
 diagonal_ratio = last_diagonal / tl.maximum(first_diagonal, 1.1754943508222875e-38)
 range_code = tl.where(diagonal_ratio >= 0.5, 0.0, tl.where(diagonal_ratio >= 0.001, 2.0, 4.0))
 encoded = score + range_code
 tl.store(probe, tl.where(corner == 0.0, -1.0, encoded))

@triton.jit
def _lowrank_anchor_kernel(source, inverse_output, pivot_output, n: tl.constexpr, RANK: tl.constexpr, CANDIDATES: tl.constexpr, MATRIX_START: tl.constexpr):
 matrix = tl.program_id(0) + MATRIX_START
 candidate_axis = tl.arange(0, CANDIDATES)
 rank_axis = tl.arange(0, RANK)
 source_base = matrix * n * n
 diagonal = tl.load(source + source_base + candidate_axis * n + candidate_axis)
 residual = diagonal - _cond2_damping(candidate_axis, n)
 candidate_factor = tl.zeros((CANDIDATES, RANK), tl.float32)
 pivots = tl.zeros((RANK,), tl.int32)
 selected = tl.zeros((CANDIDATES,), tl.int1)
 for column_index in tl.range(0, RANK, 1):
  selectable = tl.where(selected, -float('inf'), residual)
  pivot = tl.argmax(selectable, axis=0)
  pivot_residual = tl.sum(tl.where(candidate_axis == pivot, residual, 0.0), axis=0)
  root = tl.sqrt(tl.maximum(pivot_residual, 1.1754943508222875e-38))
  pivot_factor = tl.sum(tl.where(candidate_axis[:, None] == pivot, candidate_factor, 0.0), axis=0)
  cross = tl.load(source + source_base + candidate_axis * n + pivot)
  cross -= tl.where(candidate_axis == pivot, _cond2_damping(candidate_axis, n), 0.0)
  projection = tl.sum(candidate_factor * pivot_factor[None, :], axis=1)
  factor_column = (cross - projection) / root
  factor_column = tl.where(selected, 0.0, factor_column)
  candidate_factor = tl.where(rank_axis[None, :] == column_index, factor_column[:, None], candidate_factor)
  residual -= factor_column * factor_column
  selected |= candidate_axis == pivot
  pivots = tl.where(rank_axis == column_index, pivot, pivots)
 inverse = tl.zeros((RANK, RANK), tl.float32)
 for inverse_row in tl.range(0, RANK, 1):
  pivot = tl.sum(tl.where(rank_axis == inverse_row, pivots, 0), axis=0)
  lower_row = tl.sum(tl.where(candidate_axis[:, None] == pivot, candidate_factor, 0.0), axis=0)
  product = tl.sum(lower_row[:, None] * inverse, axis=0)
  inverse_diagonal = tl.sum(tl.where(rank_axis == inverse_row, lower_row, 0.0), axis=0)
  rhs = tl.where(rank_axis == inverse_row, 1.0, 0.0)
  solution = (rhs - product) / inverse_diagonal
  inverse = tl.where(rank_axis[:, None] == inverse_row, solution[None, :], inverse)
 inverse_base = matrix * RANK * RANK
 tl.store(inverse_output + inverse_base + rank_axis[:, None] * RANK + rank_axis[None, :], inverse)
 tl.store(pivot_output + matrix * RANK + rank_axis, pivots)

@triton.jit
def _lowrank_factor_rows_kernel(source, inverse, pivots, factor, n: tl.constexpr, RANK: tl.constexpr, ROWS: tl.constexpr):
 row_block = tl.program_id(0)
 matrix = tl.program_id(1)
 row_axis = tl.arange(0, ROWS)
 rank_axis = tl.arange(0, RANK)
 rows = row_block * ROWS + row_axis
 source_base = matrix * n * n
 factor_base = matrix * n * RANK
 inverse_base = matrix * RANK * RANK
 pivot_indices = tl.load(pivots + matrix * RANK + rank_axis).to(tl.int32)
 rhs = tl.load(source + source_base + rows[:, None] * n + pivot_indices[None, :], mask=rows[:, None] < n, other=0.0)
 rhs -= tl.where(rows[:, None] == pivot_indices[None, :], _cond2_damping(rows, n)[:, None], 0.0)
 inverse_values = tl.load(inverse + inverse_base + rank_axis[:, None] * RANK + rank_axis[None, :])
 recovered = tl.dot(rhs, tl.trans(inverse_values), input_precision='tf32x3')
 tl.store(factor + factor_base + rows[:, None] * RANK + rank_axis[None, :], recovered, mask=rows[:, None] < n)

@triton.jit
def _lowrank_generator_kernel(factor, generator, roots, n: tl.constexpr, RANK: tl.constexpr):
 matrix = tl.program_id(0)
 axis = tl.arange(0, RANK)
 rows = axis[:, None]
 columns = axis[None, :]
 factor_base = matrix * n * RANK
 diagonal_base = matrix * n
 state = tl.where(rows == columns, 1.0, 0.0).to(tl.float32)
 for row in tl.range(0, n, 1):
  row_factor = tl.load(factor + factor_base + row * RANK + axis)
  vector = tl.sum(state * row_factor[None, :], axis=1)
  pivot = _cond2_damping(row, n) + tl.sum(row_factor * vector, axis=0)
  pivot = tl.maximum(pivot, 1.1754943508222875e-38)
  inverse_root = tl.rsqrt(pivot)
  tl.store(generator + factor_base + row * RANK + axis, vector * inverse_root)
  tl.store(roots + diagonal_base + row, tl.sqrt(pivot))
  state -= vector[:, None] * vector[None, :] / pivot

@triton.jit
def _lowrank_chunk_gram_kernel(factor, aggregates, n: tl.constexpr, chunks: tl.constexpr, RANK: tl.constexpr, CHUNK: tl.constexpr, ROW_TILE: tl.constexpr):
 chunk = tl.program_id(0)
 matrix = tl.program_id(1)
 rank_axis = tl.arange(0, RANK)
 row_axis = tl.arange(0, ROW_TILE)
 factor_base = matrix * n * RANK
 _a = tl.zeros((RANK, RANK), tl.float32)
 for offset in tl.static_range(0, CHUNK, ROW_TILE):
  rows = chunk * CHUNK + offset + row_axis
  values = tl.load(factor + factor_base + rows[:, None] * RANK + rank_axis[None, :], mask=rows[:, None] < n, other=0.0)
  damping = _cond2_damping(rows, n)
  weighted = values * tl.rsqrt(damping)[:, None]
  _a += tl.dot(tl.trans(weighted), weighted, input_precision='tf32x3')
 base = (matrix * chunks + chunk) * RANK * RANK
 tl.store(aggregates + base + rank_axis[:, None] * RANK + rank_axis[None, :], _a)

@triton.jit
def _lowrank_chunk_prefix_kernel(aggregates, prefixes, chunks: tl.constexpr, RANK: tl.constexpr):
 element = tl.program_id(0)
 matrix = tl.program_id(1)
 value = 0.0
 matrix_base = matrix * chunks * RANK * RANK
 for chunk in tl.range(0, chunks, 1):
  offset = matrix_base + chunk * RANK * RANK + element
  tl.store(prefixes + offset, value)
  value += tl.load(aggregates + offset)

@triton.jit
def _lowrank_chunk_generator_kernel(factor, prefixes, generator, roots, n: tl.constexpr, chunks: tl.constexpr, RANK: tl.constexpr, CHUNK: tl.constexpr):
 chunk = tl.program_id(0)
 matrix = tl.program_id(1)
 axis = tl.arange(0, RANK)
 rows = axis[:, None]
 columns = axis[None, :]
 prefix_base = (matrix * chunks + chunk) * RANK * RANK
 values = tl.load(prefixes + prefix_base + rows * RANK + columns, mask=rows >= columns, other=0.0)
 values += tl.where(rows == columns, 1.0, 0.0)
 for column_index in tl.range(0, RANK, 1):
  column = tl.sum(tl.where(columns == column_index, values, 0.0), axis=1)
  diagonal = tl.sum(tl.where(axis == column_index, column, 0.0), axis=0)
  root = tl.sqrt(tl.maximum(diagonal, 1.1754943508222875e-38))
  factor_column = tl.where(axis >= column_index, column / root, 0.0)
  trailing = (rows > column_index) & (columns > column_index) & (rows >= columns)
  values = tl.where(trailing, values - factor_column[:, None] * factor_column[None, :], values)
  values = tl.where(columns == column_index, factor_column[:, None], values)
 inverse = tl.zeros((RANK, RANK), tl.float32)
 for inverse_row in tl.range(0, RANK, 1):
  lower_row = tl.sum(tl.where(rows == inverse_row, values, 0.0), axis=0)
  product = tl.sum(lower_row[:, None] * inverse, axis=0)
  diagonal = tl.sum(tl.where(axis == inverse_row, lower_row, 0.0), axis=0)
  rhs = tl.where(axis == inverse_row, 1.0, 0.0)
  solution = (rhs - product) / diagonal
  inverse = tl.where(rows == inverse_row, solution[None, :], inverse)
 state = tl.dot(tl.trans(inverse), inverse, input_precision='tf32x3')
 factor_base = matrix * n * RANK
 diagonal_base = matrix * n
 for local_row in tl.range(0, CHUNK, 1):
  row = chunk * CHUNK + local_row
  row_factor = tl.load(factor + factor_base + row * RANK + axis, mask=row < n, other=0.0)
  vector = tl.sum(state * row_factor[None, :], axis=1)
  pivot = _cond2_damping(row, n) + tl.sum(row_factor * vector, axis=0)
  pivot = tl.maximum(pivot, 1.1754943508222875e-38)
  inverse_root = tl.rsqrt(pivot)
  tl.store(generator + factor_base + row * RANK + axis, vector * inverse_root, mask=row < n)
  tl.store(roots + diagonal_base + row, tl.sqrt(pivot), mask=row < n)
  state = tl.where(row < n, state - vector[:, None] * vector[None, :] / pivot, state)

@triton.jit
def _lowrank_finish_kernel(factor, generator, roots, output, n: tl.constexpr, blocks: tl.constexpr, RANK: tl.constexpr, TILE: tl.constexpr):
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 block_row = ((tl.sqrt(8.0 * tile + 1.0) - 1.0) * 0.5).to(tl.int32)
 block_column = tile - block_row * (block_row + 1) // 2
 axis = tl.arange(0, TILE)
 rank_axis = tl.arange(0, RANK)
 rows = block_row * TILE + axis
 columns = block_column * TILE + axis
 factor_base = matrix * n * RANK
 output_base = matrix * n * n
 left = tl.load(factor + factor_base + rows[:, None] * RANK + rank_axis[None, :], mask=rows[:, None] < n, other=0.0)
 right = tl.load(generator + factor_base + columns[:, None] * RANK + rank_axis[None, :], mask=columns[:, None] < n, other=0.0)
 product = tl.dot(left, tl.trans(right), input_precision='tf32x3')
 valid = (rows[:, None] < n) & (columns[None, :] < n)
 below = rows[:, None] > columns[None, :]
 diagonal = rows[:, None] == columns[None, :]
 root = tl.load(roots + matrix * n + rows, mask=rows < n, other=0.0)
 value = tl.where(below, product, 0.0)
 value = tl.where(diagonal, root[:, None], value)
 tl.store(output + output_base + rows[:, None] * n + columns[None, :], value, mask=valid)




_POTRF32_SCHEDULED_BODY = '{.reg .pred nvp<563>;.reg .b32 nvr<715>;.reg .b32 nvf<1223>;.reg .b64 nvrd<164>;mov.u64 nvrd33,$1;mov.u32 nvr34,$3;mov.u32 nvr1,%tid.x;shr.u32 nvr2,nvr1,5;and.b32 nvr3,nvr1,31;mov.u32 nvr4,%ctaid.x;shl.b32 nvr35,nvr4,2;or.b32 nvr36,nvr35,nvr2;setp.ge.s32 nvp1,nvr36,nvr34;@nvp1 bra L__BB0_5;shl.b32 nvr37,nvr2,10;cvt.u64.u32 nvrd35,nvr37;mul.wide.u32 nvrd36,nvr37,4;mov.u64 nvrd37,global_smem;add.s64 nvrd1,nvrd37,nvrd36;mov.b32 nvr708,3;cvt.u64.u32 nvrd38,nvr1;and.b64 nvrd39,nvrd38,31;mov.b64 nvrd158,96;mov.b32 nvr707,2;mov.b64 nvrd157,64;mov.b64 nvrd156,32;mov.b32 nvr706,1;mov.b64 nvrd155,0;mov.b32 nvr705,0;mul.wide.u32 nvrd40,nvr4,4096;or.b64 nvrd41,nvrd40,nvrd35;or.b64 nvrd42,nvrd41,nvrd39;shl.b64 nvrd43,nvrd42,2;mov.u64 nvrd44,nvrd33;add.s64 nvrd45,nvrd43,nvrd44;add.s64 nvrd154,nvrd45,256;mov.u32 nvr709,nvr3;L__BB0_2:\nld.global.nc.f32 nvf1,[nvrd154+-256];xor.b32 nvr38,nvr3,nvr705;cvt.u64.u32 nvrd46,nvr38;add.s64 nvrd47,nvrd155,nvrd46;shl.b64 nvrd48,nvrd47,2;add.s64 nvrd49,nvrd1,nvrd48;st.shared.f32[nvrd49],nvf1;ld.global.nc.f32 nvf2,[nvrd154+-128];xor.b32 nvr39,nvr3,nvr706;cvt.u64.u32 nvrd50,nvr39;add.s64 nvrd51,nvrd156,nvrd50;shl.b64 nvrd52,nvrd51,2;add.s64 nvrd53,nvrd1,nvrd52;st.shared.f32[nvrd53],nvf2;ld.global.nc.f32 nvf3,[nvrd154];xor.b32 nvr40,nvr3,nvr707;cvt.u64.u32 nvrd54,nvr40;add.s64 nvrd55,nvrd157,nvrd54;shl.b64 nvrd56,nvrd55,2;add.s64 nvrd57,nvrd1,nvrd56;st.shared.f32[nvrd57],nvf3;ld.global.nc.f32 nvf4,[nvrd154+128];xor.b32 nvr41,nvr3,nvr708;cvt.u64.u32 nvrd58,nvr41;add.s64 nvrd59,nvrd158,nvrd58;shl.b64 nvrd60,nvrd59,2;add.s64 nvrd61,nvrd1,nvrd60;st.shared.f32[nvrd61],nvf4;add.s32 nvr14,nvr709,128;add.s32 nvr708,nvr708,4;add.s64 nvrd158,nvrd158,128;add.s32 nvr707,nvr707,4;add.s64 nvrd157,nvrd157,128;add.s64 nvrd156,nvrd156,128;add.s32 nvr706,nvr706,4;add.s64 nvrd155,nvrd155,128;add.s32 nvr705,nvr705,4;add.s64 nvrd154,nvrd154,512;setp.lt.u32 nvp2,nvr709,896;mov.u32 nvr709,nvr14;@nvp2 bra L__BB0_2;mov.u64 nvrd153,$2;mov.u32 nvr42,%tid.x;cvt.u64.u32 nvrd62,nvr42;and.b64 nvrd63,nvrd62,31;mov.b64 nvrd163,96;mov.b64 nvrd161,32;mov.b64 nvrd162,64;mov.b64 nvrd160,0;and.b32 nvr23,nvr42,31;mov.b32 nvr712,2;mov.b32 nvr713,3;mov.b32 nvr711,1;mov.b32 nvr710,0;bar.warp.sync -1;shl.b32 nvr43,nvr42,5;and.b32 nvr44,nvr43,992;mul.lo.s32 nvr45,nvr23,33;and.b32 nvr46,nvr43,3072;cvt.u64.u32 nvrd64,nvr46;mul.wide.u32 nvrd65,nvr46,4;mov.u64 nvrd66,global_smem;add.s64 nvrd21,nvrd66,nvrd65;mul.wide.u32 nvrd67,nvr45,4;add.s64 nvrd68,nvrd21,nvrd67;ld.shared.f32 nvf5,[nvrd68];xor.b32 nvr47,nvr23,1;or.b32 nvr48,nvr44,nvr47;mul.wide.u32 nvrd69,nvr48,4;add.s64 nvrd70,nvrd21,nvrd69;ld.shared.f32 nvf6,[nvrd70];xor.b32 nvr49,nvr23,2;or.b32 nvr50,nvr44,nvr49;mul.wide.u32 nvrd71,nvr50,4;add.s64 nvrd72,nvrd21,nvrd71;ld.shared.f32 nvf7,[nvrd72];xor.b32 nvr51,nvr23,3;or.b32 nvr52,nvr44,nvr51;mul.wide.u32 nvrd73,nvr52,4;add.s64 nvrd74,nvrd21,nvrd73;ld.shared.f32 nvf8,[nvrd74];xor.b32 nvr53,nvr23,4;or.b32 nvr54,nvr44,nvr53;mul.wide.u32 nvrd75,nvr54,4;add.s64 nvrd76,nvrd21,nvrd75;ld.shared.f32 nvf9,[nvrd76];xor.b32 nvr55,nvr23,5;or.b32 nvr56,nvr44,nvr55;mul.wide.u32 nvrd77,nvr56,4;add.s64 nvrd78,nvrd21,nvrd77;ld.shared.f32 nvf10,[nvrd78];xor.b32 nvr57,nvr23,6;or.b32 nvr58,nvr44,nvr57;mul.wide.u32 nvrd79,nvr58,4;add.s64 nvrd80,nvrd21,nvrd79;ld.shared.f32 nvf11,[nvrd80];xor.b32 nvr59,nvr23,7;or.b32 nvr60,nvr44,nvr59;mul.wide.u32 nvrd81,nvr60,4;add.s64 nvrd82,nvrd21,nvrd81;ld.shared.f32 nvf12,[nvrd82];xor.b32 nvr61,nvr23,8;or.b32 nvr62,nvr44,nvr61;mul.wide.u32 nvrd83,nvr62,4;add.s64 nvrd84,nvrd21,nvrd83;ld.shared.f32 nvf13,[nvrd84];xor.b32 nvr63,nvr23,9;or.b32 nvr64,nvr44,nvr63;mul.wide.u32 nvrd85,nvr64,4;add.s64 nvrd86,nvrd21,nvrd85;ld.shared.f32 nvf14,[nvrd86];xor.b32 nvr65,nvr23,10;or.b32 nvr66,nvr44,nvr65;mul.wide.u32 nvrd87,nvr66,4;add.s64 nvrd88,nvrd21,nvrd87;ld.shared.f32 nvf15,[nvrd88];xor.b32 nvr67,nvr23,11;or.b32 nvr68,nvr44,nvr67;mul.wide.u32 nvrd89,nvr68,4;add.s64 nvrd90,nvrd21,nvrd89;ld.shared.f32 nvf16,[nvrd90];xor.b32 nvr69,nvr23,12;or.b32 nvr70,nvr44,nvr69;mul.wide.u32 nvrd91,nvr70,4;add.s64 nvrd92,nvrd21,nvrd91;ld.shared.f32 nvf17,[nvrd92];xor.b32 nvr71,nvr23,13;or.b32 nvr72,nvr44,nvr71;mul.wide.u32 nvrd93,nvr72,4;add.s64 nvrd94,nvrd21,nvrd93;ld.shared.f32 nvf18,[nvrd94];xor.b32 nvr73,nvr23,14;or.b32 nvr74,nvr44,nvr73;mul.wide.u32 nvrd95,nvr74,4;add.s64 nvrd96,nvrd21,nvrd95;ld.shared.f32 nvf19,[nvrd96];xor.b32 nvr75,nvr23,15;or.b32 nvr76,nvr44,nvr75;mul.wide.u32 nvrd97,nvr76,4;add.s64 nvrd98,nvrd21,nvrd97;ld.shared.f32 nvf20,[nvrd98];xor.b32 nvr77,nvr23,16;or.b32 nvr78,nvr44,nvr77;mul.wide.u32 nvrd99,nvr78,4;add.s64 nvrd100,nvrd21,nvrd99;ld.shared.f32 nvf21,[nvrd100];xor.b32 nvr79,nvr23,17;or.b32 nvr80,nvr44,nvr79;mul.wide.u32 nvrd101,nvr80,4;add.s64 nvrd102,nvrd21,nvrd101;ld.shared.f32 nvf22,[nvrd102];xor.b32 nvr81,nvr23,18;or.b32 nvr82,nvr44,nvr81;mul.wide.u32 nvrd103,nvr82,4;add.s64 nvrd104,nvrd21,nvrd103;ld.shared.f32 nvf23,[nvrd104];xor.b32 nvr83,nvr23,19;or.b32 nvr84,nvr44,nvr83;mul.wide.u32 nvrd105,nvr84,4;add.s64 nvrd106,nvrd21,nvrd105;ld.shared.f32 nvf24,[nvrd106];xor.b32 nvr85,nvr23,20;or.b32 nvr86,nvr44,nvr85;mul.wide.u32 nvrd107,nvr86,4;add.s64 nvrd108,nvrd21,nvrd107;ld.shared.f32 nvf25,[nvrd108];xor.b32 nvr87,nvr23,21;or.b32 nvr88,nvr44,nvr87;mul.wide.u32 nvrd109,nvr88,4;add.s64 nvrd110,nvrd21,nvrd109;ld.shared.f32 nvf26,[nvrd110];xor.b32 nvr89,nvr23,22;or.b32 nvr90,nvr44,nvr89;mul.wide.u32 nvrd111,nvr90,4;add.s64 nvrd112,nvrd21,nvrd111;ld.shared.f32 nvf27,[nvrd112];xor.b32 nvr91,nvr23,23;or.b32 nvr92,nvr44,nvr91;mul.wide.u32 nvrd113,nvr92,4;add.s64 nvrd114,nvrd21,nvrd113;ld.shared.f32 nvf28,[nvrd114];xor.b32 nvr93,nvr23,24;or.b32 nvr94,nvr44,nvr93;mul.wide.u32 nvrd115,nvr94,4;add.s64 nvrd116,nvrd21,nvrd115;ld.shared.f32 nvf29,[nvrd116];xor.b32 nvr95,nvr23,25;or.b32 nvr96,nvr44,nvr95;mul.wide.u32 nvrd117,nvr96,4;add.s64 nvrd118,nvrd21,nvrd117;ld.shared.f32 nvf30,[nvrd118];xor.b32 nvr97,nvr23,26;or.b32 nvr98,nvr44,nvr97;mul.wide.u32 nvrd119,nvr98,4;add.s64 nvrd120,nvrd21,nvrd119;ld.shared.f32 nvf31,[nvrd120];xor.b32 nvr99,nvr23,27;or.b32 nvr100,nvr44,nvr99;mul.wide.u32 nvrd121,nvr100,4;add.s64 nvrd122,nvrd21,nvrd121;ld.shared.f32 nvf32,[nvrd122];xor.b32 nvr101,nvr23,28;or.b32 nvr102,nvr44,nvr101;mul.wide.u32 nvrd123,nvr102,4;add.s64 nvrd124,nvrd21,nvrd123;ld.shared.f32 nvf33,[nvrd124];xor.b32 nvr103,nvr23,29;or.b32 nvr104,nvr44,nvr103;mul.wide.u32 nvrd125,nvr104,4;add.s64 nvrd126,nvrd21,nvrd125;ld.shared.f32 nvf34,[nvrd126];xor.b32 nvr105,nvr23,30;or.b32 nvr106,nvr44,nvr105;mul.wide.u32 nvrd127,nvr106,4;add.s64 nvrd128,nvrd21,nvrd127;ld.shared.f32 nvf35,[nvrd128];xor.b32 nvr107,nvr23,31;or.b32 nvr108,nvr44,nvr107;mul.wide.u32 nvrd129,nvr108,4;add.s64 nvrd130,nvrd21,nvrd129;ld.shared.f32 nvf36,[nvrd130];max.ftz.f32 nvf37,nvf5,0f00800000;rsqrt.approx.ftz.f32 nvf38,nvf37;mov.b32 nvr109,nvf38;shfl.sync.idx.b32 nvr110|nvp3,nvr109,0,31,-1;mov.b32 nvf39,nvr110;mul.ftz.f32 nvf40,nvf5,nvf39;mov.b32 nvr111,nvf40;shfl.sync.idx.b32 nvr112|nvp4,nvr111,1,31,-1;mov.b32 nvf41,nvr112;neg.ftz.f32 nvf42,nvf40;fma.rn.ftz.f32 nvf43,nvf42,nvf41,nvf6;shfl.sync.idx.b32 nvr113|nvp5,nvr111,2,31,-1;mov.b32 nvf44,nvr113;fma.rn.ftz.f32 nvf45,nvf42,nvf44,nvf7;shfl.sync.idx.b32 nvr114|nvp6,nvr111,3,31,-1;mov.b32 nvf46,nvr114;fma.rn.ftz.f32 nvf47,nvf42,nvf46,nvf8;shfl.sync.idx.b32 nvr115|nvp7,nvr111,4,31,-1;mov.b32 nvf48,nvr115;fma.rn.ftz.f32 nvf49,nvf42,nvf48,nvf9;shfl.sync.idx.b32 nvr116|nvp8,nvr111,5,31,-1;mov.b32 nvf50,nvr116;fma.rn.ftz.f32 nvf51,nvf42,nvf50,nvf10;shfl.sync.idx.b32 nvr117|nvp9,nvr111,6,31,-1;mov.b32 nvf52,nvr117;fma.rn.ftz.f32 nvf53,nvf42,nvf52,nvf11;shfl.sync.idx.b32 nvr118|nvp10,nvr111,7,31,-1;mov.b32 nvf54,nvr118;fma.rn.ftz.f32 nvf55,nvf42,nvf54,nvf12;shfl.sync.idx.b32 nvr119|nvp11,nvr111,8,31,-1;mov.b32 nvf56,nvr119;fma.rn.ftz.f32 nvf57,nvf42,nvf56,nvf13;shfl.sync.idx.b32 nvr120|nvp12,nvr111,9,31,-1;mov.b32 nvf58,nvr120;fma.rn.ftz.f32 nvf59,nvf42,nvf58,nvf14;shfl.sync.idx.b32 nvr121|nvp13,nvr111,10,31,-1;mov.b32 nvf60,nvr121;fma.rn.ftz.f32 nvf61,nvf42,nvf60,nvf15;shfl.sync.idx.b32 nvr122|nvp14,nvr111,11,31,-1;mov.b32 nvf62,nvr122;fma.rn.ftz.f32 nvf63,nvf42,nvf62,nvf16;shfl.sync.idx.b32 nvr123|nvp15,nvr111,12,31,-1;mov.b32 nvf64,nvr123;fma.rn.ftz.f32 nvf65,nvf42,nvf64,nvf17;shfl.sync.idx.b32 nvr124|nvp16,nvr111,13,31,-1;mov.b32 nvf66,nvr124;fma.rn.ftz.f32 nvf67,nvf42,nvf66,nvf18;shfl.sync.idx.b32 nvr125|nvp17,nvr111,14,31,-1;mov.b32 nvf68,nvr125;fma.rn.ftz.f32 nvf69,nvf42,nvf68,nvf19;shfl.sync.idx.b32 nvr126|nvp18,nvr111,15,31,-1;mov.b32 nvf70,nvr126;fma.rn.ftz.f32 nvf71,nvf42,nvf70,nvf20;shfl.sync.idx.b32 nvr127|nvp19,nvr111,16,31,-1;mov.b32 nvf72,nvr127;fma.rn.ftz.f32 nvf73,nvf42,nvf72,nvf21;shfl.sync.idx.b32 nvr128|nvp20,nvr111,17,31,-1;mov.b32 nvf74,nvr128;fma.rn.ftz.f32 nvf75,nvf42,nvf74,nvf22;shfl.sync.idx.b32 nvr129|nvp21,nvr111,18,31,-1;mov.b32 nvf76,nvr129;fma.rn.ftz.f32 nvf77,nvf42,nvf76,nvf23;shfl.sync.idx.b32 nvr130|nvp22,nvr111,19,31,-1;mov.b32 nvf78,nvr130;fma.rn.ftz.f32 nvf79,nvf42,nvf78,nvf24;shfl.sync.idx.b32 nvr131|nvp23,nvr111,20,31,-1;mov.b32 nvf80,nvr131;fma.rn.ftz.f32 nvf81,nvf42,nvf80,nvf25;shfl.sync.idx.b32 nvr132|nvp24,nvr111,21,31,-1;mov.b32 nvf82,nvr132;fma.rn.ftz.f32 nvf83,nvf42,nvf82,nvf26;shfl.sync.idx.b32 nvr133|nvp25,nvr111,22,31,-1;mov.b32 nvf84,nvr133;fma.rn.ftz.f32 nvf85,nvf42,nvf84,nvf27;shfl.sync.idx.b32 nvr134|nvp26,nvr111,23,31,-1;mov.b32 nvf86,nvr134;fma.rn.ftz.f32 nvf87,nvf42,nvf86,nvf28;shfl.sync.idx.b32 nvr135|nvp27,nvr111,24,31,-1;mov.b32 nvf88,nvr135;fma.rn.ftz.f32 nvf89,nvf42,nvf88,nvf29;shfl.sync.idx.b32 nvr136|nvp28,nvr111,25,31,-1;mov.b32 nvf90,nvr136;fma.rn.ftz.f32 nvf91,nvf42,nvf90,nvf30;shfl.sync.idx.b32 nvr137|nvp29,nvr111,26,31,-1;mov.b32 nvf92,nvr137;fma.rn.ftz.f32 nvf93,nvf42,nvf92,nvf31;shfl.sync.idx.b32 nvr138|nvp30,nvr111,27,31,-1;mov.b32 nvf94,nvr138;fma.rn.ftz.f32 nvf95,nvf42,nvf94,nvf32;shfl.sync.idx.b32 nvr139|nvp31,nvr111,28,31,-1;mov.b32 nvf96,nvr139;fma.rn.ftz.f32 nvf97,nvf42,nvf96,nvf33;shfl.sync.idx.b32 nvr140|nvp32,nvr111,29,31,-1;mov.b32 nvf98,nvr140;fma.rn.ftz.f32 nvf99,nvf42,nvf98,nvf34;shfl.sync.idx.b32 nvr141|nvp33,nvr111,30,31,-1;mov.b32 nvf100,nvr141;fma.rn.ftz.f32 nvf101,nvf42,nvf100,nvf35;shfl.sync.idx.b32 nvr142|nvp34,nvr111,31,31,-1;mov.b32 nvf102,nvr142;fma.rn.ftz.f32 nvf103,nvf42,nvf102,nvf36;max.ftz.f32 nvf104,nvf43,0f00800000;rsqrt.approx.ftz.f32 nvf105,nvf104;mov.b32 nvr143,nvf105;shfl.sync.idx.b32 nvr144|nvp35,nvr143,1,31,-1;mov.b32 nvf106,nvr144;mul.ftz.f32 nvf107,nvf43,nvf106;mov.b32 nvr145,nvf107;shfl.sync.idx.b32 nvr146|nvp36,nvr145,2,31,-1;mov.b32 nvf108,nvr146;neg.ftz.f32 nvf109,nvf107;fma.rn.ftz.f32 nvf110,nvf109,nvf108,nvf45;shfl.sync.idx.b32 nvr147|nvp37,nvr145,3,31,-1;mov.b32 nvf111,nvr147;fma.rn.ftz.f32 nvf112,nvf109,nvf111,nvf47;shfl.sync.idx.b32 nvr148|nvp38,nvr145,4,31,-1;mov.b32 nvf113,nvr148;fma.rn.ftz.f32 nvf114,nvf109,nvf113,nvf49;shfl.sync.idx.b32 nvr149|nvp39,nvr145,5,31,-1;mov.b32 nvf115,nvr149;fma.rn.ftz.f32 nvf116,nvf109,nvf115,nvf51;shfl.sync.idx.b32 nvr150|nvp40,nvr145,6,31,-1;mov.b32 nvf117,nvr150;fma.rn.ftz.f32 nvf118,nvf109,nvf117,nvf53;shfl.sync.idx.b32 nvr151|nvp41,nvr145,7,31,-1;mov.b32 nvf119,nvr151;fma.rn.ftz.f32 nvf120,nvf109,nvf119,nvf55;shfl.sync.idx.b32 nvr152|nvp42,nvr145,8,31,-1;mov.b32 nvf121,nvr152;fma.rn.ftz.f32 nvf122,nvf109,nvf121,nvf57;shfl.sync.idx.b32 nvr153|nvp43,nvr145,9,31,-1;mov.b32 nvf123,nvr153;fma.rn.ftz.f32 nvf124,nvf109,nvf123,nvf59;shfl.sync.idx.b32 nvr154|nvp44,nvr145,10,31,-1;mov.b32 nvf125,nvr154;fma.rn.ftz.f32 nvf126,nvf109,nvf125,nvf61;shfl.sync.idx.b32 nvr155|nvp45,nvr145,11,31,-1;mov.b32 nvf127,nvr155;fma.rn.ftz.f32 nvf128,nvf109,nvf127,nvf63;shfl.sync.idx.b32 nvr156|nvp46,nvr145,12,31,-1;mov.b32 nvf129,nvr156;fma.rn.ftz.f32 nvf130,nvf109,nvf129,nvf65;shfl.sync.idx.b32 nvr157|nvp47,nvr145,13,31,-1;mov.b32 nvf131,nvr157;fma.rn.ftz.f32 nvf132,nvf109,nvf131,nvf67;shfl.sync.idx.b32 nvr158|nvp48,nvr145,14,31,-1;mov.b32 nvf133,nvr158;fma.rn.ftz.f32 nvf134,nvf109,nvf133,nvf69;shfl.sync.idx.b32 nvr159|nvp49,nvr145,15,31,-1;mov.b32 nvf135,nvr159;fma.rn.ftz.f32 nvf136,nvf109,nvf135,nvf71;shfl.sync.idx.b32 nvr160|nvp50,nvr145,16,31,-1;mov.b32 nvf137,nvr160;fma.rn.ftz.f32 nvf138,nvf109,nvf137,nvf73;shfl.sync.idx.b32 nvr161|nvp51,nvr145,17,31,-1;mov.b32 nvf139,nvr161;fma.rn.ftz.f32 nvf140,nvf109,nvf139,nvf75;shfl.sync.idx.b32 nvr162|nvp52,nvr145,18,31,-1;mov.b32 nvf141,nvr162;fma.rn.ftz.f32 nvf142,nvf109,nvf141,nvf77;shfl.sync.idx.b32 nvr163|nvp53,nvr145,19,31,-1;mov.b32 nvf143,nvr163;fma.rn.ftz.f32 nvf144,nvf109,nvf143,nvf79;shfl.sync.idx.b32 nvr164|nvp54,nvr145,20,31,-1;mov.b32 nvf145,nvr164;fma.rn.ftz.f32 nvf146,nvf109,nvf145,nvf81;shfl.sync.idx.b32 nvr165|nvp55,nvr145,21,31,-1;mov.b32 nvf147,nvr165;fma.rn.ftz.f32 nvf148,nvf109,nvf147,nvf83;shfl.sync.idx.b32 nvr166|nvp56,nvr145,22,31,-1;mov.b32 nvf149,nvr166;fma.rn.ftz.f32 nvf150,nvf109,nvf149,nvf85;shfl.sync.idx.b32 nvr167|nvp57,nvr145,23,31,-1;mov.b32 nvf151,nvr167;fma.rn.ftz.f32 nvf152,nvf109,nvf151,nvf87;shfl.sync.idx.b32 nvr168|nvp58,nvr145,24,31,-1;mov.b32 nvf153,nvr168;fma.rn.ftz.f32 nvf154,nvf109,nvf153,nvf89;shfl.sync.idx.b32 nvr169|nvp59,nvr145,25,31,-1;mov.b32 nvf155,nvr169;fma.rn.ftz.f32 nvf156,nvf109,nvf155,nvf91;shfl.sync.idx.b32 nvr170|nvp60,nvr145,26,31,-1;mov.b32 nvf157,nvr170;fma.rn.ftz.f32 nvf158,nvf109,nvf157,nvf93;shfl.sync.idx.b32 nvr171|nvp61,nvr145,27,31,-1;mov.b32 nvf159,nvr171;fma.rn.ftz.f32 nvf160,nvf109,nvf159,nvf95;shfl.sync.idx.b32 nvr172|nvp62,nvr145,28,31,-1;mov.b32 nvf161,nvr172;fma.rn.ftz.f32 nvf162,nvf109,nvf161,nvf97;shfl.sync.idx.b32 nvr173|nvp63,nvr145,29,31,-1;mov.b32 nvf163,nvr173;fma.rn.ftz.f32 nvf164,nvf109,nvf163,nvf99;shfl.sync.idx.b32 nvr174|nvp64,nvr145,30,31,-1;mov.b32 nvf165,nvr174;fma.rn.ftz.f32 nvf166,nvf109,nvf165,nvf101;shfl.sync.idx.b32 nvr175|nvp65,nvr145,31,31,-1;mov.b32 nvf167,nvr175;fma.rn.ftz.f32 nvf168,nvf109,nvf167,nvf103;max.ftz.f32 nvf169,nvf110,0f00800000;rsqrt.approx.ftz.f32 nvf170,nvf169;mov.b32 nvr176,nvf170;shfl.sync.idx.b32 nvr177|nvp66,nvr176,2,31,-1;mov.b32 nvf171,nvr177;mul.ftz.f32 nvf172,nvf110,nvf171;mov.b32 nvr178,nvf172;shfl.sync.idx.b32 nvr179|nvp67,nvr178,3,31,-1;mov.b32 nvf173,nvr179;neg.ftz.f32 nvf174,nvf172;fma.rn.ftz.f32 nvf175,nvf174,nvf173,nvf112;shfl.sync.idx.b32 nvr180|nvp68,nvr178,4,31,-1;mov.b32 nvf176,nvr180;fma.rn.ftz.f32 nvf177,nvf174,nvf176,nvf114;shfl.sync.idx.b32 nvr181|nvp69,nvr178,5,31,-1;mov.b32 nvf178,nvr181;fma.rn.ftz.f32 nvf179,nvf174,nvf178,nvf116;shfl.sync.idx.b32 nvr182|nvp70,nvr178,6,31,-1;mov.b32 nvf180,nvr182;fma.rn.ftz.f32 nvf181,nvf174,nvf180,nvf118;shfl.sync.idx.b32 nvr183|nvp71,nvr178,7,31,-1;mov.b32 nvf182,nvr183;fma.rn.ftz.f32 nvf183,nvf174,nvf182,nvf120;shfl.sync.idx.b32 nvr184|nvp72,nvr178,8,31,-1;mov.b32 nvf184,nvr184;fma.rn.ftz.f32 nvf185,nvf174,nvf184,nvf122;shfl.sync.idx.b32 nvr185|nvp73,nvr178,9,31,-1;mov.b32 nvf186,nvr185;fma.rn.ftz.f32 nvf187,nvf174,nvf186,nvf124;shfl.sync.idx.b32 nvr186|nvp74,nvr178,10,31,-1;mov.b32 nvf188,nvr186;fma.rn.ftz.f32 nvf189,nvf174,nvf188,nvf126;shfl.sync.idx.b32 nvr187|nvp75,nvr178,11,31,-1;mov.b32 nvf190,nvr187;fma.rn.ftz.f32 nvf191,nvf174,nvf190,nvf128;shfl.sync.idx.b32 nvr188|nvp76,nvr178,12,31,-1;mov.b32 nvf192,nvr188;fma.rn.ftz.f32 nvf193,nvf174,nvf192,nvf130;shfl.sync.idx.b32 nvr189|nvp77,nvr178,13,31,-1;mov.b32 nvf194,nvr189;fma.rn.ftz.f32 nvf195,nvf174,nvf194,nvf132;shfl.sync.idx.b32 nvr190|nvp78,nvr178,14,31,-1;mov.b32 nvf196,nvr190;fma.rn.ftz.f32 nvf197,nvf174,nvf196,nvf134;shfl.sync.idx.b32 nvr191|nvp79,nvr178,15,31,-1;mov.b32 nvf198,nvr191;fma.rn.ftz.f32 nvf199,nvf174,nvf198,nvf136;shfl.sync.idx.b32 nvr192|nvp80,nvr178,16,31,-1;mov.b32 nvf200,nvr192;fma.rn.ftz.f32 nvf201,nvf174,nvf200,nvf138;shfl.sync.idx.b32 nvr193|nvp81,nvr178,17,31,-1;mov.b32 nvf202,nvr193;fma.rn.ftz.f32 nvf203,nvf174,nvf202,nvf140;shfl.sync.idx.b32 nvr194|nvp82,nvr178,18,31,-1;mov.b32 nvf204,nvr194;fma.rn.ftz.f32 nvf205,nvf174,nvf204,nvf142;shfl.sync.idx.b32 nvr195|nvp83,nvr178,19,31,-1;mov.b32 nvf206,nvr195;fma.rn.ftz.f32 nvf207,nvf174,nvf206,nvf144;shfl.sync.idx.b32 nvr196|nvp84,nvr178,20,31,-1;mov.b32 nvf208,nvr196;fma.rn.ftz.f32 nvf209,nvf174,nvf208,nvf146;shfl.sync.idx.b32 nvr197|nvp85,nvr178,21,31,-1;mov.b32 nvf210,nvr197;fma.rn.ftz.f32 nvf211,nvf174,nvf210,nvf148;shfl.sync.idx.b32 nvr198|nvp86,nvr178,22,31,-1;mov.b32 nvf212,nvr198;fma.rn.ftz.f32 nvf213,nvf174,nvf212,nvf150;shfl.sync.idx.b32 nvr199|nvp87,nvr178,23,31,-1;mov.b32 nvf214,nvr199;fma.rn.ftz.f32 nvf215,nvf174,nvf214,nvf152;shfl.sync.idx.b32 nvr200|nvp88,nvr178,24,31,-1;mov.b32 nvf216,nvr200;fma.rn.ftz.f32 nvf217,nvf174,nvf216,nvf154;shfl.sync.idx.b32 nvr201|nvp89,nvr178,25,31,-1;mov.b32 nvf218,nvr201;fma.rn.ftz.f32 nvf219,nvf174,nvf218,nvf156;shfl.sync.idx.b32 nvr202|nvp90,nvr178,26,31,-1;mov.b32 nvf220,nvr202;fma.rn.ftz.f32 nvf221,nvf174,nvf220,nvf158;shfl.sync.idx.b32 nvr203|nvp91,nvr178,27,31,-1;mov.b32 nvf222,nvr203;fma.rn.ftz.f32 nvf223,nvf174,nvf222,nvf160;shfl.sync.idx.b32 nvr204|nvp92,nvr178,28,31,-1;mov.b32 nvf224,nvr204;fma.rn.ftz.f32 nvf225,nvf174,nvf224,nvf162;shfl.sync.idx.b32 nvr205|nvp93,nvr178,29,31,-1;mov.b32 nvf226,nvr205;fma.rn.ftz.f32 nvf227,nvf174,nvf226,nvf164;shfl.sync.idx.b32 nvr206|nvp94,nvr178,30,31,-1;mov.b32 nvf228,nvr206;fma.rn.ftz.f32 nvf229,nvf174,nvf228,nvf166;shfl.sync.idx.b32 nvr207|nvp95,nvr178,31,31,-1;mov.b32 nvf230,nvr207;fma.rn.ftz.f32 nvf231,nvf174,nvf230,nvf168;max.ftz.f32 nvf232,nvf175,0f00800000;rsqrt.approx.ftz.f32 nvf233,nvf232;mov.b32 nvr208,nvf233;shfl.sync.idx.b32 nvr209|nvp96,nvr208,3,31,-1;mov.b32 nvf234,nvr209;mul.ftz.f32 nvf235,nvf175,nvf234;mov.b32 nvr210,nvf235;shfl.sync.idx.b32 nvr211|nvp97,nvr210,4,31,-1;mov.b32 nvf236,nvr211;neg.ftz.f32 nvf237,nvf235;fma.rn.ftz.f32 nvf238,nvf237,nvf236,nvf177;shfl.sync.idx.b32 nvr212|nvp98,nvr210,5,31,-1;mov.b32 nvf239,nvr212;fma.rn.ftz.f32 nvf240,nvf237,nvf239,nvf179;shfl.sync.idx.b32 nvr213|nvp99,nvr210,6,31,-1;mov.b32 nvf241,nvr213;fma.rn.ftz.f32 nvf242,nvf237,nvf241,nvf181;shfl.sync.idx.b32 nvr214|nvp100,nvr210,7,31,-1;mov.b32 nvf243,nvr214;fma.rn.ftz.f32 nvf244,nvf237,nvf243,nvf183;shfl.sync.idx.b32 nvr215|nvp101,nvr210,8,31,-1;mov.b32 nvf245,nvr215;fma.rn.ftz.f32 nvf246,nvf237,nvf245,nvf185;shfl.sync.idx.b32 nvr216|nvp102,nvr210,9,31,-1;mov.b32 nvf247,nvr216;fma.rn.ftz.f32 nvf248,nvf237,nvf247,nvf187;shfl.sync.idx.b32 nvr217|nvp103,nvr210,10,31,-1;mov.b32 nvf249,nvr217;fma.rn.ftz.f32 nvf250,nvf237,nvf249,nvf189;shfl.sync.idx.b32 nvr218|nvp104,nvr210,11,31,-1;mov.b32 nvf251,nvr218;fma.rn.ftz.f32 nvf252,nvf237,nvf251,nvf191;shfl.sync.idx.b32 nvr219|nvp105,nvr210,12,31,-1;mov.b32 nvf253,nvr219;fma.rn.ftz.f32 nvf254,nvf237,nvf253,nvf193;shfl.sync.idx.b32 nvr220|nvp106,nvr210,13,31,-1;mov.b32 nvf255,nvr220;fma.rn.ftz.f32 nvf256,nvf237,nvf255,nvf195;shfl.sync.idx.b32 nvr221|nvp107,nvr210,14,31,-1;mov.b32 nvf257,nvr221;fma.rn.ftz.f32 nvf258,nvf237,nvf257,nvf197;shfl.sync.idx.b32 nvr222|nvp108,nvr210,15,31,-1;mov.b32 nvf259,nvr222;fma.rn.ftz.f32 nvf260,nvf237,nvf259,nvf199;shfl.sync.idx.b32 nvr223|nvp109,nvr210,16,31,-1;mov.b32 nvf261,nvr223;fma.rn.ftz.f32 nvf262,nvf237,nvf261,nvf201;shfl.sync.idx.b32 nvr224|nvp110,nvr210,17,31,-1;mov.b32 nvf263,nvr224;fma.rn.ftz.f32 nvf264,nvf237,nvf263,nvf203;shfl.sync.idx.b32 nvr225|nvp111,nvr210,18,31,-1;mov.b32 nvf265,nvr225;fma.rn.ftz.f32 nvf266,nvf237,nvf265,nvf205;shfl.sync.idx.b32 nvr226|nvp112,nvr210,19,31,-1;mov.b32 nvf267,nvr226;fma.rn.ftz.f32 nvf268,nvf237,nvf267,nvf207;shfl.sync.idx.b32 nvr227|nvp113,nvr210,20,31,-1;mov.b32 nvf269,nvr227;fma.rn.ftz.f32 nvf270,nvf237,nvf269,nvf209;shfl.sync.idx.b32 nvr228|nvp114,nvr210,21,31,-1;mov.b32 nvf271,nvr228;fma.rn.ftz.f32 nvf272,nvf237,nvf271,nvf211;shfl.sync.idx.b32 nvr229|nvp115,nvr210,22,31,-1;mov.b32 nvf273,nvr229;fma.rn.ftz.f32 nvf274,nvf237,nvf273,nvf213;shfl.sync.idx.b32 nvr230|nvp116,nvr210,23,31,-1;mov.b32 nvf275,nvr230;fma.rn.ftz.f32 nvf276,nvf237,nvf275,nvf215;shfl.sync.idx.b32 nvr231|nvp117,nvr210,24,31,-1;mov.b32 nvf277,nvr231;fma.rn.ftz.f32 nvf278,nvf237,nvf277,nvf217;shfl.sync.idx.b32 nvr232|nvp118,nvr210,25,31,-1;mov.b32 nvf279,nvr232;fma.rn.ftz.f32 nvf280,nvf237,nvf279,nvf219;shfl.sync.idx.b32 nvr233|nvp119,nvr210,26,31,-1;mov.b32 nvf281,nvr233;fma.rn.ftz.f32 nvf282,nvf237,nvf281,nvf221;shfl.sync.idx.b32 nvr234|nvp120,nvr210,27,31,-1;mov.b32 nvf283,nvr234;fma.rn.ftz.f32 nvf284,nvf237,nvf283,nvf223;shfl.sync.idx.b32 nvr235|nvp121,nvr210,28,31,-1;mov.b32 nvf285,nvr235;fma.rn.ftz.f32 nvf286,nvf237,nvf285,nvf225;shfl.sync.idx.b32 nvr236|nvp122,nvr210,29,31,-1;mov.b32 nvf287,nvr236;fma.rn.ftz.f32 nvf288,nvf237,nvf287,nvf227;shfl.sync.idx.b32 nvr237|nvp123,nvr210,30,31,-1;mov.b32 nvf289,nvr237;fma.rn.ftz.f32 nvf290,nvf237,nvf289,nvf229;shfl.sync.idx.b32 nvr238|nvp124,nvr210,31,31,-1;mov.b32 nvf291,nvr238;fma.rn.ftz.f32 nvf292,nvf237,nvf291,nvf231;max.ftz.f32 nvf293,nvf238,0f00800000;rsqrt.approx.ftz.f32 nvf294,nvf293;mov.b32 nvr239,nvf294;shfl.sync.idx.b32 nvr240|nvp125,nvr239,4,31,-1;mov.b32 nvf295,nvr240;mul.ftz.f32 nvf296,nvf238,nvf295;mov.b32 nvr241,nvf296;shfl.sync.idx.b32 nvr242|nvp126,nvr241,5,31,-1;mov.b32 nvf297,nvr242;neg.ftz.f32 nvf298,nvf296;fma.rn.ftz.f32 nvf299,nvf298,nvf297,nvf240;shfl.sync.idx.b32 nvr243|nvp127,nvr241,6,31,-1;mov.b32 nvf300,nvr243;fma.rn.ftz.f32 nvf301,nvf298,nvf300,nvf242;shfl.sync.idx.b32 nvr244|nvp128,nvr241,7,31,-1;mov.b32 nvf302,nvr244;fma.rn.ftz.f32 nvf303,nvf298,nvf302,nvf244;shfl.sync.idx.b32 nvr245|nvp129,nvr241,8,31,-1;mov.b32 nvf304,nvr245;fma.rn.ftz.f32 nvf305,nvf298,nvf304,nvf246;shfl.sync.idx.b32 nvr246|nvp130,nvr241,9,31,-1;mov.b32 nvf306,nvr246;fma.rn.ftz.f32 nvf307,nvf298,nvf306,nvf248;shfl.sync.idx.b32 nvr247|nvp131,nvr241,10,31,-1;mov.b32 nvf308,nvr247;fma.rn.ftz.f32 nvf309,nvf298,nvf308,nvf250;shfl.sync.idx.b32 nvr248|nvp132,nvr241,11,31,-1;mov.b32 nvf310,nvr248;fma.rn.ftz.f32 nvf311,nvf298,nvf310,nvf252;shfl.sync.idx.b32 nvr249|nvp133,nvr241,12,31,-1;mov.b32 nvf312,nvr249;fma.rn.ftz.f32 nvf313,nvf298,nvf312,nvf254;shfl.sync.idx.b32 nvr250|nvp134,nvr241,13,31,-1;mov.b32 nvf314,nvr250;fma.rn.ftz.f32 nvf315,nvf298,nvf314,nvf256;shfl.sync.idx.b32 nvr251|nvp135,nvr241,14,31,-1;mov.b32 nvf316,nvr251;fma.rn.ftz.f32 nvf317,nvf298,nvf316,nvf258;shfl.sync.idx.b32 nvr252|nvp136,nvr241,15,31,-1;mov.b32 nvf318,nvr252;fma.rn.ftz.f32 nvf319,nvf298,nvf318,nvf260;shfl.sync.idx.b32 nvr253|nvp137,nvr241,16,31,-1;mov.b32 nvf320,nvr253;fma.rn.ftz.f32 nvf321,nvf298,nvf320,nvf262;shfl.sync.idx.b32 nvr254|nvp138,nvr241,17,31,-1;mov.b32 nvf322,nvr254;fma.rn.ftz.f32 nvf323,nvf298,nvf322,nvf264;shfl.sync.idx.b32 nvr255|nvp139,nvr241,18,31,-1;mov.b32 nvf324,nvr255;fma.rn.ftz.f32 nvf325,nvf298,nvf324,nvf266;shfl.sync.idx.b32 nvr256|nvp140,nvr241,19,31,-1;mov.b32 nvf326,nvr256;fma.rn.ftz.f32 nvf327,nvf298,nvf326,nvf268;shfl.sync.idx.b32 nvr257|nvp141,nvr241,20,31,-1;mov.b32 nvf328,nvr257;fma.rn.ftz.f32 nvf329,nvf298,nvf328,nvf270;shfl.sync.idx.b32 nvr258|nvp142,nvr241,21,31,-1;mov.b32 nvf330,nvr258;fma.rn.ftz.f32 nvf331,nvf298,nvf330,nvf272;shfl.sync.idx.b32 nvr259|nvp143,nvr241,22,31,-1;mov.b32 nvf332,nvr259;fma.rn.ftz.f32 nvf333,nvf298,nvf332,nvf274;shfl.sync.idx.b32 nvr260|nvp144,nvr241,23,31,-1;mov.b32 nvf334,nvr260;fma.rn.ftz.f32 nvf335,nvf298,nvf334,nvf276;shfl.sync.idx.b32 nvr261|nvp145,nvr241,24,31,-1;mov.b32 nvf336,nvr261;fma.rn.ftz.f32 nvf337,nvf298,nvf336,nvf278;shfl.sync.idx.b32 nvr262|nvp146,nvr241,25,31,-1;mov.b32 nvf338,nvr262;fma.rn.ftz.f32 nvf339,nvf298,nvf338,nvf280;shfl.sync.idx.b32 nvr263|nvp147,nvr241,26,31,-1;mov.b32 nvf340,nvr263;fma.rn.ftz.f32 nvf341,nvf298,nvf340,nvf282;shfl.sync.idx.b32 nvr264|nvp148,nvr241,27,31,-1;mov.b32 nvf342,nvr264;fma.rn.ftz.f32 nvf343,nvf298,nvf342,nvf284;shfl.sync.idx.b32 nvr265|nvp149,nvr241,28,31,-1;mov.b32 nvf344,nvr265;fma.rn.ftz.f32 nvf345,nvf298,nvf344,nvf286;shfl.sync.idx.b32 nvr266|nvp150,nvr241,29,31,-1;mov.b32 nvf346,nvr266;fma.rn.ftz.f32 nvf347,nvf298,nvf346,nvf288;shfl.sync.idx.b32 nvr267|nvp151,nvr241,30,31,-1;mov.b32 nvf348,nvr267;fma.rn.ftz.f32 nvf349,nvf298,nvf348,nvf290;shfl.sync.idx.b32 nvr268|nvp152,nvr241,31,31,-1;mov.b32 nvf350,nvr268;fma.rn.ftz.f32 nvf351,nvf298,nvf350,nvf292;max.ftz.f32 nvf352,nvf299,0f00800000;rsqrt.approx.ftz.f32 nvf353,nvf352;mov.b32 nvr269,nvf353;shfl.sync.idx.b32 nvr270|nvp153,nvr269,5,31,-1;mov.b32 nvf354,nvr270;mul.ftz.f32 nvf355,nvf299,nvf354;mov.b32 nvr271,nvf355;shfl.sync.idx.b32 nvr272|nvp154,nvr271,6,31,-1;mov.b32 nvf356,nvr272;neg.ftz.f32 nvf357,nvf355;fma.rn.ftz.f32 nvf358,nvf357,nvf356,nvf301;shfl.sync.idx.b32 nvr273|nvp155,nvr271,7,31,-1;mov.b32 nvf359,nvr273;fma.rn.ftz.f32 nvf360,nvf357,nvf359,nvf303;shfl.sync.idx.b32 nvr274|nvp156,nvr271,8,31,-1;mov.b32 nvf361,nvr274;fma.rn.ftz.f32 nvf362,nvf357,nvf361,nvf305;shfl.sync.idx.b32 nvr275|nvp157,nvr271,9,31,-1;mov.b32 nvf363,nvr275;fma.rn.ftz.f32 nvf364,nvf357,nvf363,nvf307;shfl.sync.idx.b32 nvr276|nvp158,nvr271,10,31,-1;mov.b32 nvf365,nvr276;fma.rn.ftz.f32 nvf366,nvf357,nvf365,nvf309;shfl.sync.idx.b32 nvr277|nvp159,nvr271,11,31,-1;mov.b32 nvf367,nvr277;fma.rn.ftz.f32 nvf368,nvf357,nvf367,nvf311;shfl.sync.idx.b32 nvr278|nvp160,nvr271,12,31,-1;mov.b32 nvf369,nvr278;fma.rn.ftz.f32 nvf370,nvf357,nvf369,nvf313;shfl.sync.idx.b32 nvr279|nvp161,nvr271,13,31,-1;mov.b32 nvf371,nvr279;fma.rn.ftz.f32 nvf372,nvf357,nvf371,nvf315;shfl.sync.idx.b32 nvr280|nvp162,nvr271,14,31,-1;mov.b32 nvf373,nvr280;fma.rn.ftz.f32 nvf374,nvf357,nvf373,nvf317;shfl.sync.idx.b32 nvr281|nvp163,nvr271,15,31,-1;mov.b32 nvf375,nvr281;fma.rn.ftz.f32 nvf376,nvf357,nvf375,nvf319;shfl.sync.idx.b32 nvr282|nvp164,nvr271,16,31,-1;mov.b32 nvf377,nvr282;fma.rn.ftz.f32 nvf378,nvf357,nvf377,nvf321;shfl.sync.idx.b32 nvr283|nvp165,nvr271,17,31,-1;mov.b32 nvf379,nvr283;fma.rn.ftz.f32 nvf380,nvf357,nvf379,nvf323;shfl.sync.idx.b32 nvr284|nvp166,nvr271,18,31,-1;mov.b32 nvf381,nvr284;fma.rn.ftz.f32 nvf382,nvf357,nvf381,nvf325;shfl.sync.idx.b32 nvr285|nvp167,nvr271,19,31,-1;mov.b32 nvf383,nvr285;fma.rn.ftz.f32 nvf384,nvf357,nvf383,nvf327;shfl.sync.idx.b32 nvr286|nvp168,nvr271,20,31,-1;mov.b32 nvf385,nvr286;fma.rn.ftz.f32 nvf386,nvf357,nvf385,nvf329;shfl.sync.idx.b32 nvr287|nvp169,nvr271,21,31,-1;mov.b32 nvf387,nvr287;fma.rn.ftz.f32 nvf388,nvf357,nvf387,nvf331;shfl.sync.idx.b32 nvr288|nvp170,nvr271,22,31,-1;mov.b32 nvf389,nvr288;fma.rn.ftz.f32 nvf390,nvf357,nvf389,nvf333;shfl.sync.idx.b32 nvr289|nvp171,nvr271,23,31,-1;mov.b32 nvf391,nvr289;fma.rn.ftz.f32 nvf392,nvf357,nvf391,nvf335;shfl.sync.idx.b32 nvr290|nvp172,nvr271,24,31,-1;mov.b32 nvf393,nvr290;fma.rn.ftz.f32 nvf394,nvf357,nvf393,nvf337;shfl.sync.idx.b32 nvr291|nvp173,nvr271,25,31,-1;mov.b32 nvf395,nvr291;fma.rn.ftz.f32 nvf396,nvf357,nvf395,nvf339;shfl.sync.idx.b32 nvr292|nvp174,nvr271,26,31,-1;mov.b32 nvf397,nvr292;fma.rn.ftz.f32 nvf398,nvf357,nvf397,nvf341;shfl.sync.idx.b32 nvr293|nvp175,nvr271,27,31,-1;mov.b32 nvf399,nvr293;fma.rn.ftz.f32 nvf400,nvf357,nvf399,nvf343;shfl.sync.idx.b32 nvr294|nvp176,nvr271,28,31,-1;mov.b32 nvf401,nvr294;fma.rn.ftz.f32 nvf402,nvf357,nvf401,nvf345;shfl.sync.idx.b32 nvr295|nvp177,nvr271,29,31,-1;mov.b32 nvf403,nvr295;fma.rn.ftz.f32 nvf404,nvf357,nvf403,nvf347;shfl.sync.idx.b32 nvr296|nvp178,nvr271,30,31,-1;mov.b32 nvf405,nvr296;fma.rn.ftz.f32 nvf406,nvf357,nvf405,nvf349;shfl.sync.idx.b32 nvr297|nvp179,nvr271,31,31,-1;mov.b32 nvf407,nvr297;fma.rn.ftz.f32 nvf408,nvf357,nvf407,nvf351;max.ftz.f32 nvf409,nvf358,0f00800000;rsqrt.approx.ftz.f32 nvf410,nvf409;mov.b32 nvr298,nvf410;shfl.sync.idx.b32 nvr299|nvp180,nvr298,6,31,-1;mov.b32 nvf411,nvr299;mul.ftz.f32 nvf412,nvf358,nvf411;mov.b32 nvr300,nvf412;shfl.sync.idx.b32 nvr301|nvp181,nvr300,7,31,-1;mov.b32 nvf413,nvr301;neg.ftz.f32 nvf414,nvf412;fma.rn.ftz.f32 nvf415,nvf414,nvf413,nvf360;shfl.sync.idx.b32 nvr302|nvp182,nvr300,8,31,-1;mov.b32 nvf416,nvr302;fma.rn.ftz.f32 nvf417,nvf414,nvf416,nvf362;shfl.sync.idx.b32 nvr303|nvp183,nvr300,9,31,-1;mov.b32 nvf418,nvr303;fma.rn.ftz.f32 nvf419,nvf414,nvf418,nvf364;shfl.sync.idx.b32 nvr304|nvp184,nvr300,10,31,-1;mov.b32 nvf420,nvr304;fma.rn.ftz.f32 nvf421,nvf414,nvf420,nvf366;shfl.sync.idx.b32 nvr305|nvp185,nvr300,11,31,-1;mov.b32 nvf422,nvr305;fma.rn.ftz.f32 nvf423,nvf414,nvf422,nvf368;shfl.sync.idx.b32 nvr306|nvp186,nvr300,12,31,-1;mov.b32 nvf424,nvr306;fma.rn.ftz.f32 nvf425,nvf414,nvf424,nvf370;shfl.sync.idx.b32 nvr307|nvp187,nvr300,13,31,-1;mov.b32 nvf426,nvr307;fma.rn.ftz.f32 nvf427,nvf414,nvf426,nvf372;shfl.sync.idx.b32 nvr308|nvp188,nvr300,14,31,-1;mov.b32 nvf428,nvr308;fma.rn.ftz.f32 nvf429,nvf414,nvf428,nvf374;shfl.sync.idx.b32 nvr309|nvp189,nvr300,15,31,-1;mov.b32 nvf430,nvr309;fma.rn.ftz.f32 nvf431,nvf414,nvf430,nvf376;shfl.sync.idx.b32 nvr310|nvp190,nvr300,16,31,-1;mov.b32 nvf432,nvr310;fma.rn.ftz.f32 nvf433,nvf414,nvf432,nvf378;shfl.sync.idx.b32 nvr311|nvp191,nvr300,17,31,-1;mov.b32 nvf434,nvr311;fma.rn.ftz.f32 nvf435,nvf414,nvf434,nvf380;shfl.sync.idx.b32 nvr312|nvp192,nvr300,18,31,-1;mov.b32 nvf436,nvr312;fma.rn.ftz.f32 nvf437,nvf414,nvf436,nvf382;shfl.sync.idx.b32 nvr313|nvp193,nvr300,19,31,-1;mov.b32 nvf438,nvr313;fma.rn.ftz.f32 nvf439,nvf414,nvf438,nvf384;shfl.sync.idx.b32 nvr314|nvp194,nvr300,20,31,-1;mov.b32 nvf440,nvr314;fma.rn.ftz.f32 nvf441,nvf414,nvf440,nvf386;shfl.sync.idx.b32 nvr315|nvp195,nvr300,21,31,-1;mov.b32 nvf442,nvr315;fma.rn.ftz.f32 nvf443,nvf414,nvf442,nvf388;shfl.sync.idx.b32 nvr316|nvp196,nvr300,22,31,-1;mov.b32 nvf444,nvr316;fma.rn.ftz.f32 nvf445,nvf414,nvf444,nvf390;shfl.sync.idx.b32 nvr317|nvp197,nvr300,23,31,-1;mov.b32 nvf446,nvr317;fma.rn.ftz.f32 nvf447,nvf414,nvf446,nvf392;shfl.sync.idx.b32 nvr318|nvp198,nvr300,24,31,-1;mov.b32 nvf448,nvr318;fma.rn.ftz.f32 nvf449,nvf414,nvf448,nvf394;shfl.sync.idx.b32 nvr319|nvp199,nvr300,25,31,-1;mov.b32 nvf450,nvr319;fma.rn.ftz.f32 nvf451,nvf414,nvf450,nvf396;shfl.sync.idx.b32 nvr320|nvp200,nvr300,26,31,-1;mov.b32 nvf452,nvr320;fma.rn.ftz.f32 nvf453,nvf414,nvf452,nvf398;shfl.sync.idx.b32 nvr321|nvp201,nvr300,27,31,-1;mov.b32 nvf454,nvr321;fma.rn.ftz.f32 nvf455,nvf414,nvf454,nvf400;shfl.sync.idx.b32 nvr322|nvp202,nvr300,28,31,-1;mov.b32 nvf456,nvr322;fma.rn.ftz.f32 nvf457,nvf414,nvf456,nvf402;shfl.sync.idx.b32 nvr323|nvp203,nvr300,29,31,-1;mov.b32 nvf458,nvr323;fma.rn.ftz.f32 nvf459,nvf414,nvf458,nvf404;shfl.sync.idx.b32 nvr324|nvp204,nvr300,30,31,-1;mov.b32 nvf460,nvr324;fma.rn.ftz.f32 nvf461,nvf414,nvf460,nvf406;shfl.sync.idx.b32 nvr325|nvp205,nvr300,31,31,-1;mov.b32 nvf462,nvr325;fma.rn.ftz.f32 nvf463,nvf414,nvf462,nvf408;max.ftz.f32 nvf464,nvf415,0f00800000;rsqrt.approx.ftz.f32 nvf465,nvf464;mov.b32 nvr326,nvf465;shfl.sync.idx.b32 nvr327|nvp206,nvr326,7,31,-1;mov.b32 nvf466,nvr327;mul.ftz.f32 nvf467,nvf415,nvf466;mov.b32 nvr328,nvf467;shfl.sync.idx.b32 nvr329|nvp207,nvr328,8,31,-1;mov.b32 nvf468,nvr329;neg.ftz.f32 nvf469,nvf467;fma.rn.ftz.f32 nvf470,nvf469,nvf468,nvf417;shfl.sync.idx.b32 nvr330|nvp208,nvr328,9,31,-1;mov.b32 nvf471,nvr330;fma.rn.ftz.f32 nvf472,nvf469,nvf471,nvf419;shfl.sync.idx.b32 nvr331|nvp209,nvr328,10,31,-1;mov.b32 nvf473,nvr331;fma.rn.ftz.f32 nvf474,nvf469,nvf473,nvf421;shfl.sync.idx.b32 nvr332|nvp210,nvr328,11,31,-1;mov.b32 nvf475,nvr332;fma.rn.ftz.f32 nvf476,nvf469,nvf475,nvf423;shfl.sync.idx.b32 nvr333|nvp211,nvr328,12,31,-1;mov.b32 nvf477,nvr333;fma.rn.ftz.f32 nvf478,nvf469,nvf477,nvf425;shfl.sync.idx.b32 nvr334|nvp212,nvr328,13,31,-1;mov.b32 nvf479,nvr334;fma.rn.ftz.f32 nvf480,nvf469,nvf479,nvf427;shfl.sync.idx.b32 nvr335|nvp213,nvr328,14,31,-1;mov.b32 nvf481,nvr335;fma.rn.ftz.f32 nvf482,nvf469,nvf481,nvf429;shfl.sync.idx.b32 nvr336|nvp214,nvr328,15,31,-1;mov.b32 nvf483,nvr336;fma.rn.ftz.f32 nvf484,nvf469,nvf483,nvf431;shfl.sync.idx.b32 nvr337|nvp215,nvr328,16,31,-1;mov.b32 nvf485,nvr337;fma.rn.ftz.f32 nvf486,nvf469,nvf485,nvf433;shfl.sync.idx.b32 nvr338|nvp216,nvr328,17,31,-1;mov.b32 nvf487,nvr338;fma.rn.ftz.f32 nvf488,nvf469,nvf487,nvf435;shfl.sync.idx.b32 nvr339|nvp217,nvr328,18,31,-1;mov.b32 nvf489,nvr339;fma.rn.ftz.f32 nvf490,nvf469,nvf489,nvf437;shfl.sync.idx.b32 nvr340|nvp218,nvr328,19,31,-1;mov.b32 nvf491,nvr340;fma.rn.ftz.f32 nvf492,nvf469,nvf491,nvf439;shfl.sync.idx.b32 nvr341|nvp219,nvr328,20,31,-1;mov.b32 nvf493,nvr341;fma.rn.ftz.f32 nvf494,nvf469,nvf493,nvf441;shfl.sync.idx.b32 nvr342|nvp220,nvr328,21,31,-1;mov.b32 nvf495,nvr342;fma.rn.ftz.f32 nvf496,nvf469,nvf495,nvf443;shfl.sync.idx.b32 nvr343|nvp221,nvr328,22,31,-1;mov.b32 nvf497,nvr343;fma.rn.ftz.f32 nvf498,nvf469,nvf497,nvf445;shfl.sync.idx.b32 nvr344|nvp222,nvr328,23,31,-1;mov.b32 nvf499,nvr344;fma.rn.ftz.f32 nvf500,nvf469,nvf499,nvf447;shfl.sync.idx.b32 nvr345|nvp223,nvr328,24,31,-1;mov.b32 nvf501,nvr345;fma.rn.ftz.f32 nvf502,nvf469,nvf501,nvf449;shfl.sync.idx.b32 nvr346|nvp224,nvr328,25,31,-1;mov.b32 nvf503,nvr346;fma.rn.ftz.f32 nvf504,nvf469,nvf503,nvf451;shfl.sync.idx.b32 nvr347|nvp225,nvr328,26,31,-1;mov.b32 nvf505,nvr347;fma.rn.ftz.f32 nvf506,nvf469,nvf505,nvf453;shfl.sync.idx.b32 nvr348|nvp226,nvr328,27,31,-1;mov.b32 nvf507,nvr348;fma.rn.ftz.f32 nvf508,nvf469,nvf507,nvf455;shfl.sync.idx.b32 nvr349|nvp227,nvr328,28,31,-1;mov.b32 nvf509,nvr349;fma.rn.ftz.f32 nvf510,nvf469,nvf509,nvf457;shfl.sync.idx.b32 nvr350|nvp228,nvr328,29,31,-1;mov.b32 nvf511,nvr350;fma.rn.ftz.f32 nvf512,nvf469,nvf511,nvf459;shfl.sync.idx.b32 nvr351|nvp229,nvr328,30,31,-1;mov.b32 nvf513,nvr351;fma.rn.ftz.f32 nvf514,nvf469,nvf513,nvf461;shfl.sync.idx.b32 nvr352|nvp230,nvr328,31,31,-1;mov.b32 nvf515,nvr352;fma.rn.ftz.f32 nvf516,nvf469,nvf515,nvf463;max.ftz.f32 nvf517,nvf470,0f00800000;rsqrt.approx.ftz.f32 nvf518,nvf517;mov.b32 nvr353,nvf518;shfl.sync.idx.b32 nvr354|nvp231,nvr353,8,31,-1;mov.b32 nvf519,nvr354;mul.ftz.f32 nvf520,nvf470,nvf519;mov.b32 nvr355,nvf520;shfl.sync.idx.b32 nvr356|nvp232,nvr355,9,31,-1;mov.b32 nvf521,nvr356;neg.ftz.f32 nvf522,nvf520;fma.rn.ftz.f32 nvf523,nvf522,nvf521,nvf472;shfl.sync.idx.b32 nvr357|nvp233,nvr355,10,31,-1;mov.b32 nvf524,nvr357;fma.rn.ftz.f32 nvf525,nvf522,nvf524,nvf474;shfl.sync.idx.b32 nvr358|nvp234,nvr355,11,31,-1;mov.b32 nvf526,nvr358;fma.rn.ftz.f32 nvf527,nvf522,nvf526,nvf476;shfl.sync.idx.b32 nvr359|nvp235,nvr355,12,31,-1;mov.b32 nvf528,nvr359;fma.rn.ftz.f32 nvf529,nvf522,nvf528,nvf478;shfl.sync.idx.b32 nvr360|nvp236,nvr355,13,31,-1;mov.b32 nvf530,nvr360;fma.rn.ftz.f32 nvf531,nvf522,nvf530,nvf480;shfl.sync.idx.b32 nvr361|nvp237,nvr355,14,31,-1;mov.b32 nvf532,nvr361;fma.rn.ftz.f32 nvf533,nvf522,nvf532,nvf482;shfl.sync.idx.b32 nvr362|nvp238,nvr355,15,31,-1;mov.b32 nvf534,nvr362;fma.rn.ftz.f32 nvf535,nvf522,nvf534,nvf484;shfl.sync.idx.b32 nvr363|nvp239,nvr355,16,31,-1;mov.b32 nvf536,nvr363;fma.rn.ftz.f32 nvf537,nvf522,nvf536,nvf486;shfl.sync.idx.b32 nvr364|nvp240,nvr355,17,31,-1;mov.b32 nvf538,nvr364;fma.rn.ftz.f32 nvf539,nvf522,nvf538,nvf488;shfl.sync.idx.b32 nvr365|nvp241,nvr355,18,31,-1;mov.b32 nvf540,nvr365;fma.rn.ftz.f32 nvf541,nvf522,nvf540,nvf490;shfl.sync.idx.b32 nvr366|nvp242,nvr355,19,31,-1;mov.b32 nvf542,nvr366;fma.rn.ftz.f32 nvf543,nvf522,nvf542,nvf492;shfl.sync.idx.b32 nvr367|nvp243,nvr355,20,31,-1;mov.b32 nvf544,nvr367;fma.rn.ftz.f32 nvf545,nvf522,nvf544,nvf494;shfl.sync.idx.b32 nvr368|nvp244,nvr355,21,31,-1;mov.b32 nvf546,nvr368;fma.rn.ftz.f32 nvf547,nvf522,nvf546,nvf496;shfl.sync.idx.b32 nvr369|nvp245,nvr355,22,31,-1;mov.b32 nvf548,nvr369;fma.rn.ftz.f32 nvf549,nvf522,nvf548,nvf498;shfl.sync.idx.b32 nvr370|nvp246,nvr355,23,31,-1;mov.b32 nvf550,nvr370;fma.rn.ftz.f32 nvf551,nvf522,nvf550,nvf500;shfl.sync.idx.b32 nvr371|nvp247,nvr355,24,31,-1;mov.b32 nvf552,nvr371;fma.rn.ftz.f32 nvf553,nvf522,nvf552,nvf502;shfl.sync.idx.b32 nvr372|nvp248,nvr355,25,31,-1;mov.b32 nvf554,nvr372;fma.rn.ftz.f32 nvf555,nvf522,nvf554,nvf504;shfl.sync.idx.b32 nvr373|nvp249,nvr355,26,31,-1;mov.b32 nvf556,nvr373;fma.rn.ftz.f32 nvf557,nvf522,nvf556,nvf506;shfl.sync.idx.b32 nvr374|nvp250,nvr355,27,31,-1;mov.b32 nvf558,nvr374;fma.rn.ftz.f32 nvf559,nvf522,nvf558,nvf508;shfl.sync.idx.b32 nvr375|nvp251,nvr355,28,31,-1;mov.b32 nvf560,nvr375;fma.rn.ftz.f32 nvf561,nvf522,nvf560,nvf510;shfl.sync.idx.b32 nvr376|nvp252,nvr355,29,31,-1;mov.b32 nvf562,nvr376;fma.rn.ftz.f32 nvf563,nvf522,nvf562,nvf512;shfl.sync.idx.b32 nvr377|nvp253,nvr355,30,31,-1;mov.b32 nvf564,nvr377;fma.rn.ftz.f32 nvf565,nvf522,nvf564,nvf514;shfl.sync.idx.b32 nvr378|nvp254,nvr355,31,31,-1;mov.b32 nvf566,nvr378;fma.rn.ftz.f32 nvf567,nvf522,nvf566,nvf516;max.ftz.f32 nvf568,nvf523,0f00800000;rsqrt.approx.ftz.f32 nvf569,nvf568;mov.b32 nvr379,nvf569;shfl.sync.idx.b32 nvr380|nvp255,nvr379,9,31,-1;mov.b32 nvf570,nvr380;mul.ftz.f32 nvf571,nvf523,nvf570;mov.b32 nvr381,nvf571;shfl.sync.idx.b32 nvr382|nvp256,nvr381,10,31,-1;mov.b32 nvf572,nvr382;neg.ftz.f32 nvf573,nvf571;fma.rn.ftz.f32 nvf574,nvf573,nvf572,nvf525;shfl.sync.idx.b32 nvr383|nvp257,nvr381,11,31,-1;mov.b32 nvf575,nvr383;fma.rn.ftz.f32 nvf576,nvf573,nvf575,nvf527;shfl.sync.idx.b32 nvr384|nvp258,nvr381,12,31,-1;mov.b32 nvf577,nvr384;fma.rn.ftz.f32 nvf578,nvf573,nvf577,nvf529;shfl.sync.idx.b32 nvr385|nvp259,nvr381,13,31,-1;mov.b32 nvf579,nvr385;fma.rn.ftz.f32 nvf580,nvf573,nvf579,nvf531;shfl.sync.idx.b32 nvr386|nvp260,nvr381,14,31,-1;mov.b32 nvf581,nvr386;fma.rn.ftz.f32 nvf582,nvf573,nvf581,nvf533;shfl.sync.idx.b32 nvr387|nvp261,nvr381,15,31,-1;mov.b32 nvf583,nvr387;fma.rn.ftz.f32 nvf584,nvf573,nvf583,nvf535;shfl.sync.idx.b32 nvr388|nvp262,nvr381,16,31,-1;mov.b32 nvf585,nvr388;fma.rn.ftz.f32 nvf586,nvf573,nvf585,nvf537;shfl.sync.idx.b32 nvr389|nvp263,nvr381,17,31,-1;mov.b32 nvf587,nvr389;fma.rn.ftz.f32 nvf588,nvf573,nvf587,nvf539;shfl.sync.idx.b32 nvr390|nvp264,nvr381,18,31,-1;mov.b32 nvf589,nvr390;fma.rn.ftz.f32 nvf590,nvf573,nvf589,nvf541;shfl.sync.idx.b32 nvr391|nvp265,nvr381,19,31,-1;mov.b32 nvf591,nvr391;fma.rn.ftz.f32 nvf592,nvf573,nvf591,nvf543;shfl.sync.idx.b32 nvr392|nvp266,nvr381,20,31,-1;mov.b32 nvf593,nvr392;fma.rn.ftz.f32 nvf594,nvf573,nvf593,nvf545;shfl.sync.idx.b32 nvr393|nvp267,nvr381,21,31,-1;mov.b32 nvf595,nvr393;fma.rn.ftz.f32 nvf596,nvf573,nvf595,nvf547;shfl.sync.idx.b32 nvr394|nvp268,nvr381,22,31,-1;mov.b32 nvf597,nvr394;fma.rn.ftz.f32 nvf598,nvf573,nvf597,nvf549;shfl.sync.idx.b32 nvr395|nvp269,nvr381,23,31,-1;mov.b32 nvf599,nvr395;fma.rn.ftz.f32 nvf600,nvf573,nvf599,nvf551;shfl.sync.idx.b32 nvr396|nvp270,nvr381,24,31,-1;mov.b32 nvf601,nvr396;fma.rn.ftz.f32 nvf602,nvf573,nvf601,nvf553;shfl.sync.idx.b32 nvr397|nvp271,nvr381,25,31,-1;mov.b32 nvf603,nvr397;fma.rn.ftz.f32 nvf604,nvf573,nvf603,nvf555;shfl.sync.idx.b32 nvr398|nvp272,nvr381,26,31,-1;mov.b32 nvf605,nvr398;fma.rn.ftz.f32 nvf606,nvf573,nvf605,nvf557;shfl.sync.idx.b32 nvr399|nvp273,nvr381,27,31,-1;mov.b32 nvf607,nvr399;fma.rn.ftz.f32 nvf608,nvf573,nvf607,nvf559;shfl.sync.idx.b32 nvr400|nvp274,nvr381,28,31,-1;mov.b32 nvf609,nvr400;fma.rn.ftz.f32 nvf610,nvf573,nvf609,nvf561;shfl.sync.idx.b32 nvr401|nvp275,nvr381,29,31,-1;mov.b32 nvf611,nvr401;fma.rn.ftz.f32 nvf612,nvf573,nvf611,nvf563;shfl.sync.idx.b32 nvr402|nvp276,nvr381,30,31,-1;mov.b32 nvf613,nvr402;fma.rn.ftz.f32 nvf614,nvf573,nvf613,nvf565;shfl.sync.idx.b32 nvr403|nvp277,nvr381,31,31,-1;mov.b32 nvf615,nvr403;fma.rn.ftz.f32 nvf616,nvf573,nvf615,nvf567;max.ftz.f32 nvf617,nvf574,0f00800000;rsqrt.approx.ftz.f32 nvf618,nvf617;mov.b32 nvr404,nvf618;shfl.sync.idx.b32 nvr405|nvp278,nvr404,10,31,-1;mov.b32 nvf619,nvr405;mul.ftz.f32 nvf620,nvf574,nvf619;mov.b32 nvr406,nvf620;shfl.sync.idx.b32 nvr407|nvp279,nvr406,11,31,-1;mov.b32 nvf621,nvr407;neg.ftz.f32 nvf622,nvf620;fma.rn.ftz.f32 nvf623,nvf622,nvf621,nvf576;shfl.sync.idx.b32 nvr408|nvp280,nvr406,12,31,-1;mov.b32 nvf624,nvr408;fma.rn.ftz.f32 nvf625,nvf622,nvf624,nvf578;shfl.sync.idx.b32 nvr409|nvp281,nvr406,13,31,-1;mov.b32 nvf626,nvr409;fma.rn.ftz.f32 nvf627,nvf622,nvf626,nvf580;shfl.sync.idx.b32 nvr410|nvp282,nvr406,14,31,-1;mov.b32 nvf628,nvr410;fma.rn.ftz.f32 nvf629,nvf622,nvf628,nvf582;shfl.sync.idx.b32 nvr411|nvp283,nvr406,15,31,-1;mov.b32 nvf630,nvr411;fma.rn.ftz.f32 nvf631,nvf622,nvf630,nvf584;shfl.sync.idx.b32 nvr412|nvp284,nvr406,16,31,-1;mov.b32 nvf632,nvr412;fma.rn.ftz.f32 nvf633,nvf622,nvf632,nvf586;shfl.sync.idx.b32 nvr413|nvp285,nvr406,17,31,-1;mov.b32 nvf634,nvr413;fma.rn.ftz.f32 nvf635,nvf622,nvf634,nvf588;shfl.sync.idx.b32 nvr414|nvp286,nvr406,18,31,-1;mov.b32 nvf636,nvr414;fma.rn.ftz.f32 nvf637,nvf622,nvf636,nvf590;shfl.sync.idx.b32 nvr415|nvp287,nvr406,19,31,-1;mov.b32 nvf638,nvr415;fma.rn.ftz.f32 nvf639,nvf622,nvf638,nvf592;shfl.sync.idx.b32 nvr416|nvp288,nvr406,20,31,-1;mov.b32 nvf640,nvr416;fma.rn.ftz.f32 nvf641,nvf622,nvf640,nvf594;shfl.sync.idx.b32 nvr417|nvp289,nvr406,21,31,-1;mov.b32 nvf642,nvr417;fma.rn.ftz.f32 nvf643,nvf622,nvf642,nvf596;shfl.sync.idx.b32 nvr418|nvp290,nvr406,22,31,-1;mov.b32 nvf644,nvr418;fma.rn.ftz.f32 nvf645,nvf622,nvf644,nvf598;shfl.sync.idx.b32 nvr419|nvp291,nvr406,23,31,-1;mov.b32 nvf646,nvr419;fma.rn.ftz.f32 nvf647,nvf622,nvf646,nvf600;shfl.sync.idx.b32 nvr420|nvp292,nvr406,24,31,-1;mov.b32 nvf648,nvr420;fma.rn.ftz.f32 nvf649,nvf622,nvf648,nvf602;shfl.sync.idx.b32 nvr421|nvp293,nvr406,25,31,-1;mov.b32 nvf650,nvr421;fma.rn.ftz.f32 nvf651,nvf622,nvf650,nvf604;shfl.sync.idx.b32 nvr422|nvp294,nvr406,26,31,-1;mov.b32 nvf652,nvr422;fma.rn.ftz.f32 nvf653,nvf622,nvf652,nvf606;shfl.sync.idx.b32 nvr423|nvp295,nvr406,27,31,-1;mov.b32 nvf654,nvr423;fma.rn.ftz.f32 nvf655,nvf622,nvf654,nvf608;shfl.sync.idx.b32 nvr424|nvp296,nvr406,28,31,-1;mov.b32 nvf656,nvr424;fma.rn.ftz.f32 nvf657,nvf622,nvf656,nvf610;shfl.sync.idx.b32 nvr425|nvp297,nvr406,29,31,-1;mov.b32 nvf658,nvr425;fma.rn.ftz.f32 nvf659,nvf622,nvf658,nvf612;shfl.sync.idx.b32 nvr426|nvp298,nvr406,30,31,-1;mov.b32 nvf660,nvr426;fma.rn.ftz.f32 nvf661,nvf622,nvf660,nvf614;shfl.sync.idx.b32 nvr427|nvp299,nvr406,31,31,-1;mov.b32 nvf662,nvr427;fma.rn.ftz.f32 nvf663,nvf622,nvf662,nvf616;max.ftz.f32 nvf664,nvf623,0f00800000;rsqrt.approx.ftz.f32 nvf665,nvf664;mov.b32 nvr428,nvf665;shfl.sync.idx.b32 nvr429|nvp300,nvr428,11,31,-1;mov.b32 nvf666,nvr429;mul.ftz.f32 nvf667,nvf623,nvf666;mov.b32 nvr430,nvf667;shfl.sync.idx.b32 nvr431|nvp301,nvr430,12,31,-1;mov.b32 nvf668,nvr431;neg.ftz.f32 nvf669,nvf667;fma.rn.ftz.f32 nvf670,nvf669,nvf668,nvf625;shfl.sync.idx.b32 nvr432|nvp302,nvr430,13,31,-1;mov.b32 nvf671,nvr432;fma.rn.ftz.f32 nvf672,nvf669,nvf671,nvf627;shfl.sync.idx.b32 nvr433|nvp303,nvr430,14,31,-1;mov.b32 nvf673,nvr433;fma.rn.ftz.f32 nvf674,nvf669,nvf673,nvf629;shfl.sync.idx.b32 nvr434|nvp304,nvr430,15,31,-1;mov.b32 nvf675,nvr434;fma.rn.ftz.f32 nvf676,nvf669,nvf675,nvf631;shfl.sync.idx.b32 nvr435|nvp305,nvr430,16,31,-1;mov.b32 nvf677,nvr435;fma.rn.ftz.f32 nvf678,nvf669,nvf677,nvf633;shfl.sync.idx.b32 nvr436|nvp306,nvr430,17,31,-1;mov.b32 nvf679,nvr436;fma.rn.ftz.f32 nvf680,nvf669,nvf679,nvf635;shfl.sync.idx.b32 nvr437|nvp307,nvr430,18,31,-1;mov.b32 nvf681,nvr437;fma.rn.ftz.f32 nvf682,nvf669,nvf681,nvf637;shfl.sync.idx.b32 nvr438|nvp308,nvr430,19,31,-1;mov.b32 nvf683,nvr438;fma.rn.ftz.f32 nvf684,nvf669,nvf683,nvf639;shfl.sync.idx.b32 nvr439|nvp309,nvr430,20,31,-1;mov.b32 nvf685,nvr439;fma.rn.ftz.f32 nvf686,nvf669,nvf685,nvf641;shfl.sync.idx.b32 nvr440|nvp310,nvr430,21,31,-1;mov.b32 nvf687,nvr440;fma.rn.ftz.f32 nvf688,nvf669,nvf687,nvf643;shfl.sync.idx.b32 nvr441|nvp311,nvr430,22,31,-1;mov.b32 nvf689,nvr441;fma.rn.ftz.f32 nvf690,nvf669,nvf689,nvf645;shfl.sync.idx.b32 nvr442|nvp312,nvr430,23,31,-1;mov.b32 nvf691,nvr442;fma.rn.ftz.f32 nvf692,nvf669,nvf691,nvf647;shfl.sync.idx.b32 nvr443|nvp313,nvr430,24,31,-1;mov.b32 nvf693,nvr443;fma.rn.ftz.f32 nvf694,nvf669,nvf693,nvf649;shfl.sync.idx.b32 nvr444|nvp314,nvr430,25,31,-1;mov.b32 nvf695,nvr444;fma.rn.ftz.f32 nvf696,nvf669,nvf695,nvf651;shfl.sync.idx.b32 nvr445|nvp315,nvr430,26,31,-1;mov.b32 nvf697,nvr445;fma.rn.ftz.f32 nvf698,nvf669,nvf697,nvf653;shfl.sync.idx.b32 nvr446|nvp316,nvr430,27,31,-1;mov.b32 nvf699,nvr446;fma.rn.ftz.f32 nvf700,nvf669,nvf699,nvf655;shfl.sync.idx.b32 nvr447|nvp317,nvr430,28,31,-1;mov.b32 nvf701,nvr447;fma.rn.ftz.f32 nvf702,nvf669,nvf701,nvf657;shfl.sync.idx.b32 nvr448|nvp318,nvr430,29,31,-1;mov.b32 nvf703,nvr448;fma.rn.ftz.f32 nvf704,nvf669,nvf703,nvf659;shfl.sync.idx.b32 nvr449|nvp319,nvr430,30,31,-1;mov.b32 nvf705,nvr449;fma.rn.ftz.f32 nvf706,nvf669,nvf705,nvf661;shfl.sync.idx.b32 nvr450|nvp320,nvr430,31,31,-1;mov.b32 nvf707,nvr450;fma.rn.ftz.f32 nvf708,nvf669,nvf707,nvf663;max.ftz.f32 nvf709,nvf670,0f00800000;rsqrt.approx.ftz.f32 nvf710,nvf709;mov.b32 nvr451,nvf710;shfl.sync.idx.b32 nvr452|nvp321,nvr451,12,31,-1;mov.b32 nvf711,nvr452;mul.ftz.f32 nvf712,nvf670,nvf711;mov.b32 nvr453,nvf712;shfl.sync.idx.b32 nvr454|nvp322,nvr453,13,31,-1;mov.b32 nvf713,nvr454;neg.ftz.f32 nvf714,nvf712;fma.rn.ftz.f32 nvf715,nvf714,nvf713,nvf672;shfl.sync.idx.b32 nvr455|nvp323,nvr453,14,31,-1;mov.b32 nvf716,nvr455;fma.rn.ftz.f32 nvf717,nvf714,nvf716,nvf674;shfl.sync.idx.b32 nvr456|nvp324,nvr453,15,31,-1;mov.b32 nvf718,nvr456;fma.rn.ftz.f32 nvf719,nvf714,nvf718,nvf676;shfl.sync.idx.b32 nvr457|nvp325,nvr453,16,31,-1;mov.b32 nvf720,nvr457;fma.rn.ftz.f32 nvf721,nvf714,nvf720,nvf678;shfl.sync.idx.b32 nvr458|nvp326,nvr453,17,31,-1;mov.b32 nvf722,nvr458;fma.rn.ftz.f32 nvf723,nvf714,nvf722,nvf680;shfl.sync.idx.b32 nvr459|nvp327,nvr453,18,31,-1;mov.b32 nvf724,nvr459;fma.rn.ftz.f32 nvf725,nvf714,nvf724,nvf682;shfl.sync.idx.b32 nvr460|nvp328,nvr453,19,31,-1;mov.b32 nvf726,nvr460;fma.rn.ftz.f32 nvf727,nvf714,nvf726,nvf684;shfl.sync.idx.b32 nvr461|nvp329,nvr453,20,31,-1;mov.b32 nvf728,nvr461;fma.rn.ftz.f32 nvf729,nvf714,nvf728,nvf686;shfl.sync.idx.b32 nvr462|nvp330,nvr453,21,31,-1;mov.b32 nvf730,nvr462;fma.rn.ftz.f32 nvf731,nvf714,nvf730,nvf688;shfl.sync.idx.b32 nvr463|nvp331,nvr453,22,31,-1;mov.b32 nvf732,nvr463;fma.rn.ftz.f32 nvf733,nvf714,nvf732,nvf690;shfl.sync.idx.b32 nvr464|nvp332,nvr453,23,31,-1;mov.b32 nvf734,nvr464;fma.rn.ftz.f32 nvf735,nvf714,nvf734,nvf692;shfl.sync.idx.b32 nvr465|nvp333,nvr453,24,31,-1;mov.b32 nvf736,nvr465;fma.rn.ftz.f32 nvf737,nvf714,nvf736,nvf694;shfl.sync.idx.b32 nvr466|nvp334,nvr453,25,31,-1;mov.b32 nvf738,nvr466;fma.rn.ftz.f32 nvf739,nvf714,nvf738,nvf696;shfl.sync.idx.b32 nvr467|nvp335,nvr453,26,31,-1;mov.b32 nvf740,nvr467;fma.rn.ftz.f32 nvf741,nvf714,nvf740,nvf698;shfl.sync.idx.b32 nvr468|nvp336,nvr453,27,31,-1;mov.b32 nvf742,nvr468;fma.rn.ftz.f32 nvf743,nvf714,nvf742,nvf700;shfl.sync.idx.b32 nvr469|nvp337,nvr453,28,31,-1;mov.b32 nvf744,nvr469;fma.rn.ftz.f32 nvf745,nvf714,nvf744,nvf702;shfl.sync.idx.b32 nvr470|nvp338,nvr453,29,31,-1;mov.b32 nvf746,nvr470;fma.rn.ftz.f32 nvf747,nvf714,nvf746,nvf704;shfl.sync.idx.b32 nvr471|nvp339,nvr453,30,31,-1;mov.b32 nvf748,nvr471;fma.rn.ftz.f32 nvf749,nvf714,nvf748,nvf706;shfl.sync.idx.b32 nvr472|nvp340,nvr453,31,31,-1;mov.b32 nvf750,nvr472;fma.rn.ftz.f32 nvf751,nvf714,nvf750,nvf708;max.ftz.f32 nvf752,nvf715,0f00800000;rsqrt.approx.ftz.f32 nvf753,nvf752;mov.b32 nvr473,nvf753;shfl.sync.idx.b32 nvr474|nvp341,nvr473,13,31,-1;mov.b32 nvf754,nvr474;mul.ftz.f32 nvf755,nvf715,nvf754;mov.b32 nvr475,nvf755;shfl.sync.idx.b32 nvr476|nvp342,nvr475,14,31,-1;mov.b32 nvf756,nvr476;neg.ftz.f32 nvf757,nvf755;fma.rn.ftz.f32 nvf758,nvf757,nvf756,nvf717;shfl.sync.idx.b32 nvr477|nvp343,nvr475,15,31,-1;mov.b32 nvf759,nvr477;fma.rn.ftz.f32 nvf760,nvf757,nvf759,nvf719;shfl.sync.idx.b32 nvr478|nvp344,nvr475,16,31,-1;mov.b32 nvf761,nvr478;fma.rn.ftz.f32 nvf762,nvf757,nvf761,nvf721;shfl.sync.idx.b32 nvr479|nvp345,nvr475,17,31,-1;mov.b32 nvf763,nvr479;fma.rn.ftz.f32 nvf764,nvf757,nvf763,nvf723;shfl.sync.idx.b32 nvr480|nvp346,nvr475,18,31,-1;mov.b32 nvf765,nvr480;fma.rn.ftz.f32 nvf766,nvf757,nvf765,nvf725;shfl.sync.idx.b32 nvr481|nvp347,nvr475,19,31,-1;mov.b32 nvf767,nvr481;fma.rn.ftz.f32 nvf768,nvf757,nvf767,nvf727;shfl.sync.idx.b32 nvr482|nvp348,nvr475,20,31,-1;mov.b32 nvf769,nvr482;fma.rn.ftz.f32 nvf770,nvf757,nvf769,nvf729;shfl.sync.idx.b32 nvr483|nvp349,nvr475,21,31,-1;mov.b32 nvf771,nvr483;fma.rn.ftz.f32 nvf772,nvf757,nvf771,nvf731;shfl.sync.idx.b32 nvr484|nvp350,nvr475,22,31,-1;mov.b32 nvf773,nvr484;fma.rn.ftz.f32 nvf774,nvf757,nvf773,nvf733;shfl.sync.idx.b32 nvr485|nvp351,nvr475,23,31,-1;mov.b32 nvf775,nvr485;fma.rn.ftz.f32 nvf776,nvf757,nvf775,nvf735;shfl.sync.idx.b32 nvr486|nvp352,nvr475,24,31,-1;mov.b32 nvf777,nvr486;fma.rn.ftz.f32 nvf778,nvf757,nvf777,nvf737;shfl.sync.idx.b32 nvr487|nvp353,nvr475,25,31,-1;mov.b32 nvf779,nvr487;fma.rn.ftz.f32 nvf780,nvf757,nvf779,nvf739;shfl.sync.idx.b32 nvr488|nvp354,nvr475,26,31,-1;mov.b32 nvf781,nvr488;fma.rn.ftz.f32 nvf782,nvf757,nvf781,nvf741;shfl.sync.idx.b32 nvr489|nvp355,nvr475,27,31,-1;mov.b32 nvf783,nvr489;fma.rn.ftz.f32 nvf784,nvf757,nvf783,nvf743;shfl.sync.idx.b32 nvr490|nvp356,nvr475,28,31,-1;mov.b32 nvf785,nvr490;fma.rn.ftz.f32 nvf786,nvf757,nvf785,nvf745;shfl.sync.idx.b32 nvr491|nvp357,nvr475,29,31,-1;mov.b32 nvf787,nvr491;fma.rn.ftz.f32 nvf788,nvf757,nvf787,nvf747;shfl.sync.idx.b32 nvr492|nvp358,nvr475,30,31,-1;mov.b32 nvf789,nvr492;fma.rn.ftz.f32 nvf790,nvf757,nvf789,nvf749;shfl.sync.idx.b32 nvr493|nvp359,nvr475,31,31,-1;mov.b32 nvf791,nvr493;fma.rn.ftz.f32 nvf792,nvf757,nvf791,nvf751;max.ftz.f32 nvf793,nvf758,0f00800000;rsqrt.approx.ftz.f32 nvf794,nvf793;mov.b32 nvr494,nvf794;shfl.sync.idx.b32 nvr495|nvp360,nvr494,14,31,-1;mov.b32 nvf795,nvr495;mul.ftz.f32 nvf796,nvf758,nvf795;mov.b32 nvr496,nvf796;shfl.sync.idx.b32 nvr497|nvp361,nvr496,15,31,-1;mov.b32 nvf797,nvr497;neg.ftz.f32 nvf798,nvf796;fma.rn.ftz.f32 nvf799,nvf798,nvf797,nvf760;shfl.sync.idx.b32 nvr498|nvp362,nvr496,16,31,-1;mov.b32 nvf800,nvr498;fma.rn.ftz.f32 nvf801,nvf798,nvf800,nvf762;shfl.sync.idx.b32 nvr499|nvp363,nvr496,17,31,-1;mov.b32 nvf802,nvr499;fma.rn.ftz.f32 nvf803,nvf798,nvf802,nvf764;shfl.sync.idx.b32 nvr500|nvp364,nvr496,18,31,-1;mov.b32 nvf804,nvr500;fma.rn.ftz.f32 nvf805,nvf798,nvf804,nvf766;shfl.sync.idx.b32 nvr501|nvp365,nvr496,19,31,-1;mov.b32 nvf806,nvr501;fma.rn.ftz.f32 nvf807,nvf798,nvf806,nvf768;shfl.sync.idx.b32 nvr502|nvp366,nvr496,20,31,-1;mov.b32 nvf808,nvr502;fma.rn.ftz.f32 nvf809,nvf798,nvf808,nvf770;shfl.sync.idx.b32 nvr503|nvp367,nvr496,21,31,-1;mov.b32 nvf810,nvr503;fma.rn.ftz.f32 nvf811,nvf798,nvf810,nvf772;shfl.sync.idx.b32 nvr504|nvp368,nvr496,22,31,-1;mov.b32 nvf812,nvr504;fma.rn.ftz.f32 nvf813,nvf798,nvf812,nvf774;shfl.sync.idx.b32 nvr505|nvp369,nvr496,23,31,-1;mov.b32 nvf814,nvr505;fma.rn.ftz.f32 nvf815,nvf798,nvf814,nvf776;shfl.sync.idx.b32 nvr506|nvp370,nvr496,24,31,-1;mov.b32 nvf816,nvr506;fma.rn.ftz.f32 nvf817,nvf798,nvf816,nvf778;shfl.sync.idx.b32 nvr507|nvp371,nvr496,25,31,-1;mov.b32 nvf818,nvr507;fma.rn.ftz.f32 nvf819,nvf798,nvf818,nvf780;shfl.sync.idx.b32 nvr508|nvp372,nvr496,26,31,-1;mov.b32 nvf820,nvr508;fma.rn.ftz.f32 nvf821,nvf798,nvf820,nvf782;shfl.sync.idx.b32 nvr509|nvp373,nvr496,27,31,-1;mov.b32 nvf822,nvr509;fma.rn.ftz.f32 nvf823,nvf798,nvf822,nvf784;shfl.sync.idx.b32 nvr510|nvp374,nvr496,28,31,-1;mov.b32 nvf824,nvr510;fma.rn.ftz.f32 nvf825,nvf798,nvf824,nvf786;shfl.sync.idx.b32 nvr511|nvp375,nvr496,29,31,-1;mov.b32 nvf826,nvr511;fma.rn.ftz.f32 nvf827,nvf798,nvf826,nvf788;shfl.sync.idx.b32 nvr512|nvp376,nvr496,30,31,-1;mov.b32 nvf828,nvr512;fma.rn.ftz.f32 nvf829,nvf798,nvf828,nvf790;shfl.sync.idx.b32 nvr513|nvp377,nvr496,31,31,-1;mov.b32 nvf830,nvr513;fma.rn.ftz.f32 nvf831,nvf798,nvf830,nvf792;max.ftz.f32 nvf832,nvf799,0f00800000;rsqrt.approx.ftz.f32 nvf833,nvf832;mov.b32 nvr514,nvf833;shfl.sync.idx.b32 nvr515|nvp378,nvr514,15,31,-1;mov.b32 nvf834,nvr515;mul.ftz.f32 nvf835,nvf799,nvf834;mov.b32 nvr516,nvf835;shfl.sync.idx.b32 nvr517|nvp379,nvr516,16,31,-1;mov.b32 nvf836,nvr517;neg.ftz.f32 nvf837,nvf835;fma.rn.ftz.f32 nvf838,nvf837,nvf836,nvf801;shfl.sync.idx.b32 nvr518|nvp380,nvr516,17,31,-1;mov.b32 nvf839,nvr518;fma.rn.ftz.f32 nvf840,nvf837,nvf839,nvf803;shfl.sync.idx.b32 nvr519|nvp381,nvr516,18,31,-1;mov.b32 nvf841,nvr519;fma.rn.ftz.f32 nvf842,nvf837,nvf841,nvf805;shfl.sync.idx.b32 nvr520|nvp382,nvr516,19,31,-1;mov.b32 nvf843,nvr520;fma.rn.ftz.f32 nvf844,nvf837,nvf843,nvf807;shfl.sync.idx.b32 nvr521|nvp383,nvr516,20,31,-1;mov.b32 nvf845,nvr521;fma.rn.ftz.f32 nvf846,nvf837,nvf845,nvf809;shfl.sync.idx.b32 nvr522|nvp384,nvr516,21,31,-1;mov.b32 nvf847,nvr522;fma.rn.ftz.f32 nvf848,nvf837,nvf847,nvf811;shfl.sync.idx.b32 nvr523|nvp385,nvr516,22,31,-1;mov.b32 nvf849,nvr523;fma.rn.ftz.f32 nvf850,nvf837,nvf849,nvf813;shfl.sync.idx.b32 nvr524|nvp386,nvr516,23,31,-1;mov.b32 nvf851,nvr524;fma.rn.ftz.f32 nvf852,nvf837,nvf851,nvf815;shfl.sync.idx.b32 nvr525|nvp387,nvr516,24,31,-1;mov.b32 nvf853,nvr525;fma.rn.ftz.f32 nvf854,nvf837,nvf853,nvf817;shfl.sync.idx.b32 nvr526|nvp388,nvr516,25,31,-1;mov.b32 nvf855,nvr526;fma.rn.ftz.f32 nvf856,nvf837,nvf855,nvf819;shfl.sync.idx.b32 nvr527|nvp389,nvr516,26,31,-1;mov.b32 nvf857,nvr527;fma.rn.ftz.f32 nvf858,nvf837,nvf857,nvf821;shfl.sync.idx.b32 nvr528|nvp390,nvr516,27,31,-1;mov.b32 nvf859,nvr528;fma.rn.ftz.f32 nvf860,nvf837,nvf859,nvf823;shfl.sync.idx.b32 nvr529|nvp391,nvr516,28,31,-1;mov.b32 nvf861,nvr529;fma.rn.ftz.f32 nvf862,nvf837,nvf861,nvf825;shfl.sync.idx.b32 nvr530|nvp392,nvr516,29,31,-1;mov.b32 nvf863,nvr530;fma.rn.ftz.f32 nvf864,nvf837,nvf863,nvf827;shfl.sync.idx.b32 nvr531|nvp393,nvr516,30,31,-1;mov.b32 nvf865,nvr531;fma.rn.ftz.f32 nvf866,nvf837,nvf865,nvf829;shfl.sync.idx.b32 nvr532|nvp394,nvr516,31,31,-1;mov.b32 nvf867,nvr532;fma.rn.ftz.f32 nvf868,nvf837,nvf867,nvf831;max.ftz.f32 nvf869,nvf838,0f00800000;rsqrt.approx.ftz.f32 nvf870,nvf869;mov.b32 nvr533,nvf870;shfl.sync.idx.b32 nvr534|nvp395,nvr533,16,31,-1;mov.b32 nvf871,nvr534;mul.ftz.f32 nvf872,nvf838,nvf871;mov.b32 nvr535,nvf872;shfl.sync.idx.b32 nvr536|nvp396,nvr535,17,31,-1;mov.b32 nvf873,nvr536;neg.ftz.f32 nvf874,nvf872;fma.rn.ftz.f32 nvf875,nvf874,nvf873,nvf840;shfl.sync.idx.b32 nvr537|nvp397,nvr535,18,31,-1;mov.b32 nvf876,nvr537;fma.rn.ftz.f32 nvf877,nvf874,nvf876,nvf842;shfl.sync.idx.b32 nvr538|nvp398,nvr535,19,31,-1;mov.b32 nvf878,nvr538;fma.rn.ftz.f32 nvf879,nvf874,nvf878,nvf844;shfl.sync.idx.b32 nvr539|nvp399,nvr535,20,31,-1;mov.b32 nvf880,nvr539;fma.rn.ftz.f32 nvf881,nvf874,nvf880,nvf846;shfl.sync.idx.b32 nvr540|nvp400,nvr535,21,31,-1;mov.b32 nvf882,nvr540;fma.rn.ftz.f32 nvf883,nvf874,nvf882,nvf848;shfl.sync.idx.b32 nvr541|nvp401,nvr535,22,31,-1;mov.b32 nvf884,nvr541;fma.rn.ftz.f32 nvf885,nvf874,nvf884,nvf850;shfl.sync.idx.b32 nvr542|nvp402,nvr535,23,31,-1;mov.b32 nvf886,nvr542;fma.rn.ftz.f32 nvf887,nvf874,nvf886,nvf852;shfl.sync.idx.b32 nvr543|nvp403,nvr535,24,31,-1;mov.b32 nvf888,nvr543;fma.rn.ftz.f32 nvf889,nvf874,nvf888,nvf854;shfl.sync.idx.b32 nvr544|nvp404,nvr535,25,31,-1;mov.b32 nvf890,nvr544;fma.rn.ftz.f32 nvf891,nvf874,nvf890,nvf856;shfl.sync.idx.b32 nvr545|nvp405,nvr535,26,31,-1;mov.b32 nvf892,nvr545;fma.rn.ftz.f32 nvf893,nvf874,nvf892,nvf858;shfl.sync.idx.b32 nvr546|nvp406,nvr535,27,31,-1;mov.b32 nvf894,nvr546;fma.rn.ftz.f32 nvf895,nvf874,nvf894,nvf860;shfl.sync.idx.b32 nvr547|nvp407,nvr535,28,31,-1;mov.b32 nvf896,nvr547;fma.rn.ftz.f32 nvf897,nvf874,nvf896,nvf862;shfl.sync.idx.b32 nvr548|nvp408,nvr535,29,31,-1;mov.b32 nvf898,nvr548;fma.rn.ftz.f32 nvf899,nvf874,nvf898,nvf864;shfl.sync.idx.b32 nvr549|nvp409,nvr535,30,31,-1;mov.b32 nvf900,nvr549;fma.rn.ftz.f32 nvf901,nvf874,nvf900,nvf866;shfl.sync.idx.b32 nvr550|nvp410,nvr535,31,31,-1;mov.b32 nvf902,nvr550;fma.rn.ftz.f32 nvf903,nvf874,nvf902,nvf868;max.ftz.f32 nvf904,nvf875,0f00800000;rsqrt.approx.ftz.f32 nvf905,nvf904;mov.b32 nvr551,nvf905;shfl.sync.idx.b32 nvr552|nvp411,nvr551,17,31,-1;mov.b32 nvf906,nvr552;mul.ftz.f32 nvf907,nvf875,nvf906;mov.b32 nvr553,nvf907;shfl.sync.idx.b32 nvr554|nvp412,nvr553,18,31,-1;mov.b32 nvf908,nvr554;neg.ftz.f32 nvf909,nvf907;fma.rn.ftz.f32 nvf910,nvf909,nvf908,nvf877;shfl.sync.idx.b32 nvr555|nvp413,nvr553,19,31,-1;mov.b32 nvf911,nvr555;fma.rn.ftz.f32 nvf912,nvf909,nvf911,nvf879;shfl.sync.idx.b32 nvr556|nvp414,nvr553,20,31,-1;mov.b32 nvf913,nvr556;fma.rn.ftz.f32 nvf914,nvf909,nvf913,nvf881;shfl.sync.idx.b32 nvr557|nvp415,nvr553,21,31,-1;mov.b32 nvf915,nvr557;fma.rn.ftz.f32 nvf916,nvf909,nvf915,nvf883;shfl.sync.idx.b32 nvr558|nvp416,nvr553,22,31,-1;mov.b32 nvf917,nvr558;fma.rn.ftz.f32 nvf918,nvf909,nvf917,nvf885;shfl.sync.idx.b32 nvr559|nvp417,nvr553,23,31,-1;mov.b32 nvf919,nvr559;fma.rn.ftz.f32 nvf920,nvf909,nvf919,nvf887;shfl.sync.idx.b32 nvr560|nvp418,nvr553,24,31,-1;mov.b32 nvf921,nvr560;fma.rn.ftz.f32 nvf922,nvf909,nvf921,nvf889;shfl.sync.idx.b32 nvr561|nvp419,nvr553,25,31,-1;mov.b32 nvf923,nvr561;fma.rn.ftz.f32 nvf924,nvf909,nvf923,nvf891;shfl.sync.idx.b32 nvr562|nvp420,nvr553,26,31,-1;mov.b32 nvf925,nvr562;fma.rn.ftz.f32 nvf926,nvf909,nvf925,nvf893;shfl.sync.idx.b32 nvr563|nvp421,nvr553,27,31,-1;mov.b32 nvf927,nvr563;fma.rn.ftz.f32 nvf928,nvf909,nvf927,nvf895;shfl.sync.idx.b32 nvr564|nvp422,nvr553,28,31,-1;mov.b32 nvf929,nvr564;fma.rn.ftz.f32 nvf930,nvf909,nvf929,nvf897;shfl.sync.idx.b32 nvr565|nvp423,nvr553,29,31,-1;mov.b32 nvf931,nvr565;fma.rn.ftz.f32 nvf932,nvf909,nvf931,nvf899;shfl.sync.idx.b32 nvr566|nvp424,nvr553,30,31,-1;mov.b32 nvf933,nvr566;fma.rn.ftz.f32 nvf934,nvf909,nvf933,nvf901;shfl.sync.idx.b32 nvr567|nvp425,nvr553,31,31,-1;mov.b32 nvf935,nvr567;fma.rn.ftz.f32 nvf936,nvf909,nvf935,nvf903;max.ftz.f32 nvf937,nvf910,0f00800000;rsqrt.approx.ftz.f32 nvf938,nvf937;mov.b32 nvr568,nvf938;shfl.sync.idx.b32 nvr569|nvp426,nvr568,18,31,-1;mov.b32 nvf939,nvr569;mul.ftz.f32 nvf940,nvf910,nvf939;mov.b32 nvr570,nvf940;shfl.sync.idx.b32 nvr571|nvp427,nvr570,19,31,-1;mov.b32 nvf941,nvr571;neg.ftz.f32 nvf942,nvf940;fma.rn.ftz.f32 nvf943,nvf942,nvf941,nvf912;shfl.sync.idx.b32 nvr572|nvp428,nvr570,20,31,-1;mov.b32 nvf944,nvr572;fma.rn.ftz.f32 nvf945,nvf942,nvf944,nvf914;shfl.sync.idx.b32 nvr573|nvp429,nvr570,21,31,-1;mov.b32 nvf946,nvr573;fma.rn.ftz.f32 nvf947,nvf942,nvf946,nvf916;shfl.sync.idx.b32 nvr574|nvp430,nvr570,22,31,-1;mov.b32 nvf948,nvr574;fma.rn.ftz.f32 nvf949,nvf942,nvf948,nvf918;shfl.sync.idx.b32 nvr575|nvp431,nvr570,23,31,-1;mov.b32 nvf950,nvr575;fma.rn.ftz.f32 nvf951,nvf942,nvf950,nvf920;shfl.sync.idx.b32 nvr576|nvp432,nvr570,24,31,-1;mov.b32 nvf952,nvr576;fma.rn.ftz.f32 nvf953,nvf942,nvf952,nvf922;shfl.sync.idx.b32 nvr577|nvp433,nvr570,25,31,-1;mov.b32 nvf954,nvr577;fma.rn.ftz.f32 nvf955,nvf942,nvf954,nvf924;shfl.sync.idx.b32 nvr578|nvp434,nvr570,26,31,-1;mov.b32 nvf956,nvr578;fma.rn.ftz.f32 nvf957,nvf942,nvf956,nvf926;shfl.sync.idx.b32 nvr579|nvp435,nvr570,27,31,-1;mov.b32 nvf958,nvr579;fma.rn.ftz.f32 nvf959,nvf942,nvf958,nvf928;shfl.sync.idx.b32 nvr580|nvp436,nvr570,28,31,-1;mov.b32 nvf960,nvr580;fma.rn.ftz.f32 nvf961,nvf942,nvf960,nvf930;shfl.sync.idx.b32 nvr581|nvp437,nvr570,29,31,-1;mov.b32 nvf962,nvr581;fma.rn.ftz.f32 nvf963,nvf942,nvf962,nvf932;shfl.sync.idx.b32 nvr582|nvp438,nvr570,30,31,-1;mov.b32 nvf964,nvr582;fma.rn.ftz.f32 nvf965,nvf942,nvf964,nvf934;shfl.sync.idx.b32 nvr583|nvp439,nvr570,31,31,-1;mov.b32 nvf966,nvr583;fma.rn.ftz.f32 nvf967,nvf942,nvf966,nvf936;max.ftz.f32 nvf968,nvf943,0f00800000;rsqrt.approx.ftz.f32 nvf969,nvf968;mov.b32 nvr584,nvf969;shfl.sync.idx.b32 nvr585|nvp440,nvr584,19,31,-1;mov.b32 nvf970,nvr585;mul.ftz.f32 nvf971,nvf943,nvf970;mov.b32 nvr586,nvf971;shfl.sync.idx.b32 nvr587|nvp441,nvr586,20,31,-1;mov.b32 nvf972,nvr587;neg.ftz.f32 nvf973,nvf971;fma.rn.ftz.f32 nvf974,nvf973,nvf972,nvf945;shfl.sync.idx.b32 nvr588|nvp442,nvr586,21,31,-1;mov.b32 nvf975,nvr588;fma.rn.ftz.f32 nvf976,nvf973,nvf975,nvf947;shfl.sync.idx.b32 nvr589|nvp443,nvr586,22,31,-1;mov.b32 nvf977,nvr589;fma.rn.ftz.f32 nvf978,nvf973,nvf977,nvf949;shfl.sync.idx.b32 nvr590|nvp444,nvr586,23,31,-1;mov.b32 nvf979,nvr590;fma.rn.ftz.f32 nvf980,nvf973,nvf979,nvf951;shfl.sync.idx.b32 nvr591|nvp445,nvr586,24,31,-1;mov.b32 nvf981,nvr591;fma.rn.ftz.f32 nvf982,nvf973,nvf981,nvf953;shfl.sync.idx.b32 nvr592|nvp446,nvr586,25,31,-1;mov.b32 nvf983,nvr592;fma.rn.ftz.f32 nvf984,nvf973,nvf983,nvf955;shfl.sync.idx.b32 nvr593|nvp447,nvr586,26,31,-1;mov.b32 nvf985,nvr593;fma.rn.ftz.f32 nvf986,nvf973,nvf985,nvf957;shfl.sync.idx.b32 nvr594|nvp448,nvr586,27,31,-1;mov.b32 nvf987,nvr594;fma.rn.ftz.f32 nvf988,nvf973,nvf987,nvf959;shfl.sync.idx.b32 nvr595|nvp449,nvr586,28,31,-1;mov.b32 nvf989,nvr595;fma.rn.ftz.f32 nvf990,nvf973,nvf989,nvf961;shfl.sync.idx.b32 nvr596|nvp450,nvr586,29,31,-1;mov.b32 nvf991,nvr596;fma.rn.ftz.f32 nvf992,nvf973,nvf991,nvf963;shfl.sync.idx.b32 nvr597|nvp451,nvr586,30,31,-1;mov.b32 nvf993,nvr597;fma.rn.ftz.f32 nvf994,nvf973,nvf993,nvf965;shfl.sync.idx.b32 nvr598|nvp452,nvr586,31,31,-1;mov.b32 nvf995,nvr598;fma.rn.ftz.f32 nvf996,nvf973,nvf995,nvf967;max.ftz.f32 nvf997,nvf974,0f00800000;rsqrt.approx.ftz.f32 nvf998,nvf997;mov.b32 nvr599,nvf998;shfl.sync.idx.b32 nvr600|nvp453,nvr599,20,31,-1;mov.b32 nvf999,nvr600;mul.ftz.f32 nvf1000,nvf974,nvf999;mov.b32 nvr601,nvf1000;shfl.sync.idx.b32 nvr602|nvp454,nvr601,21,31,-1;mov.b32 nvf1001,nvr602;neg.ftz.f32 nvf1002,nvf1000;fma.rn.ftz.f32 nvf1003,nvf1002,nvf1001,nvf976;shfl.sync.idx.b32 nvr603|nvp455,nvr601,22,31,-1;mov.b32 nvf1004,nvr603;fma.rn.ftz.f32 nvf1005,nvf1002,nvf1004,nvf978;shfl.sync.idx.b32 nvr604|nvp456,nvr601,23,31,-1;mov.b32 nvf1006,nvr604;fma.rn.ftz.f32 nvf1007,nvf1002,nvf1006,nvf980;shfl.sync.idx.b32 nvr605|nvp457,nvr601,24,31,-1;mov.b32 nvf1008,nvr605;fma.rn.ftz.f32 nvf1009,nvf1002,nvf1008,nvf982;shfl.sync.idx.b32 nvr606|nvp458,nvr601,25,31,-1;mov.b32 nvf1010,nvr606;fma.rn.ftz.f32 nvf1011,nvf1002,nvf1010,nvf984;shfl.sync.idx.b32 nvr607|nvp459,nvr601,26,31,-1;mov.b32 nvf1012,nvr607;fma.rn.ftz.f32 nvf1013,nvf1002,nvf1012,nvf986;shfl.sync.idx.b32 nvr608|nvp460,nvr601,27,31,-1;mov.b32 nvf1014,nvr608;fma.rn.ftz.f32 nvf1015,nvf1002,nvf1014,nvf988;shfl.sync.idx.b32 nvr609|nvp461,nvr601,28,31,-1;mov.b32 nvf1016,nvr609;fma.rn.ftz.f32 nvf1017,nvf1002,nvf1016,nvf990;shfl.sync.idx.b32 nvr610|nvp462,nvr601,29,31,-1;mov.b32 nvf1018,nvr610;fma.rn.ftz.f32 nvf1019,nvf1002,nvf1018,nvf992;shfl.sync.idx.b32 nvr611|nvp463,nvr601,30,31,-1;mov.b32 nvf1020,nvr611;fma.rn.ftz.f32 nvf1021,nvf1002,nvf1020,nvf994;shfl.sync.idx.b32 nvr612|nvp464,nvr601,31,31,-1;mov.b32 nvf1022,nvr612;fma.rn.ftz.f32 nvf1023,nvf1002,nvf1022,nvf996;max.ftz.f32 nvf1024,nvf1003,0f00800000;rsqrt.approx.ftz.f32 nvf1025,nvf1024;mov.b32 nvr613,nvf1025;shfl.sync.idx.b32 nvr614|nvp465,nvr613,21,31,-1;mov.b32 nvf1026,nvr614;mul.ftz.f32 nvf1027,nvf1003,nvf1026;mov.b32 nvr615,nvf1027;shfl.sync.idx.b32 nvr616|nvp466,nvr615,22,31,-1;mov.b32 nvf1028,nvr616;neg.ftz.f32 nvf1029,nvf1027;fma.rn.ftz.f32 nvf1030,nvf1029,nvf1028,nvf1005;shfl.sync.idx.b32 nvr617|nvp467,nvr615,23,31,-1;mov.b32 nvf1031,nvr617;fma.rn.ftz.f32 nvf1032,nvf1029,nvf1031,nvf1007;shfl.sync.idx.b32 nvr618|nvp468,nvr615,24,31,-1;mov.b32 nvf1033,nvr618;fma.rn.ftz.f32 nvf1034,nvf1029,nvf1033,nvf1009;shfl.sync.idx.b32 nvr619|nvp469,nvr615,25,31,-1;mov.b32 nvf1035,nvr619;fma.rn.ftz.f32 nvf1036,nvf1029,nvf1035,nvf1011;shfl.sync.idx.b32 nvr620|nvp470,nvr615,26,31,-1;mov.b32 nvf1037,nvr620;fma.rn.ftz.f32 nvf1038,nvf1029,nvf1037,nvf1013;shfl.sync.idx.b32 nvr621|nvp471,nvr615,27,31,-1;mov.b32 nvf1039,nvr621;fma.rn.ftz.f32 nvf1040,nvf1029,nvf1039,nvf1015;shfl.sync.idx.b32 nvr622|nvp472,nvr615,28,31,-1;mov.b32 nvf1041,nvr622;fma.rn.ftz.f32 nvf1042,nvf1029,nvf1041,nvf1017;shfl.sync.idx.b32 nvr623|nvp473,nvr615,29,31,-1;mov.b32 nvf1043,nvr623;fma.rn.ftz.f32 nvf1044,nvf1029,nvf1043,nvf1019;shfl.sync.idx.b32 nvr624|nvp474,nvr615,30,31,-1;mov.b32 nvf1045,nvr624;fma.rn.ftz.f32 nvf1046,nvf1029,nvf1045,nvf1021;shfl.sync.idx.b32 nvr625|nvp475,nvr615,31,31,-1;mov.b32 nvf1047,nvr625;fma.rn.ftz.f32 nvf1048,nvf1029,nvf1047,nvf1023;max.ftz.f32 nvf1049,nvf1030,0f00800000;rsqrt.approx.ftz.f32 nvf1050,nvf1049;mov.b32 nvr626,nvf1050;shfl.sync.idx.b32 nvr627|nvp476,nvr626,22,31,-1;mov.b32 nvf1051,nvr627;mul.ftz.f32 nvf1052,nvf1030,nvf1051;mov.b32 nvr628,nvf1052;shfl.sync.idx.b32 nvr629|nvp477,nvr628,23,31,-1;mov.b32 nvf1053,nvr629;neg.ftz.f32 nvf1054,nvf1052;fma.rn.ftz.f32 nvf1055,nvf1054,nvf1053,nvf1032;shfl.sync.idx.b32 nvr630|nvp478,nvr628,24,31,-1;mov.b32 nvf1056,nvr630;fma.rn.ftz.f32 nvf1057,nvf1054,nvf1056,nvf1034;shfl.sync.idx.b32 nvr631|nvp479,nvr628,25,31,-1;mov.b32 nvf1058,nvr631;fma.rn.ftz.f32 nvf1059,nvf1054,nvf1058,nvf1036;shfl.sync.idx.b32 nvr632|nvp480,nvr628,26,31,-1;mov.b32 nvf1060,nvr632;fma.rn.ftz.f32 nvf1061,nvf1054,nvf1060,nvf1038;shfl.sync.idx.b32 nvr633|nvp481,nvr628,27,31,-1;mov.b32 nvf1062,nvr633;fma.rn.ftz.f32 nvf1063,nvf1054,nvf1062,nvf1040;shfl.sync.idx.b32 nvr634|nvp482,nvr628,28,31,-1;mov.b32 nvf1064,nvr634;fma.rn.ftz.f32 nvf1065,nvf1054,nvf1064,nvf1042;shfl.sync.idx.b32 nvr635|nvp483,nvr628,29,31,-1;mov.b32 nvf1066,nvr635;fma.rn.ftz.f32 nvf1067,nvf1054,nvf1066,nvf1044;shfl.sync.idx.b32 nvr636|nvp484,nvr628,30,31,-1;mov.b32 nvf1068,nvr636;fma.rn.ftz.f32 nvf1069,nvf1054,nvf1068,nvf1046;shfl.sync.idx.b32 nvr637|nvp485,nvr628,31,31,-1;mov.b32 nvf1070,nvr637;fma.rn.ftz.f32 nvf1071,nvf1054,nvf1070,nvf1048;max.ftz.f32 nvf1072,nvf1055,0f00800000;rsqrt.approx.ftz.f32 nvf1073,nvf1072;mov.b32 nvr638,nvf1073;shfl.sync.idx.b32 nvr639|nvp486,nvr638,23,31,-1;mov.b32 nvf1074,nvr639;mul.ftz.f32 nvf1075,nvf1055,nvf1074;mov.b32 nvr640,nvf1075;shfl.sync.idx.b32 nvr641|nvp487,nvr640,24,31,-1;mov.b32 nvf1076,nvr641;neg.ftz.f32 nvf1077,nvf1075;fma.rn.ftz.f32 nvf1078,nvf1077,nvf1076,nvf1057;shfl.sync.idx.b32 nvr642|nvp488,nvr640,25,31,-1;mov.b32 nvf1079,nvr642;fma.rn.ftz.f32 nvf1080,nvf1077,nvf1079,nvf1059;shfl.sync.idx.b32 nvr643|nvp489,nvr640,26,31,-1;mov.b32 nvf1081,nvr643;fma.rn.ftz.f32 nvf1082,nvf1077,nvf1081,nvf1061;shfl.sync.idx.b32 nvr644|nvp490,nvr640,27,31,-1;mov.b32 nvf1083,nvr644;fma.rn.ftz.f32 nvf1084,nvf1077,nvf1083,nvf1063;shfl.sync.idx.b32 nvr645|nvp491,nvr640,28,31,-1;mov.b32 nvf1085,nvr645;fma.rn.ftz.f32 nvf1086,nvf1077,nvf1085,nvf1065;shfl.sync.idx.b32 nvr646|nvp492,nvr640,29,31,-1;mov.b32 nvf1087,nvr646;fma.rn.ftz.f32 nvf1088,nvf1077,nvf1087,nvf1067;shfl.sync.idx.b32 nvr647|nvp493,nvr640,30,31,-1;mov.b32 nvf1089,nvr647;fma.rn.ftz.f32 nvf1090,nvf1077,nvf1089,nvf1069;shfl.sync.idx.b32 nvr648|nvp494,nvr640,31,31,-1;mov.b32 nvf1091,nvr648;fma.rn.ftz.f32 nvf1092,nvf1077,nvf1091,nvf1071;max.ftz.f32 nvf1093,nvf1078,0f00800000;rsqrt.approx.ftz.f32 nvf1094,nvf1093;mov.b32 nvr649,nvf1094;shfl.sync.idx.b32 nvr650|nvp495,nvr649,24,31,-1;mov.b32 nvf1095,nvr650;mul.ftz.f32 nvf1096,nvf1078,nvf1095;mov.b32 nvr651,nvf1096;shfl.sync.idx.b32 nvr652|nvp496,nvr651,25,31,-1;mov.b32 nvf1097,nvr652;neg.ftz.f32 nvf1098,nvf1096;fma.rn.ftz.f32 nvf1099,nvf1098,nvf1097,nvf1080;shfl.sync.idx.b32 nvr653|nvp497,nvr651,26,31,-1;mov.b32 nvf1100,nvr653;fma.rn.ftz.f32 nvf1101,nvf1098,nvf1100,nvf1082;shfl.sync.idx.b32 nvr654|nvp498,nvr651,27,31,-1;mov.b32 nvf1102,nvr654;fma.rn.ftz.f32 nvf1103,nvf1098,nvf1102,nvf1084;shfl.sync.idx.b32 nvr655|nvp499,nvr651,28,31,-1;mov.b32 nvf1104,nvr655;fma.rn.ftz.f32 nvf1105,nvf1098,nvf1104,nvf1086;shfl.sync.idx.b32 nvr656|nvp500,nvr651,29,31,-1;mov.b32 nvf1106,nvr656;fma.rn.ftz.f32 nvf1107,nvf1098,nvf1106,nvf1088;shfl.sync.idx.b32 nvr657|nvp501,nvr651,30,31,-1;mov.b32 nvf1108,nvr657;fma.rn.ftz.f32 nvf1109,nvf1098,nvf1108,nvf1090;shfl.sync.idx.b32 nvr658|nvp502,nvr651,31,31,-1;mov.b32 nvf1110,nvr658;fma.rn.ftz.f32 nvf1111,nvf1098,nvf1110,nvf1092;max.ftz.f32 nvf1112,nvf1099,0f00800000;rsqrt.approx.ftz.f32 nvf1113,nvf1112;mov.b32 nvr659,nvf1113;shfl.sync.idx.b32 nvr660|nvp503,nvr659,25,31,-1;mov.b32 nvf1114,nvr660;mul.ftz.f32 nvf1115,nvf1099,nvf1114;mov.b32 nvr661,nvf1115;shfl.sync.idx.b32 nvr662|nvp504,nvr661,26,31,-1;mov.b32 nvf1116,nvr662;neg.ftz.f32 nvf1117,nvf1115;fma.rn.ftz.f32 nvf1118,nvf1117,nvf1116,nvf1101;shfl.sync.idx.b32 nvr663|nvp505,nvr661,27,31,-1;mov.b32 nvf1119,nvr663;fma.rn.ftz.f32 nvf1120,nvf1117,nvf1119,nvf1103;shfl.sync.idx.b32 nvr664|nvp506,nvr661,28,31,-1;mov.b32 nvf1121,nvr664;fma.rn.ftz.f32 nvf1122,nvf1117,nvf1121,nvf1105;shfl.sync.idx.b32 nvr665|nvp507,nvr661,29,31,-1;mov.b32 nvf1123,nvr665;fma.rn.ftz.f32 nvf1124,nvf1117,nvf1123,nvf1107;shfl.sync.idx.b32 nvr666|nvp508,nvr661,30,31,-1;mov.b32 nvf1125,nvr666;fma.rn.ftz.f32 nvf1126,nvf1117,nvf1125,nvf1109;shfl.sync.idx.b32 nvr667|nvp509,nvr661,31,31,-1;mov.b32 nvf1127,nvr667;fma.rn.ftz.f32 nvf1128,nvf1117,nvf1127,nvf1111;max.ftz.f32 nvf1129,nvf1118,0f00800000;rsqrt.approx.ftz.f32 nvf1130,nvf1129;mov.b32 nvr668,nvf1130;shfl.sync.idx.b32 nvr669|nvp510,nvr668,26,31,-1;mov.b32 nvf1131,nvr669;mul.ftz.f32 nvf1132,nvf1118,nvf1131;mov.b32 nvr670,nvf1132;shfl.sync.idx.b32 nvr671|nvp511,nvr670,27,31,-1;mov.b32 nvf1133,nvr671;neg.ftz.f32 nvf1134,nvf1132;fma.rn.ftz.f32 nvf1135,nvf1134,nvf1133,nvf1120;shfl.sync.idx.b32 nvr672|nvp512,nvr670,28,31,-1;mov.b32 nvf1136,nvr672;fma.rn.ftz.f32 nvf1137,nvf1134,nvf1136,nvf1122;shfl.sync.idx.b32 nvr673|nvp513,nvr670,29,31,-1;mov.b32 nvf1138,nvr673;fma.rn.ftz.f32 nvf1139,nvf1134,nvf1138,nvf1124;shfl.sync.idx.b32 nvr674|nvp514,nvr670,30,31,-1;mov.b32 nvf1140,nvr674;fma.rn.ftz.f32 nvf1141,nvf1134,nvf1140,nvf1126;shfl.sync.idx.b32 nvr675|nvp515,nvr670,31,31,-1;mov.b32 nvf1142,nvr675;fma.rn.ftz.f32 nvf1143,nvf1134,nvf1142,nvf1128;max.ftz.f32 nvf1144,nvf1135,0f00800000;rsqrt.approx.ftz.f32 nvf1145,nvf1144;mov.b32 nvr676,nvf1145;shfl.sync.idx.b32 nvr677|nvp516,nvr676,27,31,-1;mov.b32 nvf1146,nvr677;mul.ftz.f32 nvf1147,nvf1135,nvf1146;mov.b32 nvr678,nvf1147;shfl.sync.idx.b32 nvr679|nvp517,nvr678,28,31,-1;mov.b32 nvf1148,nvr679;neg.ftz.f32 nvf1149,nvf1147;fma.rn.ftz.f32 nvf1150,nvf1149,nvf1148,nvf1137;shfl.sync.idx.b32 nvr680|nvp518,nvr678,29,31,-1;mov.b32 nvf1151,nvr680;fma.rn.ftz.f32 nvf1152,nvf1149,nvf1151,nvf1139;shfl.sync.idx.b32 nvr681|nvp519,nvr678,30,31,-1;mov.b32 nvf1153,nvr681;fma.rn.ftz.f32 nvf1154,nvf1149,nvf1153,nvf1141;shfl.sync.idx.b32 nvr682|nvp520,nvr678,31,31,-1;mov.b32 nvf1155,nvr682;fma.rn.ftz.f32 nvf1156,nvf1149,nvf1155,nvf1143;max.ftz.f32 nvf1157,nvf1150,0f00800000;rsqrt.approx.ftz.f32 nvf1158,nvf1157;mov.b32 nvr683,nvf1158;shfl.sync.idx.b32 nvr684|nvp521,nvr683,28,31,-1;mov.b32 nvf1159,nvr684;mul.ftz.f32 nvf1160,nvf1150,nvf1159;mov.b32 nvr685,nvf1160;shfl.sync.idx.b32 nvr686|nvp522,nvr685,29,31,-1;mov.b32 nvf1161,nvr686;neg.ftz.f32 nvf1162,nvf1160;fma.rn.ftz.f32 nvf1163,nvf1162,nvf1161,nvf1152;shfl.sync.idx.b32 nvr687|nvp523,nvr685,30,31,-1;mov.b32 nvf1164,nvr687;fma.rn.ftz.f32 nvf1165,nvf1162,nvf1164,nvf1154;shfl.sync.idx.b32 nvr688|nvp524,nvr685,31,31,-1;mov.b32 nvf1166,nvr688;fma.rn.ftz.f32 nvf1167,nvf1162,nvf1166,nvf1156;max.ftz.f32 nvf1168,nvf1163,0f00800000;rsqrt.approx.ftz.f32 nvf1169,nvf1168;mov.b32 nvr689,nvf1169;shfl.sync.idx.b32 nvr690|nvp525,nvr689,29,31,-1;mov.b32 nvf1170,nvr690;mul.ftz.f32 nvf1171,nvf1163,nvf1170;mov.b32 nvr691,nvf1171;shfl.sync.idx.b32 nvr692|nvp526,nvr691,30,31,-1;mov.b32 nvf1172,nvr692;neg.ftz.f32 nvf1173,nvf1171;fma.rn.ftz.f32 nvf1174,nvf1173,nvf1172,nvf1165;shfl.sync.idx.b32 nvr693|nvp527,nvr691,31,31,-1;mov.b32 nvf1175,nvr693;fma.rn.ftz.f32 nvf1176,nvf1173,nvf1175,nvf1167;max.ftz.f32 nvf1177,nvf1174,0f00800000;rsqrt.approx.ftz.f32 nvf1178,nvf1177;mov.b32 nvr694,nvf1178;shfl.sync.idx.b32 nvr695|nvp528,nvr694,30,31,-1;mov.b32 nvf1179,nvr695;mul.ftz.f32 nvf1180,nvf1174,nvf1179;mov.b32 nvr696,nvf1180;shfl.sync.idx.b32 nvr697|nvp529,nvr696,31,31,-1;mov.b32 nvf1181,nvr697;neg.ftz.f32 nvf1182,nvf1180;fma.rn.ftz.f32 nvf1183,nvf1182,nvf1181,nvf1176;max.ftz.f32 nvf1184,nvf1183,0f00800000;rsqrt.approx.ftz.f32 nvf1185,nvf1184;mov.b32 nvr698,nvf1185;shfl.sync.idx.b32 nvr699|nvp530,nvr698,31,31,-1;mov.b32 nvf1186,nvr699;mul.ftz.f32 nvf1187,nvf1183,nvf1186;st.shared.f32[nvrd68],nvf40;setp.eq.s32 nvp531,nvr23,0;selp.f32 nvf1188,0f00000000,nvf107,nvp531;st.shared.f32[nvrd70],nvf1188;setp.gt.u32 nvp532,nvr23,1;selp.f32 nvf1189,nvf172,0f00000000,nvp532;st.shared.f32[nvrd72],nvf1189;setp.gt.u32 nvp533,nvr23,2;selp.f32 nvf1190,nvf235,0f00000000,nvp533;st.shared.f32[nvrd74],nvf1190;setp.gt.u32 nvp534,nvr23,3;selp.f32 nvf1191,nvf296,0f00000000,nvp534;st.shared.f32[nvrd76],nvf1191;setp.gt.u32 nvp535,nvr23,4;selp.f32 nvf1192,nvf355,0f00000000,nvp535;st.shared.f32[nvrd78],nvf1192;setp.gt.u32 nvp536,nvr23,5;selp.f32 nvf1193,nvf412,0f00000000,nvp536;st.shared.f32[nvrd80],nvf1193;setp.gt.u32 nvp537,nvr23,6;selp.f32 nvf1194,nvf467,0f00000000,nvp537;st.shared.f32[nvrd82],nvf1194;setp.gt.u32 nvp538,nvr23,7;selp.f32 nvf1195,nvf520,0f00000000,nvp538;st.shared.f32[nvrd84],nvf1195;setp.gt.u32 nvp539,nvr23,8;selp.f32 nvf1196,nvf571,0f00000000,nvp539;st.shared.f32[nvrd86],nvf1196;setp.gt.u32 nvp540,nvr23,9;selp.f32 nvf1197,nvf620,0f00000000,nvp540;st.shared.f32[nvrd88],nvf1197;setp.gt.u32 nvp541,nvr23,10;selp.f32 nvf1198,nvf667,0f00000000,nvp541;st.shared.f32[nvrd90],nvf1198;setp.gt.u32 nvp542,nvr23,11;selp.f32 nvf1199,nvf712,0f00000000,nvp542;st.shared.f32[nvrd92],nvf1199;setp.gt.u32 nvp543,nvr23,12;selp.f32 nvf1200,nvf755,0f00000000,nvp543;st.shared.f32[nvrd94],nvf1200;setp.gt.u32 nvp544,nvr23,13;selp.f32 nvf1201,nvf796,0f00000000,nvp544;st.shared.f32[nvrd96],nvf1201;setp.gt.u32 nvp545,nvr23,14;selp.f32 nvf1202,nvf835,0f00000000,nvp545;st.shared.f32[nvrd98],nvf1202;setp.gt.u32 nvp546,nvr23,15;selp.f32 nvf1203,nvf872,0f00000000,nvp546;st.shared.f32[nvrd100],nvf1203;setp.gt.u32 nvp547,nvr23,16;selp.f32 nvf1204,nvf907,0f00000000,nvp547;st.shared.f32[nvrd102],nvf1204;setp.gt.u32 nvp548,nvr23,17;selp.f32 nvf1205,nvf940,0f00000000,nvp548;st.shared.f32[nvrd104],nvf1205;setp.gt.u32 nvp549,nvr23,18;selp.f32 nvf1206,nvf971,0f00000000,nvp549;st.shared.f32[nvrd106],nvf1206;setp.gt.u32 nvp550,nvr23,19;selp.f32 nvf1207,nvf1000,0f00000000,nvp550;st.shared.f32[nvrd108],nvf1207;setp.gt.u32 nvp551,nvr23,20;selp.f32 nvf1208,nvf1027,0f00000000,nvp551;st.shared.f32[nvrd110],nvf1208;setp.gt.u32 nvp552,nvr23,21;selp.f32 nvf1209,nvf1052,0f00000000,nvp552;st.shared.f32[nvrd112],nvf1209;setp.gt.u32 nvp553,nvr23,22;selp.f32 nvf1210,nvf1075,0f00000000,nvp553;st.shared.f32[nvrd114],nvf1210;setp.gt.u32 nvp554,nvr23,23;selp.f32 nvf1211,nvf1096,0f00000000,nvp554;st.shared.f32[nvrd116],nvf1211;setp.gt.u32 nvp555,nvr23,24;selp.f32 nvf1212,nvf1115,0f00000000,nvp555;st.shared.f32[nvrd118],nvf1212;setp.gt.u32 nvp556,nvr23,25;selp.f32 nvf1213,nvf1132,0f00000000,nvp556;st.shared.f32[nvrd120],nvf1213;setp.gt.u32 nvp557,nvr23,26;selp.f32 nvf1214,nvf1147,0f00000000,nvp557;st.shared.f32[nvrd122],nvf1214;setp.gt.u32 nvp558,nvr23,27;selp.f32 nvf1215,nvf1160,0f00000000,nvp558;st.shared.f32[nvrd124],nvf1215;setp.gt.u32 nvp559,nvr23,28;selp.f32 nvf1216,nvf1171,0f00000000,nvp559;st.shared.f32[nvrd126],nvf1216;setp.gt.u32 nvp560,nvr23,29;selp.f32 nvf1217,nvf1180,0f00000000,nvp560;st.shared.f32[nvrd128],nvf1217;setp.eq.s32 nvp561,nvr23,31;selp.f32 nvf1218,nvf1187,0f00000000,nvp561;st.shared.f32[nvrd130],nvf1218;bar.warp.sync -1;mov.u64 nvrd131,nvrd153;mov.u32 nvr700,%ctaid.x;mul.wide.u32 nvrd132,nvr700,4096;or.b64 nvrd133,nvrd132,nvrd64;or.b64 nvrd134,nvrd133,nvrd63;shl.b64 nvrd135,nvrd134,2;add.s64 nvrd136,nvrd135,nvrd131;add.s64 nvrd159,nvrd136,256;mov.u32 nvr714,nvr23;L__BB0_4:\nxor.b32 nvr701,nvr23,nvr710;cvt.u64.u32 nvrd137,nvr701;add.s64 nvrd138,nvrd160,nvrd137;shl.b64 nvrd139,nvrd138,2;add.s64 nvrd140,nvrd21,nvrd139;ld.shared.f32 nvf1219,[nvrd140];st.global.f32[nvrd159+-256],nvf1219;xor.b32 nvr702,nvr23,nvr711;cvt.u64.u32 nvrd141,nvr702;add.s64 nvrd142,nvrd161,nvrd141;shl.b64 nvrd143,nvrd142,2;add.s64 nvrd144,nvrd21,nvrd143;ld.shared.f32 nvf1220,[nvrd144];st.global.f32[nvrd159+-128],nvf1220;xor.b32 nvr703,nvr23,nvr712;cvt.u64.u32 nvrd145,nvr703;add.s64 nvrd146,nvrd162,nvrd145;shl.b64 nvrd147,nvrd146,2;add.s64 nvrd148,nvrd21,nvrd147;ld.shared.f32 nvf1221,[nvrd148];st.global.f32[nvrd159],nvf1221;xor.b32 nvr704,nvr23,nvr713;cvt.u64.u32 nvrd149,nvr704;add.s64 nvrd150,nvrd163,nvrd149;shl.b64 nvrd151,nvrd150,2;add.s64 nvrd152,nvrd21,nvrd151;ld.shared.f32 nvf1222,[nvrd152];st.global.f32[nvrd159+128],nvf1222;add.s32 nvr29,nvr714,128;add.s32 nvr713,nvr713,4;add.s64 nvrd163,nvrd163,128;add.s32 nvr712,nvr712,4;add.s64 nvrd162,nvrd162,128;add.s64 nvrd161,nvrd161,128;add.s32 nvr711,nvr711,4;add.s32 nvr710,nvr710,4;add.s64 nvrd160,nvrd160,128;add.s64 nvrd159,nvrd159,512;setp.lt.u32 nvp562,nvr714,896;mov.u32 nvr714,nvr29;@nvp562 bra L__BB0_4;L__BB0_5:\nret;}'
_POTRF64_NVVM_ASM = '{.reg .pred nvp<1139>;.reg .b32 nvr<1548>;.reg .b32 nvf<5692>;.reg .b64 nvrd<498>;mov.u64 nvrd125,$1;mov.u64 nvrd126,$2;mov.u64 nvrd1,nvrd125;mov.u64 nvrd2,nvrd126;mov.u32 nvr1,%tid.x;mov.u32 nvr2,%ctaid.x;shl.b32 nvr89,nvr2,12;mov.b32 nvr1521,3;or.b32 nvr90,nvr1,nvr89;or.b32 nvr91,nvr90,192;mul.wide.u32 nvrd3,nvr91,4;add.s64 nvrd461,nvrd2,nvrd3;mov.b32 nvr1520,2;mov.b32 nvr1518,0;or.b32 nvr92,nvr90,128;mul.wide.u32 nvrd5,nvr92,4;add.s64 nvrd460,nvrd2,nvrd5;mul.wide.u32 nvrd7,nvr90,4;add.s64 nvrd459,nvrd2,nvrd7;mov.b32 nvr1519,1;or.b32 nvr93,nvr90,64;mul.wide.u32 nvrd9,nvr93,4;add.s64 nvrd458,nvrd2,nvrd9;mov.u32 nvr1513,nvr1519;mov.u32 nvr1514,nvr1518;mov.u32 nvr1515,nvr1520;mov.u32 nvr1516,nvr1521;mov.u32 nvr1517,nvr1518;L__BB0_2:\n.pragma "nounroll";mov.b32 nvr94,0;st.global.u32[nvrd459+128],nvr94;setp.le.u32 nvp2,nvr1,nvr1514;@nvp2 bra L__BB0_4;mov.b32 nvr95,0;st.global.u32[nvrd459],nvr95;st.global.u32[nvrd459+8320],nvr95;L__BB0_4:\nmov.b32 nvr96,0;st.global.u32[nvrd458+128],nvr96;setp.le.u32 nvp3,nvr1,nvr1513;@nvp3 bra L__BB0_6;mov.b32 nvr97,0;st.global.u32[nvrd458],nvr97;st.global.u32[nvrd458+8320],nvr97;L__BB0_6:\nmov.b32 nvr98,0;st.global.u32[nvrd460+128],nvr98;setp.le.u32 nvp4,nvr1,nvr1515;@nvp4 bra L__BB0_8;mov.b32 nvr99,0;st.global.u32[nvrd460],nvr99;st.global.u32[nvrd460+8320],nvr99;L__BB0_8:\nmov.b32 nvr100,0;st.global.u32[nvrd461+128],nvr100;setp.le.u32 nvp5,nvr1,nvr1516;@nvp5 bra L__BB0_10;mov.b32 nvr101,0;st.global.u32[nvrd461],nvr101;st.global.u32[nvrd461+8320],nvr101;L__BB0_10:\nadd.s32 nvr1517,nvr1517,4;add.s32 nvr1516,nvr1516,4;add.s64 nvrd461,nvrd461,1024;add.s32 nvr1515,nvr1515,4;add.s32 nvr1514,nvr1514,4;add.s64 nvrd460,nvrd460,1024;add.s64 nvrd459,nvrd459,1024;add.s32 nvr1513,nvr1513,4;add.s64 nvrd458,nvrd458,1024;setp.ne.s32 nvp6,nvr1517,32;@nvp6 bra L__BB0_2;add.s64 nvrd469,nvrd1,nvrd3;mov.b64 nvrd468,96;add.s64 nvrd467,nvrd1,nvrd5;mov.b64 nvrd466,64;mov.b64 nvrd465,32;add.s64 nvrd464,nvrd1,nvrd9;mov.b64 nvrd463,0;add.s64 nvrd462,nvrd1,nvrd7;mov.u32 nvr1522,nvr1;L__BB0_12:\nld.global.nc.f32 nvf1,[nvrd462];xor.b32 nvr102,nvr1,nvr1518;cvt.u64.u32 nvrd127,nvr102;add.s64 nvrd128,nvrd463,nvrd127;shl.b64 nvrd129,nvrd128,2;mov.u64 nvrd130,global_smem;add.s64 nvrd131,nvrd130,nvrd129;st.shared.f32[nvrd131],nvf1;ld.global.nc.f32 nvf2,[nvrd464];xor.b32 nvr103,nvr1,nvr1519;cvt.u64.u32 nvrd132,nvr103;add.s64 nvrd133,nvrd465,nvrd132;shl.b64 nvrd134,nvrd133,2;add.s64 nvrd135,nvrd130,nvrd134;st.shared.f32[nvrd135],nvf2;ld.global.nc.f32 nvf3,[nvrd467];xor.b32 nvr104,nvr1,nvr1520;cvt.u64.u32 nvrd136,nvr104;add.s64 nvrd137,nvrd466,nvrd136;shl.b64 nvrd138,nvrd137,2;add.s64 nvrd139,nvrd130,nvrd138;st.shared.f32[nvrd139],nvf3;ld.global.nc.f32 nvf4,[nvrd469];xor.b32 nvr105,nvr1,nvr1521;cvt.u64.u32 nvrd140,nvr105;add.s64 nvrd141,nvrd468,nvrd140;shl.b64 nvrd142,nvrd141,2;add.s64 nvrd143,nvrd130,nvrd142;st.shared.f32[nvrd143],nvf4;add.s32 nvr22,nvr1522,128;add.s32 nvr1521,nvr1521,4;add.s64 nvrd469,nvrd469,1024;add.s64 nvrd468,nvrd468,128;add.s32 nvr1520,nvr1520,4;add.s64 nvrd467,nvrd467,1024;add.s64 nvrd466,nvrd466,128;add.s64 nvrd465,nvrd465,128;add.s32 nvr1519,nvr1519,4;add.s64 nvrd464,nvrd464,1024;add.s64 nvrd463,nvrd463,128;add.s64 nvrd462,nvrd462,1024;add.s32 nvr1518,nvr1518,4;setp.lt.u32 nvp7,nvr1522,896;mov.u32 nvr1522,nvr22;@nvp7 bra L__BB0_12;mov.u64 nvrd455,$2;mov.u32 nvr27,%tid.x;mov.b32 nvr1531,3;mov.b32 nvr1530,2;mov.b32 nvr1529,1;mov.b64 nvrd479,32;mov.b64 nvrd480,64;mov.b64 nvrd481,96;mov.b64 nvrd478,0;mov.b32 nvr1528,0;mov.u64 nvrd144,nvrd455;mov.u32 nvr106,%ctaid.x;shl.b32 nvr107,nvr106,12;or.b32 nvr108,nvr27,nvr107;or.b32 nvr109,nvr108,192;mul.wide.u32 nvrd47,nvr109,4;add.s64 nvrd477,nvrd144,nvrd47;mul.wide.u32 nvrd49,nvr108,4;add.s64 nvrd470,nvrd144,nvrd49;or.b32 nvr110,nvr108,128;mul.wide.u32 nvrd51,nvr110,4;add.s64 nvrd475,nvrd144,nvrd51;or.b32 nvr111,nvr108,64;mul.wide.u32 nvrd53,nvr111,4;add.s64 nvrd471,nvrd144,nvrd53;bar.warp.sync -1;shl.b32 nvr112,nvr27,5;mul.lo.s32 nvr113,nvr27,33;mul.wide.u32 nvrd145,nvr113,4;mov.u64 nvrd146,global_smem;add.s64 nvrd147,nvrd146,nvrd145;ld.shared.f32 nvf5,[nvrd147];xor.b32 nvr114,nvr27,1;or.b32 nvr115,nvr112,nvr114;mul.wide.u32 nvrd148,nvr115,4;add.s64 nvrd149,nvrd146,nvrd148;ld.shared.f32 nvf6,[nvrd149];xor.b32 nvr116,nvr27,2;or.b32 nvr117,nvr112,nvr116;mul.wide.u32 nvrd150,nvr117,4;add.s64 nvrd151,nvrd146,nvrd150;ld.shared.f32 nvf7,[nvrd151];xor.b32 nvr118,nvr27,3;or.b32 nvr119,nvr112,nvr118;mul.wide.u32 nvrd152,nvr119,4;add.s64 nvrd153,nvrd146,nvrd152;ld.shared.f32 nvf8,[nvrd153];xor.b32 nvr120,nvr27,4;or.b32 nvr121,nvr112,nvr120;mul.wide.u32 nvrd154,nvr121,4;add.s64 nvrd155,nvrd146,nvrd154;ld.shared.f32 nvf9,[nvrd155];xor.b32 nvr122,nvr27,5;or.b32 nvr123,nvr112,nvr122;mul.wide.u32 nvrd156,nvr123,4;add.s64 nvrd157,nvrd146,nvrd156;ld.shared.f32 nvf10,[nvrd157];xor.b32 nvr124,nvr27,6;or.b32 nvr125,nvr112,nvr124;mul.wide.u32 nvrd158,nvr125,4;add.s64 nvrd159,nvrd146,nvrd158;ld.shared.f32 nvf11,[nvrd159];xor.b32 nvr126,nvr27,7;or.b32 nvr127,nvr112,nvr126;mul.wide.u32 nvrd160,nvr127,4;add.s64 nvrd161,nvrd146,nvrd160;ld.shared.f32 nvf12,[nvrd161];xor.b32 nvr128,nvr27,8;or.b32 nvr129,nvr112,nvr128;mul.wide.u32 nvrd162,nvr129,4;add.s64 nvrd163,nvrd146,nvrd162;ld.shared.f32 nvf13,[nvrd163];xor.b32 nvr130,nvr27,9;or.b32 nvr131,nvr112,nvr130;mul.wide.u32 nvrd164,nvr131,4;add.s64 nvrd165,nvrd146,nvrd164;ld.shared.f32 nvf14,[nvrd165];xor.b32 nvr132,nvr27,10;or.b32 nvr133,nvr112,nvr132;mul.wide.u32 nvrd166,nvr133,4;add.s64 nvrd167,nvrd146,nvrd166;ld.shared.f32 nvf15,[nvrd167];xor.b32 nvr134,nvr27,11;or.b32 nvr135,nvr112,nvr134;mul.wide.u32 nvrd168,nvr135,4;add.s64 nvrd169,nvrd146,nvrd168;ld.shared.f32 nvf16,[nvrd169];xor.b32 nvr136,nvr27,12;or.b32 nvr137,nvr112,nvr136;mul.wide.u32 nvrd170,nvr137,4;add.s64 nvrd171,nvrd146,nvrd170;ld.shared.f32 nvf17,[nvrd171];xor.b32 nvr138,nvr27,13;or.b32 nvr139,nvr112,nvr138;mul.wide.u32 nvrd172,nvr139,4;add.s64 nvrd173,nvrd146,nvrd172;ld.shared.f32 nvf18,[nvrd173];xor.b32 nvr140,nvr27,14;or.b32 nvr141,nvr112,nvr140;mul.wide.u32 nvrd174,nvr141,4;add.s64 nvrd175,nvrd146,nvrd174;ld.shared.f32 nvf19,[nvrd175];xor.b32 nvr142,nvr27,15;or.b32 nvr143,nvr112,nvr142;mul.wide.u32 nvrd176,nvr143,4;add.s64 nvrd177,nvrd146,nvrd176;ld.shared.f32 nvf20,[nvrd177];xor.b32 nvr144,nvr27,16;or.b32 nvr145,nvr112,nvr144;mul.wide.u32 nvrd178,nvr145,4;add.s64 nvrd179,nvrd146,nvrd178;ld.shared.f32 nvf21,[nvrd179];xor.b32 nvr146,nvr27,17;or.b32 nvr147,nvr112,nvr146;mul.wide.u32 nvrd180,nvr147,4;add.s64 nvrd181,nvrd146,nvrd180;ld.shared.f32 nvf22,[nvrd181];xor.b32 nvr148,nvr27,18;or.b32 nvr149,nvr112,nvr148;mul.wide.u32 nvrd182,nvr149,4;add.s64 nvrd183,nvrd146,nvrd182;ld.shared.f32 nvf23,[nvrd183];xor.b32 nvr150,nvr27,19;or.b32 nvr151,nvr112,nvr150;mul.wide.u32 nvrd184,nvr151,4;add.s64 nvrd185,nvrd146,nvrd184;ld.shared.f32 nvf24,[nvrd185];xor.b32 nvr152,nvr27,20;or.b32 nvr153,nvr112,nvr152;mul.wide.u32 nvrd186,nvr153,4;add.s64 nvrd187,nvrd146,nvrd186;ld.shared.f32 nvf25,[nvrd187];xor.b32 nvr154,nvr27,21;or.b32 nvr155,nvr112,nvr154;mul.wide.u32 nvrd188,nvr155,4;add.s64 nvrd189,nvrd146,nvrd188;ld.shared.f32 nvf26,[nvrd189];xor.b32 nvr156,nvr27,22;or.b32 nvr157,nvr112,nvr156;mul.wide.u32 nvrd190,nvr157,4;add.s64 nvrd191,nvrd146,nvrd190;ld.shared.f32 nvf27,[nvrd191];xor.b32 nvr158,nvr27,23;or.b32 nvr159,nvr112,nvr158;mul.wide.u32 nvrd192,nvr159,4;add.s64 nvrd193,nvrd146,nvrd192;ld.shared.f32 nvf28,[nvrd193];xor.b32 nvr160,nvr27,24;or.b32 nvr161,nvr112,nvr160;mul.wide.u32 nvrd194,nvr161,4;add.s64 nvrd195,nvrd146,nvrd194;ld.shared.f32 nvf29,[nvrd195];xor.b32 nvr162,nvr27,25;or.b32 nvr163,nvr112,nvr162;mul.wide.u32 nvrd196,nvr163,4;add.s64 nvrd197,nvrd146,nvrd196;ld.shared.f32 nvf30,[nvrd197];xor.b32 nvr164,nvr27,26;or.b32 nvr165,nvr112,nvr164;mul.wide.u32 nvrd198,nvr165,4;add.s64 nvrd199,nvrd146,nvrd198;ld.shared.f32 nvf31,[nvrd199];xor.b32 nvr166,nvr27,27;or.b32 nvr167,nvr112,nvr166;mul.wide.u32 nvrd200,nvr167,4;add.s64 nvrd201,nvrd146,nvrd200;ld.shared.f32 nvf32,[nvrd201];xor.b32 nvr168,nvr27,28;or.b32 nvr169,nvr112,nvr168;mul.wide.u32 nvrd202,nvr169,4;add.s64 nvrd203,nvrd146,nvrd202;ld.shared.f32 nvf33,[nvrd203];xor.b32 nvr170,nvr27,29;or.b32 nvr171,nvr112,nvr170;mul.wide.u32 nvrd204,nvr171,4;add.s64 nvrd205,nvrd146,nvrd204;ld.shared.f32 nvf34,[nvrd205];xor.b32 nvr172,nvr27,30;or.b32 nvr173,nvr112,nvr172;mul.wide.u32 nvrd206,nvr173,4;add.s64 nvrd207,nvrd146,nvrd206;ld.shared.f32 nvf35,[nvrd207];xor.b32 nvr174,nvr27,31;or.b32 nvr175,nvr112,nvr174;mul.wide.u32 nvrd208,nvr175,4;add.s64 nvrd209,nvrd146,nvrd208;ld.shared.f32 nvf36,[nvrd209];max.ftz.f32 nvf37,nvf5,0f00800000;sqrt.approx.ftz.f32 nvf38,nvf37;mov.b32 nvr176,nvf38;shfl.sync.idx.b32 nvr177|nvp8,nvr176,0,31,-1;mov.b32 nvf39,nvr177;div.approx.ftz.f32 nvf40,nvf5,nvf39;mov.b32 nvr178,nvf40;shfl.sync.idx.b32 nvr179|nvp9,nvr178,1,31,-1;mov.b32 nvf41,nvr179;neg.ftz.f32 nvf42,nvf40;fma.rn.ftz.f32 nvf43,nvf42,nvf41,nvf6;shfl.sync.idx.b32 nvr180|nvp10,nvr178,2,31,-1;mov.b32 nvf44,nvr180;fma.rn.ftz.f32 nvf45,nvf42,nvf44,nvf7;shfl.sync.idx.b32 nvr181|nvp11,nvr178,3,31,-1;mov.b32 nvf46,nvr181;fma.rn.ftz.f32 nvf47,nvf42,nvf46,nvf8;shfl.sync.idx.b32 nvr182|nvp12,nvr178,4,31,-1;mov.b32 nvf48,nvr182;fma.rn.ftz.f32 nvf49,nvf42,nvf48,nvf9;shfl.sync.idx.b32 nvr183|nvp13,nvr178,5,31,-1;mov.b32 nvf50,nvr183;fma.rn.ftz.f32 nvf51,nvf42,nvf50,nvf10;shfl.sync.idx.b32 nvr184|nvp14,nvr178,6,31,-1;mov.b32 nvf52,nvr184;fma.rn.ftz.f32 nvf53,nvf42,nvf52,nvf11;shfl.sync.idx.b32 nvr185|nvp15,nvr178,7,31,-1;mov.b32 nvf54,nvr185;fma.rn.ftz.f32 nvf55,nvf42,nvf54,nvf12;shfl.sync.idx.b32 nvr186|nvp16,nvr178,8,31,-1;mov.b32 nvf56,nvr186;fma.rn.ftz.f32 nvf57,nvf42,nvf56,nvf13;shfl.sync.idx.b32 nvr187|nvp17,nvr178,9,31,-1;mov.b32 nvf58,nvr187;fma.rn.ftz.f32 nvf59,nvf42,nvf58,nvf14;shfl.sync.idx.b32 nvr188|nvp18,nvr178,10,31,-1;mov.b32 nvf60,nvr188;fma.rn.ftz.f32 nvf61,nvf42,nvf60,nvf15;shfl.sync.idx.b32 nvr189|nvp19,nvr178,11,31,-1;mov.b32 nvf62,nvr189;fma.rn.ftz.f32 nvf63,nvf42,nvf62,nvf16;shfl.sync.idx.b32 nvr190|nvp20,nvr178,12,31,-1;mov.b32 nvf64,nvr190;fma.rn.ftz.f32 nvf65,nvf42,nvf64,nvf17;shfl.sync.idx.b32 nvr191|nvp21,nvr178,13,31,-1;mov.b32 nvf66,nvr191;fma.rn.ftz.f32 nvf67,nvf42,nvf66,nvf18;shfl.sync.idx.b32 nvr192|nvp22,nvr178,14,31,-1;mov.b32 nvf68,nvr192;fma.rn.ftz.f32 nvf69,nvf42,nvf68,nvf19;shfl.sync.idx.b32 nvr193|nvp23,nvr178,15,31,-1;mov.b32 nvf70,nvr193;fma.rn.ftz.f32 nvf71,nvf42,nvf70,nvf20;shfl.sync.idx.b32 nvr194|nvp24,nvr178,16,31,-1;mov.b32 nvf72,nvr194;fma.rn.ftz.f32 nvf73,nvf42,nvf72,nvf21;shfl.sync.idx.b32 nvr195|nvp25,nvr178,17,31,-1;mov.b32 nvf74,nvr195;fma.rn.ftz.f32 nvf75,nvf42,nvf74,nvf22;shfl.sync.idx.b32 nvr196|nvp26,nvr178,18,31,-1;mov.b32 nvf76,nvr196;fma.rn.ftz.f32 nvf77,nvf42,nvf76,nvf23;shfl.sync.idx.b32 nvr197|nvp27,nvr178,19,31,-1;mov.b32 nvf78,nvr197;fma.rn.ftz.f32 nvf79,nvf42,nvf78,nvf24;shfl.sync.idx.b32 nvr198|nvp28,nvr178,20,31,-1;mov.b32 nvf80,nvr198;fma.rn.ftz.f32 nvf81,nvf42,nvf80,nvf25;shfl.sync.idx.b32 nvr199|nvp29,nvr178,21,31,-1;mov.b32 nvf82,nvr199;fma.rn.ftz.f32 nvf83,nvf42,nvf82,nvf26;shfl.sync.idx.b32 nvr200|nvp30,nvr178,22,31,-1;mov.b32 nvf84,nvr200;fma.rn.ftz.f32 nvf85,nvf42,nvf84,nvf27;shfl.sync.idx.b32 nvr201|nvp31,nvr178,23,31,-1;mov.b32 nvf86,nvr201;fma.rn.ftz.f32 nvf87,nvf42,nvf86,nvf28;shfl.sync.idx.b32 nvr202|nvp32,nvr178,24,31,-1;mov.b32 nvf88,nvr202;fma.rn.ftz.f32 nvf89,nvf42,nvf88,nvf29;shfl.sync.idx.b32 nvr203|nvp33,nvr178,25,31,-1;mov.b32 nvf90,nvr203;fma.rn.ftz.f32 nvf91,nvf42,nvf90,nvf30;shfl.sync.idx.b32 nvr204|nvp34,nvr178,26,31,-1;mov.b32 nvf92,nvr204;fma.rn.ftz.f32 nvf93,nvf42,nvf92,nvf31;shfl.sync.idx.b32 nvr205|nvp35,nvr178,27,31,-1;mov.b32 nvf94,nvr205;fma.rn.ftz.f32 nvf95,nvf42,nvf94,nvf32;shfl.sync.idx.b32 nvr206|nvp36,nvr178,28,31,-1;mov.b32 nvf96,nvr206;fma.rn.ftz.f32 nvf97,nvf42,nvf96,nvf33;shfl.sync.idx.b32 nvr207|nvp37,nvr178,29,31,-1;mov.b32 nvf98,nvr207;fma.rn.ftz.f32 nvf99,nvf42,nvf98,nvf34;shfl.sync.idx.b32 nvr208|nvp38,nvr178,30,31,-1;mov.b32 nvf100,nvr208;fma.rn.ftz.f32 nvf101,nvf42,nvf100,nvf35;shfl.sync.idx.b32 nvr209|nvp39,nvr178,31,31,-1;mov.b32 nvf102,nvr209;fma.rn.ftz.f32 nvf103,nvf42,nvf102,nvf36;max.ftz.f32 nvf104,nvf43,0f00800000;sqrt.approx.ftz.f32 nvf105,nvf104;mov.b32 nvr210,nvf105;shfl.sync.idx.b32 nvr211|nvp40,nvr210,1,31,-1;mov.b32 nvf106,nvr211;div.approx.ftz.f32 nvf107,nvf43,nvf106;mov.b32 nvr212,nvf107;shfl.sync.idx.b32 nvr213|nvp41,nvr212,2,31,-1;mov.b32 nvf108,nvr213;neg.ftz.f32 nvf109,nvf107;fma.rn.ftz.f32 nvf110,nvf109,nvf108,nvf45;shfl.sync.idx.b32 nvr214|nvp42,nvr212,3,31,-1;mov.b32 nvf111,nvr214;fma.rn.ftz.f32 nvf112,nvf109,nvf111,nvf47;shfl.sync.idx.b32 nvr215|nvp43,nvr212,4,31,-1;mov.b32 nvf113,nvr215;fma.rn.ftz.f32 nvf114,nvf109,nvf113,nvf49;shfl.sync.idx.b32 nvr216|nvp44,nvr212,5,31,-1;mov.b32 nvf115,nvr216;fma.rn.ftz.f32 nvf116,nvf109,nvf115,nvf51;shfl.sync.idx.b32 nvr217|nvp45,nvr212,6,31,-1;mov.b32 nvf117,nvr217;fma.rn.ftz.f32 nvf118,nvf109,nvf117,nvf53;shfl.sync.idx.b32 nvr218|nvp46,nvr212,7,31,-1;mov.b32 nvf119,nvr218;fma.rn.ftz.f32 nvf120,nvf109,nvf119,nvf55;shfl.sync.idx.b32 nvr219|nvp47,nvr212,8,31,-1;mov.b32 nvf121,nvr219;fma.rn.ftz.f32 nvf122,nvf109,nvf121,nvf57;shfl.sync.idx.b32 nvr220|nvp48,nvr212,9,31,-1;mov.b32 nvf123,nvr220;fma.rn.ftz.f32 nvf124,nvf109,nvf123,nvf59;shfl.sync.idx.b32 nvr221|nvp49,nvr212,10,31,-1;mov.b32 nvf125,nvr221;fma.rn.ftz.f32 nvf126,nvf109,nvf125,nvf61;shfl.sync.idx.b32 nvr222|nvp50,nvr212,11,31,-1;mov.b32 nvf127,nvr222;fma.rn.ftz.f32 nvf128,nvf109,nvf127,nvf63;shfl.sync.idx.b32 nvr223|nvp51,nvr212,12,31,-1;mov.b32 nvf129,nvr223;fma.rn.ftz.f32 nvf130,nvf109,nvf129,nvf65;shfl.sync.idx.b32 nvr224|nvp52,nvr212,13,31,-1;mov.b32 nvf131,nvr224;fma.rn.ftz.f32 nvf132,nvf109,nvf131,nvf67;shfl.sync.idx.b32 nvr225|nvp53,nvr212,14,31,-1;mov.b32 nvf133,nvr225;fma.rn.ftz.f32 nvf134,nvf109,nvf133,nvf69;shfl.sync.idx.b32 nvr226|nvp54,nvr212,15,31,-1;mov.b32 nvf135,nvr226;fma.rn.ftz.f32 nvf136,nvf109,nvf135,nvf71;shfl.sync.idx.b32 nvr227|nvp55,nvr212,16,31,-1;mov.b32 nvf137,nvr227;fma.rn.ftz.f32 nvf138,nvf109,nvf137,nvf73;shfl.sync.idx.b32 nvr228|nvp56,nvr212,17,31,-1;mov.b32 nvf139,nvr228;fma.rn.ftz.f32 nvf140,nvf109,nvf139,nvf75;shfl.sync.idx.b32 nvr229|nvp57,nvr212,18,31,-1;mov.b32 nvf141,nvr229;fma.rn.ftz.f32 nvf142,nvf109,nvf141,nvf77;shfl.sync.idx.b32 nvr230|nvp58,nvr212,19,31,-1;mov.b32 nvf143,nvr230;fma.rn.ftz.f32 nvf144,nvf109,nvf143,nvf79;shfl.sync.idx.b32 nvr231|nvp59,nvr212,20,31,-1;mov.b32 nvf145,nvr231;fma.rn.ftz.f32 nvf146,nvf109,nvf145,nvf81;shfl.sync.idx.b32 nvr232|nvp60,nvr212,21,31,-1;mov.b32 nvf147,nvr232;fma.rn.ftz.f32 nvf148,nvf109,nvf147,nvf83;shfl.sync.idx.b32 nvr233|nvp61,nvr212,22,31,-1;mov.b32 nvf149,nvr233;fma.rn.ftz.f32 nvf150,nvf109,nvf149,nvf85;shfl.sync.idx.b32 nvr234|nvp62,nvr212,23,31,-1;mov.b32 nvf151,nvr234;fma.rn.ftz.f32 nvf152,nvf109,nvf151,nvf87;shfl.sync.idx.b32 nvr235|nvp63,nvr212,24,31,-1;mov.b32 nvf153,nvr235;fma.rn.ftz.f32 nvf154,nvf109,nvf153,nvf89;shfl.sync.idx.b32 nvr236|nvp64,nvr212,25,31,-1;mov.b32 nvf155,nvr236;fma.rn.ftz.f32 nvf156,nvf109,nvf155,nvf91;shfl.sync.idx.b32 nvr237|nvp65,nvr212,26,31,-1;mov.b32 nvf157,nvr237;fma.rn.ftz.f32 nvf158,nvf109,nvf157,nvf93;shfl.sync.idx.b32 nvr238|nvp66,nvr212,27,31,-1;mov.b32 nvf159,nvr238;fma.rn.ftz.f32 nvf160,nvf109,nvf159,nvf95;shfl.sync.idx.b32 nvr239|nvp67,nvr212,28,31,-1;mov.b32 nvf161,nvr239;fma.rn.ftz.f32 nvf162,nvf109,nvf161,nvf97;shfl.sync.idx.b32 nvr240|nvp68,nvr212,29,31,-1;mov.b32 nvf163,nvr240;fma.rn.ftz.f32 nvf164,nvf109,nvf163,nvf99;shfl.sync.idx.b32 nvr241|nvp69,nvr212,30,31,-1;mov.b32 nvf165,nvr241;fma.rn.ftz.f32 nvf166,nvf109,nvf165,nvf101;shfl.sync.idx.b32 nvr242|nvp70,nvr212,31,31,-1;mov.b32 nvf167,nvr242;fma.rn.ftz.f32 nvf168,nvf109,nvf167,nvf103;max.ftz.f32 nvf169,nvf110,0f00800000;sqrt.approx.ftz.f32 nvf170,nvf169;mov.b32 nvr243,nvf170;shfl.sync.idx.b32 nvr244|nvp71,nvr243,2,31,-1;mov.b32 nvf171,nvr244;div.approx.ftz.f32 nvf172,nvf110,nvf171;mov.b32 nvr245,nvf172;shfl.sync.idx.b32 nvr246|nvp72,nvr245,3,31,-1;mov.b32 nvf173,nvr246;neg.ftz.f32 nvf174,nvf172;fma.rn.ftz.f32 nvf175,nvf174,nvf173,nvf112;shfl.sync.idx.b32 nvr247|nvp73,nvr245,4,31,-1;mov.b32 nvf176,nvr247;fma.rn.ftz.f32 nvf177,nvf174,nvf176,nvf114;shfl.sync.idx.b32 nvr248|nvp74,nvr245,5,31,-1;mov.b32 nvf178,nvr248;fma.rn.ftz.f32 nvf179,nvf174,nvf178,nvf116;shfl.sync.idx.b32 nvr249|nvp75,nvr245,6,31,-1;mov.b32 nvf180,nvr249;fma.rn.ftz.f32 nvf181,nvf174,nvf180,nvf118;shfl.sync.idx.b32 nvr250|nvp76,nvr245,7,31,-1;mov.b32 nvf182,nvr250;fma.rn.ftz.f32 nvf183,nvf174,nvf182,nvf120;shfl.sync.idx.b32 nvr251|nvp77,nvr245,8,31,-1;mov.b32 nvf184,nvr251;fma.rn.ftz.f32 nvf185,nvf174,nvf184,nvf122;shfl.sync.idx.b32 nvr252|nvp78,nvr245,9,31,-1;mov.b32 nvf186,nvr252;fma.rn.ftz.f32 nvf187,nvf174,nvf186,nvf124;shfl.sync.idx.b32 nvr253|nvp79,nvr245,10,31,-1;mov.b32 nvf188,nvr253;fma.rn.ftz.f32 nvf189,nvf174,nvf188,nvf126;shfl.sync.idx.b32 nvr254|nvp80,nvr245,11,31,-1;mov.b32 nvf190,nvr254;fma.rn.ftz.f32 nvf191,nvf174,nvf190,nvf128;shfl.sync.idx.b32 nvr255|nvp81,nvr245,12,31,-1;mov.b32 nvf192,nvr255;fma.rn.ftz.f32 nvf193,nvf174,nvf192,nvf130;shfl.sync.idx.b32 nvr256|nvp82,nvr245,13,31,-1;mov.b32 nvf194,nvr256;fma.rn.ftz.f32 nvf195,nvf174,nvf194,nvf132;shfl.sync.idx.b32 nvr257|nvp83,nvr245,14,31,-1;mov.b32 nvf196,nvr257;fma.rn.ftz.f32 nvf197,nvf174,nvf196,nvf134;shfl.sync.idx.b32 nvr258|nvp84,nvr245,15,31,-1;mov.b32 nvf198,nvr258;fma.rn.ftz.f32 nvf199,nvf174,nvf198,nvf136;shfl.sync.idx.b32 nvr259|nvp85,nvr245,16,31,-1;mov.b32 nvf200,nvr259;fma.rn.ftz.f32 nvf201,nvf174,nvf200,nvf138;shfl.sync.idx.b32 nvr260|nvp86,nvr245,17,31,-1;mov.b32 nvf202,nvr260;fma.rn.ftz.f32 nvf203,nvf174,nvf202,nvf140;shfl.sync.idx.b32 nvr261|nvp87,nvr245,18,31,-1;mov.b32 nvf204,nvr261;fma.rn.ftz.f32 nvf205,nvf174,nvf204,nvf142;shfl.sync.idx.b32 nvr262|nvp88,nvr245,19,31,-1;mov.b32 nvf206,nvr262;fma.rn.ftz.f32 nvf207,nvf174,nvf206,nvf144;shfl.sync.idx.b32 nvr263|nvp89,nvr245,20,31,-1;mov.b32 nvf208,nvr263;fma.rn.ftz.f32 nvf209,nvf174,nvf208,nvf146;shfl.sync.idx.b32 nvr264|nvp90,nvr245,21,31,-1;mov.b32 nvf210,nvr264;fma.rn.ftz.f32 nvf211,nvf174,nvf210,nvf148;shfl.sync.idx.b32 nvr265|nvp91,nvr245,22,31,-1;mov.b32 nvf212,nvr265;fma.rn.ftz.f32 nvf213,nvf174,nvf212,nvf150;shfl.sync.idx.b32 nvr266|nvp92,nvr245,23,31,-1;mov.b32 nvf214,nvr266;fma.rn.ftz.f32 nvf215,nvf174,nvf214,nvf152;shfl.sync.idx.b32 nvr267|nvp93,nvr245,24,31,-1;mov.b32 nvf216,nvr267;fma.rn.ftz.f32 nvf217,nvf174,nvf216,nvf154;shfl.sync.idx.b32 nvr268|nvp94,nvr245,25,31,-1;mov.b32 nvf218,nvr268;fma.rn.ftz.f32 nvf219,nvf174,nvf218,nvf156;shfl.sync.idx.b32 nvr269|nvp95,nvr245,26,31,-1;mov.b32 nvf220,nvr269;fma.rn.ftz.f32 nvf221,nvf174,nvf220,nvf158;shfl.sync.idx.b32 nvr270|nvp96,nvr245,27,31,-1;mov.b32 nvf222,nvr270;fma.rn.ftz.f32 nvf223,nvf174,nvf222,nvf160;shfl.sync.idx.b32 nvr271|nvp97,nvr245,28,31,-1;mov.b32 nvf224,nvr271;fma.rn.ftz.f32 nvf225,nvf174,nvf224,nvf162;shfl.sync.idx.b32 nvr272|nvp98,nvr245,29,31,-1;mov.b32 nvf226,nvr272;fma.rn.ftz.f32 nvf227,nvf174,nvf226,nvf164;shfl.sync.idx.b32 nvr273|nvp99,nvr245,30,31,-1;mov.b32 nvf228,nvr273;fma.rn.ftz.f32 nvf229,nvf174,nvf228,nvf166;shfl.sync.idx.b32 nvr274|nvp100,nvr245,31,31,-1;mov.b32 nvf230,nvr274;fma.rn.ftz.f32 nvf231,nvf174,nvf230,nvf168;max.ftz.f32 nvf232,nvf175,0f00800000;sqrt.approx.ftz.f32 nvf233,nvf232;mov.b32 nvr275,nvf233;shfl.sync.idx.b32 nvr276|nvp101,nvr275,3,31,-1;mov.b32 nvf234,nvr276;div.approx.ftz.f32 nvf235,nvf175,nvf234;mov.b32 nvr277,nvf235;shfl.sync.idx.b32 nvr278|nvp102,nvr277,4,31,-1;mov.b32 nvf236,nvr278;neg.ftz.f32 nvf237,nvf235;fma.rn.ftz.f32 nvf238,nvf237,nvf236,nvf177;shfl.sync.idx.b32 nvr279|nvp103,nvr277,5,31,-1;mov.b32 nvf239,nvr279;fma.rn.ftz.f32 nvf240,nvf237,nvf239,nvf179;shfl.sync.idx.b32 nvr280|nvp104,nvr277,6,31,-1;mov.b32 nvf241,nvr280;fma.rn.ftz.f32 nvf242,nvf237,nvf241,nvf181;shfl.sync.idx.b32 nvr281|nvp105,nvr277,7,31,-1;mov.b32 nvf243,nvr281;fma.rn.ftz.f32 nvf244,nvf237,nvf243,nvf183;shfl.sync.idx.b32 nvr282|nvp106,nvr277,8,31,-1;mov.b32 nvf245,nvr282;fma.rn.ftz.f32 nvf246,nvf237,nvf245,nvf185;shfl.sync.idx.b32 nvr283|nvp107,nvr277,9,31,-1;mov.b32 nvf247,nvr283;fma.rn.ftz.f32 nvf248,nvf237,nvf247,nvf187;shfl.sync.idx.b32 nvr284|nvp108,nvr277,10,31,-1;mov.b32 nvf249,nvr284;fma.rn.ftz.f32 nvf250,nvf237,nvf249,nvf189;shfl.sync.idx.b32 nvr285|nvp109,nvr277,11,31,-1;mov.b32 nvf251,nvr285;fma.rn.ftz.f32 nvf252,nvf237,nvf251,nvf191;shfl.sync.idx.b32 nvr286|nvp110,nvr277,12,31,-1;mov.b32 nvf253,nvr286;fma.rn.ftz.f32 nvf254,nvf237,nvf253,nvf193;shfl.sync.idx.b32 nvr287|nvp111,nvr277,13,31,-1;mov.b32 nvf255,nvr287;fma.rn.ftz.f32 nvf256,nvf237,nvf255,nvf195;shfl.sync.idx.b32 nvr288|nvp112,nvr277,14,31,-1;mov.b32 nvf257,nvr288;fma.rn.ftz.f32 nvf258,nvf237,nvf257,nvf197;shfl.sync.idx.b32 nvr289|nvp113,nvr277,15,31,-1;mov.b32 nvf259,nvr289;fma.rn.ftz.f32 nvf260,nvf237,nvf259,nvf199;shfl.sync.idx.b32 nvr290|nvp114,nvr277,16,31,-1;mov.b32 nvf261,nvr290;fma.rn.ftz.f32 nvf262,nvf237,nvf261,nvf201;shfl.sync.idx.b32 nvr291|nvp115,nvr277,17,31,-1;mov.b32 nvf263,nvr291;fma.rn.ftz.f32 nvf264,nvf237,nvf263,nvf203;shfl.sync.idx.b32 nvr292|nvp116,nvr277,18,31,-1;mov.b32 nvf265,nvr292;fma.rn.ftz.f32 nvf266,nvf237,nvf265,nvf205;shfl.sync.idx.b32 nvr293|nvp117,nvr277,19,31,-1;mov.b32 nvf267,nvr293;fma.rn.ftz.f32 nvf268,nvf237,nvf267,nvf207;shfl.sync.idx.b32 nvr294|nvp118,nvr277,20,31,-1;mov.b32 nvf269,nvr294;fma.rn.ftz.f32 nvf270,nvf237,nvf269,nvf209;shfl.sync.idx.b32 nvr295|nvp119,nvr277,21,31,-1;mov.b32 nvf271,nvr295;fma.rn.ftz.f32 nvf272,nvf237,nvf271,nvf211;shfl.sync.idx.b32 nvr296|nvp120,nvr277,22,31,-1;mov.b32 nvf273,nvr296;fma.rn.ftz.f32 nvf274,nvf237,nvf273,nvf213;shfl.sync.idx.b32 nvr297|nvp121,nvr277,23,31,-1;mov.b32 nvf275,nvr297;fma.rn.ftz.f32 nvf276,nvf237,nvf275,nvf215;shfl.sync.idx.b32 nvr298|nvp122,nvr277,24,31,-1;mov.b32 nvf277,nvr298;fma.rn.ftz.f32 nvf278,nvf237,nvf277,nvf217;shfl.sync.idx.b32 nvr299|nvp123,nvr277,25,31,-1;mov.b32 nvf279,nvr299;fma.rn.ftz.f32 nvf280,nvf237,nvf279,nvf219;shfl.sync.idx.b32 nvr300|nvp124,nvr277,26,31,-1;mov.b32 nvf281,nvr300;fma.rn.ftz.f32 nvf282,nvf237,nvf281,nvf221;shfl.sync.idx.b32 nvr301|nvp125,nvr277,27,31,-1;mov.b32 nvf283,nvr301;fma.rn.ftz.f32 nvf284,nvf237,nvf283,nvf223;shfl.sync.idx.b32 nvr302|nvp126,nvr277,28,31,-1;mov.b32 nvf285,nvr302;fma.rn.ftz.f32 nvf286,nvf237,nvf285,nvf225;shfl.sync.idx.b32 nvr303|nvp127,nvr277,29,31,-1;mov.b32 nvf287,nvr303;fma.rn.ftz.f32 nvf288,nvf237,nvf287,nvf227;shfl.sync.idx.b32 nvr304|nvp128,nvr277,30,31,-1;mov.b32 nvf289,nvr304;fma.rn.ftz.f32 nvf290,nvf237,nvf289,nvf229;shfl.sync.idx.b32 nvr305|nvp129,nvr277,31,31,-1;mov.b32 nvf291,nvr305;fma.rn.ftz.f32 nvf292,nvf237,nvf291,nvf231;max.ftz.f32 nvf293,nvf238,0f00800000;sqrt.approx.ftz.f32 nvf294,nvf293;mov.b32 nvr306,nvf294;shfl.sync.idx.b32 nvr307|nvp130,nvr306,4,31,-1;mov.b32 nvf295,nvr307;div.approx.ftz.f32 nvf296,nvf238,nvf295;mov.b32 nvr308,nvf296;shfl.sync.idx.b32 nvr309|nvp131,nvr308,5,31,-1;mov.b32 nvf297,nvr309;neg.ftz.f32 nvf298,nvf296;fma.rn.ftz.f32 nvf299,nvf298,nvf297,nvf240;shfl.sync.idx.b32 nvr310|nvp132,nvr308,6,31,-1;mov.b32 nvf300,nvr310;fma.rn.ftz.f32 nvf301,nvf298,nvf300,nvf242;shfl.sync.idx.b32 nvr311|nvp133,nvr308,7,31,-1;mov.b32 nvf302,nvr311;fma.rn.ftz.f32 nvf303,nvf298,nvf302,nvf244;shfl.sync.idx.b32 nvr312|nvp134,nvr308,8,31,-1;mov.b32 nvf304,nvr312;fma.rn.ftz.f32 nvf305,nvf298,nvf304,nvf246;shfl.sync.idx.b32 nvr313|nvp135,nvr308,9,31,-1;mov.b32 nvf306,nvr313;fma.rn.ftz.f32 nvf307,nvf298,nvf306,nvf248;shfl.sync.idx.b32 nvr314|nvp136,nvr308,10,31,-1;mov.b32 nvf308,nvr314;fma.rn.ftz.f32 nvf309,nvf298,nvf308,nvf250;shfl.sync.idx.b32 nvr315|nvp137,nvr308,11,31,-1;mov.b32 nvf310,nvr315;fma.rn.ftz.f32 nvf311,nvf298,nvf310,nvf252;shfl.sync.idx.b32 nvr316|nvp138,nvr308,12,31,-1;mov.b32 nvf312,nvr316;fma.rn.ftz.f32 nvf313,nvf298,nvf312,nvf254;shfl.sync.idx.b32 nvr317|nvp139,nvr308,13,31,-1;mov.b32 nvf314,nvr317;fma.rn.ftz.f32 nvf315,nvf298,nvf314,nvf256;shfl.sync.idx.b32 nvr318|nvp140,nvr308,14,31,-1;mov.b32 nvf316,nvr318;fma.rn.ftz.f32 nvf317,nvf298,nvf316,nvf258;shfl.sync.idx.b32 nvr319|nvp141,nvr308,15,31,-1;mov.b32 nvf318,nvr319;fma.rn.ftz.f32 nvf319,nvf298,nvf318,nvf260;shfl.sync.idx.b32 nvr320|nvp142,nvr308,16,31,-1;mov.b32 nvf320,nvr320;fma.rn.ftz.f32 nvf321,nvf298,nvf320,nvf262;shfl.sync.idx.b32 nvr321|nvp143,nvr308,17,31,-1;mov.b32 nvf322,nvr321;fma.rn.ftz.f32 nvf323,nvf298,nvf322,nvf264;shfl.sync.idx.b32 nvr322|nvp144,nvr308,18,31,-1;mov.b32 nvf324,nvr322;fma.rn.ftz.f32 nvf325,nvf298,nvf324,nvf266;shfl.sync.idx.b32 nvr323|nvp145,nvr308,19,31,-1;mov.b32 nvf326,nvr323;fma.rn.ftz.f32 nvf327,nvf298,nvf326,nvf268;shfl.sync.idx.b32 nvr324|nvp146,nvr308,20,31,-1;mov.b32 nvf328,nvr324;fma.rn.ftz.f32 nvf329,nvf298,nvf328,nvf270;shfl.sync.idx.b32 nvr325|nvp147,nvr308,21,31,-1;mov.b32 nvf330,nvr325;fma.rn.ftz.f32 nvf331,nvf298,nvf330,nvf272;shfl.sync.idx.b32 nvr326|nvp148,nvr308,22,31,-1;mov.b32 nvf332,nvr326;fma.rn.ftz.f32 nvf333,nvf298,nvf332,nvf274;shfl.sync.idx.b32 nvr327|nvp149,nvr308,23,31,-1;mov.b32 nvf334,nvr327;fma.rn.ftz.f32 nvf335,nvf298,nvf334,nvf276;shfl.sync.idx.b32 nvr328|nvp150,nvr308,24,31,-1;mov.b32 nvf336,nvr328;fma.rn.ftz.f32 nvf337,nvf298,nvf336,nvf278;shfl.sync.idx.b32 nvr329|nvp151,nvr308,25,31,-1;mov.b32 nvf338,nvr329;fma.rn.ftz.f32 nvf339,nvf298,nvf338,nvf280;shfl.sync.idx.b32 nvr330|nvp152,nvr308,26,31,-1;mov.b32 nvf340,nvr330;fma.rn.ftz.f32 nvf341,nvf298,nvf340,nvf282;shfl.sync.idx.b32 nvr331|nvp153,nvr308,27,31,-1;mov.b32 nvf342,nvr331;fma.rn.ftz.f32 nvf343,nvf298,nvf342,nvf284;shfl.sync.idx.b32 nvr332|nvp154,nvr308,28,31,-1;mov.b32 nvf344,nvr332;fma.rn.ftz.f32 nvf345,nvf298,nvf344,nvf286;shfl.sync.idx.b32 nvr333|nvp155,nvr308,29,31,-1;mov.b32 nvf346,nvr333;fma.rn.ftz.f32 nvf347,nvf298,nvf346,nvf288;shfl.sync.idx.b32 nvr334|nvp156,nvr308,30,31,-1;mov.b32 nvf348,nvr334;fma.rn.ftz.f32 nvf349,nvf298,nvf348,nvf290;shfl.sync.idx.b32 nvr335|nvp157,nvr308,31,31,-1;mov.b32 nvf350,nvr335;fma.rn.ftz.f32 nvf351,nvf298,nvf350,nvf292;max.ftz.f32 nvf352,nvf299,0f00800000;sqrt.approx.ftz.f32 nvf353,nvf352;mov.b32 nvr336,nvf353;shfl.sync.idx.b32 nvr337|nvp158,nvr336,5,31,-1;mov.b32 nvf354,nvr337;div.approx.ftz.f32 nvf355,nvf299,nvf354;mov.b32 nvr338,nvf355;shfl.sync.idx.b32 nvr339|nvp159,nvr338,6,31,-1;mov.b32 nvf356,nvr339;neg.ftz.f32 nvf357,nvf355;fma.rn.ftz.f32 nvf358,nvf357,nvf356,nvf301;shfl.sync.idx.b32 nvr340|nvp160,nvr338,7,31,-1;mov.b32 nvf359,nvr340;fma.rn.ftz.f32 nvf360,nvf357,nvf359,nvf303;shfl.sync.idx.b32 nvr341|nvp161,nvr338,8,31,-1;mov.b32 nvf361,nvr341;fma.rn.ftz.f32 nvf362,nvf357,nvf361,nvf305;shfl.sync.idx.b32 nvr342|nvp162,nvr338,9,31,-1;mov.b32 nvf363,nvr342;fma.rn.ftz.f32 nvf364,nvf357,nvf363,nvf307;shfl.sync.idx.b32 nvr343|nvp163,nvr338,10,31,-1;mov.b32 nvf365,nvr343;fma.rn.ftz.f32 nvf366,nvf357,nvf365,nvf309;shfl.sync.idx.b32 nvr344|nvp164,nvr338,11,31,-1;mov.b32 nvf367,nvr344;fma.rn.ftz.f32 nvf368,nvf357,nvf367,nvf311;shfl.sync.idx.b32 nvr345|nvp165,nvr338,12,31,-1;mov.b32 nvf369,nvr345;fma.rn.ftz.f32 nvf370,nvf357,nvf369,nvf313;shfl.sync.idx.b32 nvr346|nvp166,nvr338,13,31,-1;mov.b32 nvf371,nvr346;fma.rn.ftz.f32 nvf372,nvf357,nvf371,nvf315;shfl.sync.idx.b32 nvr347|nvp167,nvr338,14,31,-1;mov.b32 nvf373,nvr347;fma.rn.ftz.f32 nvf374,nvf357,nvf373,nvf317;shfl.sync.idx.b32 nvr348|nvp168,nvr338,15,31,-1;mov.b32 nvf375,nvr348;fma.rn.ftz.f32 nvf376,nvf357,nvf375,nvf319;shfl.sync.idx.b32 nvr349|nvp169,nvr338,16,31,-1;mov.b32 nvf377,nvr349;fma.rn.ftz.f32 nvf378,nvf357,nvf377,nvf321;shfl.sync.idx.b32 nvr350|nvp170,nvr338,17,31,-1;mov.b32 nvf379,nvr350;fma.rn.ftz.f32 nvf380,nvf357,nvf379,nvf323;shfl.sync.idx.b32 nvr351|nvp171,nvr338,18,31,-1;mov.b32 nvf381,nvr351;fma.rn.ftz.f32 nvf382,nvf357,nvf381,nvf325;shfl.sync.idx.b32 nvr352|nvp172,nvr338,19,31,-1;mov.b32 nvf383,nvr352;fma.rn.ftz.f32 nvf384,nvf357,nvf383,nvf327;shfl.sync.idx.b32 nvr353|nvp173,nvr338,20,31,-1;mov.b32 nvf385,nvr353;fma.rn.ftz.f32 nvf386,nvf357,nvf385,nvf329;shfl.sync.idx.b32 nvr354|nvp174,nvr338,21,31,-1;mov.b32 nvf387,nvr354;fma.rn.ftz.f32 nvf388,nvf357,nvf387,nvf331;shfl.sync.idx.b32 nvr355|nvp175,nvr338,22,31,-1;mov.b32 nvf389,nvr355;fma.rn.ftz.f32 nvf390,nvf357,nvf389,nvf333;shfl.sync.idx.b32 nvr356|nvp176,nvr338,23,31,-1;mov.b32 nvf391,nvr356;fma.rn.ftz.f32 nvf392,nvf357,nvf391,nvf335;shfl.sync.idx.b32 nvr357|nvp177,nvr338,24,31,-1;mov.b32 nvf393,nvr357;fma.rn.ftz.f32 nvf394,nvf357,nvf393,nvf337;shfl.sync.idx.b32 nvr358|nvp178,nvr338,25,31,-1;mov.b32 nvf395,nvr358;fma.rn.ftz.f32 nvf396,nvf357,nvf395,nvf339;shfl.sync.idx.b32 nvr359|nvp179,nvr338,26,31,-1;mov.b32 nvf397,nvr359;fma.rn.ftz.f32 nvf398,nvf357,nvf397,nvf341;shfl.sync.idx.b32 nvr360|nvp180,nvr338,27,31,-1;mov.b32 nvf399,nvr360;fma.rn.ftz.f32 nvf400,nvf357,nvf399,nvf343;shfl.sync.idx.b32 nvr361|nvp181,nvr338,28,31,-1;mov.b32 nvf401,nvr361;fma.rn.ftz.f32 nvf402,nvf357,nvf401,nvf345;shfl.sync.idx.b32 nvr362|nvp182,nvr338,29,31,-1;mov.b32 nvf403,nvr362;fma.rn.ftz.f32 nvf404,nvf357,nvf403,nvf347;shfl.sync.idx.b32 nvr363|nvp183,nvr338,30,31,-1;mov.b32 nvf405,nvr363;fma.rn.ftz.f32 nvf406,nvf357,nvf405,nvf349;shfl.sync.idx.b32 nvr364|nvp184,nvr338,31,31,-1;mov.b32 nvf407,nvr364;fma.rn.ftz.f32 nvf408,nvf357,nvf407,nvf351;max.ftz.f32 nvf409,nvf358,0f00800000;sqrt.approx.ftz.f32 nvf410,nvf409;mov.b32 nvr365,nvf410;shfl.sync.idx.b32 nvr366|nvp185,nvr365,6,31,-1;mov.b32 nvf411,nvr366;div.approx.ftz.f32 nvf412,nvf358,nvf411;mov.b32 nvr367,nvf412;shfl.sync.idx.b32 nvr368|nvp186,nvr367,7,31,-1;mov.b32 nvf413,nvr368;neg.ftz.f32 nvf414,nvf412;fma.rn.ftz.f32 nvf415,nvf414,nvf413,nvf360;shfl.sync.idx.b32 nvr369|nvp187,nvr367,8,31,-1;mov.b32 nvf416,nvr369;fma.rn.ftz.f32 nvf417,nvf414,nvf416,nvf362;shfl.sync.idx.b32 nvr370|nvp188,nvr367,9,31,-1;mov.b32 nvf418,nvr370;fma.rn.ftz.f32 nvf419,nvf414,nvf418,nvf364;shfl.sync.idx.b32 nvr371|nvp189,nvr367,10,31,-1;mov.b32 nvf420,nvr371;fma.rn.ftz.f32 nvf421,nvf414,nvf420,nvf366;shfl.sync.idx.b32 nvr372|nvp190,nvr367,11,31,-1;mov.b32 nvf422,nvr372;fma.rn.ftz.f32 nvf423,nvf414,nvf422,nvf368;shfl.sync.idx.b32 nvr373|nvp191,nvr367,12,31,-1;mov.b32 nvf424,nvr373;fma.rn.ftz.f32 nvf425,nvf414,nvf424,nvf370;shfl.sync.idx.b32 nvr374|nvp192,nvr367,13,31,-1;mov.b32 nvf426,nvr374;fma.rn.ftz.f32 nvf427,nvf414,nvf426,nvf372;shfl.sync.idx.b32 nvr375|nvp193,nvr367,14,31,-1;mov.b32 nvf428,nvr375;fma.rn.ftz.f32 nvf429,nvf414,nvf428,nvf374;shfl.sync.idx.b32 nvr376|nvp194,nvr367,15,31,-1;mov.b32 nvf430,nvr376;fma.rn.ftz.f32 nvf431,nvf414,nvf430,nvf376;shfl.sync.idx.b32 nvr377|nvp195,nvr367,16,31,-1;mov.b32 nvf432,nvr377;fma.rn.ftz.f32 nvf433,nvf414,nvf432,nvf378;shfl.sync.idx.b32 nvr378|nvp196,nvr367,17,31,-1;mov.b32 nvf434,nvr378;fma.rn.ftz.f32 nvf435,nvf414,nvf434,nvf380;shfl.sync.idx.b32 nvr379|nvp197,nvr367,18,31,-1;mov.b32 nvf436,nvr379;fma.rn.ftz.f32 nvf437,nvf414,nvf436,nvf382;shfl.sync.idx.b32 nvr380|nvp198,nvr367,19,31,-1;mov.b32 nvf438,nvr380;fma.rn.ftz.f32 nvf439,nvf414,nvf438,nvf384;shfl.sync.idx.b32 nvr381|nvp199,nvr367,20,31,-1;mov.b32 nvf440,nvr381;fma.rn.ftz.f32 nvf441,nvf414,nvf440,nvf386;shfl.sync.idx.b32 nvr382|nvp200,nvr367,21,31,-1;mov.b32 nvf442,nvr382;fma.rn.ftz.f32 nvf443,nvf414,nvf442,nvf388;shfl.sync.idx.b32 nvr383|nvp201,nvr367,22,31,-1;mov.b32 nvf444,nvr383;fma.rn.ftz.f32 nvf445,nvf414,nvf444,nvf390;shfl.sync.idx.b32 nvr384|nvp202,nvr367,23,31,-1;mov.b32 nvf446,nvr384;fma.rn.ftz.f32 nvf447,nvf414,nvf446,nvf392;shfl.sync.idx.b32 nvr385|nvp203,nvr367,24,31,-1;mov.b32 nvf448,nvr385;fma.rn.ftz.f32 nvf449,nvf414,nvf448,nvf394;shfl.sync.idx.b32 nvr386|nvp204,nvr367,25,31,-1;mov.b32 nvf450,nvr386;fma.rn.ftz.f32 nvf451,nvf414,nvf450,nvf396;shfl.sync.idx.b32 nvr387|nvp205,nvr367,26,31,-1;mov.b32 nvf452,nvr387;fma.rn.ftz.f32 nvf453,nvf414,nvf452,nvf398;shfl.sync.idx.b32 nvr388|nvp206,nvr367,27,31,-1;mov.b32 nvf454,nvr388;fma.rn.ftz.f32 nvf455,nvf414,nvf454,nvf400;shfl.sync.idx.b32 nvr389|nvp207,nvr367,28,31,-1;mov.b32 nvf456,nvr389;fma.rn.ftz.f32 nvf457,nvf414,nvf456,nvf402;shfl.sync.idx.b32 nvr390|nvp208,nvr367,29,31,-1;mov.b32 nvf458,nvr390;fma.rn.ftz.f32 nvf459,nvf414,nvf458,nvf404;shfl.sync.idx.b32 nvr391|nvp209,nvr367,30,31,-1;mov.b32 nvf460,nvr391;fma.rn.ftz.f32 nvf461,nvf414,nvf460,nvf406;shfl.sync.idx.b32 nvr392|nvp210,nvr367,31,31,-1;mov.b32 nvf462,nvr392;fma.rn.ftz.f32 nvf463,nvf414,nvf462,nvf408;max.ftz.f32 nvf464,nvf415,0f00800000;sqrt.approx.ftz.f32 nvf465,nvf464;mov.b32 nvr393,nvf465;shfl.sync.idx.b32 nvr394|nvp211,nvr393,7,31,-1;mov.b32 nvf466,nvr394;div.approx.ftz.f32 nvf467,nvf415,nvf466;mov.b32 nvr395,nvf467;shfl.sync.idx.b32 nvr396|nvp212,nvr395,8,31,-1;mov.b32 nvf468,nvr396;neg.ftz.f32 nvf469,nvf467;fma.rn.ftz.f32 nvf470,nvf469,nvf468,nvf417;shfl.sync.idx.b32 nvr397|nvp213,nvr395,9,31,-1;mov.b32 nvf471,nvr397;fma.rn.ftz.f32 nvf472,nvf469,nvf471,nvf419;shfl.sync.idx.b32 nvr398|nvp214,nvr395,10,31,-1;mov.b32 nvf473,nvr398;fma.rn.ftz.f32 nvf474,nvf469,nvf473,nvf421;shfl.sync.idx.b32 nvr399|nvp215,nvr395,11,31,-1;mov.b32 nvf475,nvr399;fma.rn.ftz.f32 nvf476,nvf469,nvf475,nvf423;shfl.sync.idx.b32 nvr400|nvp216,nvr395,12,31,-1;mov.b32 nvf477,nvr400;fma.rn.ftz.f32 nvf478,nvf469,nvf477,nvf425;shfl.sync.idx.b32 nvr401|nvp217,nvr395,13,31,-1;mov.b32 nvf479,nvr401;fma.rn.ftz.f32 nvf480,nvf469,nvf479,nvf427;shfl.sync.idx.b32 nvr402|nvp218,nvr395,14,31,-1;mov.b32 nvf481,nvr402;fma.rn.ftz.f32 nvf482,nvf469,nvf481,nvf429;shfl.sync.idx.b32 nvr403|nvp219,nvr395,15,31,-1;mov.b32 nvf483,nvr403;fma.rn.ftz.f32 nvf484,nvf469,nvf483,nvf431;shfl.sync.idx.b32 nvr404|nvp220,nvr395,16,31,-1;mov.b32 nvf485,nvr404;fma.rn.ftz.f32 nvf486,nvf469,nvf485,nvf433;shfl.sync.idx.b32 nvr405|nvp221,nvr395,17,31,-1;mov.b32 nvf487,nvr405;fma.rn.ftz.f32 nvf488,nvf469,nvf487,nvf435;shfl.sync.idx.b32 nvr406|nvp222,nvr395,18,31,-1;mov.b32 nvf489,nvr406;fma.rn.ftz.f32 nvf490,nvf469,nvf489,nvf437;shfl.sync.idx.b32 nvr407|nvp223,nvr395,19,31,-1;mov.b32 nvf491,nvr407;fma.rn.ftz.f32 nvf492,nvf469,nvf491,nvf439;shfl.sync.idx.b32 nvr408|nvp224,nvr395,20,31,-1;mov.b32 nvf493,nvr408;fma.rn.ftz.f32 nvf494,nvf469,nvf493,nvf441;shfl.sync.idx.b32 nvr409|nvp225,nvr395,21,31,-1;mov.b32 nvf495,nvr409;fma.rn.ftz.f32 nvf496,nvf469,nvf495,nvf443;shfl.sync.idx.b32 nvr410|nvp226,nvr395,22,31,-1;mov.b32 nvf497,nvr410;fma.rn.ftz.f32 nvf498,nvf469,nvf497,nvf445;shfl.sync.idx.b32 nvr411|nvp227,nvr395,23,31,-1;mov.b32 nvf499,nvr411;fma.rn.ftz.f32 nvf500,nvf469,nvf499,nvf447;shfl.sync.idx.b32 nvr412|nvp228,nvr395,24,31,-1;mov.b32 nvf501,nvr412;fma.rn.ftz.f32 nvf502,nvf469,nvf501,nvf449;shfl.sync.idx.b32 nvr413|nvp229,nvr395,25,31,-1;mov.b32 nvf503,nvr413;fma.rn.ftz.f32 nvf504,nvf469,nvf503,nvf451;shfl.sync.idx.b32 nvr414|nvp230,nvr395,26,31,-1;mov.b32 nvf505,nvr414;fma.rn.ftz.f32 nvf506,nvf469,nvf505,nvf453;shfl.sync.idx.b32 nvr415|nvp231,nvr395,27,31,-1;mov.b32 nvf507,nvr415;fma.rn.ftz.f32 nvf508,nvf469,nvf507,nvf455;shfl.sync.idx.b32 nvr416|nvp232,nvr395,28,31,-1;mov.b32 nvf509,nvr416;fma.rn.ftz.f32 nvf510,nvf469,nvf509,nvf457;shfl.sync.idx.b32 nvr417|nvp233,nvr395,29,31,-1;mov.b32 nvf511,nvr417;fma.rn.ftz.f32 nvf512,nvf469,nvf511,nvf459;shfl.sync.idx.b32 nvr418|nvp234,nvr395,30,31,-1;mov.b32 nvf513,nvr418;fma.rn.ftz.f32 nvf514,nvf469,nvf513,nvf461;shfl.sync.idx.b32 nvr419|nvp235,nvr395,31,31,-1;mov.b32 nvf515,nvr419;fma.rn.ftz.f32 nvf516,nvf469,nvf515,nvf463;max.ftz.f32 nvf517,nvf470,0f00800000;sqrt.approx.ftz.f32 nvf518,nvf517;mov.b32 nvr420,nvf518;shfl.sync.idx.b32 nvr421|nvp236,nvr420,8,31,-1;mov.b32 nvf519,nvr421;div.approx.ftz.f32 nvf520,nvf470,nvf519;mov.b32 nvr422,nvf520;shfl.sync.idx.b32 nvr423|nvp237,nvr422,9,31,-1;mov.b32 nvf521,nvr423;neg.ftz.f32 nvf522,nvf520;fma.rn.ftz.f32 nvf523,nvf522,nvf521,nvf472;shfl.sync.idx.b32 nvr424|nvp238,nvr422,10,31,-1;mov.b32 nvf524,nvr424;fma.rn.ftz.f32 nvf525,nvf522,nvf524,nvf474;shfl.sync.idx.b32 nvr425|nvp239,nvr422,11,31,-1;mov.b32 nvf526,nvr425;fma.rn.ftz.f32 nvf527,nvf522,nvf526,nvf476;shfl.sync.idx.b32 nvr426|nvp240,nvr422,12,31,-1;mov.b32 nvf528,nvr426;fma.rn.ftz.f32 nvf529,nvf522,nvf528,nvf478;shfl.sync.idx.b32 nvr427|nvp241,nvr422,13,31,-1;mov.b32 nvf530,nvr427;fma.rn.ftz.f32 nvf531,nvf522,nvf530,nvf480;shfl.sync.idx.b32 nvr428|nvp242,nvr422,14,31,-1;mov.b32 nvf532,nvr428;fma.rn.ftz.f32 nvf533,nvf522,nvf532,nvf482;shfl.sync.idx.b32 nvr429|nvp243,nvr422,15,31,-1;mov.b32 nvf534,nvr429;fma.rn.ftz.f32 nvf535,nvf522,nvf534,nvf484;shfl.sync.idx.b32 nvr430|nvp244,nvr422,16,31,-1;mov.b32 nvf536,nvr430;fma.rn.ftz.f32 nvf537,nvf522,nvf536,nvf486;shfl.sync.idx.b32 nvr431|nvp245,nvr422,17,31,-1;mov.b32 nvf538,nvr431;fma.rn.ftz.f32 nvf539,nvf522,nvf538,nvf488;shfl.sync.idx.b32 nvr432|nvp246,nvr422,18,31,-1;mov.b32 nvf540,nvr432;fma.rn.ftz.f32 nvf541,nvf522,nvf540,nvf490;shfl.sync.idx.b32 nvr433|nvp247,nvr422,19,31,-1;mov.b32 nvf542,nvr433;fma.rn.ftz.f32 nvf543,nvf522,nvf542,nvf492;shfl.sync.idx.b32 nvr434|nvp248,nvr422,20,31,-1;mov.b32 nvf544,nvr434;fma.rn.ftz.f32 nvf545,nvf522,nvf544,nvf494;shfl.sync.idx.b32 nvr435|nvp249,nvr422,21,31,-1;mov.b32 nvf546,nvr435;fma.rn.ftz.f32 nvf547,nvf522,nvf546,nvf496;shfl.sync.idx.b32 nvr436|nvp250,nvr422,22,31,-1;mov.b32 nvf548,nvr436;fma.rn.ftz.f32 nvf549,nvf522,nvf548,nvf498;shfl.sync.idx.b32 nvr437|nvp251,nvr422,23,31,-1;mov.b32 nvf550,nvr437;fma.rn.ftz.f32 nvf551,nvf522,nvf550,nvf500;shfl.sync.idx.b32 nvr438|nvp252,nvr422,24,31,-1;mov.b32 nvf552,nvr438;fma.rn.ftz.f32 nvf553,nvf522,nvf552,nvf502;shfl.sync.idx.b32 nvr439|nvp253,nvr422,25,31,-1;mov.b32 nvf554,nvr439;fma.rn.ftz.f32 nvf555,nvf522,nvf554,nvf504;shfl.sync.idx.b32 nvr440|nvp254,nvr422,26,31,-1;mov.b32 nvf556,nvr440;fma.rn.ftz.f32 nvf557,nvf522,nvf556,nvf506;shfl.sync.idx.b32 nvr441|nvp255,nvr422,27,31,-1;mov.b32 nvf558,nvr441;fma.rn.ftz.f32 nvf559,nvf522,nvf558,nvf508;shfl.sync.idx.b32 nvr442|nvp256,nvr422,28,31,-1;mov.b32 nvf560,nvr442;fma.rn.ftz.f32 nvf561,nvf522,nvf560,nvf510;shfl.sync.idx.b32 nvr443|nvp257,nvr422,29,31,-1;mov.b32 nvf562,nvr443;fma.rn.ftz.f32 nvf563,nvf522,nvf562,nvf512;shfl.sync.idx.b32 nvr444|nvp258,nvr422,30,31,-1;mov.b32 nvf564,nvr444;fma.rn.ftz.f32 nvf565,nvf522,nvf564,nvf514;shfl.sync.idx.b32 nvr445|nvp259,nvr422,31,31,-1;mov.b32 nvf566,nvr445;fma.rn.ftz.f32 nvf567,nvf522,nvf566,nvf516;max.ftz.f32 nvf568,nvf523,0f00800000;sqrt.approx.ftz.f32 nvf569,nvf568;mov.b32 nvr446,nvf569;shfl.sync.idx.b32 nvr447|nvp260,nvr446,9,31,-1;mov.b32 nvf570,nvr447;div.approx.ftz.f32 nvf571,nvf523,nvf570;mov.b32 nvr448,nvf571;shfl.sync.idx.b32 nvr449|nvp261,nvr448,10,31,-1;mov.b32 nvf572,nvr449;neg.ftz.f32 nvf573,nvf571;fma.rn.ftz.f32 nvf574,nvf573,nvf572,nvf525;shfl.sync.idx.b32 nvr450|nvp262,nvr448,11,31,-1;mov.b32 nvf575,nvr450;fma.rn.ftz.f32 nvf576,nvf573,nvf575,nvf527;shfl.sync.idx.b32 nvr451|nvp263,nvr448,12,31,-1;mov.b32 nvf577,nvr451;fma.rn.ftz.f32 nvf578,nvf573,nvf577,nvf529;shfl.sync.idx.b32 nvr452|nvp264,nvr448,13,31,-1;mov.b32 nvf579,nvr452;fma.rn.ftz.f32 nvf580,nvf573,nvf579,nvf531;shfl.sync.idx.b32 nvr453|nvp265,nvr448,14,31,-1;mov.b32 nvf581,nvr453;fma.rn.ftz.f32 nvf582,nvf573,nvf581,nvf533;shfl.sync.idx.b32 nvr454|nvp266,nvr448,15,31,-1;mov.b32 nvf583,nvr454;fma.rn.ftz.f32 nvf584,nvf573,nvf583,nvf535;shfl.sync.idx.b32 nvr455|nvp267,nvr448,16,31,-1;mov.b32 nvf585,nvr455;fma.rn.ftz.f32 nvf586,nvf573,nvf585,nvf537;shfl.sync.idx.b32 nvr456|nvp268,nvr448,17,31,-1;mov.b32 nvf587,nvr456;fma.rn.ftz.f32 nvf588,nvf573,nvf587,nvf539;shfl.sync.idx.b32 nvr457|nvp269,nvr448,18,31,-1;mov.b32 nvf589,nvr457;fma.rn.ftz.f32 nvf590,nvf573,nvf589,nvf541;shfl.sync.idx.b32 nvr458|nvp270,nvr448,19,31,-1;mov.b32 nvf591,nvr458;fma.rn.ftz.f32 nvf592,nvf573,nvf591,nvf543;shfl.sync.idx.b32 nvr459|nvp271,nvr448,20,31,-1;mov.b32 nvf593,nvr459;fma.rn.ftz.f32 nvf594,nvf573,nvf593,nvf545;shfl.sync.idx.b32 nvr460|nvp272,nvr448,21,31,-1;mov.b32 nvf595,nvr460;fma.rn.ftz.f32 nvf596,nvf573,nvf595,nvf547;shfl.sync.idx.b32 nvr461|nvp273,nvr448,22,31,-1;mov.b32 nvf597,nvr461;fma.rn.ftz.f32 nvf598,nvf573,nvf597,nvf549;shfl.sync.idx.b32 nvr462|nvp274,nvr448,23,31,-1;mov.b32 nvf599,nvr462;fma.rn.ftz.f32 nvf600,nvf573,nvf599,nvf551;shfl.sync.idx.b32 nvr463|nvp275,nvr448,24,31,-1;mov.b32 nvf601,nvr463;fma.rn.ftz.f32 nvf602,nvf573,nvf601,nvf553;shfl.sync.idx.b32 nvr464|nvp276,nvr448,25,31,-1;mov.b32 nvf603,nvr464;fma.rn.ftz.f32 nvf604,nvf573,nvf603,nvf555;shfl.sync.idx.b32 nvr465|nvp277,nvr448,26,31,-1;mov.b32 nvf605,nvr465;fma.rn.ftz.f32 nvf606,nvf573,nvf605,nvf557;shfl.sync.idx.b32 nvr466|nvp278,nvr448,27,31,-1;mov.b32 nvf607,nvr466;fma.rn.ftz.f32 nvf608,nvf573,nvf607,nvf559;shfl.sync.idx.b32 nvr467|nvp279,nvr448,28,31,-1;mov.b32 nvf609,nvr467;fma.rn.ftz.f32 nvf610,nvf573,nvf609,nvf561;shfl.sync.idx.b32 nvr468|nvp280,nvr448,29,31,-1;mov.b32 nvf611,nvr468;fma.rn.ftz.f32 nvf612,nvf573,nvf611,nvf563;shfl.sync.idx.b32 nvr469|nvp281,nvr448,30,31,-1;mov.b32 nvf613,nvr469;fma.rn.ftz.f32 nvf614,nvf573,nvf613,nvf565;shfl.sync.idx.b32 nvr470|nvp282,nvr448,31,31,-1;mov.b32 nvf615,nvr470;fma.rn.ftz.f32 nvf616,nvf573,nvf615,nvf567;max.ftz.f32 nvf617,nvf574,0f00800000;sqrt.approx.ftz.f32 nvf618,nvf617;mov.b32 nvr471,nvf618;shfl.sync.idx.b32 nvr472|nvp283,nvr471,10,31,-1;mov.b32 nvf619,nvr472;div.approx.ftz.f32 nvf620,nvf574,nvf619;mov.b32 nvr473,nvf620;shfl.sync.idx.b32 nvr474|nvp284,nvr473,11,31,-1;mov.b32 nvf621,nvr474;neg.ftz.f32 nvf622,nvf620;fma.rn.ftz.f32 nvf623,nvf622,nvf621,nvf576;shfl.sync.idx.b32 nvr475|nvp285,nvr473,12,31,-1;mov.b32 nvf624,nvr475;fma.rn.ftz.f32 nvf625,nvf622,nvf624,nvf578;shfl.sync.idx.b32 nvr476|nvp286,nvr473,13,31,-1;mov.b32 nvf626,nvr476;fma.rn.ftz.f32 nvf627,nvf622,nvf626,nvf580;shfl.sync.idx.b32 nvr477|nvp287,nvr473,14,31,-1;mov.b32 nvf628,nvr477;fma.rn.ftz.f32 nvf629,nvf622,nvf628,nvf582;shfl.sync.idx.b32 nvr478|nvp288,nvr473,15,31,-1;mov.b32 nvf630,nvr478;fma.rn.ftz.f32 nvf631,nvf622,nvf630,nvf584;shfl.sync.idx.b32 nvr479|nvp289,nvr473,16,31,-1;mov.b32 nvf632,nvr479;fma.rn.ftz.f32 nvf633,nvf622,nvf632,nvf586;shfl.sync.idx.b32 nvr480|nvp290,nvr473,17,31,-1;mov.b32 nvf634,nvr480;fma.rn.ftz.f32 nvf635,nvf622,nvf634,nvf588;shfl.sync.idx.b32 nvr481|nvp291,nvr473,18,31,-1;mov.b32 nvf636,nvr481;fma.rn.ftz.f32 nvf637,nvf622,nvf636,nvf590;shfl.sync.idx.b32 nvr482|nvp292,nvr473,19,31,-1;mov.b32 nvf638,nvr482;fma.rn.ftz.f32 nvf639,nvf622,nvf638,nvf592;shfl.sync.idx.b32 nvr483|nvp293,nvr473,20,31,-1;mov.b32 nvf640,nvr483;fma.rn.ftz.f32 nvf641,nvf622,nvf640,nvf594;shfl.sync.idx.b32 nvr484|nvp294,nvr473,21,31,-1;mov.b32 nvf642,nvr484;fma.rn.ftz.f32 nvf643,nvf622,nvf642,nvf596;shfl.sync.idx.b32 nvr485|nvp295,nvr473,22,31,-1;mov.b32 nvf644,nvr485;fma.rn.ftz.f32 nvf645,nvf622,nvf644,nvf598;shfl.sync.idx.b32 nvr486|nvp296,nvr473,23,31,-1;mov.b32 nvf646,nvr486;fma.rn.ftz.f32 nvf647,nvf622,nvf646,nvf600;shfl.sync.idx.b32 nvr487|nvp297,nvr473,24,31,-1;mov.b32 nvf648,nvr487;fma.rn.ftz.f32 nvf649,nvf622,nvf648,nvf602;shfl.sync.idx.b32 nvr488|nvp298,nvr473,25,31,-1;mov.b32 nvf650,nvr488;fma.rn.ftz.f32 nvf651,nvf622,nvf650,nvf604;shfl.sync.idx.b32 nvr489|nvp299,nvr473,26,31,-1;mov.b32 nvf652,nvr489;fma.rn.ftz.f32 nvf653,nvf622,nvf652,nvf606;shfl.sync.idx.b32 nvr490|nvp300,nvr473,27,31,-1;mov.b32 nvf654,nvr490;fma.rn.ftz.f32 nvf655,nvf622,nvf654,nvf608;shfl.sync.idx.b32 nvr491|nvp301,nvr473,28,31,-1;mov.b32 nvf656,nvr491;fma.rn.ftz.f32 nvf657,nvf622,nvf656,nvf610;shfl.sync.idx.b32 nvr492|nvp302,nvr473,29,31,-1;mov.b32 nvf658,nvr492;fma.rn.ftz.f32 nvf659,nvf622,nvf658,nvf612;shfl.sync.idx.b32 nvr493|nvp303,nvr473,30,31,-1;mov.b32 nvf660,nvr493;fma.rn.ftz.f32 nvf661,nvf622,nvf660,nvf614;shfl.sync.idx.b32 nvr494|nvp304,nvr473,31,31,-1;mov.b32 nvf662,nvr494;fma.rn.ftz.f32 nvf663,nvf622,nvf662,nvf616;max.ftz.f32 nvf664,nvf623,0f00800000;sqrt.approx.ftz.f32 nvf665,nvf664;mov.b32 nvr495,nvf665;shfl.sync.idx.b32 nvr496|nvp305,nvr495,11,31,-1;mov.b32 nvf666,nvr496;div.approx.ftz.f32 nvf667,nvf623,nvf666;mov.b32 nvr497,nvf667;shfl.sync.idx.b32 nvr498|nvp306,nvr497,12,31,-1;mov.b32 nvf668,nvr498;neg.ftz.f32 nvf669,nvf667;fma.rn.ftz.f32 nvf670,nvf669,nvf668,nvf625;shfl.sync.idx.b32 nvr499|nvp307,nvr497,13,31,-1;mov.b32 nvf671,nvr499;fma.rn.ftz.f32 nvf672,nvf669,nvf671,nvf627;shfl.sync.idx.b32 nvr500|nvp308,nvr497,14,31,-1;mov.b32 nvf673,nvr500;fma.rn.ftz.f32 nvf674,nvf669,nvf673,nvf629;shfl.sync.idx.b32 nvr501|nvp309,nvr497,15,31,-1;mov.b32 nvf675,nvr501;fma.rn.ftz.f32 nvf676,nvf669,nvf675,nvf631;shfl.sync.idx.b32 nvr502|nvp310,nvr497,16,31,-1;mov.b32 nvf677,nvr502;fma.rn.ftz.f32 nvf678,nvf669,nvf677,nvf633;shfl.sync.idx.b32 nvr503|nvp311,nvr497,17,31,-1;mov.b32 nvf679,nvr503;fma.rn.ftz.f32 nvf680,nvf669,nvf679,nvf635;shfl.sync.idx.b32 nvr504|nvp312,nvr497,18,31,-1;mov.b32 nvf681,nvr504;fma.rn.ftz.f32 nvf682,nvf669,nvf681,nvf637;shfl.sync.idx.b32 nvr505|nvp313,nvr497,19,31,-1;mov.b32 nvf683,nvr505;fma.rn.ftz.f32 nvf684,nvf669,nvf683,nvf639;shfl.sync.idx.b32 nvr506|nvp314,nvr497,20,31,-1;mov.b32 nvf685,nvr506;fma.rn.ftz.f32 nvf686,nvf669,nvf685,nvf641;shfl.sync.idx.b32 nvr507|nvp315,nvr497,21,31,-1;mov.b32 nvf687,nvr507;fma.rn.ftz.f32 nvf688,nvf669,nvf687,nvf643;shfl.sync.idx.b32 nvr508|nvp316,nvr497,22,31,-1;mov.b32 nvf689,nvr508;fma.rn.ftz.f32 nvf690,nvf669,nvf689,nvf645;shfl.sync.idx.b32 nvr509|nvp317,nvr497,23,31,-1;mov.b32 nvf691,nvr509;fma.rn.ftz.f32 nvf692,nvf669,nvf691,nvf647;shfl.sync.idx.b32 nvr510|nvp318,nvr497,24,31,-1;mov.b32 nvf693,nvr510;fma.rn.ftz.f32 nvf694,nvf669,nvf693,nvf649;shfl.sync.idx.b32 nvr511|nvp319,nvr497,25,31,-1;mov.b32 nvf695,nvr511;fma.rn.ftz.f32 nvf696,nvf669,nvf695,nvf651;shfl.sync.idx.b32 nvr512|nvp320,nvr497,26,31,-1;mov.b32 nvf697,nvr512;fma.rn.ftz.f32 nvf698,nvf669,nvf697,nvf653;shfl.sync.idx.b32 nvr513|nvp321,nvr497,27,31,-1;mov.b32 nvf699,nvr513;fma.rn.ftz.f32 nvf700,nvf669,nvf699,nvf655;shfl.sync.idx.b32 nvr514|nvp322,nvr497,28,31,-1;mov.b32 nvf701,nvr514;fma.rn.ftz.f32 nvf702,nvf669,nvf701,nvf657;shfl.sync.idx.b32 nvr515|nvp323,nvr497,29,31,-1;mov.b32 nvf703,nvr515;fma.rn.ftz.f32 nvf704,nvf669,nvf703,nvf659;shfl.sync.idx.b32 nvr516|nvp324,nvr497,30,31,-1;mov.b32 nvf705,nvr516;fma.rn.ftz.f32 nvf706,nvf669,nvf705,nvf661;shfl.sync.idx.b32 nvr517|nvp325,nvr497,31,31,-1;mov.b32 nvf707,nvr517;fma.rn.ftz.f32 nvf708,nvf669,nvf707,nvf663;max.ftz.f32 nvf709,nvf670,0f00800000;sqrt.approx.ftz.f32 nvf710,nvf709;mov.b32 nvr518,nvf710;shfl.sync.idx.b32 nvr519|nvp326,nvr518,12,31,-1;mov.b32 nvf711,nvr519;div.approx.ftz.f32 nvf712,nvf670,nvf711;mov.b32 nvr520,nvf712;shfl.sync.idx.b32 nvr521|nvp327,nvr520,13,31,-1;mov.b32 nvf713,nvr521;neg.ftz.f32 nvf714,nvf712;fma.rn.ftz.f32 nvf715,nvf714,nvf713,nvf672;shfl.sync.idx.b32 nvr522|nvp328,nvr520,14,31,-1;mov.b32 nvf716,nvr522;fma.rn.ftz.f32 nvf717,nvf714,nvf716,nvf674;shfl.sync.idx.b32 nvr523|nvp329,nvr520,15,31,-1;mov.b32 nvf718,nvr523;fma.rn.ftz.f32 nvf719,nvf714,nvf718,nvf676;shfl.sync.idx.b32 nvr524|nvp330,nvr520,16,31,-1;mov.b32 nvf720,nvr524;fma.rn.ftz.f32 nvf721,nvf714,nvf720,nvf678;shfl.sync.idx.b32 nvr525|nvp331,nvr520,17,31,-1;mov.b32 nvf722,nvr525;fma.rn.ftz.f32 nvf723,nvf714,nvf722,nvf680;shfl.sync.idx.b32 nvr526|nvp332,nvr520,18,31,-1;mov.b32 nvf724,nvr526;fma.rn.ftz.f32 nvf725,nvf714,nvf724,nvf682;shfl.sync.idx.b32 nvr527|nvp333,nvr520,19,31,-1;mov.b32 nvf726,nvr527;fma.rn.ftz.f32 nvf727,nvf714,nvf726,nvf684;shfl.sync.idx.b32 nvr528|nvp334,nvr520,20,31,-1;mov.b32 nvf728,nvr528;fma.rn.ftz.f32 nvf729,nvf714,nvf728,nvf686;shfl.sync.idx.b32 nvr529|nvp335,nvr520,21,31,-1;mov.b32 nvf730,nvr529;fma.rn.ftz.f32 nvf731,nvf714,nvf730,nvf688;shfl.sync.idx.b32 nvr530|nvp336,nvr520,22,31,-1;mov.b32 nvf732,nvr530;fma.rn.ftz.f32 nvf733,nvf714,nvf732,nvf690;shfl.sync.idx.b32 nvr531|nvp337,nvr520,23,31,-1;mov.b32 nvf734,nvr531;fma.rn.ftz.f32 nvf735,nvf714,nvf734,nvf692;shfl.sync.idx.b32 nvr532|nvp338,nvr520,24,31,-1;mov.b32 nvf736,nvr532;fma.rn.ftz.f32 nvf737,nvf714,nvf736,nvf694;shfl.sync.idx.b32 nvr533|nvp339,nvr520,25,31,-1;mov.b32 nvf738,nvr533;fma.rn.ftz.f32 nvf739,nvf714,nvf738,nvf696;shfl.sync.idx.b32 nvr534|nvp340,nvr520,26,31,-1;mov.b32 nvf740,nvr534;fma.rn.ftz.f32 nvf741,nvf714,nvf740,nvf698;shfl.sync.idx.b32 nvr535|nvp341,nvr520,27,31,-1;mov.b32 nvf742,nvr535;fma.rn.ftz.f32 nvf743,nvf714,nvf742,nvf700;shfl.sync.idx.b32 nvr536|nvp342,nvr520,28,31,-1;mov.b32 nvf744,nvr536;fma.rn.ftz.f32 nvf745,nvf714,nvf744,nvf702;shfl.sync.idx.b32 nvr537|nvp343,nvr520,29,31,-1;mov.b32 nvf746,nvr537;fma.rn.ftz.f32 nvf747,nvf714,nvf746,nvf704;shfl.sync.idx.b32 nvr538|nvp344,nvr520,30,31,-1;mov.b32 nvf748,nvr538;fma.rn.ftz.f32 nvf749,nvf714,nvf748,nvf706;shfl.sync.idx.b32 nvr539|nvp345,nvr520,31,31,-1;mov.b32 nvf750,nvr539;fma.rn.ftz.f32 nvf751,nvf714,nvf750,nvf708;max.ftz.f32 nvf752,nvf715,0f00800000;sqrt.approx.ftz.f32 nvf753,nvf752;mov.b32 nvr540,nvf753;shfl.sync.idx.b32 nvr541|nvp346,nvr540,13,31,-1;mov.b32 nvf754,nvr541;div.approx.ftz.f32 nvf755,nvf715,nvf754;mov.b32 nvr542,nvf755;shfl.sync.idx.b32 nvr543|nvp347,nvr542,14,31,-1;mov.b32 nvf756,nvr543;neg.ftz.f32 nvf757,nvf755;fma.rn.ftz.f32 nvf758,nvf757,nvf756,nvf717;shfl.sync.idx.b32 nvr544|nvp348,nvr542,15,31,-1;mov.b32 nvf759,nvr544;fma.rn.ftz.f32 nvf760,nvf757,nvf759,nvf719;shfl.sync.idx.b32 nvr545|nvp349,nvr542,16,31,-1;mov.b32 nvf761,nvr545;fma.rn.ftz.f32 nvf762,nvf757,nvf761,nvf721;shfl.sync.idx.b32 nvr546|nvp350,nvr542,17,31,-1;mov.b32 nvf763,nvr546;fma.rn.ftz.f32 nvf764,nvf757,nvf763,nvf723;shfl.sync.idx.b32 nvr547|nvp351,nvr542,18,31,-1;mov.b32 nvf765,nvr547;fma.rn.ftz.f32 nvf766,nvf757,nvf765,nvf725;shfl.sync.idx.b32 nvr548|nvp352,nvr542,19,31,-1;mov.b32 nvf767,nvr548;fma.rn.ftz.f32 nvf768,nvf757,nvf767,nvf727;shfl.sync.idx.b32 nvr549|nvp353,nvr542,20,31,-1;mov.b32 nvf769,nvr549;fma.rn.ftz.f32 nvf770,nvf757,nvf769,nvf729;shfl.sync.idx.b32 nvr550|nvp354,nvr542,21,31,-1;mov.b32 nvf771,nvr550;fma.rn.ftz.f32 nvf772,nvf757,nvf771,nvf731;shfl.sync.idx.b32 nvr551|nvp355,nvr542,22,31,-1;mov.b32 nvf773,nvr551;fma.rn.ftz.f32 nvf774,nvf757,nvf773,nvf733;shfl.sync.idx.b32 nvr552|nvp356,nvr542,23,31,-1;mov.b32 nvf775,nvr552;fma.rn.ftz.f32 nvf776,nvf757,nvf775,nvf735;shfl.sync.idx.b32 nvr553|nvp357,nvr542,24,31,-1;mov.b32 nvf777,nvr553;fma.rn.ftz.f32 nvf778,nvf757,nvf777,nvf737;shfl.sync.idx.b32 nvr554|nvp358,nvr542,25,31,-1;mov.b32 nvf779,nvr554;fma.rn.ftz.f32 nvf780,nvf757,nvf779,nvf739;shfl.sync.idx.b32 nvr555|nvp359,nvr542,26,31,-1;mov.b32 nvf781,nvr555;fma.rn.ftz.f32 nvf782,nvf757,nvf781,nvf741;shfl.sync.idx.b32 nvr556|nvp360,nvr542,27,31,-1;mov.b32 nvf783,nvr556;fma.rn.ftz.f32 nvf784,nvf757,nvf783,nvf743;shfl.sync.idx.b32 nvr557|nvp361,nvr542,28,31,-1;mov.b32 nvf785,nvr557;fma.rn.ftz.f32 nvf786,nvf757,nvf785,nvf745;shfl.sync.idx.b32 nvr558|nvp362,nvr542,29,31,-1;mov.b32 nvf787,nvr558;fma.rn.ftz.f32 nvf788,nvf757,nvf787,nvf747;shfl.sync.idx.b32 nvr559|nvp363,nvr542,30,31,-1;mov.b32 nvf789,nvr559;fma.rn.ftz.f32 nvf790,nvf757,nvf789,nvf749;shfl.sync.idx.b32 nvr560|nvp364,nvr542,31,31,-1;mov.b32 nvf791,nvr560;fma.rn.ftz.f32 nvf792,nvf757,nvf791,nvf751;max.ftz.f32 nvf793,nvf758,0f00800000;sqrt.approx.ftz.f32 nvf794,nvf793;mov.b32 nvr561,nvf794;shfl.sync.idx.b32 nvr562|nvp365,nvr561,14,31,-1;mov.b32 nvf795,nvr562;div.approx.ftz.f32 nvf796,nvf758,nvf795;mov.b32 nvr563,nvf796;shfl.sync.idx.b32 nvr564|nvp366,nvr563,15,31,-1;mov.b32 nvf797,nvr564;neg.ftz.f32 nvf798,nvf796;fma.rn.ftz.f32 nvf799,nvf798,nvf797,nvf760;shfl.sync.idx.b32 nvr565|nvp367,nvr563,16,31,-1;mov.b32 nvf800,nvr565;fma.rn.ftz.f32 nvf801,nvf798,nvf800,nvf762;shfl.sync.idx.b32 nvr566|nvp368,nvr563,17,31,-1;mov.b32 nvf802,nvr566;fma.rn.ftz.f32 nvf803,nvf798,nvf802,nvf764;shfl.sync.idx.b32 nvr567|nvp369,nvr563,18,31,-1;mov.b32 nvf804,nvr567;fma.rn.ftz.f32 nvf805,nvf798,nvf804,nvf766;shfl.sync.idx.b32 nvr568|nvp370,nvr563,19,31,-1;mov.b32 nvf806,nvr568;fma.rn.ftz.f32 nvf807,nvf798,nvf806,nvf768;shfl.sync.idx.b32 nvr569|nvp371,nvr563,20,31,-1;mov.b32 nvf808,nvr569;fma.rn.ftz.f32 nvf809,nvf798,nvf808,nvf770;shfl.sync.idx.b32 nvr570|nvp372,nvr563,21,31,-1;mov.b32 nvf810,nvr570;fma.rn.ftz.f32 nvf811,nvf798,nvf810,nvf772;shfl.sync.idx.b32 nvr571|nvp373,nvr563,22,31,-1;mov.b32 nvf812,nvr571;fma.rn.ftz.f32 nvf813,nvf798,nvf812,nvf774;shfl.sync.idx.b32 nvr572|nvp374,nvr563,23,31,-1;mov.b32 nvf814,nvr572;fma.rn.ftz.f32 nvf815,nvf798,nvf814,nvf776;shfl.sync.idx.b32 nvr573|nvp375,nvr563,24,31,-1;mov.b32 nvf816,nvr573;fma.rn.ftz.f32 nvf817,nvf798,nvf816,nvf778;shfl.sync.idx.b32 nvr574|nvp376,nvr563,25,31,-1;mov.b32 nvf818,nvr574;fma.rn.ftz.f32 nvf819,nvf798,nvf818,nvf780;shfl.sync.idx.b32 nvr575|nvp377,nvr563,26,31,-1;mov.b32 nvf820,nvr575;fma.rn.ftz.f32 nvf821,nvf798,nvf820,nvf782;shfl.sync.idx.b32 nvr576|nvp378,nvr563,27,31,-1;mov.b32 nvf822,nvr576;fma.rn.ftz.f32 nvf823,nvf798,nvf822,nvf784;shfl.sync.idx.b32 nvr577|nvp379,nvr563,28,31,-1;mov.b32 nvf824,nvr577;fma.rn.ftz.f32 nvf825,nvf798,nvf824,nvf786;shfl.sync.idx.b32 nvr578|nvp380,nvr563,29,31,-1;mov.b32 nvf826,nvr578;fma.rn.ftz.f32 nvf827,nvf798,nvf826,nvf788;shfl.sync.idx.b32 nvr579|nvp381,nvr563,30,31,-1;mov.b32 nvf828,nvr579;fma.rn.ftz.f32 nvf829,nvf798,nvf828,nvf790;shfl.sync.idx.b32 nvr580|nvp382,nvr563,31,31,-1;mov.b32 nvf830,nvr580;fma.rn.ftz.f32 nvf831,nvf798,nvf830,nvf792;max.ftz.f32 nvf832,nvf799,0f00800000;sqrt.approx.ftz.f32 nvf833,nvf832;mov.b32 nvr581,nvf833;shfl.sync.idx.b32 nvr582|nvp383,nvr581,15,31,-1;mov.b32 nvf834,nvr582;div.approx.ftz.f32 nvf835,nvf799,nvf834;mov.b32 nvr583,nvf835;shfl.sync.idx.b32 nvr584|nvp384,nvr583,16,31,-1;mov.b32 nvf836,nvr584;neg.ftz.f32 nvf837,nvf835;fma.rn.ftz.f32 nvf838,nvf837,nvf836,nvf801;shfl.sync.idx.b32 nvr585|nvp385,nvr583,17,31,-1;mov.b32 nvf839,nvr585;fma.rn.ftz.f32 nvf840,nvf837,nvf839,nvf803;shfl.sync.idx.b32 nvr586|nvp386,nvr583,18,31,-1;mov.b32 nvf841,nvr586;fma.rn.ftz.f32 nvf842,nvf837,nvf841,nvf805;shfl.sync.idx.b32 nvr587|nvp387,nvr583,19,31,-1;mov.b32 nvf843,nvr587;fma.rn.ftz.f32 nvf844,nvf837,nvf843,nvf807;shfl.sync.idx.b32 nvr588|nvp388,nvr583,20,31,-1;mov.b32 nvf845,nvr588;fma.rn.ftz.f32 nvf846,nvf837,nvf845,nvf809;shfl.sync.idx.b32 nvr589|nvp389,nvr583,21,31,-1;mov.b32 nvf847,nvr589;fma.rn.ftz.f32 nvf848,nvf837,nvf847,nvf811;shfl.sync.idx.b32 nvr590|nvp390,nvr583,22,31,-1;mov.b32 nvf849,nvr590;fma.rn.ftz.f32 nvf850,nvf837,nvf849,nvf813;shfl.sync.idx.b32 nvr591|nvp391,nvr583,23,31,-1;mov.b32 nvf851,nvr591;fma.rn.ftz.f32 nvf852,nvf837,nvf851,nvf815;shfl.sync.idx.b32 nvr592|nvp392,nvr583,24,31,-1;mov.b32 nvf853,nvr592;fma.rn.ftz.f32 nvf854,nvf837,nvf853,nvf817;shfl.sync.idx.b32 nvr593|nvp393,nvr583,25,31,-1;mov.b32 nvf855,nvr593;fma.rn.ftz.f32 nvf856,nvf837,nvf855,nvf819;shfl.sync.idx.b32 nvr594|nvp394,nvr583,26,31,-1;mov.b32 nvf857,nvr594;fma.rn.ftz.f32 nvf858,nvf837,nvf857,nvf821;shfl.sync.idx.b32 nvr595|nvp395,nvr583,27,31,-1;mov.b32 nvf859,nvr595;fma.rn.ftz.f32 nvf860,nvf837,nvf859,nvf823;shfl.sync.idx.b32 nvr596|nvp396,nvr583,28,31,-1;mov.b32 nvf861,nvr596;fma.rn.ftz.f32 nvf862,nvf837,nvf861,nvf825;shfl.sync.idx.b32 nvr597|nvp397,nvr583,29,31,-1;mov.b32 nvf863,nvr597;fma.rn.ftz.f32 nvf864,nvf837,nvf863,nvf827;shfl.sync.idx.b32 nvr598|nvp398,nvr583,30,31,-1;mov.b32 nvf865,nvr598;fma.rn.ftz.f32 nvf866,nvf837,nvf865,nvf829;shfl.sync.idx.b32 nvr599|nvp399,nvr583,31,31,-1;mov.b32 nvf867,nvr599;fma.rn.ftz.f32 nvf868,nvf837,nvf867,nvf831;max.ftz.f32 nvf869,nvf838,0f00800000;sqrt.approx.ftz.f32 nvf870,nvf869;mov.b32 nvr600,nvf870;shfl.sync.idx.b32 nvr601|nvp400,nvr600,16,31,-1;mov.b32 nvf871,nvr601;div.approx.ftz.f32 nvf872,nvf838,nvf871;mov.b32 nvr602,nvf872;shfl.sync.idx.b32 nvr603|nvp401,nvr602,17,31,-1;mov.b32 nvf873,nvr603;neg.ftz.f32 nvf874,nvf872;fma.rn.ftz.f32 nvf875,nvf874,nvf873,nvf840;shfl.sync.idx.b32 nvr604|nvp402,nvr602,18,31,-1;mov.b32 nvf876,nvr604;fma.rn.ftz.f32 nvf877,nvf874,nvf876,nvf842;shfl.sync.idx.b32 nvr605|nvp403,nvr602,19,31,-1;mov.b32 nvf878,nvr605;fma.rn.ftz.f32 nvf879,nvf874,nvf878,nvf844;shfl.sync.idx.b32 nvr606|nvp404,nvr602,20,31,-1;mov.b32 nvf880,nvr606;fma.rn.ftz.f32 nvf881,nvf874,nvf880,nvf846;shfl.sync.idx.b32 nvr607|nvp405,nvr602,21,31,-1;mov.b32 nvf882,nvr607;fma.rn.ftz.f32 nvf883,nvf874,nvf882,nvf848;shfl.sync.idx.b32 nvr608|nvp406,nvr602,22,31,-1;mov.b32 nvf884,nvr608;fma.rn.ftz.f32 nvf885,nvf874,nvf884,nvf850;shfl.sync.idx.b32 nvr609|nvp407,nvr602,23,31,-1;mov.b32 nvf886,nvr609;fma.rn.ftz.f32 nvf887,nvf874,nvf886,nvf852;shfl.sync.idx.b32 nvr610|nvp408,nvr602,24,31,-1;mov.b32 nvf888,nvr610;fma.rn.ftz.f32 nvf889,nvf874,nvf888,nvf854;shfl.sync.idx.b32 nvr611|nvp409,nvr602,25,31,-1;mov.b32 nvf890,nvr611;fma.rn.ftz.f32 nvf891,nvf874,nvf890,nvf856;shfl.sync.idx.b32 nvr612|nvp410,nvr602,26,31,-1;mov.b32 nvf892,nvr612;fma.rn.ftz.f32 nvf893,nvf874,nvf892,nvf858;shfl.sync.idx.b32 nvr613|nvp411,nvr602,27,31,-1;mov.b32 nvf894,nvr613;fma.rn.ftz.f32 nvf895,nvf874,nvf894,nvf860;shfl.sync.idx.b32 nvr614|nvp412,nvr602,28,31,-1;mov.b32 nvf896,nvr614;fma.rn.ftz.f32 nvf897,nvf874,nvf896,nvf862;shfl.sync.idx.b32 nvr615|nvp413,nvr602,29,31,-1;mov.b32 nvf898,nvr615;fma.rn.ftz.f32 nvf899,nvf874,nvf898,nvf864;shfl.sync.idx.b32 nvr616|nvp414,nvr602,30,31,-1;mov.b32 nvf900,nvr616;fma.rn.ftz.f32 nvf901,nvf874,nvf900,nvf866;shfl.sync.idx.b32 nvr617|nvp415,nvr602,31,31,-1;mov.b32 nvf902,nvr617;fma.rn.ftz.f32 nvf903,nvf874,nvf902,nvf868;max.ftz.f32 nvf904,nvf875,0f00800000;sqrt.approx.ftz.f32 nvf905,nvf904;mov.b32 nvr618,nvf905;shfl.sync.idx.b32 nvr619|nvp416,nvr618,17,31,-1;mov.b32 nvf906,nvr619;div.approx.ftz.f32 nvf907,nvf875,nvf906;mov.b32 nvr620,nvf907;shfl.sync.idx.b32 nvr621|nvp417,nvr620,18,31,-1;mov.b32 nvf908,nvr621;neg.ftz.f32 nvf909,nvf907;fma.rn.ftz.f32 nvf910,nvf909,nvf908,nvf877;shfl.sync.idx.b32 nvr622|nvp418,nvr620,19,31,-1;mov.b32 nvf911,nvr622;fma.rn.ftz.f32 nvf912,nvf909,nvf911,nvf879;shfl.sync.idx.b32 nvr623|nvp419,nvr620,20,31,-1;mov.b32 nvf913,nvr623;fma.rn.ftz.f32 nvf914,nvf909,nvf913,nvf881;shfl.sync.idx.b32 nvr624|nvp420,nvr620,21,31,-1;mov.b32 nvf915,nvr624;fma.rn.ftz.f32 nvf916,nvf909,nvf915,nvf883;shfl.sync.idx.b32 nvr625|nvp421,nvr620,22,31,-1;mov.b32 nvf917,nvr625;fma.rn.ftz.f32 nvf918,nvf909,nvf917,nvf885;shfl.sync.idx.b32 nvr626|nvp422,nvr620,23,31,-1;mov.b32 nvf919,nvr626;fma.rn.ftz.f32 nvf920,nvf909,nvf919,nvf887;shfl.sync.idx.b32 nvr627|nvp423,nvr620,24,31,-1;mov.b32 nvf921,nvr627;fma.rn.ftz.f32 nvf922,nvf909,nvf921,nvf889;shfl.sync.idx.b32 nvr628|nvp424,nvr620,25,31,-1;mov.b32 nvf923,nvr628;fma.rn.ftz.f32 nvf924,nvf909,nvf923,nvf891;shfl.sync.idx.b32 nvr629|nvp425,nvr620,26,31,-1;mov.b32 nvf925,nvr629;fma.rn.ftz.f32 nvf926,nvf909,nvf925,nvf893;shfl.sync.idx.b32 nvr630|nvp426,nvr620,27,31,-1;mov.b32 nvf927,nvr630;fma.rn.ftz.f32 nvf928,nvf909,nvf927,nvf895;shfl.sync.idx.b32 nvr631|nvp427,nvr620,28,31,-1;mov.b32 nvf929,nvr631;fma.rn.ftz.f32 nvf930,nvf909,nvf929,nvf897;shfl.sync.idx.b32 nvr632|nvp428,nvr620,29,31,-1;mov.b32 nvf931,nvr632;fma.rn.ftz.f32 nvf932,nvf909,nvf931,nvf899;shfl.sync.idx.b32 nvr633|nvp429,nvr620,30,31,-1;mov.b32 nvf933,nvr633;fma.rn.ftz.f32 nvf934,nvf909,nvf933,nvf901;shfl.sync.idx.b32 nvr634|nvp430,nvr620,31,31,-1;mov.b32 nvf935,nvr634;fma.rn.ftz.f32 nvf936,nvf909,nvf935,nvf903;max.ftz.f32 nvf937,nvf910,0f00800000;sqrt.approx.ftz.f32 nvf938,nvf937;mov.b32 nvr635,nvf938;shfl.sync.idx.b32 nvr636|nvp431,nvr635,18,31,-1;mov.b32 nvf939,nvr636;div.approx.ftz.f32 nvf940,nvf910,nvf939;mov.b32 nvr637,nvf940;shfl.sync.idx.b32 nvr638|nvp432,nvr637,19,31,-1;mov.b32 nvf941,nvr638;neg.ftz.f32 nvf942,nvf940;fma.rn.ftz.f32 nvf943,nvf942,nvf941,nvf912;shfl.sync.idx.b32 nvr639|nvp433,nvr637,20,31,-1;mov.b32 nvf944,nvr639;fma.rn.ftz.f32 nvf945,nvf942,nvf944,nvf914;shfl.sync.idx.b32 nvr640|nvp434,nvr637,21,31,-1;mov.b32 nvf946,nvr640;fma.rn.ftz.f32 nvf947,nvf942,nvf946,nvf916;shfl.sync.idx.b32 nvr641|nvp435,nvr637,22,31,-1;mov.b32 nvf948,nvr641;fma.rn.ftz.f32 nvf949,nvf942,nvf948,nvf918;shfl.sync.idx.b32 nvr642|nvp436,nvr637,23,31,-1;mov.b32 nvf950,nvr642;fma.rn.ftz.f32 nvf951,nvf942,nvf950,nvf920;shfl.sync.idx.b32 nvr643|nvp437,nvr637,24,31,-1;mov.b32 nvf952,nvr643;fma.rn.ftz.f32 nvf953,nvf942,nvf952,nvf922;shfl.sync.idx.b32 nvr644|nvp438,nvr637,25,31,-1;mov.b32 nvf954,nvr644;fma.rn.ftz.f32 nvf955,nvf942,nvf954,nvf924;shfl.sync.idx.b32 nvr645|nvp439,nvr637,26,31,-1;mov.b32 nvf956,nvr645;fma.rn.ftz.f32 nvf957,nvf942,nvf956,nvf926;shfl.sync.idx.b32 nvr646|nvp440,nvr637,27,31,-1;mov.b32 nvf958,nvr646;fma.rn.ftz.f32 nvf959,nvf942,nvf958,nvf928;shfl.sync.idx.b32 nvr647|nvp441,nvr637,28,31,-1;mov.b32 nvf960,nvr647;fma.rn.ftz.f32 nvf961,nvf942,nvf960,nvf930;shfl.sync.idx.b32 nvr648|nvp442,nvr637,29,31,-1;mov.b32 nvf962,nvr648;fma.rn.ftz.f32 nvf963,nvf942,nvf962,nvf932;shfl.sync.idx.b32 nvr649|nvp443,nvr637,30,31,-1;mov.b32 nvf964,nvr649;fma.rn.ftz.f32 nvf965,nvf942,nvf964,nvf934;shfl.sync.idx.b32 nvr650|nvp444,nvr637,31,31,-1;mov.b32 nvf966,nvr650;fma.rn.ftz.f32 nvf967,nvf942,nvf966,nvf936;max.ftz.f32 nvf968,nvf943,0f00800000;sqrt.approx.ftz.f32 nvf969,nvf968;mov.b32 nvr651,nvf969;shfl.sync.idx.b32 nvr652|nvp445,nvr651,19,31,-1;mov.b32 nvf970,nvr652;div.approx.ftz.f32 nvf971,nvf943,nvf970;mov.b32 nvr653,nvf971;shfl.sync.idx.b32 nvr654|nvp446,nvr653,20,31,-1;mov.b32 nvf972,nvr654;neg.ftz.f32 nvf973,nvf971;fma.rn.ftz.f32 nvf974,nvf973,nvf972,nvf945;shfl.sync.idx.b32 nvr655|nvp447,nvr653,21,31,-1;mov.b32 nvf975,nvr655;fma.rn.ftz.f32 nvf976,nvf973,nvf975,nvf947;shfl.sync.idx.b32 nvr656|nvp448,nvr653,22,31,-1;mov.b32 nvf977,nvr656;fma.rn.ftz.f32 nvf978,nvf973,nvf977,nvf949;shfl.sync.idx.b32 nvr657|nvp449,nvr653,23,31,-1;mov.b32 nvf979,nvr657;fma.rn.ftz.f32 nvf980,nvf973,nvf979,nvf951;shfl.sync.idx.b32 nvr658|nvp450,nvr653,24,31,-1;mov.b32 nvf981,nvr658;fma.rn.ftz.f32 nvf982,nvf973,nvf981,nvf953;shfl.sync.idx.b32 nvr659|nvp451,nvr653,25,31,-1;mov.b32 nvf983,nvr659;fma.rn.ftz.f32 nvf984,nvf973,nvf983,nvf955;shfl.sync.idx.b32 nvr660|nvp452,nvr653,26,31,-1;mov.b32 nvf985,nvr660;fma.rn.ftz.f32 nvf986,nvf973,nvf985,nvf957;shfl.sync.idx.b32 nvr661|nvp453,nvr653,27,31,-1;mov.b32 nvf987,nvr661;fma.rn.ftz.f32 nvf988,nvf973,nvf987,nvf959;shfl.sync.idx.b32 nvr662|nvp454,nvr653,28,31,-1;mov.b32 nvf989,nvr662;fma.rn.ftz.f32 nvf990,nvf973,nvf989,nvf961;shfl.sync.idx.b32 nvr663|nvp455,nvr653,29,31,-1;mov.b32 nvf991,nvr663;fma.rn.ftz.f32 nvf992,nvf973,nvf991,nvf963;shfl.sync.idx.b32 nvr664|nvp456,nvr653,30,31,-1;mov.b32 nvf993,nvr664;fma.rn.ftz.f32 nvf994,nvf973,nvf993,nvf965;shfl.sync.idx.b32 nvr665|nvp457,nvr653,31,31,-1;mov.b32 nvf995,nvr665;fma.rn.ftz.f32 nvf996,nvf973,nvf995,nvf967;max.ftz.f32 nvf997,nvf974,0f00800000;sqrt.approx.ftz.f32 nvf998,nvf997;mov.b32 nvr666,nvf998;shfl.sync.idx.b32 nvr667|nvp458,nvr666,20,31,-1;mov.b32 nvf999,nvr667;div.approx.ftz.f32 nvf1000,nvf974,nvf999;mov.b32 nvr668,nvf1000;shfl.sync.idx.b32 nvr669|nvp459,nvr668,21,31,-1;mov.b32 nvf1001,nvr669;neg.ftz.f32 nvf1002,nvf1000;fma.rn.ftz.f32 nvf1003,nvf1002,nvf1001,nvf976;shfl.sync.idx.b32 nvr670|nvp460,nvr668,22,31,-1;mov.b32 nvf1004,nvr670;fma.rn.ftz.f32 nvf1005,nvf1002,nvf1004,nvf978;shfl.sync.idx.b32 nvr671|nvp461,nvr668,23,31,-1;mov.b32 nvf1006,nvr671;fma.rn.ftz.f32 nvf1007,nvf1002,nvf1006,nvf980;shfl.sync.idx.b32 nvr672|nvp462,nvr668,24,31,-1;mov.b32 nvf1008,nvr672;fma.rn.ftz.f32 nvf1009,nvf1002,nvf1008,nvf982;shfl.sync.idx.b32 nvr673|nvp463,nvr668,25,31,-1;mov.b32 nvf1010,nvr673;fma.rn.ftz.f32 nvf1011,nvf1002,nvf1010,nvf984;shfl.sync.idx.b32 nvr674|nvp464,nvr668,26,31,-1;mov.b32 nvf1012,nvr674;fma.rn.ftz.f32 nvf1013,nvf1002,nvf1012,nvf986;shfl.sync.idx.b32 nvr675|nvp465,nvr668,27,31,-1;mov.b32 nvf1014,nvr675;fma.rn.ftz.f32 nvf1015,nvf1002,nvf1014,nvf988;shfl.sync.idx.b32 nvr676|nvp466,nvr668,28,31,-1;mov.b32 nvf1016,nvr676;fma.rn.ftz.f32 nvf1017,nvf1002,nvf1016,nvf990;shfl.sync.idx.b32 nvr677|nvp467,nvr668,29,31,-1;mov.b32 nvf1018,nvr677;fma.rn.ftz.f32 nvf1019,nvf1002,nvf1018,nvf992;shfl.sync.idx.b32 nvr678|nvp468,nvr668,30,31,-1;mov.b32 nvf1020,nvr678;fma.rn.ftz.f32 nvf1021,nvf1002,nvf1020,nvf994;shfl.sync.idx.b32 nvr679|nvp469,nvr668,31,31,-1;mov.b32 nvf1022,nvr679;fma.rn.ftz.f32 nvf1023,nvf1002,nvf1022,nvf996;max.ftz.f32 nvf1024,nvf1003,0f00800000;sqrt.approx.ftz.f32 nvf1025,nvf1024;mov.b32 nvr680,nvf1025;shfl.sync.idx.b32 nvr681|nvp470,nvr680,21,31,-1;mov.b32 nvf1026,nvr681;div.approx.ftz.f32 nvf1027,nvf1003,nvf1026;mov.b32 nvr682,nvf1027;shfl.sync.idx.b32 nvr683|nvp471,nvr682,22,31,-1;mov.b32 nvf1028,nvr683;neg.ftz.f32 nvf1029,nvf1027;fma.rn.ftz.f32 nvf1030,nvf1029,nvf1028,nvf1005;shfl.sync.idx.b32 nvr684|nvp472,nvr682,23,31,-1;mov.b32 nvf1031,nvr684;fma.rn.ftz.f32 nvf1032,nvf1029,nvf1031,nvf1007;shfl.sync.idx.b32 nvr685|nvp473,nvr682,24,31,-1;mov.b32 nvf1033,nvr685;fma.rn.ftz.f32 nvf1034,nvf1029,nvf1033,nvf1009;shfl.sync.idx.b32 nvr686|nvp474,nvr682,25,31,-1;mov.b32 nvf1035,nvr686;fma.rn.ftz.f32 nvf1036,nvf1029,nvf1035,nvf1011;shfl.sync.idx.b32 nvr687|nvp475,nvr682,26,31,-1;mov.b32 nvf1037,nvr687;fma.rn.ftz.f32 nvf1038,nvf1029,nvf1037,nvf1013;shfl.sync.idx.b32 nvr688|nvp476,nvr682,27,31,-1;mov.b32 nvf1039,nvr688;fma.rn.ftz.f32 nvf1040,nvf1029,nvf1039,nvf1015;shfl.sync.idx.b32 nvr689|nvp477,nvr682,28,31,-1;mov.b32 nvf1041,nvr689;fma.rn.ftz.f32 nvf1042,nvf1029,nvf1041,nvf1017;shfl.sync.idx.b32 nvr690|nvp478,nvr682,29,31,-1;mov.b32 nvf1043,nvr690;fma.rn.ftz.f32 nvf1044,nvf1029,nvf1043,nvf1019;shfl.sync.idx.b32 nvr691|nvp479,nvr682,30,31,-1;mov.b32 nvf1045,nvr691;fma.rn.ftz.f32 nvf1046,nvf1029,nvf1045,nvf1021;shfl.sync.idx.b32 nvr692|nvp480,nvr682,31,31,-1;mov.b32 nvf1047,nvr692;fma.rn.ftz.f32 nvf1048,nvf1029,nvf1047,nvf1023;max.ftz.f32 nvf1049,nvf1030,0f00800000;sqrt.approx.ftz.f32 nvf1050,nvf1049;mov.b32 nvr693,nvf1050;shfl.sync.idx.b32 nvr694|nvp481,nvr693,22,31,-1;mov.b32 nvf1051,nvr694;div.approx.ftz.f32 nvf1052,nvf1030,nvf1051;mov.b32 nvr695,nvf1052;shfl.sync.idx.b32 nvr696|nvp482,nvr695,23,31,-1;mov.b32 nvf1053,nvr696;neg.ftz.f32 nvf1054,nvf1052;fma.rn.ftz.f32 nvf1055,nvf1054,nvf1053,nvf1032;shfl.sync.idx.b32 nvr697|nvp483,nvr695,24,31,-1;mov.b32 nvf1056,nvr697;fma.rn.ftz.f32 nvf1057,nvf1054,nvf1056,nvf1034;shfl.sync.idx.b32 nvr698|nvp484,nvr695,25,31,-1;mov.b32 nvf1058,nvr698;fma.rn.ftz.f32 nvf1059,nvf1054,nvf1058,nvf1036;shfl.sync.idx.b32 nvr699|nvp485,nvr695,26,31,-1;mov.b32 nvf1060,nvr699;fma.rn.ftz.f32 nvf1061,nvf1054,nvf1060,nvf1038;shfl.sync.idx.b32 nvr700|nvp486,nvr695,27,31,-1;mov.b32 nvf1062,nvr700;fma.rn.ftz.f32 nvf1063,nvf1054,nvf1062,nvf1040;shfl.sync.idx.b32 nvr701|nvp487,nvr695,28,31,-1;mov.b32 nvf1064,nvr701;fma.rn.ftz.f32 nvf1065,nvf1054,nvf1064,nvf1042;shfl.sync.idx.b32 nvr702|nvp488,nvr695,29,31,-1;mov.b32 nvf1066,nvr702;fma.rn.ftz.f32 nvf1067,nvf1054,nvf1066,nvf1044;shfl.sync.idx.b32 nvr703|nvp489,nvr695,30,31,-1;mov.b32 nvf1068,nvr703;fma.rn.ftz.f32 nvf1069,nvf1054,nvf1068,nvf1046;shfl.sync.idx.b32 nvr704|nvp490,nvr695,31,31,-1;mov.b32 nvf1070,nvr704;fma.rn.ftz.f32 nvf1071,nvf1054,nvf1070,nvf1048;max.ftz.f32 nvf1072,nvf1055,0f00800000;sqrt.approx.ftz.f32 nvf1073,nvf1072;mov.b32 nvr705,nvf1073;shfl.sync.idx.b32 nvr706|nvp491,nvr705,23,31,-1;mov.b32 nvf1074,nvr706;div.approx.ftz.f32 nvf1075,nvf1055,nvf1074;mov.b32 nvr707,nvf1075;shfl.sync.idx.b32 nvr708|nvp492,nvr707,24,31,-1;mov.b32 nvf1076,nvr708;neg.ftz.f32 nvf1077,nvf1075;fma.rn.ftz.f32 nvf1078,nvf1077,nvf1076,nvf1057;shfl.sync.idx.b32 nvr709|nvp493,nvr707,25,31,-1;mov.b32 nvf1079,nvr709;fma.rn.ftz.f32 nvf1080,nvf1077,nvf1079,nvf1059;shfl.sync.idx.b32 nvr710|nvp494,nvr707,26,31,-1;mov.b32 nvf1081,nvr710;fma.rn.ftz.f32 nvf1082,nvf1077,nvf1081,nvf1061;shfl.sync.idx.b32 nvr711|nvp495,nvr707,27,31,-1;mov.b32 nvf1083,nvr711;fma.rn.ftz.f32 nvf1084,nvf1077,nvf1083,nvf1063;shfl.sync.idx.b32 nvr712|nvp496,nvr707,28,31,-1;mov.b32 nvf1085,nvr712;fma.rn.ftz.f32 nvf1086,nvf1077,nvf1085,nvf1065;shfl.sync.idx.b32 nvr713|nvp497,nvr707,29,31,-1;mov.b32 nvf1087,nvr713;fma.rn.ftz.f32 nvf1088,nvf1077,nvf1087,nvf1067;shfl.sync.idx.b32 nvr714|nvp498,nvr707,30,31,-1;mov.b32 nvf1089,nvr714;fma.rn.ftz.f32 nvf1090,nvf1077,nvf1089,nvf1069;shfl.sync.idx.b32 nvr715|nvp499,nvr707,31,31,-1;mov.b32 nvf1091,nvr715;fma.rn.ftz.f32 nvf1092,nvf1077,nvf1091,nvf1071;max.ftz.f32 nvf1093,nvf1078,0f00800000;sqrt.approx.ftz.f32 nvf1094,nvf1093;mov.b32 nvr716,nvf1094;shfl.sync.idx.b32 nvr717|nvp500,nvr716,24,31,-1;mov.b32 nvf1095,nvr717;div.approx.ftz.f32 nvf1096,nvf1078,nvf1095;mov.b32 nvr718,nvf1096;shfl.sync.idx.b32 nvr719|nvp501,nvr718,25,31,-1;mov.b32 nvf1097,nvr719;neg.ftz.f32 nvf1098,nvf1096;fma.rn.ftz.f32 nvf1099,nvf1098,nvf1097,nvf1080;shfl.sync.idx.b32 nvr720|nvp502,nvr718,26,31,-1;mov.b32 nvf1100,nvr720;fma.rn.ftz.f32 nvf1101,nvf1098,nvf1100,nvf1082;shfl.sync.idx.b32 nvr721|nvp503,nvr718,27,31,-1;mov.b32 nvf1102,nvr721;fma.rn.ftz.f32 nvf1103,nvf1098,nvf1102,nvf1084;shfl.sync.idx.b32 nvr722|nvp504,nvr718,28,31,-1;mov.b32 nvf1104,nvr722;fma.rn.ftz.f32 nvf1105,nvf1098,nvf1104,nvf1086;shfl.sync.idx.b32 nvr723|nvp505,nvr718,29,31,-1;mov.b32 nvf1106,nvr723;fma.rn.ftz.f32 nvf1107,nvf1098,nvf1106,nvf1088;shfl.sync.idx.b32 nvr724|nvp506,nvr718,30,31,-1;mov.b32 nvf1108,nvr724;fma.rn.ftz.f32 nvf1109,nvf1098,nvf1108,nvf1090;shfl.sync.idx.b32 nvr725|nvp507,nvr718,31,31,-1;mov.b32 nvf1110,nvr725;fma.rn.ftz.f32 nvf1111,nvf1098,nvf1110,nvf1092;max.ftz.f32 nvf1112,nvf1099,0f00800000;sqrt.approx.ftz.f32 nvf1113,nvf1112;mov.b32 nvr726,nvf1113;shfl.sync.idx.b32 nvr727|nvp508,nvr726,25,31,-1;mov.b32 nvf1114,nvr727;div.approx.ftz.f32 nvf1115,nvf1099,nvf1114;mov.b32 nvr728,nvf1115;shfl.sync.idx.b32 nvr729|nvp509,nvr728,26,31,-1;mov.b32 nvf1116,nvr729;neg.ftz.f32 nvf1117,nvf1115;fma.rn.ftz.f32 nvf1118,nvf1117,nvf1116,nvf1101;shfl.sync.idx.b32 nvr730|nvp510,nvr728,27,31,-1;mov.b32 nvf1119,nvr730;fma.rn.ftz.f32 nvf1120,nvf1117,nvf1119,nvf1103;shfl.sync.idx.b32 nvr731|nvp511,nvr728,28,31,-1;mov.b32 nvf1121,nvr731;fma.rn.ftz.f32 nvf1122,nvf1117,nvf1121,nvf1105;shfl.sync.idx.b32 nvr732|nvp512,nvr728,29,31,-1;mov.b32 nvf1123,nvr732;fma.rn.ftz.f32 nvf1124,nvf1117,nvf1123,nvf1107;shfl.sync.idx.b32 nvr733|nvp513,nvr728,30,31,-1;mov.b32 nvf1125,nvr733;fma.rn.ftz.f32 nvf1126,nvf1117,nvf1125,nvf1109;shfl.sync.idx.b32 nvr734|nvp514,nvr728,31,31,-1;mov.b32 nvf1127,nvr734;fma.rn.ftz.f32 nvf1128,nvf1117,nvf1127,nvf1111;max.ftz.f32 nvf1129,nvf1118,0f00800000;sqrt.approx.ftz.f32 nvf1130,nvf1129;mov.b32 nvr735,nvf1130;shfl.sync.idx.b32 nvr736|nvp515,nvr735,26,31,-1;mov.b32 nvf1131,nvr736;div.approx.ftz.f32 nvf1132,nvf1118,nvf1131;mov.b32 nvr737,nvf1132;shfl.sync.idx.b32 nvr738|nvp516,nvr737,27,31,-1;mov.b32 nvf1133,nvr738;neg.ftz.f32 nvf1134,nvf1132;fma.rn.ftz.f32 nvf1135,nvf1134,nvf1133,nvf1120;shfl.sync.idx.b32 nvr739|nvp517,nvr737,28,31,-1;mov.b32 nvf1136,nvr739;fma.rn.ftz.f32 nvf1137,nvf1134,nvf1136,nvf1122;shfl.sync.idx.b32 nvr740|nvp518,nvr737,29,31,-1;mov.b32 nvf1138,nvr740;fma.rn.ftz.f32 nvf1139,nvf1134,nvf1138,nvf1124;shfl.sync.idx.b32 nvr741|nvp519,nvr737,30,31,-1;mov.b32 nvf1140,nvr741;fma.rn.ftz.f32 nvf1141,nvf1134,nvf1140,nvf1126;shfl.sync.idx.b32 nvr742|nvp520,nvr737,31,31,-1;mov.b32 nvf1142,nvr742;fma.rn.ftz.f32 nvf1143,nvf1134,nvf1142,nvf1128;max.ftz.f32 nvf1144,nvf1135,0f00800000;sqrt.approx.ftz.f32 nvf1145,nvf1144;mov.b32 nvr743,nvf1145;shfl.sync.idx.b32 nvr744|nvp521,nvr743,27,31,-1;mov.b32 nvf1146,nvr744;div.approx.ftz.f32 nvf1147,nvf1135,nvf1146;mov.b32 nvr745,nvf1147;shfl.sync.idx.b32 nvr746|nvp522,nvr745,28,31,-1;mov.b32 nvf1148,nvr746;neg.ftz.f32 nvf1149,nvf1147;fma.rn.ftz.f32 nvf1150,nvf1149,nvf1148,nvf1137;shfl.sync.idx.b32 nvr747|nvp523,nvr745,29,31,-1;mov.b32 nvf1151,nvr747;fma.rn.ftz.f32 nvf1152,nvf1149,nvf1151,nvf1139;shfl.sync.idx.b32 nvr748|nvp524,nvr745,30,31,-1;mov.b32 nvf1153,nvr748;fma.rn.ftz.f32 nvf1154,nvf1149,nvf1153,nvf1141;shfl.sync.idx.b32 nvr749|nvp525,nvr745,31,31,-1;mov.b32 nvf1155,nvr749;fma.rn.ftz.f32 nvf1156,nvf1149,nvf1155,nvf1143;max.ftz.f32 nvf1157,nvf1150,0f00800000;sqrt.approx.ftz.f32 nvf1158,nvf1157;mov.b32 nvr750,nvf1158;shfl.sync.idx.b32 nvr751|nvp526,nvr750,28,31,-1;mov.b32 nvf1159,nvr751;div.approx.ftz.f32 nvf1160,nvf1150,nvf1159;mov.b32 nvr752,nvf1160;shfl.sync.idx.b32 nvr753|nvp527,nvr752,29,31,-1;mov.b32 nvf1161,nvr753;neg.ftz.f32 nvf1162,nvf1160;fma.rn.ftz.f32 nvf1163,nvf1162,nvf1161,nvf1152;shfl.sync.idx.b32 nvr754|nvp528,nvr752,30,31,-1;mov.b32 nvf1164,nvr754;fma.rn.ftz.f32 nvf1165,nvf1162,nvf1164,nvf1154;shfl.sync.idx.b32 nvr755|nvp529,nvr752,31,31,-1;mov.b32 nvf1166,nvr755;fma.rn.ftz.f32 nvf1167,nvf1162,nvf1166,nvf1156;max.ftz.f32 nvf1168,nvf1163,0f00800000;sqrt.approx.ftz.f32 nvf1169,nvf1168;mov.b32 nvr756,nvf1169;shfl.sync.idx.b32 nvr757|nvp530,nvr756,29,31,-1;mov.b32 nvf1170,nvr757;div.approx.ftz.f32 nvf1171,nvf1163,nvf1170;mov.b32 nvr758,nvf1171;shfl.sync.idx.b32 nvr759|nvp531,nvr758,30,31,-1;mov.b32 nvf1172,nvr759;neg.ftz.f32 nvf1173,nvf1171;fma.rn.ftz.f32 nvf1174,nvf1173,nvf1172,nvf1165;shfl.sync.idx.b32 nvr760|nvp532,nvr758,31,31,-1;mov.b32 nvf1175,nvr760;fma.rn.ftz.f32 nvf1176,nvf1173,nvf1175,nvf1167;max.ftz.f32 nvf1177,nvf1174,0f00800000;sqrt.approx.ftz.f32 nvf1178,nvf1177;mov.b32 nvr761,nvf1178;shfl.sync.idx.b32 nvr762|nvp533,nvr761,30,31,-1;mov.b32 nvf1179,nvr762;div.approx.ftz.f32 nvf1180,nvf1174,nvf1179;mov.b32 nvr763,nvf1180;shfl.sync.idx.b32 nvr764|nvp534,nvr763,31,31,-1;mov.b32 nvf1181,nvr764;neg.ftz.f32 nvf1182,nvf1180;fma.rn.ftz.f32 nvf1183,nvf1182,nvf1181,nvf1176;max.ftz.f32 nvf1184,nvf1183,0f00800000;sqrt.approx.ftz.f32 nvf1185,nvf1184;mov.b32 nvr765,nvf1185;shfl.sync.idx.b32 nvr766|nvp535,nvr765,31,31,-1;mov.b32 nvf1186,nvr766;div.approx.ftz.f32 nvf1187,nvf1183,nvf1186;st.shared.f32[nvrd147],nvf40;setp.eq.s32 nvp536,nvr27,0;selp.f32 nvf1188,0f00000000,nvf107,nvp536;st.shared.f32[nvrd149],nvf1188;setp.gt.u32 nvp537,nvr27,1;selp.f32 nvf1189,nvf172,0f00000000,nvp537;st.shared.f32[nvrd151],nvf1189;setp.gt.u32 nvp538,nvr27,2;selp.f32 nvf1190,nvf235,0f00000000,nvp538;st.shared.f32[nvrd153],nvf1190;setp.gt.u32 nvp539,nvr27,3;selp.f32 nvf1191,nvf296,0f00000000,nvp539;st.shared.f32[nvrd155],nvf1191;setp.gt.u32 nvp540,nvr27,4;selp.f32 nvf1192,nvf355,0f00000000,nvp540;st.shared.f32[nvrd157],nvf1192;setp.gt.u32 nvp541,nvr27,5;selp.f32 nvf1193,nvf412,0f00000000,nvp541;st.shared.f32[nvrd159],nvf1193;setp.gt.u32 nvp542,nvr27,6;selp.f32 nvf1194,nvf467,0f00000000,nvp542;st.shared.f32[nvrd161],nvf1194;setp.gt.u32 nvp543,nvr27,7;selp.f32 nvf1195,nvf520,0f00000000,nvp543;st.shared.f32[nvrd163],nvf1195;setp.gt.u32 nvp544,nvr27,8;selp.f32 nvf1196,nvf571,0f00000000,nvp544;st.shared.f32[nvrd165],nvf1196;setp.gt.u32 nvp545,nvr27,9;selp.f32 nvf1197,nvf620,0f00000000,nvp545;st.shared.f32[nvrd167],nvf1197;setp.gt.u32 nvp546,nvr27,10;selp.f32 nvf1198,nvf667,0f00000000,nvp546;st.shared.f32[nvrd169],nvf1198;setp.gt.u32 nvp547,nvr27,11;selp.f32 nvf1199,nvf712,0f00000000,nvp547;st.shared.f32[nvrd171],nvf1199;setp.gt.u32 nvp548,nvr27,12;selp.f32 nvf1200,nvf755,0f00000000,nvp548;st.shared.f32[nvrd173],nvf1200;setp.gt.u32 nvp549,nvr27,13;selp.f32 nvf1201,nvf796,0f00000000,nvp549;st.shared.f32[nvrd175],nvf1201;setp.gt.u32 nvp550,nvr27,14;selp.f32 nvf1202,nvf835,0f00000000,nvp550;st.shared.f32[nvrd177],nvf1202;setp.gt.u32 nvp551,nvr27,15;selp.f32 nvf1203,nvf872,0f00000000,nvp551;st.shared.f32[nvrd179],nvf1203;setp.gt.u32 nvp552,nvr27,16;selp.f32 nvf1204,nvf907,0f00000000,nvp552;st.shared.f32[nvrd181],nvf1204;setp.gt.u32 nvp553,nvr27,17;selp.f32 nvf1205,nvf940,0f00000000,nvp553;st.shared.f32[nvrd183],nvf1205;setp.gt.u32 nvp554,nvr27,18;selp.f32 nvf1206,nvf971,0f00000000,nvp554;st.shared.f32[nvrd185],nvf1206;setp.gt.u32 nvp555,nvr27,19;selp.f32 nvf1207,nvf1000,0f00000000,nvp555;st.shared.f32[nvrd187],nvf1207;setp.gt.u32 nvp556,nvr27,20;selp.f32 nvf1208,nvf1027,0f00000000,nvp556;st.shared.f32[nvrd189],nvf1208;setp.gt.u32 nvp557,nvr27,21;selp.f32 nvf1209,nvf1052,0f00000000,nvp557;st.shared.f32[nvrd191],nvf1209;setp.gt.u32 nvp558,nvr27,22;selp.f32 nvf1210,nvf1075,0f00000000,nvp558;st.shared.f32[nvrd193],nvf1210;setp.gt.u32 nvp559,nvr27,23;selp.f32 nvf1211,nvf1096,0f00000000,nvp559;st.shared.f32[nvrd195],nvf1211;setp.gt.u32 nvp560,nvr27,24;selp.f32 nvf1212,nvf1115,0f00000000,nvp560;st.shared.f32[nvrd197],nvf1212;setp.gt.u32 nvp561,nvr27,25;selp.f32 nvf1213,nvf1132,0f00000000,nvp561;st.shared.f32[nvrd199],nvf1213;setp.gt.u32 nvp562,nvr27,26;selp.f32 nvf1214,nvf1147,0f00000000,nvp562;st.shared.f32[nvrd201],nvf1214;setp.gt.u32 nvp563,nvr27,27;selp.f32 nvf1215,nvf1160,0f00000000,nvp563;st.shared.f32[nvrd203],nvf1215;setp.gt.u32 nvp564,nvr27,28;selp.f32 nvf1216,nvf1171,0f00000000,nvp564;st.shared.f32[nvrd205],nvf1216;setp.gt.u32 nvp565,nvr27,29;selp.f32 nvf1217,nvf1180,0f00000000,nvp565;st.shared.f32[nvrd207],nvf1217;setp.eq.s32 nvp566,nvr27,31;selp.f32 nvf1218,nvf1187,0f00000000,nvp566;st.shared.f32[nvrd209],nvf1218;bar.warp.sync -1;mov.u32 nvr1523,nvr1528;mov.u32 nvr1524,nvr1529;mov.u64 nvrd472,nvrd479;mov.u64 nvrd473,nvrd478;mov.u64 nvrd474,nvrd480;mov.u32 nvr1525,nvr1530;mov.u64 nvrd476,nvrd481;mov.u32 nvr1526,nvr1531;mov.u32 nvr1527,nvr27;L__BB0_14:\nsetp.lt.u32 nvp567,nvr1523,nvr27;@nvp567 bra L__BB0_16;xor.b32 nvr767,nvr27,nvr1523;cvt.u64.u32 nvrd210,nvr767;add.s64 nvrd211,nvrd473,nvrd210;shl.b64 nvrd212,nvrd211,2;add.s64 nvrd214,nvrd146,nvrd212;ld.shared.f32 nvf1219,[nvrd214];st.global.f32[nvrd470],nvf1219;L__BB0_16:\nsetp.lt.u32 nvp568,nvr1524,nvr27;@nvp568 bra L__BB0_18;xor.b32 nvr768,nvr27,nvr1524;cvt.u64.u32 nvrd215,nvr768;add.s64 nvrd216,nvrd472,nvrd215;shl.b64 nvrd217,nvrd216,2;add.s64 nvrd219,nvrd146,nvrd217;ld.shared.f32 nvf1220,[nvrd219];st.global.f32[nvrd471],nvf1220;L__BB0_18:\nsetp.lt.u32 nvp569,nvr1525,nvr27;@nvp569 bra L__BB0_20;xor.b32 nvr769,nvr27,nvr1525;cvt.u64.u32 nvrd220,nvr769;add.s64 nvrd221,nvrd474,nvrd220;shl.b64 nvrd222,nvrd221,2;add.s64 nvrd224,nvrd146,nvrd222;ld.shared.f32 nvf1221,[nvrd224];st.global.f32[nvrd475],nvf1221;L__BB0_20:\nsetp.lt.u32 nvp570,nvr1526,nvr27;@nvp570 bra L__BB0_22;xor.b32 nvr770,nvr27,nvr1526;cvt.u64.u32 nvrd225,nvr770;add.s64 nvrd226,nvrd476,nvrd225;shl.b64 nvrd227,nvrd226,2;add.s64 nvrd229,nvrd146,nvrd227;ld.shared.f32 nvf1222,[nvrd229];st.global.f32[nvrd477],nvf1222;L__BB0_22:\nadd.s32 nvr37,nvr1527,128;add.s64 nvrd477,nvrd477,1024;add.s32 nvr1526,nvr1526,4;add.s64 nvrd476,nvrd476,128;add.s64 nvrd475,nvrd475,1024;add.s32 nvr1525,nvr1525,4;add.s64 nvrd474,nvrd474,128;add.s64 nvrd473,nvrd473,128;add.s64 nvrd472,nvrd472,128;add.s64 nvrd471,nvrd471,1024;add.s32 nvr1524,nvr1524,4;add.s32 nvr1523,nvr1523,4;add.s64 nvrd470,nvrd470,1024;setp.lt.u32 nvp571,nvr1527,896;mov.u32 nvr1527,nvr37;@nvp571 bra L__BB0_14;mov.u64 nvrd453,$1;mov.u64 nvrd230,nvrd453;add.s64 nvrd482,nvrd230,8192;mov.u32 nvr1532,nvr27;L__BB0_24:\nadd.s64 nvrd231,nvrd482,nvrd49;ld.global.nc.f32 nvf1223,[nvrd231];xor.b32 nvr771,nvr27,nvr1528;cvt.u64.u32 nvrd232,nvr771;add.s64 nvrd233,nvrd478,nvrd232;shl.b64 nvrd234,nvrd233,2;add.s64 nvrd236,nvrd146,4096;add.s64 nvrd237,nvrd236,nvrd234;st.shared.f32[nvrd237],nvf1223;add.s64 nvrd238,nvrd482,nvrd53;ld.global.nc.f32 nvf1224,[nvrd238];xor.b32 nvr772,nvr27,nvr1529;cvt.u64.u32 nvrd239,nvr772;add.s64 nvrd240,nvrd479,nvrd239;shl.b64 nvrd241,nvrd240,2;add.s64 nvrd242,nvrd236,nvrd241;st.shared.f32[nvrd242],nvf1224;add.s64 nvrd243,nvrd482,nvrd51;ld.global.nc.f32 nvf1225,[nvrd243];xor.b32 nvr773,nvr27,nvr1530;cvt.u64.u32 nvrd244,nvr773;add.s64 nvrd245,nvrd480,nvrd244;shl.b64 nvrd246,nvrd245,2;add.s64 nvrd247,nvrd236,nvrd246;st.shared.f32[nvrd247],nvf1225;add.s64 nvrd248,nvrd482,nvrd47;ld.global.nc.f32 nvf1226,[nvrd248];xor.b32 nvr774,nvr27,nvr1531;cvt.u64.u32 nvrd249,nvr774;add.s64 nvrd250,nvrd481,nvrd249;shl.b64 nvrd251,nvrd250,2;add.s64 nvrd252,nvrd236,nvrd251;st.shared.f32[nvrd252],nvf1226;add.s32 nvr47,nvr1532,128;add.s32 nvr1531,nvr1531,4;add.s64 nvrd482,nvrd482,1024;add.s64 nvrd481,nvrd481,128;add.s32 nvr1530,nvr1530,4;add.s64 nvrd480,nvrd480,128;add.s64 nvrd479,nvrd479,128;add.s32 nvr1529,nvr1529,4;add.s64 nvrd478,nvrd478,128;add.s32 nvr1528,nvr1528,4;setp.lt.u32 nvp572,nvr1532,896;mov.u32 nvr1532,nvr47;@nvp572 bra L__BB0_24;mov.u64 nvrd456,$2;mov.u32 nvr52,%tid.x;mov.b32 nvr1546,3;mov.b32 nvr1545,2;mov.b32 nvr1544,1;mov.b64 nvrd493,32;mov.b64 nvrd495,64;mov.b64 nvrd491,96;mov.b64 nvrd488,0;mov.b32 nvr1538,0;bar.warp.sync -1;mul.lo.s32 nvr775,nvr52,33;mul.wide.u32 nvrd253,nvr775,4;mov.u64 nvrd254,global_smem;add.s64 nvrd255,nvrd254,nvrd253;ld.shared.f32 nvf1227,[nvrd255+4096];shl.b32 nvr776,nvr52,5;xor.b32 nvr777,nvr52,1;or.b32 nvr778,nvr776,nvr777;mul.wide.u32 nvrd256,nvr778,4;add.s64 nvrd257,nvrd254,nvrd256;ld.shared.f32 nvf1228,[nvrd257+4096];xor.b32 nvr779,nvr52,2;or.b32 nvr780,nvr776,nvr779;mul.wide.u32 nvrd258,nvr780,4;add.s64 nvrd259,nvrd254,nvrd258;ld.shared.f32 nvf1229,[nvrd259+4096];xor.b32 nvr781,nvr52,3;or.b32 nvr782,nvr776,nvr781;mul.wide.u32 nvrd260,nvr782,4;add.s64 nvrd261,nvrd254,nvrd260;ld.shared.f32 nvf1230,[nvrd261+4096];xor.b32 nvr783,nvr52,4;or.b32 nvr784,nvr776,nvr783;mul.wide.u32 nvrd262,nvr784,4;add.s64 nvrd263,nvrd254,nvrd262;ld.shared.f32 nvf1231,[nvrd263+4096];xor.b32 nvr785,nvr52,5;or.b32 nvr786,nvr776,nvr785;mul.wide.u32 nvrd264,nvr786,4;add.s64 nvrd265,nvrd254,nvrd264;ld.shared.f32 nvf1232,[nvrd265+4096];xor.b32 nvr787,nvr52,6;or.b32 nvr788,nvr776,nvr787;mul.wide.u32 nvrd266,nvr788,4;add.s64 nvrd267,nvrd254,nvrd266;ld.shared.f32 nvf1233,[nvrd267+4096];xor.b32 nvr789,nvr52,7;or.b32 nvr790,nvr776,nvr789;mul.wide.u32 nvrd268,nvr790,4;add.s64 nvrd269,nvrd254,nvrd268;ld.shared.f32 nvf1234,[nvrd269+4096];xor.b32 nvr791,nvr52,8;or.b32 nvr792,nvr776,nvr791;mul.wide.u32 nvrd270,nvr792,4;add.s64 nvrd271,nvrd254,nvrd270;ld.shared.f32 nvf1235,[nvrd271+4096];xor.b32 nvr793,nvr52,9;or.b32 nvr794,nvr776,nvr793;mul.wide.u32 nvrd272,nvr794,4;add.s64 nvrd273,nvrd254,nvrd272;ld.shared.f32 nvf1236,[nvrd273+4096];xor.b32 nvr795,nvr52,10;or.b32 nvr796,nvr776,nvr795;mul.wide.u32 nvrd274,nvr796,4;add.s64 nvrd275,nvrd254,nvrd274;ld.shared.f32 nvf1237,[nvrd275+4096];xor.b32 nvr797,nvr52,11;or.b32 nvr798,nvr776,nvr797;mul.wide.u32 nvrd276,nvr798,4;add.s64 nvrd277,nvrd254,nvrd276;ld.shared.f32 nvf1238,[nvrd277+4096];xor.b32 nvr799,nvr52,12;or.b32 nvr800,nvr776,nvr799;mul.wide.u32 nvrd278,nvr800,4;add.s64 nvrd279,nvrd254,nvrd278;ld.shared.f32 nvf1239,[nvrd279+4096];xor.b32 nvr801,nvr52,13;or.b32 nvr802,nvr776,nvr801;mul.wide.u32 nvrd280,nvr802,4;add.s64 nvrd281,nvrd254,nvrd280;ld.shared.f32 nvf1240,[nvrd281+4096];xor.b32 nvr803,nvr52,14;or.b32 nvr804,nvr776,nvr803;mul.wide.u32 nvrd282,nvr804,4;add.s64 nvrd283,nvrd254,nvrd282;ld.shared.f32 nvf1241,[nvrd283+4096];xor.b32 nvr805,nvr52,15;or.b32 nvr806,nvr776,nvr805;mul.wide.u32 nvrd284,nvr806,4;add.s64 nvrd285,nvrd254,nvrd284;ld.shared.f32 nvf1242,[nvrd285+4096];xor.b32 nvr807,nvr52,16;or.b32 nvr808,nvr776,nvr807;mul.wide.u32 nvrd286,nvr808,4;add.s64 nvrd287,nvrd254,nvrd286;ld.shared.f32 nvf1243,[nvrd287+4096];xor.b32 nvr809,nvr52,17;or.b32 nvr810,nvr776,nvr809;mul.wide.u32 nvrd288,nvr810,4;add.s64 nvrd289,nvrd254,nvrd288;ld.shared.f32 nvf1244,[nvrd289+4096];xor.b32 nvr811,nvr52,18;or.b32 nvr812,nvr776,nvr811;mul.wide.u32 nvrd290,nvr812,4;add.s64 nvrd291,nvrd254,nvrd290;ld.shared.f32 nvf1245,[nvrd291+4096];xor.b32 nvr813,nvr52,19;or.b32 nvr814,nvr776,nvr813;mul.wide.u32 nvrd292,nvr814,4;add.s64 nvrd293,nvrd254,nvrd292;ld.shared.f32 nvf1246,[nvrd293+4096];xor.b32 nvr815,nvr52,20;or.b32 nvr816,nvr776,nvr815;mul.wide.u32 nvrd294,nvr816,4;add.s64 nvrd295,nvrd254,nvrd294;ld.shared.f32 nvf1247,[nvrd295+4096];xor.b32 nvr817,nvr52,21;or.b32 nvr818,nvr776,nvr817;mul.wide.u32 nvrd296,nvr818,4;add.s64 nvrd297,nvrd254,nvrd296;ld.shared.f32 nvf1248,[nvrd297+4096];xor.b32 nvr819,nvr52,22;or.b32 nvr820,nvr776,nvr819;mul.wide.u32 nvrd298,nvr820,4;add.s64 nvrd299,nvrd254,nvrd298;ld.shared.f32 nvf1249,[nvrd299+4096];xor.b32 nvr821,nvr52,23;or.b32 nvr822,nvr776,nvr821;mul.wide.u32 nvrd300,nvr822,4;add.s64 nvrd301,nvrd254,nvrd300;ld.shared.f32 nvf1250,[nvrd301+4096];xor.b32 nvr823,nvr52,24;or.b32 nvr824,nvr776,nvr823;mul.wide.u32 nvrd302,nvr824,4;add.s64 nvrd303,nvrd254,nvrd302;ld.shared.f32 nvf1251,[nvrd303+4096];xor.b32 nvr825,nvr52,25;or.b32 nvr826,nvr776,nvr825;mul.wide.u32 nvrd304,nvr826,4;add.s64 nvrd305,nvrd254,nvrd304;ld.shared.f32 nvf1252,[nvrd305+4096];xor.b32 nvr827,nvr52,26;or.b32 nvr828,nvr776,nvr827;mul.wide.u32 nvrd306,nvr828,4;add.s64 nvrd307,nvrd254,nvrd306;ld.shared.f32 nvf1253,[nvrd307+4096];xor.b32 nvr829,nvr52,27;or.b32 nvr830,nvr776,nvr829;mul.wide.u32 nvrd308,nvr830,4;add.s64 nvrd309,nvrd254,nvrd308;ld.shared.f32 nvf1254,[nvrd309+4096];xor.b32 nvr831,nvr52,28;or.b32 nvr832,nvr776,nvr831;mul.wide.u32 nvrd310,nvr832,4;add.s64 nvrd311,nvrd254,nvrd310;ld.shared.f32 nvf1255,[nvrd311+4096];xor.b32 nvr833,nvr52,29;or.b32 nvr834,nvr776,nvr833;mul.wide.u32 nvrd312,nvr834,4;add.s64 nvrd313,nvrd254,nvrd312;ld.shared.f32 nvf1256,[nvrd313+4096];xor.b32 nvr835,nvr52,30;or.b32 nvr836,nvr776,nvr835;mul.wide.u32 nvrd314,nvr836,4;add.s64 nvrd315,nvrd254,nvrd314;ld.shared.f32 nvf1257,[nvrd315+4096];xor.b32 nvr837,nvr52,31;or.b32 nvr838,nvr776,nvr837;mul.wide.u32 nvrd316,nvr838,4;add.s64 nvrd317,nvrd254,nvrd316;ld.shared.f32 nvf1258,[nvrd317+4096];ld.shared.f32 nvf1259,[global_smem];div.approx.ftz.f32 nvf1260,nvf1227,nvf1259;neg.ftz.f32 nvf1261,nvf1260;ld.shared.v2.f32{nvf1262,nvf1263},[global_smem+128];fma.rn.ftz.f32 nvf1264,nvf1261,nvf1263,nvf1228;div.approx.ftz.f32 nvf1265,nvf1264,nvf1262;.pragma "used_bytes_mask 65295";ld.shared.v4.f32{nvf1266,nvf1267,nvf1268,nvf1269},[global_smem+256];fma.rn.ftz.f32 nvf1270,nvf1261,nvf1268,nvf1229;neg.ftz.f32 nvf1271,nvf1265;fma.rn.ftz.f32 nvf1272,nvf1271,nvf1269,nvf1270;div.approx.ftz.f32 nvf1273,nvf1272,nvf1266;ld.shared.v4.f32{nvf1274,nvf1275,nvf1276,nvf1277},[global_smem+384];fma.rn.ftz.f32 nvf1278,nvf1261,nvf1277,nvf1230;fma.rn.ftz.f32 nvf1279,nvf1271,nvf1276,nvf1278;neg.ftz.f32 nvf1280,nvf1273;fma.rn.ftz.f32 nvf1281,nvf1280,nvf1275,nvf1279;div.approx.ftz.f32 nvf1282,nvf1281,nvf1274;ld.shared.v4.f32{nvf1283,nvf1284,nvf1285,nvf1286},[global_smem+528];fma.rn.ftz.f32 nvf1287,nvf1261,nvf1283,nvf1231;fma.rn.ftz.f32 nvf1288,nvf1271,nvf1284,nvf1287;fma.rn.ftz.f32 nvf1289,nvf1280,nvf1285,nvf1288;neg.ftz.f32 nvf1290,nvf1282;fma.rn.ftz.f32 nvf1291,nvf1290,nvf1286,nvf1289;ld.shared.f32 nvf1292,[global_smem+512];div.approx.ftz.f32 nvf1293,nvf1291,nvf1292;ld.shared.v4.f32{nvf1294,nvf1295,nvf1296,nvf1297},[global_smem+656];fma.rn.ftz.f32 nvf1298,nvf1261,nvf1295,nvf1232;fma.rn.ftz.f32 nvf1299,nvf1271,nvf1294,nvf1298;fma.rn.ftz.f32 nvf1300,nvf1280,nvf1297,nvf1299;fma.rn.ftz.f32 nvf1301,nvf1290,nvf1296,nvf1300;neg.ftz.f32 nvf1302,nvf1293;ld.shared.v2.f32{nvf1303,nvf1304},[global_smem+640];fma.rn.ftz.f32 nvf1305,nvf1302,nvf1304,nvf1301;div.approx.ftz.f32 nvf1306,nvf1305,nvf1303;ld.shared.v4.f32{nvf1307,nvf1308,nvf1309,nvf1310},[global_smem+784];fma.rn.ftz.f32 nvf1311,nvf1261,nvf1309,nvf1233;fma.rn.ftz.f32 nvf1312,nvf1271,nvf1310,nvf1311;fma.rn.ftz.f32 nvf1313,nvf1280,nvf1307,nvf1312;fma.rn.ftz.f32 nvf1314,nvf1290,nvf1308,nvf1313;.pragma "used_bytes_mask 65295";ld.shared.v4.f32{nvf1315,nvf1316,nvf1317,nvf1318},[global_smem+768];fma.rn.ftz.f32 nvf1319,nvf1302,nvf1317,nvf1314;neg.ftz.f32 nvf1320,nvf1306;fma.rn.ftz.f32 nvf1321,nvf1320,nvf1318,nvf1319;div.approx.ftz.f32 nvf1322,nvf1321,nvf1315;ld.shared.v4.f32{nvf1323,nvf1324,nvf1325,nvf1326},[global_smem+912];fma.rn.ftz.f32 nvf1327,nvf1261,nvf1326,nvf1234;fma.rn.ftz.f32 nvf1328,nvf1271,nvf1325,nvf1327;fma.rn.ftz.f32 nvf1329,nvf1280,nvf1324,nvf1328;fma.rn.ftz.f32 nvf1330,nvf1290,nvf1323,nvf1329;ld.shared.v4.f32{nvf1331,nvf1332,nvf1333,nvf1334},[global_smem+896];fma.rn.ftz.f32 nvf1335,nvf1302,nvf1334,nvf1330;fma.rn.ftz.f32 nvf1336,nvf1320,nvf1333,nvf1335;neg.ftz.f32 nvf1337,nvf1322;fma.rn.ftz.f32 nvf1338,nvf1337,nvf1332,nvf1336;div.approx.ftz.f32 nvf1339,nvf1338,nvf1331;ld.shared.v4.f32{nvf1340,nvf1341,nvf1342,nvf1343},[global_smem+1056];fma.rn.ftz.f32 nvf1344,nvf1261,nvf1340,nvf1235;fma.rn.ftz.f32 nvf1345,nvf1271,nvf1341,nvf1344;fma.rn.ftz.f32 nvf1346,nvf1280,nvf1342,nvf1345;fma.rn.ftz.f32 nvf1347,nvf1290,nvf1343,nvf1346;ld.shared.v4.f32{nvf1348,nvf1349,nvf1350,nvf1351},[global_smem+1072];fma.rn.ftz.f32 nvf1352,nvf1302,nvf1348,nvf1347;fma.rn.ftz.f32 nvf1353,nvf1320,nvf1349,nvf1352;fma.rn.ftz.f32 nvf1354,nvf1337,nvf1350,nvf1353;neg.ftz.f32 nvf1355,nvf1339;fma.rn.ftz.f32 nvf1356,nvf1355,nvf1351,nvf1354;ld.shared.f32 nvf1357,[global_smem+1024];div.approx.ftz.f32 nvf1358,nvf1356,nvf1357;ld.shared.v4.f32{nvf1359,nvf1360,nvf1361,nvf1362},[global_smem+1184];fma.rn.ftz.f32 nvf1363,nvf1261,nvf1360,nvf1236;fma.rn.ftz.f32 nvf1364,nvf1271,nvf1359,nvf1363;fma.rn.ftz.f32 nvf1365,nvf1280,nvf1362,nvf1364;fma.rn.ftz.f32 nvf1366,nvf1290,nvf1361,nvf1365;ld.shared.v4.f32{nvf1367,nvf1368,nvf1369,nvf1370},[global_smem+1200];fma.rn.ftz.f32 nvf1371,nvf1302,nvf1368,nvf1366;fma.rn.ftz.f32 nvf1372,nvf1320,nvf1367,nvf1371;fma.rn.ftz.f32 nvf1373,nvf1337,nvf1370,nvf1372;fma.rn.ftz.f32 nvf1374,nvf1355,nvf1369,nvf1373;neg.ftz.f32 nvf1375,nvf1358;ld.shared.v2.f32{nvf1376,nvf1377},[global_smem+1152];fma.rn.ftz.f32 nvf1378,nvf1375,nvf1377,nvf1374;div.approx.ftz.f32 nvf1379,nvf1378,nvf1376;ld.shared.v4.f32{nvf1380,nvf1381,nvf1382,nvf1383},[global_smem+1312];fma.rn.ftz.f32 nvf1384,nvf1261,nvf1382,nvf1237;fma.rn.ftz.f32 nvf1385,nvf1271,nvf1383,nvf1384;fma.rn.ftz.f32 nvf1386,nvf1280,nvf1380,nvf1385;fma.rn.ftz.f32 nvf1387,nvf1290,nvf1381,nvf1386;ld.shared.v4.f32{nvf1388,nvf1389,nvf1390,nvf1391},[global_smem+1328];fma.rn.ftz.f32 nvf1392,nvf1302,nvf1390,nvf1387;fma.rn.ftz.f32 nvf1393,nvf1320,nvf1391,nvf1392;fma.rn.ftz.f32 nvf1394,nvf1337,nvf1388,nvf1393;fma.rn.ftz.f32 nvf1395,nvf1355,nvf1389,nvf1394;.pragma "used_bytes_mask 65295";ld.shared.v4.f32{nvf1396,nvf1397,nvf1398,nvf1399},[global_smem+1280];fma.rn.ftz.f32 nvf1400,nvf1375,nvf1398,nvf1395;neg.ftz.f32 nvf1401,nvf1379;fma.rn.ftz.f32 nvf1402,nvf1401,nvf1399,nvf1400;div.approx.ftz.f32 nvf1403,nvf1402,nvf1396;ld.shared.v4.f32{nvf1404,nvf1405,nvf1406,nvf1407},[global_smem+1440];fma.rn.ftz.f32 nvf1408,nvf1261,nvf1407,nvf1238;fma.rn.ftz.f32 nvf1409,nvf1271,nvf1406,nvf1408;fma.rn.ftz.f32 nvf1410,nvf1280,nvf1405,nvf1409;fma.rn.ftz.f32 nvf1411,nvf1290,nvf1404,nvf1410;ld.shared.v4.f32{nvf1412,nvf1413,nvf1414,nvf1415},[global_smem+1456];fma.rn.ftz.f32 nvf1416,nvf1302,nvf1415,nvf1411;fma.rn.ftz.f32 nvf1417,nvf1320,nvf1414,nvf1416;fma.rn.ftz.f32 nvf1418,nvf1337,nvf1413,nvf1417;fma.rn.ftz.f32 nvf1419,nvf1355,nvf1412,nvf1418;ld.shared.v4.f32{nvf1420,nvf1421,nvf1422,nvf1423},[global_smem+1408];fma.rn.ftz.f32 nvf1424,nvf1375,nvf1423,nvf1419;fma.rn.ftz.f32 nvf1425,nvf1401,nvf1422,nvf1424;neg.ftz.f32 nvf1426,nvf1403;fma.rn.ftz.f32 nvf1427,nvf1426,nvf1421,nvf1425;div.approx.ftz.f32 nvf1428,nvf1427,nvf1420;ld.shared.v4.f32{nvf1429,nvf1430,nvf1431,nvf1432},[global_smem+1584];fma.rn.ftz.f32 nvf1433,nvf1261,nvf1429,nvf1239;fma.rn.ftz.f32 nvf1434,nvf1271,nvf1430,nvf1433;fma.rn.ftz.f32 nvf1435,nvf1280,nvf1431,nvf1434;fma.rn.ftz.f32 nvf1436,nvf1290,nvf1432,nvf1435;ld.shared.v4.f32{nvf1437,nvf1438,nvf1439,nvf1440},[global_smem+1568];fma.rn.ftz.f32 nvf1441,nvf1302,nvf1437,nvf1436;fma.rn.ftz.f32 nvf1442,nvf1320,nvf1438,nvf1441;fma.rn.ftz.f32 nvf1443,nvf1337,nvf1439,nvf1442;fma.rn.ftz.f32 nvf1444,nvf1355,nvf1440,nvf1443;ld.shared.v4.f32{nvf1445,nvf1446,nvf1447,nvf1448},[global_smem+1552];fma.rn.ftz.f32 nvf1449,nvf1375,nvf1445,nvf1444;fma.rn.ftz.f32 nvf1450,nvf1401,nvf1446,nvf1449;fma.rn.ftz.f32 nvf1451,nvf1426,nvf1447,nvf1450;neg.ftz.f32 nvf1452,nvf1428;fma.rn.ftz.f32 nvf1453,nvf1452,nvf1448,nvf1451;ld.shared.f32 nvf1454,[global_smem+1536];div.approx.ftz.f32 nvf1455,nvf1453,nvf1454;ld.shared.v4.f32{nvf1456,nvf1457,nvf1458,nvf1459},[global_smem+1712];fma.rn.ftz.f32 nvf1460,nvf1261,nvf1457,nvf1240;fma.rn.ftz.f32 nvf1461,nvf1271,nvf1456,nvf1460;fma.rn.ftz.f32 nvf1462,nvf1280,nvf1459,nvf1461;fma.rn.ftz.f32 nvf1463,nvf1290,nvf1458,nvf1462;ld.shared.v4.f32{nvf1464,nvf1465,nvf1466,nvf1467},[global_smem+1696];fma.rn.ftz.f32 nvf1468,nvf1302,nvf1465,nvf1463;fma.rn.ftz.f32 nvf1469,nvf1320,nvf1464,nvf1468;fma.rn.ftz.f32 nvf1470,nvf1337,nvf1467,nvf1469;fma.rn.ftz.f32 nvf1471,nvf1355,nvf1466,nvf1470;ld.shared.v4.f32{nvf1472,nvf1473,nvf1474,nvf1475},[global_smem+1680];fma.rn.ftz.f32 nvf1476,nvf1375,nvf1473,nvf1471;fma.rn.ftz.f32 nvf1477,nvf1401,nvf1472,nvf1476;fma.rn.ftz.f32 nvf1478,nvf1426,nvf1475,nvf1477;fma.rn.ftz.f32 nvf1479,nvf1452,nvf1474,nvf1478;neg.ftz.f32 nvf1480,nvf1455;ld.shared.v2.f32{nvf1481,nvf1482},[global_smem+1664];fma.rn.ftz.f32 nvf1483,nvf1480,nvf1482,nvf1479;div.approx.ftz.f32 nvf1484,nvf1483,nvf1481;ld.shared.v4.f32{nvf1485,nvf1486,nvf1487,nvf1488},[global_smem+1840];fma.rn.ftz.f32 nvf1489,nvf1261,nvf1487,nvf1241;fma.rn.ftz.f32 nvf1490,nvf1271,nvf1488,nvf1489;fma.rn.ftz.f32 nvf1491,nvf1280,nvf1485,nvf1490;fma.rn.ftz.f32 nvf1492,nvf1290,nvf1486,nvf1491;ld.shared.v4.f32{nvf1493,nvf1494,nvf1495,nvf1496},[global_smem+1824];fma.rn.ftz.f32 nvf1497,nvf1302,nvf1495,nvf1492;fma.rn.ftz.f32 nvf1498,nvf1320,nvf1496,nvf1497;fma.rn.ftz.f32 nvf1499,nvf1337,nvf1493,nvf1498;fma.rn.ftz.f32 nvf1500,nvf1355,nvf1494,nvf1499;ld.shared.v4.f32{nvf1501,nvf1502,nvf1503,nvf1504},[global_smem+1808];fma.rn.ftz.f32 nvf1505,nvf1375,nvf1503,nvf1500;fma.rn.ftz.f32 nvf1506,nvf1401,nvf1504,nvf1505;fma.rn.ftz.f32 nvf1507,nvf1426,nvf1501,nvf1506;fma.rn.ftz.f32 nvf1508,nvf1452,nvf1502,nvf1507;.pragma "used_bytes_mask 65295";ld.shared.v4.f32{nvf1509,nvf1510,nvf1511,nvf1512},[global_smem+1792];fma.rn.ftz.f32 nvf1513,nvf1480,nvf1511,nvf1508;neg.ftz.f32 nvf1514,nvf1484;fma.rn.ftz.f32 nvf1515,nvf1514,nvf1512,nvf1513;div.approx.ftz.f32 nvf1516,nvf1515,nvf1509;ld.shared.v4.f32{nvf1517,nvf1518,nvf1519,nvf1520},[global_smem+1968];fma.rn.ftz.f32 nvf1521,nvf1261,nvf1520,nvf1242;fma.rn.ftz.f32 nvf1522,nvf1271,nvf1519,nvf1521;fma.rn.ftz.f32 nvf1523,nvf1280,nvf1518,nvf1522;fma.rn.ftz.f32 nvf1524,nvf1290,nvf1517,nvf1523;ld.shared.v4.f32{nvf1525,nvf1526,nvf1527,nvf1528},[global_smem+1952];fma.rn.ftz.f32 nvf1529,nvf1302,nvf1528,nvf1524;fma.rn.ftz.f32 nvf1530,nvf1320,nvf1527,nvf1529;fma.rn.ftz.f32 nvf1531,nvf1337,nvf1526,nvf1530;fma.rn.ftz.f32 nvf1532,nvf1355,nvf1525,nvf1531;ld.shared.v4.f32{nvf1533,nvf1534,nvf1535,nvf1536},[global_smem+1936];fma.rn.ftz.f32 nvf1537,nvf1375,nvf1536,nvf1532;fma.rn.ftz.f32 nvf1538,nvf1401,nvf1535,nvf1537;fma.rn.ftz.f32 nvf1539,nvf1426,nvf1534,nvf1538;fma.rn.ftz.f32 nvf1540,nvf1452,nvf1533,nvf1539;ld.shared.v4.f32{nvf1541,nvf1542,nvf1543,nvf1544},[global_smem+1920];fma.rn.ftz.f32 nvf1545,nvf1480,nvf1544,nvf1540;fma.rn.ftz.f32 nvf1546,nvf1514,nvf1543,nvf1545;neg.ftz.f32 nvf1547,nvf1516;fma.rn.ftz.f32 nvf1548,nvf1547,nvf1542,nvf1546;div.approx.ftz.f32 nvf1549,nvf1548,nvf1541;ld.shared.v4.f32{nvf1550,nvf1551,nvf1552,nvf1553},[global_smem+2112];fma.rn.ftz.f32 nvf1554,nvf1261,nvf1550,nvf1243;fma.rn.ftz.f32 nvf1555,nvf1271,nvf1551,nvf1554;fma.rn.ftz.f32 nvf1556,nvf1280,nvf1552,nvf1555;fma.rn.ftz.f32 nvf1557,nvf1290,nvf1553,nvf1556;ld.shared.v4.f32{nvf1558,nvf1559,nvf1560,nvf1561},[global_smem+2128];fma.rn.ftz.f32 nvf1562,nvf1302,nvf1558,nvf1557;fma.rn.ftz.f32 nvf1563,nvf1320,nvf1559,nvf1562;fma.rn.ftz.f32 nvf1564,nvf1337,nvf1560,nvf1563;fma.rn.ftz.f32 nvf1565,nvf1355,nvf1561,nvf1564;ld.shared.v4.f32{nvf1566,nvf1567,nvf1568,nvf1569},[global_smem+2144];fma.rn.ftz.f32 nvf1570,nvf1375,nvf1566,nvf1565;fma.rn.ftz.f32 nvf1571,nvf1401,nvf1567,nvf1570;fma.rn.ftz.f32 nvf1572,nvf1426,nvf1568,nvf1571;fma.rn.ftz.f32 nvf1573,nvf1452,nvf1569,nvf1572;ld.shared.v4.f32{nvf1574,nvf1575,nvf1576,nvf1577},[global_smem+2160];fma.rn.ftz.f32 nvf1578,nvf1480,nvf1574,nvf1573;fma.rn.ftz.f32 nvf1579,nvf1514,nvf1575,nvf1578;fma.rn.ftz.f32 nvf1580,nvf1547,nvf1576,nvf1579;neg.ftz.f32 nvf1581,nvf1549;fma.rn.ftz.f32 nvf1582,nvf1581,nvf1577,nvf1580;ld.shared.f32 nvf1583,[global_smem+2048];div.approx.ftz.f32 nvf1584,nvf1582,nvf1583;ld.shared.v4.f32{nvf1585,nvf1586,nvf1587,nvf1588},[global_smem+2240];fma.rn.ftz.f32 nvf1589,nvf1261,nvf1586,nvf1244;fma.rn.ftz.f32 nvf1590,nvf1271,nvf1585,nvf1589;fma.rn.ftz.f32 nvf1591,nvf1280,nvf1588,nvf1590;fma.rn.ftz.f32 nvf1592,nvf1290,nvf1587,nvf1591;ld.shared.v4.f32{nvf1593,nvf1594,nvf1595,nvf1596},[global_smem+2256];fma.rn.ftz.f32 nvf1597,nvf1302,nvf1594,nvf1592;fma.rn.ftz.f32 nvf1598,nvf1320,nvf1593,nvf1597;fma.rn.ftz.f32 nvf1599,nvf1337,nvf1596,nvf1598;fma.rn.ftz.f32 nvf1600,nvf1355,nvf1595,nvf1599;ld.shared.v4.f32{nvf1601,nvf1602,nvf1603,nvf1604},[global_smem+2272];fma.rn.ftz.f32 nvf1605,nvf1375,nvf1602,nvf1600;fma.rn.ftz.f32 nvf1606,nvf1401,nvf1601,nvf1605;fma.rn.ftz.f32 nvf1607,nvf1426,nvf1604,nvf1606;fma.rn.ftz.f32 nvf1608,nvf1452,nvf1603,nvf1607;ld.shared.v4.f32{nvf1609,nvf1610,nvf1611,nvf1612},[global_smem+2288];fma.rn.ftz.f32 nvf1613,nvf1480,nvf1610,nvf1608;fma.rn.ftz.f32 nvf1614,nvf1514,nvf1609,nvf1613;fma.rn.ftz.f32 nvf1615,nvf1547,nvf1612,nvf1614;fma.rn.ftz.f32 nvf1616,nvf1581,nvf1611,nvf1615;neg.ftz.f32 nvf1617,nvf1584;ld.shared.v2.f32{nvf1618,nvf1619},[global_smem+2176];fma.rn.ftz.f32 nvf1620,nvf1617,nvf1619,nvf1616;div.approx.ftz.f32 nvf1621,nvf1620,nvf1618;ld.shared.v4.f32{nvf1622,nvf1623,nvf1624,nvf1625},[global_smem+2368];fma.rn.ftz.f32 nvf1626,nvf1261,nvf1624,nvf1245;fma.rn.ftz.f32 nvf1627,nvf1271,nvf1625,nvf1626;fma.rn.ftz.f32 nvf1628,nvf1280,nvf1622,nvf1627;fma.rn.ftz.f32 nvf1629,nvf1290,nvf1623,nvf1628;ld.shared.v4.f32{nvf1630,nvf1631,nvf1632,nvf1633},[global_smem+2384];fma.rn.ftz.f32 nvf1634,nvf1302,nvf1632,nvf1629;fma.rn.ftz.f32 nvf1635,nvf1320,nvf1633,nvf1634;fma.rn.ftz.f32 nvf1636,nvf1337,nvf1630,nvf1635;fma.rn.ftz.f32 nvf1637,nvf1355,nvf1631,nvf1636;ld.shared.v4.f32{nvf1638,nvf1639,nvf1640,nvf1641},[global_smem+2400];fma.rn.ftz.f32 nvf1642,nvf1375,nvf1640,nvf1637;fma.rn.ftz.f32 nvf1643,nvf1401,nvf1641,nvf1642;fma.rn.ftz.f32 nvf1644,nvf1426,nvf1638,nvf1643;fma.rn.ftz.f32 nvf1645,nvf1452,nvf1639,nvf1644;ld.shared.v4.f32{nvf1646,nvf1647,nvf1648,nvf1649},[global_smem+2416];fma.rn.ftz.f32 nvf1650,nvf1480,nvf1648,nvf1645;fma.rn.ftz.f32 nvf1651,nvf1514,nvf1649,nvf1650;fma.rn.ftz.f32 nvf1652,nvf1547,nvf1646,nvf1651;fma.rn.ftz.f32 nvf1653,nvf1581,nvf1647,nvf1652;.pragma "used_bytes_mask 65295";ld.shared.v4.f32{nvf1654,nvf1655,nvf1656,nvf1657},[global_smem+2304];fma.rn.ftz.f32 nvf1658,nvf1617,nvf1656,nvf1653;neg.ftz.f32 nvf1659,nvf1621;fma.rn.ftz.f32 nvf1660,nvf1659,nvf1657,nvf1658;div.approx.ftz.f32 nvf1661,nvf1660,nvf1654;ld.shared.v4.f32{nvf1662,nvf1663,nvf1664,nvf1665},[global_smem+2496];fma.rn.ftz.f32 nvf1666,nvf1261,nvf1665,nvf1246;fma.rn.ftz.f32 nvf1667,nvf1271,nvf1664,nvf1666;fma.rn.ftz.f32 nvf1668,nvf1280,nvf1663,nvf1667;fma.rn.ftz.f32 nvf1669,nvf1290,nvf1662,nvf1668;ld.shared.v4.f32{nvf1670,nvf1671,nvf1672,nvf1673},[global_smem+2512];fma.rn.ftz.f32 nvf1674,nvf1302,nvf1673,nvf1669;fma.rn.ftz.f32 nvf1675,nvf1320,nvf1672,nvf1674;fma.rn.ftz.f32 nvf1676,nvf1337,nvf1671,nvf1675;fma.rn.ftz.f32 nvf1677,nvf1355,nvf1670,nvf1676;ld.shared.v4.f32{nvf1678,nvf1679,nvf1680,nvf1681},[global_smem+2528];fma.rn.ftz.f32 nvf1682,nvf1375,nvf1681,nvf1677;fma.rn.ftz.f32 nvf1683,nvf1401,nvf1680,nvf1682;fma.rn.ftz.f32 nvf1684,nvf1426,nvf1679,nvf1683;fma.rn.ftz.f32 nvf1685,nvf1452,nvf1678,nvf1684;ld.shared.v4.f32{nvf1686,nvf1687,nvf1688,nvf1689},[global_smem+2544];fma.rn.ftz.f32 nvf1690,nvf1480,nvf1689,nvf1685;fma.rn.ftz.f32 nvf1691,nvf1514,nvf1688,nvf1690;fma.rn.ftz.f32 nvf1692,nvf1547,nvf1687,nvf1691;fma.rn.ftz.f32 nvf1693,nvf1581,nvf1686,nvf1692;ld.shared.v4.f32{nvf1694,nvf1695,nvf1696,nvf1697},[global_smem+2432];fma.rn.ftz.f32 nvf1698,nvf1617,nvf1697,nvf1693;fma.rn.ftz.f32 nvf1699,nvf1659,nvf1696,nvf1698;neg.ftz.f32 nvf1700,nvf1661;fma.rn.ftz.f32 nvf1701,nvf1700,nvf1695,nvf1699;div.approx.ftz.f32 nvf1702,nvf1701,nvf1694;ld.shared.v4.f32{nvf1703,nvf1704,nvf1705,nvf1706},[global_smem+2640];fma.rn.ftz.f32 nvf1707,nvf1261,nvf1703,nvf1247;fma.rn.ftz.f32 nvf1708,nvf1271,nvf1704,nvf1707;fma.rn.ftz.f32 nvf1709,nvf1280,nvf1705,nvf1708;fma.rn.ftz.f32 nvf1710,nvf1290,nvf1706,nvf1709;ld.shared.v4.f32{nvf1711,nvf1712,nvf1713,nvf1714},[global_smem+2624];fma.rn.ftz.f32 nvf1715,nvf1302,nvf1711,nvf1710;fma.rn.ftz.f32 nvf1716,nvf1320,nvf1712,nvf1715;fma.rn.ftz.f32 nvf1717,nvf1337,nvf1713,nvf1716;fma.rn.ftz.f32 nvf1718,nvf1355,nvf1714,nvf1717;ld.shared.v4.f32{nvf1719,nvf1720,nvf1721,nvf1722},[global_smem+2672];fma.rn.ftz.f32 nvf1723,nvf1375,nvf1719,nvf1718;fma.rn.ftz.f32 nvf1724,nvf1401,nvf1720,nvf1723;fma.rn.ftz.f32 nvf1725,nvf1426,nvf1721,nvf1724;fma.rn.ftz.f32 nvf1726,nvf1452,nvf1722,nvf1725;ld.shared.v4.f32{nvf1727,nvf1728,nvf1729,nvf1730},[global_smem+2656];fma.rn.ftz.f32 nvf1731,nvf1480,nvf1727,nvf1726;fma.rn.ftz.f32 nvf1732,nvf1514,nvf1728,nvf1731;fma.rn.ftz.f32 nvf1733,nvf1547,nvf1729,nvf1732;fma.rn.ftz.f32 nvf1734,nvf1581,nvf1730,nvf1733;ld.shared.v4.f32{nvf1735,nvf1736,nvf1737,nvf1738},[global_smem+2576];fma.rn.ftz.f32 nvf1739,nvf1617,nvf1735,nvf1734;fma.rn.ftz.f32 nvf1740,nvf1659,nvf1736,nvf1739;fma.rn.ftz.f32 nvf1741,nvf1700,nvf1737,nvf1740;neg.ftz.f32 nvf1742,nvf1702;fma.rn.ftz.f32 nvf1743,nvf1742,nvf1738,nvf1741;ld.shared.f32 nvf1744,[global_smem+2560];div.approx.ftz.f32 nvf1745,nvf1743,nvf1744;ld.shared.v4.f32{nvf1746,nvf1747,nvf1748,nvf1749},[global_smem+2768];fma.rn.ftz.f32 nvf1750,nvf1261,nvf1747,nvf1248;fma.rn.ftz.f32 nvf1751,nvf1271,nvf1746,nvf1750;fma.rn.ftz.f32 nvf1752,nvf1280,nvf1749,nvf1751;fma.rn.ftz.f32 nvf1753,nvf1290,nvf1748,nvf1752;ld.shared.v4.f32{nvf1754,nvf1755,nvf1756,nvf1757},[global_smem+2752];fma.rn.ftz.f32 nvf1758,nvf1302,nvf1755,nvf1753;fma.rn.ftz.f32 nvf1759,nvf1320,nvf1754,nvf1758;fma.rn.ftz.f32 nvf1760,nvf1337,nvf1757,nvf1759;fma.rn.ftz.f32 nvf1761,nvf1355,nvf1756,nvf1760;ld.shared.v4.f32{nvf1762,nvf1763,nvf1764,nvf1765},[global_smem+2800];fma.rn.ftz.f32 nvf1766,nvf1375,nvf1763,nvf1761;fma.rn.ftz.f32 nvf1767,nvf1401,nvf1762,nvf1766;fma.rn.ftz.f32 nvf1768,nvf1426,nvf1765,nvf1767;fma.rn.ftz.f32 nvf1769,nvf1452,nvf1764,nvf1768;ld.shared.v4.f32{nvf1770,nvf1771,nvf1772,nvf1773},[global_smem+2784];fma.rn.ftz.f32 nvf1774,nvf1480,nvf1771,nvf1769;fma.rn.ftz.f32 nvf1775,nvf1514,nvf1770,nvf1774;fma.rn.ftz.f32 nvf1776,nvf1547,nvf1773,nvf1775;fma.rn.ftz.f32 nvf1777,nvf1581,nvf1772,nvf1776;ld.shared.v4.f32{nvf1778,nvf1779,nvf1780,nvf1781},[global_smem+2704];fma.rn.ftz.f32 nvf1782,nvf1617,nvf1779,nvf1777;fma.rn.ftz.f32 nvf1783,nvf1659,nvf1778,nvf1782;fma.rn.ftz.f32 nvf1784,nvf1700,nvf1781,nvf1783;fma.rn.ftz.f32 nvf1785,nvf1742,nvf1780,nvf1784;neg.ftz.f32 nvf1786,nvf1745;ld.shared.v2.f32{nvf1787,nvf1788},[global_smem+2688];fma.rn.ftz.f32 nvf1789,nvf1786,nvf1788,nvf1785;div.approx.ftz.f32 nvf1790,nvf1789,nvf1787;ld.shared.v4.f32{nvf1791,nvf1792,nvf1793,nvf1794},[global_smem+2896];fma.rn.ftz.f32 nvf1795,nvf1261,nvf1793,nvf1249;fma.rn.ftz.f32 nvf1796,nvf1271,nvf1794,nvf1795;fma.rn.ftz.f32 nvf1797,nvf1280,nvf1791,nvf1796;fma.rn.ftz.f32 nvf1798,nvf1290,nvf1792,nvf1797;ld.shared.v4.f32{nvf1799,nvf1800,nvf1801,nvf1802},[global_smem+2880];fma.rn.ftz.f32 nvf1803,nvf1302,nvf1801,nvf1798;fma.rn.ftz.f32 nvf1804,nvf1320,nvf1802,nvf1803;fma.rn.ftz.f32 nvf1805,nvf1337,nvf1799,nvf1804;fma.rn.ftz.f32 nvf1806,nvf1355,nvf1800,nvf1805;ld.shared.v4.f32{nvf1807,nvf1808,nvf1809,nvf1810},[global_smem+2928];fma.rn.ftz.f32 nvf1811,nvf1375,nvf1809,nvf1806;fma.rn.ftz.f32 nvf1812,nvf1401,nvf1810,nvf1811;fma.rn.ftz.f32 nvf1813,nvf1426,nvf1807,nvf1812;fma.rn.ftz.f32 nvf1814,nvf1452,nvf1808,nvf1813;ld.shared.v4.f32{nvf1815,nvf1816,nvf1817,nvf1818},[global_smem+2912];fma.rn.ftz.f32 nvf1819,nvf1480,nvf1817,nvf1814;fma.rn.ftz.f32 nvf1820,nvf1514,nvf1818,nvf1819;fma.rn.ftz.f32 nvf1821,nvf1547,nvf1815,nvf1820;fma.rn.ftz.f32 nvf1822,nvf1581,nvf1816,nvf1821;ld.shared.v4.f32{nvf1823,nvf1824,nvf1825,nvf1826},[global_smem+2832];fma.rn.ftz.f32 nvf1827,nvf1617,nvf1825,nvf1822;fma.rn.ftz.f32 nvf1828,nvf1659,nvf1826,nvf1827;fma.rn.ftz.f32 nvf1829,nvf1700,nvf1823,nvf1828;fma.rn.ftz.f32 nvf1830,nvf1742,nvf1824,nvf1829;.pragma "used_bytes_mask 65295";ld.shared.v4.f32{nvf1831,nvf1832,nvf1833,nvf1834},[global_smem+2816];fma.rn.ftz.f32 nvf1835,nvf1786,nvf1833,nvf1830;neg.ftz.f32 nvf1836,nvf1790;fma.rn.ftz.f32 nvf1837,nvf1836,nvf1834,nvf1835;div.approx.ftz.f32 nvf1838,nvf1837,nvf1831;ld.shared.v4.f32{nvf1839,nvf1840,nvf1841,nvf1842},[global_smem+3024];fma.rn.ftz.f32 nvf1843,nvf1261,nvf1842,nvf1250;fma.rn.ftz.f32 nvf1844,nvf1271,nvf1841,nvf1843;fma.rn.ftz.f32 nvf1845,nvf1280,nvf1840,nvf1844;fma.rn.ftz.f32 nvf1846,nvf1290,nvf1839,nvf1845;ld.shared.v4.f32{nvf1847,nvf1848,nvf1849,nvf1850},[global_smem+3008];fma.rn.ftz.f32 nvf1851,nvf1302,nvf1850,nvf1846;fma.rn.ftz.f32 nvf1852,nvf1320,nvf1849,nvf1851;fma.rn.ftz.f32 nvf1853,nvf1337,nvf1848,nvf1852;fma.rn.ftz.f32 nvf1854,nvf1355,nvf1847,nvf1853;ld.shared.v4.f32{nvf1855,nvf1856,nvf1857,nvf1858},[global_smem+3056];fma.rn.ftz.f32 nvf1859,nvf1375,nvf1858,nvf1854;fma.rn.ftz.f32 nvf1860,nvf1401,nvf1857,nvf1859;fma.rn.ftz.f32 nvf1861,nvf1426,nvf1856,nvf1860;fma.rn.ftz.f32 nvf1862,nvf1452,nvf1855,nvf1861;ld.shared.v4.f32{nvf1863,nvf1864,nvf1865,nvf1866},[global_smem+3040];fma.rn.ftz.f32 nvf1867,nvf1480,nvf1866,nvf1862;fma.rn.ftz.f32 nvf1868,nvf1514,nvf1865,nvf1867;fma.rn.ftz.f32 nvf1869,nvf1547,nvf1864,nvf1868;fma.rn.ftz.f32 nvf1870,nvf1581,nvf1863,nvf1869;ld.shared.v4.f32{nvf1871,nvf1872,nvf1873,nvf1874},[global_smem+2960];fma.rn.ftz.f32 nvf1875,nvf1617,nvf1874,nvf1870;fma.rn.ftz.f32 nvf1876,nvf1659,nvf1873,nvf1875;fma.rn.ftz.f32 nvf1877,nvf1700,nvf1872,nvf1876;fma.rn.ftz.f32 nvf1878,nvf1742,nvf1871,nvf1877;ld.shared.v4.f32{nvf1879,nvf1880,nvf1881,nvf1882},[global_smem+2944];fma.rn.ftz.f32 nvf1883,nvf1786,nvf1882,nvf1878;fma.rn.ftz.f32 nvf1884,nvf1836,nvf1881,nvf1883;neg.ftz.f32 nvf1885,nvf1838;fma.rn.ftz.f32 nvf1886,nvf1885,nvf1880,nvf1884;div.approx.ftz.f32 nvf1887,nvf1886,nvf1879;ld.shared.v4.f32{nvf1888,nvf1889,nvf1890,nvf1891},[global_smem+3168];fma.rn.ftz.f32 nvf1892,nvf1261,nvf1888,nvf1251;fma.rn.ftz.f32 nvf1893,nvf1271,nvf1889,nvf1892;fma.rn.ftz.f32 nvf1894,nvf1280,nvf1890,nvf1893;fma.rn.ftz.f32 nvf1895,nvf1290,nvf1891,nvf1894;ld.shared.v4.f32{nvf1896,nvf1897,nvf1898,nvf1899},[global_smem+3184];fma.rn.ftz.f32 nvf1900,nvf1302,nvf1896,nvf1895;fma.rn.ftz.f32 nvf1901,nvf1320,nvf1897,nvf1900;fma.rn.ftz.f32 nvf1902,nvf1337,nvf1898,nvf1901;fma.rn.ftz.f32 nvf1903,nvf1355,nvf1899,nvf1902;ld.shared.v4.f32{nvf1904,nvf1905,nvf1906,nvf1907},[global_smem+3136];fma.rn.ftz.f32 nvf1908,nvf1375,nvf1904,nvf1903;fma.rn.ftz.f32 nvf1909,nvf1401,nvf1905,nvf1908;fma.rn.ftz.f32 nvf1910,nvf1426,nvf1906,nvf1909;fma.rn.ftz.f32 nvf1911,nvf1452,nvf1907,nvf1910;ld.shared.v4.f32{nvf1912,nvf1913,nvf1914,nvf1915},[global_smem+3152];fma.rn.ftz.f32 nvf1916,nvf1480,nvf1912,nvf1911;fma.rn.ftz.f32 nvf1917,nvf1514,nvf1913,nvf1916;fma.rn.ftz.f32 nvf1918,nvf1547,nvf1914,nvf1917;fma.rn.ftz.f32 nvf1919,nvf1581,nvf1915,nvf1918;ld.shared.v4.f32{nvf1920,nvf1921,nvf1922,nvf1923},[global_smem+3104];fma.rn.ftz.f32 nvf1924,nvf1617,nvf1920,nvf1919;fma.rn.ftz.f32 nvf1925,nvf1659,nvf1921,nvf1924;fma.rn.ftz.f32 nvf1926,nvf1700,nvf1922,nvf1925;fma.rn.ftz.f32 nvf1927,nvf1742,nvf1923,nvf1926;ld.shared.v4.f32{nvf1928,nvf1929,nvf1930,nvf1931},[global_smem+3120];fma.rn.ftz.f32 nvf1932,nvf1786,nvf1928,nvf1927;fma.rn.ftz.f32 nvf1933,nvf1836,nvf1929,nvf1932;fma.rn.ftz.f32 nvf1934,nvf1885,nvf1930,nvf1933;neg.ftz.f32 nvf1935,nvf1887;fma.rn.ftz.f32 nvf1936,nvf1935,nvf1931,nvf1934;ld.shared.f32 nvf1937,[global_smem+3072];div.approx.ftz.f32 nvf1938,nvf1936,nvf1937;ld.shared.v4.f32{nvf1939,nvf1940,nvf1941,nvf1942},[global_smem+3296];fma.rn.ftz.f32 nvf1943,nvf1261,nvf1940,nvf1252;fma.rn.ftz.f32 nvf1944,nvf1271,nvf1939,nvf1943;fma.rn.ftz.f32 nvf1945,nvf1280,nvf1942,nvf1944;fma.rn.ftz.f32 nvf1946,nvf1290,nvf1941,nvf1945;ld.shared.v4.f32{nvf1947,nvf1948,nvf1949,nvf1950},[global_smem+3312];fma.rn.ftz.f32 nvf1951,nvf1302,nvf1948,nvf1946;fma.rn.ftz.f32 nvf1952,nvf1320,nvf1947,nvf1951;fma.rn.ftz.f32 nvf1953,nvf1337,nvf1950,nvf1952;fma.rn.ftz.f32 nvf1954,nvf1355,nvf1949,nvf1953;ld.shared.v4.f32{nvf1955,nvf1956,nvf1957,nvf1958},[global_smem+3264];fma.rn.ftz.f32 nvf1959,nvf1375,nvf1956,nvf1954;fma.rn.ftz.f32 nvf1960,nvf1401,nvf1955,nvf1959;fma.rn.ftz.f32 nvf1961,nvf1426,nvf1958,nvf1960;fma.rn.ftz.f32 nvf1962,nvf1452,nvf1957,nvf1961;ld.shared.v4.f32{nvf1963,nvf1964,nvf1965,nvf1966},[global_smem+3280];fma.rn.ftz.f32 nvf1967,nvf1480,nvf1964,nvf1962;fma.rn.ftz.f32 nvf1968,nvf1514,nvf1963,nvf1967;fma.rn.ftz.f32 nvf1969,nvf1547,nvf1966,nvf1968;fma.rn.ftz.f32 nvf1970,nvf1581,nvf1965,nvf1969;ld.shared.v4.f32{nvf1971,nvf1972,nvf1973,nvf1974},[global_smem+3232];fma.rn.ftz.f32 nvf1975,nvf1617,nvf1972,nvf1970;fma.rn.ftz.f32 nvf1976,nvf1659,nvf1971,nvf1975;fma.rn.ftz.f32 nvf1977,nvf1700,nvf1974,nvf1976;fma.rn.ftz.f32 nvf1978,nvf1742,nvf1973,nvf1977;ld.shared.v4.f32{nvf1979,nvf1980,nvf1981,nvf1982},[global_smem+3248];fma.rn.ftz.f32 nvf1983,nvf1786,nvf1980,nvf1978;fma.rn.ftz.f32 nvf1984,nvf1836,nvf1979,nvf1983;fma.rn.ftz.f32 nvf1985,nvf1885,nvf1982,nvf1984;fma.rn.ftz.f32 nvf1986,nvf1935,nvf1981,nvf1985;neg.ftz.f32 nvf1987,nvf1938;ld.shared.v2.f32{nvf1988,nvf1989},[global_smem+3200];fma.rn.ftz.f32 nvf1990,nvf1987,nvf1989,nvf1986;div.approx.ftz.f32 nvf1991,nvf1990,nvf1988;ld.shared.v4.f32{nvf1992,nvf1993,nvf1994,nvf1995},[global_smem+3424];fma.rn.ftz.f32 nvf1996,nvf1261,nvf1994,nvf1253;fma.rn.ftz.f32 nvf1997,nvf1271,nvf1995,nvf1996;fma.rn.ftz.f32 nvf1998,nvf1280,nvf1992,nvf1997;fma.rn.ftz.f32 nvf1999,nvf1290,nvf1993,nvf1998;ld.shared.v4.f32{nvf2000,nvf2001,nvf2002,nvf2003},[global_smem+3440];fma.rn.ftz.f32 nvf2004,nvf1302,nvf2002,nvf1999;fma.rn.ftz.f32 nvf2005,nvf1320,nvf2003,nvf2004;fma.rn.ftz.f32 nvf2006,nvf1337,nvf2000,nvf2005;fma.rn.ftz.f32 nvf2007,nvf1355,nvf2001,nvf2006;ld.shared.v4.f32{nvf2008,nvf2009,nvf2010,nvf2011},[global_smem+3392];fma.rn.ftz.f32 nvf2012,nvf1375,nvf2010,nvf2007;fma.rn.ftz.f32 nvf2013,nvf1401,nvf2011,nvf2012;fma.rn.ftz.f32 nvf2014,nvf1426,nvf2008,nvf2013;fma.rn.ftz.f32 nvf2015,nvf1452,nvf2009,nvf2014;ld.shared.v4.f32{nvf2016,nvf2017,nvf2018,nvf2019},[global_smem+3408];fma.rn.ftz.f32 nvf2020,nvf1480,nvf2018,nvf2015;fma.rn.ftz.f32 nvf2021,nvf1514,nvf2019,nvf2020;fma.rn.ftz.f32 nvf2022,nvf1547,nvf2016,nvf2021;fma.rn.ftz.f32 nvf2023,nvf1581,nvf2017,nvf2022;ld.shared.v4.f32{nvf2024,nvf2025,nvf2026,nvf2027},[global_smem+3360];fma.rn.ftz.f32 nvf2028,nvf1617,nvf2026,nvf2023;fma.rn.ftz.f32 nvf2029,nvf1659,nvf2027,nvf2028;fma.rn.ftz.f32 nvf2030,nvf1700,nvf2024,nvf2029;fma.rn.ftz.f32 nvf2031,nvf1742,nvf2025,nvf2030;ld.shared.v4.f32{nvf2032,nvf2033,nvf2034,nvf2035},[global_smem+3376];fma.rn.ftz.f32 nvf2036,nvf1786,nvf2034,nvf2031;fma.rn.ftz.f32 nvf2037,nvf1836,nvf2035,nvf2036;fma.rn.ftz.f32 nvf2038,nvf1885,nvf2032,nvf2037;fma.rn.ftz.f32 nvf2039,nvf1935,nvf2033,nvf2038;.pragma "used_bytes_mask 65295";ld.shared.v4.f32{nvf2040,nvf2041,nvf2042,nvf2043},[global_smem+3328];fma.rn.ftz.f32 nvf2044,nvf1987,nvf2042,nvf2039;neg.ftz.f32 nvf2045,nvf1991;fma.rn.ftz.f32 nvf2046,nvf2045,nvf2043,nvf2044;div.approx.ftz.f32 nvf2047,nvf2046,nvf2040;ld.shared.v4.f32{nvf2048,nvf2049,nvf2050,nvf2051},[global_smem+3552];fma.rn.ftz.f32 nvf2052,nvf1261,nvf2051,nvf1254;fma.rn.ftz.f32 nvf2053,nvf1271,nvf2050,nvf2052;fma.rn.ftz.f32 nvf2054,nvf1280,nvf2049,nvf2053;fma.rn.ftz.f32 nvf2055,nvf1290,nvf2048,nvf2054;ld.shared.v4.f32{nvf2056,nvf2057,nvf2058,nvf2059},[global_smem+3568];fma.rn.ftz.f32 nvf2060,nvf1302,nvf2059,nvf2055;fma.rn.ftz.f32 nvf2061,nvf1320,nvf2058,nvf2060;fma.rn.ftz.f32 nvf2062,nvf1337,nvf2057,nvf2061;fma.rn.ftz.f32 nvf2063,nvf1355,nvf2056,nvf2062;ld.shared.v4.f32{nvf2064,nvf2065,nvf2066,nvf2067},[global_smem+3520];fma.rn.ftz.f32 nvf2068,nvf1375,nvf2067,nvf2063;fma.rn.ftz.f32 nvf2069,nvf1401,nvf2066,nvf2068;fma.rn.ftz.f32 nvf2070,nvf1426,nvf2065,nvf2069;fma.rn.ftz.f32 nvf2071,nvf1452,nvf2064,nvf2070;ld.shared.v4.f32{nvf2072,nvf2073,nvf2074,nvf2075},[global_smem+3536];fma.rn.ftz.f32 nvf2076,nvf1480,nvf2075,nvf2071;fma.rn.ftz.f32 nvf2077,nvf1514,nvf2074,nvf2076;fma.rn.ftz.f32 nvf2078,nvf1547,nvf2073,nvf2077;fma.rn.ftz.f32 nvf2079,nvf1581,nvf2072,nvf2078;ld.shared.v4.f32{nvf2080,nvf2081,nvf2082,nvf2083},[global_smem+3488];fma.rn.ftz.f32 nvf2084,nvf1617,nvf2083,nvf2079;fma.rn.ftz.f32 nvf2085,nvf1659,nvf2082,nvf2084;fma.rn.ftz.f32 nvf2086,nvf1700,nvf2081,nvf2085;fma.rn.ftz.f32 nvf2087,nvf1742,nvf2080,nvf2086;ld.shared.v4.f32{nvf2088,nvf2089,nvf2090,nvf2091},[global_smem+3504];fma.rn.ftz.f32 nvf2092,nvf1786,nvf2091,nvf2087;fma.rn.ftz.f32 nvf2093,nvf1836,nvf2090,nvf2092;fma.rn.ftz.f32 nvf2094,nvf1885,nvf2089,nvf2093;fma.rn.ftz.f32 nvf2095,nvf1935,nvf2088,nvf2094;ld.shared.v4.f32{nvf2096,nvf2097,nvf2098,nvf2099},[global_smem+3456];fma.rn.ftz.f32 nvf2100,nvf1987,nvf2099,nvf2095;fma.rn.ftz.f32 nvf2101,nvf2045,nvf2098,nvf2100;neg.ftz.f32 nvf2102,nvf2047;fma.rn.ftz.f32 nvf2103,nvf2102,nvf2097,nvf2101;div.approx.ftz.f32 nvf2104,nvf2103,nvf2096;ld.shared.v4.f32{nvf2105,nvf2106,nvf2107,nvf2108},[global_smem+3696];fma.rn.ftz.f32 nvf2109,nvf1261,nvf2105,nvf1255;fma.rn.ftz.f32 nvf2110,nvf1271,nvf2106,nvf2109;fma.rn.ftz.f32 nvf2111,nvf1280,nvf2107,nvf2110;fma.rn.ftz.f32 nvf2112,nvf1290,nvf2108,nvf2111;ld.shared.v4.f32{nvf2113,nvf2114,nvf2115,nvf2116},[global_smem+3680];fma.rn.ftz.f32 nvf2117,nvf1302,nvf2113,nvf2112;fma.rn.ftz.f32 nvf2118,nvf1320,nvf2114,nvf2117;fma.rn.ftz.f32 nvf2119,nvf1337,nvf2115,nvf2118;fma.rn.ftz.f32 nvf2120,nvf1355,nvf2116,nvf2119;ld.shared.v4.f32{nvf2121,nvf2122,nvf2123,nvf2124},[global_smem+3664];fma.rn.ftz.f32 nvf2125,nvf1375,nvf2121,nvf2120;fma.rn.ftz.f32 nvf2126,nvf1401,nvf2122,nvf2125;fma.rn.ftz.f32 nvf2127,nvf1426,nvf2123,nvf2126;fma.rn.ftz.f32 nvf2128,nvf1452,nvf2124,nvf2127;ld.shared.v4.f32{nvf2129,nvf2130,nvf2131,nvf2132},[global_smem+3648];fma.rn.ftz.f32 nvf2133,nvf1480,nvf2129,nvf2128;fma.rn.ftz.f32 nvf2134,nvf1514,nvf2130,nvf2133;fma.rn.ftz.f32 nvf2135,nvf1547,nvf2131,nvf2134;fma.rn.ftz.f32 nvf2136,nvf1581,nvf2132,nvf2135;ld.shared.v4.f32{nvf2137,nvf2138,nvf2139,nvf2140},[global_smem+3632];fma.rn.ftz.f32 nvf2141,nvf1617,nvf2137,nvf2136;fma.rn.ftz.f32 nvf2142,nvf1659,nvf2138,nvf2141;fma.rn.ftz.f32 nvf2143,nvf1700,nvf2139,nvf2142;fma.rn.ftz.f32 nvf2144,nvf1742,nvf2140,nvf2143;ld.shared.v4.f32{nvf2145,nvf2146,nvf2147,nvf2148},[global_smem+3616];fma.rn.ftz.f32 nvf2149,nvf1786,nvf2145,nvf2144;fma.rn.ftz.f32 nvf2150,nvf1836,nvf2146,nvf2149;fma.rn.ftz.f32 nvf2151,nvf1885,nvf2147,nvf2150;fma.rn.ftz.f32 nvf2152,nvf1935,nvf2148,nvf2151;ld.shared.v4.f32{nvf2153,nvf2154,nvf2155,nvf2156},[global_smem+3600];fma.rn.ftz.f32 nvf2157,nvf1987,nvf2153,nvf2152;fma.rn.ftz.f32 nvf2158,nvf2045,nvf2154,nvf2157;fma.rn.ftz.f32 nvf2159,nvf2102,nvf2155,nvf2158;neg.ftz.f32 nvf2160,nvf2104;fma.rn.ftz.f32 nvf2161,nvf2160,nvf2156,nvf2159;ld.shared.f32 nvf2162,[global_smem+3584];div.approx.ftz.f32 nvf2163,nvf2161,nvf2162;ld.shared.v4.f32{nvf2164,nvf2165,nvf2166,nvf2167},[global_smem+3824];fma.rn.ftz.f32 nvf2168,nvf1261,nvf2165,nvf1256;fma.rn.ftz.f32 nvf2169,nvf1271,nvf2164,nvf2168;fma.rn.ftz.f32 nvf2170,nvf1280,nvf2167,nvf2169;fma.rn.ftz.f32 nvf2171,nvf1290,nvf2166,nvf2170;ld.shared.v4.f32{nvf2172,nvf2173,nvf2174,nvf2175},[global_smem+3808];fma.rn.ftz.f32 nvf2176,nvf1302,nvf2173,nvf2171;fma.rn.ftz.f32 nvf2177,nvf1320,nvf2172,nvf2176;fma.rn.ftz.f32 nvf2178,nvf1337,nvf2175,nvf2177;fma.rn.ftz.f32 nvf2179,nvf1355,nvf2174,nvf2178;ld.shared.v4.f32{nvf2180,nvf2181,nvf2182,nvf2183},[global_smem+3792];fma.rn.ftz.f32 nvf2184,nvf1375,nvf2181,nvf2179;fma.rn.ftz.f32 nvf2185,nvf1401,nvf2180,nvf2184;fma.rn.ftz.f32 nvf2186,nvf1426,nvf2183,nvf2185;fma.rn.ftz.f32 nvf2187,nvf1452,nvf2182,nvf2186;ld.shared.v4.f32{nvf2188,nvf2189,nvf2190,nvf2191},[global_smem+3776];fma.rn.ftz.f32 nvf2192,nvf1480,nvf2189,nvf2187;fma.rn.ftz.f32 nvf2193,nvf1514,nvf2188,nvf2192;fma.rn.ftz.f32 nvf2194,nvf1547,nvf2191,nvf2193;fma.rn.ftz.f32 nvf2195,nvf1581,nvf2190,nvf2194;ld.shared.v4.f32{nvf2196,nvf2197,nvf2198,nvf2199},[global_smem+3760];fma.rn.ftz.f32 nvf2200,nvf1617,nvf2197,nvf2195;fma.rn.ftz.f32 nvf2201,nvf1659,nvf2196,nvf2200;fma.rn.ftz.f32 nvf2202,nvf1700,nvf2199,nvf2201;fma.rn.ftz.f32 nvf2203,nvf1742,nvf2198,nvf2202;ld.shared.v4.f32{nvf2204,nvf2205,nvf2206,nvf2207},[global_smem+3744];fma.rn.ftz.f32 nvf2208,nvf1786,nvf2205,nvf2203;fma.rn.ftz.f32 nvf2209,nvf1836,nvf2204,nvf2208;fma.rn.ftz.f32 nvf2210,nvf1885,nvf2207,nvf2209;fma.rn.ftz.f32 nvf2211,nvf1935,nvf2206,nvf2210;ld.shared.v4.f32{nvf2212,nvf2213,nvf2214,nvf2215},[global_smem+3728];fma.rn.ftz.f32 nvf2216,nvf1987,nvf2213,nvf2211;fma.rn.ftz.f32 nvf2217,nvf2045,nvf2212,nvf2216;fma.rn.ftz.f32 nvf2218,nvf2102,nvf2215,nvf2217;fma.rn.ftz.f32 nvf2219,nvf2160,nvf2214,nvf2218;neg.ftz.f32 nvf2220,nvf2163;ld.shared.v2.f32{nvf2221,nvf2222},[global_smem+3712];fma.rn.ftz.f32 nvf2223,nvf2220,nvf2222,nvf2219;div.approx.ftz.f32 nvf2224,nvf2223,nvf2221;ld.shared.v4.f32{nvf2225,nvf2226,nvf2227,nvf2228},[global_smem+3952];fma.rn.ftz.f32 nvf2229,nvf1261,nvf2227,nvf1257;fma.rn.ftz.f32 nvf2230,nvf1271,nvf2228,nvf2229;fma.rn.ftz.f32 nvf2231,nvf1280,nvf2225,nvf2230;fma.rn.ftz.f32 nvf2232,nvf1290,nvf2226,nvf2231;ld.shared.v4.f32{nvf2233,nvf2234,nvf2235,nvf2236},[global_smem+3936];fma.rn.ftz.f32 nvf2237,nvf1302,nvf2235,nvf2232;fma.rn.ftz.f32 nvf2238,nvf1320,nvf2236,nvf2237;fma.rn.ftz.f32 nvf2239,nvf1337,nvf2233,nvf2238;fma.rn.ftz.f32 nvf2240,nvf1355,nvf2234,nvf2239;ld.shared.v4.f32{nvf2241,nvf2242,nvf2243,nvf2244},[global_smem+3920];fma.rn.ftz.f32 nvf2245,nvf1375,nvf2243,nvf2240;fma.rn.ftz.f32 nvf2246,nvf1401,nvf2244,nvf2245;fma.rn.ftz.f32 nvf2247,nvf1426,nvf2241,nvf2246;fma.rn.ftz.f32 nvf2248,nvf1452,nvf2242,nvf2247;ld.shared.v4.f32{nvf2249,nvf2250,nvf2251,nvf2252},[global_smem+3904];fma.rn.ftz.f32 nvf2253,nvf1480,nvf2251,nvf2248;fma.rn.ftz.f32 nvf2254,nvf1514,nvf2252,nvf2253;fma.rn.ftz.f32 nvf2255,nvf1547,nvf2249,nvf2254;fma.rn.ftz.f32 nvf2256,nvf1581,nvf2250,nvf2255;ld.shared.v4.f32{nvf2257,nvf2258,nvf2259,nvf2260},[global_smem+3888];fma.rn.ftz.f32 nvf2261,nvf1617,nvf2259,nvf2256;fma.rn.ftz.f32 nvf2262,nvf1659,nvf2260,nvf2261;fma.rn.ftz.f32 nvf2263,nvf1700,nvf2257,nvf2262;fma.rn.ftz.f32 nvf2264,nvf1742,nvf2258,nvf2263;ld.shared.v4.f32{nvf2265,nvf2266,nvf2267,nvf2268},[global_smem+3872];fma.rn.ftz.f32 nvf2269,nvf1786,nvf2267,nvf2264;fma.rn.ftz.f32 nvf2270,nvf1836,nvf2268,nvf2269;fma.rn.ftz.f32 nvf2271,nvf1885,nvf2265,nvf2270;fma.rn.ftz.f32 nvf2272,nvf1935,nvf2266,nvf2271;ld.shared.v4.f32{nvf2273,nvf2274,nvf2275,nvf2276},[global_smem+3856];fma.rn.ftz.f32 nvf2277,nvf1987,nvf2275,nvf2272;fma.rn.ftz.f32 nvf2278,nvf2045,nvf2276,nvf2277;fma.rn.ftz.f32 nvf2279,nvf2102,nvf2273,nvf2278;fma.rn.ftz.f32 nvf2280,nvf2160,nvf2274,nvf2279;.pragma "used_bytes_mask 65295";ld.shared.v4.f32{nvf2281,nvf2282,nvf2283,nvf2284},[global_smem+3840];fma.rn.ftz.f32 nvf2285,nvf2220,nvf2283,nvf2280;neg.ftz.f32 nvf2286,nvf2224;fma.rn.ftz.f32 nvf2287,nvf2286,nvf2284,nvf2285;div.approx.ftz.f32 nvf2288,nvf2287,nvf2281;ld.shared.v4.f32{nvf2289,nvf2290,nvf2291,nvf2292},[global_smem+4080];fma.rn.ftz.f32 nvf2293,nvf1261,nvf2292,nvf1258;fma.rn.ftz.f32 nvf2294,nvf1271,nvf2291,nvf2293;fma.rn.ftz.f32 nvf2295,nvf1280,nvf2290,nvf2294;fma.rn.ftz.f32 nvf2296,nvf1290,nvf2289,nvf2295;ld.shared.v4.f32{nvf2297,nvf2298,nvf2299,nvf2300},[global_smem+4064];fma.rn.ftz.f32 nvf2301,nvf1302,nvf2300,nvf2296;fma.rn.ftz.f32 nvf2302,nvf1320,nvf2299,nvf2301;fma.rn.ftz.f32 nvf2303,nvf1337,nvf2298,nvf2302;fma.rn.ftz.f32 nvf2304,nvf1355,nvf2297,nvf2303;ld.shared.v4.f32{nvf2305,nvf2306,nvf2307,nvf2308},[global_smem+4048];fma.rn.ftz.f32 nvf2309,nvf1375,nvf2308,nvf2304;fma.rn.ftz.f32 nvf2310,nvf1401,nvf2307,nvf2309;fma.rn.ftz.f32 nvf2311,nvf1426,nvf2306,nvf2310;fma.rn.ftz.f32 nvf2312,nvf1452,nvf2305,nvf2311;ld.shared.v4.f32{nvf2313,nvf2314,nvf2315,nvf2316},[global_smem+4032];fma.rn.ftz.f32 nvf2317,nvf1480,nvf2316,nvf2312;fma.rn.ftz.f32 nvf2318,nvf1514,nvf2315,nvf2317;fma.rn.ftz.f32 nvf2319,nvf1547,nvf2314,nvf2318;fma.rn.ftz.f32 nvf2320,nvf1581,nvf2313,nvf2319;ld.shared.v4.f32{nvf2321,nvf2322,nvf2323,nvf2324},[global_smem+4016];fma.rn.ftz.f32 nvf2325,nvf1617,nvf2324,nvf2320;fma.rn.ftz.f32 nvf2326,nvf1659,nvf2323,nvf2325;fma.rn.ftz.f32 nvf2327,nvf1700,nvf2322,nvf2326;fma.rn.ftz.f32 nvf2328,nvf1742,nvf2321,nvf2327;ld.shared.v4.f32{nvf2329,nvf2330,nvf2331,nvf2332},[global_smem+4000];fma.rn.ftz.f32 nvf2333,nvf1786,nvf2332,nvf2328;fma.rn.ftz.f32 nvf2334,nvf1836,nvf2331,nvf2333;fma.rn.ftz.f32 nvf2335,nvf1885,nvf2330,nvf2334;fma.rn.ftz.f32 nvf2336,nvf1935,nvf2329,nvf2335;ld.shared.v4.f32{nvf2337,nvf2338,nvf2339,nvf2340},[global_smem+3984];fma.rn.ftz.f32 nvf2341,nvf1987,nvf2340,nvf2336;fma.rn.ftz.f32 nvf2342,nvf2045,nvf2339,nvf2341;fma.rn.ftz.f32 nvf2343,nvf2102,nvf2338,nvf2342;fma.rn.ftz.f32 nvf2344,nvf2160,nvf2337,nvf2343;ld.shared.v4.f32{nvf2345,nvf2346,nvf2347,nvf2348},[global_smem+3968];fma.rn.ftz.f32 nvf2349,nvf2220,nvf2348,nvf2344;fma.rn.ftz.f32 nvf2350,nvf2286,nvf2347,nvf2349;neg.ftz.f32 nvf2351,nvf2288;fma.rn.ftz.f32 nvf2352,nvf2351,nvf2346,nvf2350;div.approx.ftz.f32 nvf2353,nvf2352,nvf2345;st.shared.f32[nvrd255+4096],nvf1260;st.shared.f32[nvrd257+4096],nvf1265;st.shared.f32[nvrd259+4096],nvf1273;st.shared.f32[nvrd261+4096],nvf1282;st.shared.f32[nvrd263+4096],nvf1293;st.shared.f32[nvrd265+4096],nvf1306;st.shared.f32[nvrd267+4096],nvf1322;st.shared.f32[nvrd269+4096],nvf1339;st.shared.f32[nvrd271+4096],nvf1358;st.shared.f32[nvrd273+4096],nvf1379;st.shared.f32[nvrd275+4096],nvf1403;st.shared.f32[nvrd277+4096],nvf1428;st.shared.f32[nvrd279+4096],nvf1455;st.shared.f32[nvrd281+4096],nvf1484;st.shared.f32[nvrd283+4096],nvf1516;st.shared.f32[nvrd285+4096],nvf1549;st.shared.f32[nvrd287+4096],nvf1584;st.shared.f32[nvrd289+4096],nvf1621;st.shared.f32[nvrd291+4096],nvf1661;st.shared.f32[nvrd293+4096],nvf1702;st.shared.f32[nvrd295+4096],nvf1745;st.shared.f32[nvrd297+4096],nvf1790;st.shared.f32[nvrd299+4096],nvf1838;st.shared.f32[nvrd301+4096],nvf1887;st.shared.f32[nvrd303+4096],nvf1938;st.shared.f32[nvrd305+4096],nvf1991;st.shared.f32[nvrd307+4096],nvf2047;st.shared.f32[nvrd309+4096],nvf2104;st.shared.f32[nvrd311+4096],nvf2163;st.shared.f32[nvrd313+4096],nvf2224;st.shared.f32[nvrd315+4096],nvf2288;st.shared.f32[nvrd317+4096],nvf2353;bar.warp.sync -1;mov.u64 nvrd318,nvrd456;add.s64 nvrd487,nvrd318,8192;mov.u32 nvr1533,nvr1538;mov.u64 nvrd483,nvrd488;mov.u32 nvr1534,nvr1544;mov.u64 nvrd484,nvrd493;mov.u64 nvrd485,nvrd495;mov.u32 nvr1535,nvr1545;mov.u64 nvrd486,nvrd491;mov.u32 nvr1536,nvr1546;mov.u32 nvr1537,nvr52;L__BB0_26:\nxor.b32 nvr839,nvr52,nvr1533;cvt.u64.u32 nvrd319,nvr839;add.s64 nvrd320,nvrd483,nvrd319;shl.b64 nvrd321,nvrd320,2;add.s64 nvrd323,nvrd254,4096;add.s64 nvrd324,nvrd323,nvrd321;ld.shared.f32 nvf2354,[nvrd324];mov.u32 nvr841,%ctaid.x;shl.b32 nvr842,nvr841,12;or.b32 nvr843,nvr52,nvr842;mul.wide.u32 nvrd92,nvr843,4;add.s64 nvrd325,nvrd487,nvrd92;st.global.f32[nvrd325],nvf2354;xor.b32 nvr844,nvr52,nvr1534;cvt.u64.u32 nvrd326,nvr844;add.s64 nvrd327,nvrd484,nvrd326;shl.b64 nvrd328,nvrd327,2;add.s64 nvrd329,nvrd323,nvrd328;ld.shared.f32 nvf2355,[nvrd329];or.b32 nvr845,nvr843,64;mul.wide.u32 nvrd93,nvr845,4;add.s64 nvrd330,nvrd487,nvrd93;st.global.f32[nvrd330],nvf2355;xor.b32 nvr846,nvr52,nvr1535;cvt.u64.u32 nvrd331,nvr846;add.s64 nvrd332,nvrd485,nvrd331;shl.b64 nvrd333,nvrd332,2;add.s64 nvrd334,nvrd323,nvrd333;ld.shared.f32 nvf2356,[nvrd334];or.b32 nvr847,nvr843,128;mul.wide.u32 nvrd94,nvr847,4;add.s64 nvrd335,nvrd487,nvrd94;st.global.f32[nvrd335],nvf2356;xor.b32 nvr848,nvr52,nvr1536;cvt.u64.u32 nvrd336,nvr848;add.s64 nvrd337,nvrd486,nvrd336;shl.b64 nvrd338,nvrd337,2;add.s64 nvrd339,nvrd323,nvrd338;ld.shared.f32 nvf2357,[nvrd339];or.b32 nvr849,nvr843,192;mul.wide.u32 nvrd95,nvr849,4;add.s64 nvrd340,nvrd487,nvrd95;st.global.f32[nvrd340],nvf2357;add.s32 nvr62,nvr1537,128;add.s64 nvrd487,nvrd487,1024;add.s32 nvr1536,nvr1536,4;add.s64 nvrd486,nvrd486,128;add.s32 nvr1535,nvr1535,4;add.s64 nvrd485,nvrd485,128;add.s64 nvrd484,nvrd484,128;add.s32 nvr1534,nvr1534,4;add.s64 nvrd483,nvrd483,128;add.s32 nvr1533,nvr1533,4;setp.lt.u32 nvp573,nvr1537,896;mov.u32 nvr1537,nvr62;@nvp573 bra L__BB0_26;mov.u64 nvrd454,$1;mov.u64 nvrd341,nvrd454;add.s64 nvrd492,nvrd341,8320;mov.u32 nvr1539,nvr1544;mov.u64 nvrd489,nvrd493;mov.u64 nvrd490,nvrd495;mov.u32 nvr1540,nvr1545;mov.u32 nvr1541,nvr1546;mov.u32 nvr1542,nvr52;L__BB0_28:\nadd.s64 nvrd342,nvrd492,nvrd92;ld.global.nc.f32 nvf2358,[nvrd342];xor.b32 nvr850,nvr52,nvr1538;cvt.u64.u32 nvrd343,nvr850;add.s64 nvrd344,nvrd488,nvrd343;shl.b64 nvrd345,nvrd344,2;add.s64 nvrd347,nvrd254,nvrd345;st.shared.f32[nvrd347],nvf2358;add.s64 nvrd348,nvrd492,nvrd93;ld.global.nc.f32 nvf2359,[nvrd348];xor.b32 nvr851,nvr52,nvr1539;cvt.u64.u32 nvrd349,nvr851;add.s64 nvrd350,nvrd489,nvrd349;shl.b64 nvrd351,nvrd350,2;add.s64 nvrd352,nvrd254,nvrd351;st.shared.f32[nvrd352],nvf2359;add.s64 nvrd353,nvrd492,nvrd94;ld.global.nc.f32 nvf2360,[nvrd353];xor.b32 nvr852,nvr52,nvr1540;cvt.u64.u32 nvrd354,nvr852;add.s64 nvrd355,nvrd490,nvrd354;shl.b64 nvrd356,nvrd355,2;add.s64 nvrd357,nvrd254,nvrd356;st.shared.f32[nvrd357],nvf2360;add.s64 nvrd358,nvrd492,nvrd95;ld.global.nc.f32 nvf2361,[nvrd358];xor.b32 nvr853,nvr52,nvr1541;cvt.u64.u32 nvrd359,nvr853;add.s64 nvrd360,nvrd491,nvrd359;shl.b64 nvrd361,nvrd360,2;add.s64 nvrd362,nvrd254,nvrd361;st.shared.f32[nvrd362],nvf2361;add.s32 nvr72,nvr1542,128;add.s32 nvr1541,nvr1541,4;add.s64 nvrd492,nvrd492,1024;add.s64 nvrd491,nvrd491,128;add.s32 nvr1540,nvr1540,4;add.s64 nvrd490,nvrd490,128;add.s64 nvrd489,nvrd489,128;add.s32 nvr1539,nvr1539,4;add.s64 nvrd488,nvrd488,128;add.s32 nvr1538,nvr1538,4;setp.lt.u32 nvp574,nvr1542,896;mov.u32 nvr1542,nvr72;@nvp574 bra L__BB0_28;mov.u64 nvrd457,$2;mov.b64 nvrd496,96;mov.b64 nvrd494,0;mov.b32 nvr1543,0;setp.eq.s32 nvp575,nvr52,31;setp.gt.u32 nvp576,nvr52,29;setp.gt.u32 nvp577,nvr52,28;setp.gt.u32 nvp578,nvr52,27;setp.gt.u32 nvp579,nvr52,26;setp.gt.u32 nvp580,nvr52,25;setp.gt.u32 nvp581,nvr52,24;setp.gt.u32 nvp582,nvr52,23;setp.gt.u32 nvp583,nvr52,22;setp.gt.u32 nvp584,nvr52,21;setp.gt.u32 nvp585,nvr52,20;setp.gt.u32 nvp586,nvr52,19;setp.gt.u32 nvp587,nvr52,18;setp.gt.u32 nvp588,nvr52,17;setp.gt.u32 nvp589,nvr52,16;setp.gt.u32 nvp590,nvr52,15;setp.gt.u32 nvp591,nvr52,14;setp.gt.u32 nvp592,nvr52,13;setp.gt.u32 nvp593,nvr52,12;setp.gt.u32 nvp594,nvr52,11;setp.gt.u32 nvp595,nvr52,10;setp.gt.u32 nvp596,nvr52,9;setp.gt.u32 nvp597,nvr52,8;setp.gt.u32 nvp598,nvr52,7;setp.gt.u32 nvp599,nvr52,6;setp.gt.u32 nvp600,nvr52,5;setp.gt.u32 nvp601,nvr52,4;setp.gt.u32 nvp602,nvr52,3;setp.gt.u32 nvp603,nvr52,2;setp.gt.u32 nvp604,nvr52,1;setp.eq.s32 nvp605,nvr52,0;bar.warp.sync -1;mul.lo.s32 nvr854,nvr52,33;mul.wide.u32 nvrd363,nvr854,4;mov.u64 nvrd364,global_smem;add.s64 nvrd365,nvrd364,nvrd363;ld.shared.f32 nvf2362,[nvrd365];shl.b32 nvr855,nvr52,5;xor.b32 nvr856,nvr52,1;or.b32 nvr857,nvr855,nvr856;mul.wide.u32 nvrd366,nvr857,4;add.s64 nvrd367,nvrd364,nvrd366;ld.shared.f32 nvf2363,[nvrd367];xor.b32 nvr858,nvr52,2;or.b32 nvr859,nvr855,nvr858;mul.wide.u32 nvrd368,nvr859,4;add.s64 nvrd369,nvrd364,nvrd368;ld.shared.f32 nvf2364,[nvrd369];xor.b32 nvr860,nvr52,3;or.b32 nvr861,nvr855,nvr860;mul.wide.u32 nvrd370,nvr861,4;add.s64 nvrd371,nvrd364,nvrd370;ld.shared.f32 nvf2365,[nvrd371];xor.b32 nvr862,nvr52,4;or.b32 nvr863,nvr855,nvr862;mul.wide.u32 nvrd372,nvr863,4;add.s64 nvrd373,nvrd364,nvrd372;ld.shared.f32 nvf2366,[nvrd373];xor.b32 nvr864,nvr52,5;or.b32 nvr865,nvr855,nvr864;mul.wide.u32 nvrd374,nvr865,4;add.s64 nvrd375,nvrd364,nvrd374;ld.shared.f32 nvf2367,[nvrd375];xor.b32 nvr866,nvr52,6;or.b32 nvr867,nvr855,nvr866;mul.wide.u32 nvrd376,nvr867,4;add.s64 nvrd377,nvrd364,nvrd376;ld.shared.f32 nvf2368,[nvrd377];xor.b32 nvr868,nvr52,7;or.b32 nvr869,nvr855,nvr868;mul.wide.u32 nvrd378,nvr869,4;add.s64 nvrd379,nvrd364,nvrd378;ld.shared.f32 nvf2369,[nvrd379];xor.b32 nvr870,nvr52,8;or.b32 nvr871,nvr855,nvr870;mul.wide.u32 nvrd380,nvr871,4;add.s64 nvrd381,nvrd364,nvrd380;ld.shared.f32 nvf2370,[nvrd381];xor.b32 nvr872,nvr52,9;or.b32 nvr873,nvr855,nvr872;mul.wide.u32 nvrd382,nvr873,4;add.s64 nvrd383,nvrd364,nvrd382;ld.shared.f32 nvf2371,[nvrd383];xor.b32 nvr874,nvr52,10;or.b32 nvr875,nvr855,nvr874;mul.wide.u32 nvrd384,nvr875,4;add.s64 nvrd385,nvrd364,nvrd384;ld.shared.f32 nvf2372,[nvrd385];xor.b32 nvr876,nvr52,11;or.b32 nvr877,nvr855,nvr876;mul.wide.u32 nvrd386,nvr877,4;add.s64 nvrd387,nvrd364,nvrd386;ld.shared.f32 nvf2373,[nvrd387];xor.b32 nvr878,nvr52,12;or.b32 nvr879,nvr855,nvr878;mul.wide.u32 nvrd388,nvr879,4;add.s64 nvrd389,nvrd364,nvrd388;ld.shared.f32 nvf2374,[nvrd389];xor.b32 nvr880,nvr52,13;or.b32 nvr881,nvr855,nvr880;mul.wide.u32 nvrd390,nvr881,4;add.s64 nvrd391,nvrd364,nvrd390;ld.shared.f32 nvf2375,[nvrd391];xor.b32 nvr882,nvr52,14;or.b32 nvr883,nvr855,nvr882;mul.wide.u32 nvrd392,nvr883,4;add.s64 nvrd393,nvrd364,nvrd392;ld.shared.f32 nvf2376,[nvrd393];xor.b32 nvr884,nvr52,15;or.b32 nvr885,nvr855,nvr884;mul.wide.u32 nvrd394,nvr885,4;add.s64 nvrd395,nvrd364,nvrd394;ld.shared.f32 nvf2377,[nvrd395];xor.b32 nvr886,nvr52,16;or.b32 nvr887,nvr855,nvr886;mul.wide.u32 nvrd396,nvr887,4;add.s64 nvrd397,nvrd364,nvrd396;ld.shared.f32 nvf2378,[nvrd397];xor.b32 nvr888,nvr52,17;or.b32 nvr889,nvr855,nvr888;mul.wide.u32 nvrd398,nvr889,4;add.s64 nvrd399,nvrd364,nvrd398;ld.shared.f32 nvf2379,[nvrd399];xor.b32 nvr890,nvr52,18;or.b32 nvr891,nvr855,nvr890;mul.wide.u32 nvrd400,nvr891,4;add.s64 nvrd401,nvrd364,nvrd400;ld.shared.f32 nvf2380,[nvrd401];xor.b32 nvr892,nvr52,19;or.b32 nvr893,nvr855,nvr892;mul.wide.u32 nvrd402,nvr893,4;add.s64 nvrd403,nvrd364,nvrd402;ld.shared.f32 nvf2381,[nvrd403];xor.b32 nvr894,nvr52,20;or.b32 nvr895,nvr855,nvr894;mul.wide.u32 nvrd404,nvr895,4;add.s64 nvrd405,nvrd364,nvrd404;ld.shared.f32 nvf2382,[nvrd405];xor.b32 nvr896,nvr52,21;or.b32 nvr897,nvr855,nvr896;mul.wide.u32 nvrd406,nvr897,4;add.s64 nvrd407,nvrd364,nvrd406;ld.shared.f32 nvf2383,[nvrd407];xor.b32 nvr898,nvr52,22;or.b32 nvr899,nvr855,nvr898;mul.wide.u32 nvrd408,nvr899,4;add.s64 nvrd409,nvrd364,nvrd408;ld.shared.f32 nvf2384,[nvrd409];xor.b32 nvr900,nvr52,23;or.b32 nvr901,nvr855,nvr900;mul.wide.u32 nvrd410,nvr901,4;add.s64 nvrd411,nvrd364,nvrd410;ld.shared.f32 nvf2385,[nvrd411];xor.b32 nvr902,nvr52,24;or.b32 nvr903,nvr855,nvr902;mul.wide.u32 nvrd412,nvr903,4;add.s64 nvrd413,nvrd364,nvrd412;ld.shared.f32 nvf2386,[nvrd413];xor.b32 nvr904,nvr52,25;or.b32 nvr905,nvr855,nvr904;mul.wide.u32 nvrd414,nvr905,4;add.s64 nvrd415,nvrd364,nvrd414;ld.shared.f32 nvf2387,[nvrd415];xor.b32 nvr906,nvr52,26;or.b32 nvr907,nvr855,nvr906;mul.wide.u32 nvrd416,nvr907,4;add.s64 nvrd417,nvrd364,nvrd416;ld.shared.f32 nvf2388,[nvrd417];xor.b32 nvr908,nvr52,27;or.b32 nvr909,nvr855,nvr908;mul.wide.u32 nvrd418,nvr909,4;add.s64 nvrd419,nvrd364,nvrd418;ld.shared.f32 nvf2389,[nvrd419];xor.b32 nvr910,nvr52,28;or.b32 nvr911,nvr855,nvr910;mul.wide.u32 nvrd420,nvr911,4;add.s64 nvrd421,nvrd364,nvrd420;ld.shared.f32 nvf2390,[nvrd421];xor.b32 nvr912,nvr52,29;or.b32 nvr913,nvr855,nvr912;mul.wide.u32 nvrd422,nvr913,4;add.s64 nvrd423,nvrd364,nvrd422;ld.shared.f32 nvf2391,[nvrd423];xor.b32 nvr914,nvr52,30;or.b32 nvr915,nvr855,nvr914;mul.wide.u32 nvrd424,nvr915,4;add.s64 nvrd425,nvrd364,nvrd424;ld.shared.f32 nvf2392,[nvrd425];xor.b32 nvr916,nvr52,31;or.b32 nvr917,nvr855,nvr916;mul.wide.u32 nvrd426,nvr917,4;add.s64 nvrd427,nvrd364,nvrd426;ld.shared.f32 nvf2393,[nvrd427];ld.shared.f32 nvf2394,[nvrd365+4096];ld.shared.v4.f32{nvf2395,nvf2396,nvf2397,nvf2398},[global_smem+4096];neg.ftz.f32 nvf2399,nvf2394;fma.rn.ftz.f32 nvf2400,nvf2399,nvf2395,nvf2362;ld.shared.f32 nvf2401,[nvrd367+4096];neg.ftz.f32 nvf2402,nvf2401;fma.rn.ftz.f32 nvf2403,nvf2402,nvf2396,nvf2400;ld.shared.f32 nvf2404,[nvrd369+4096];neg.ftz.f32 nvf2405,nvf2404;fma.rn.ftz.f32 nvf2406,nvf2405,nvf2397,nvf2403;ld.shared.f32 nvf2407,[nvrd371+4096];neg.ftz.f32 nvf2408,nvf2407;fma.rn.ftz.f32 nvf2409,nvf2408,nvf2398,nvf2406;ld.shared.f32 nvf2410,[nvrd373+4096];ld.shared.v4.f32{nvf2411,nvf2412,nvf2413,nvf2414},[global_smem+4112];neg.ftz.f32 nvf2415,nvf2410;fma.rn.ftz.f32 nvf2416,nvf2415,nvf2411,nvf2409;ld.shared.f32 nvf2417,[nvrd375+4096];neg.ftz.f32 nvf2418,nvf2417;fma.rn.ftz.f32 nvf2419,nvf2418,nvf2412,nvf2416;ld.shared.f32 nvf2420,[nvrd377+4096];neg.ftz.f32 nvf2421,nvf2420;fma.rn.ftz.f32 nvf2422,nvf2421,nvf2413,nvf2419;ld.shared.f32 nvf2423,[nvrd379+4096];neg.ftz.f32 nvf2424,nvf2423;fma.rn.ftz.f32 nvf2425,nvf2424,nvf2414,nvf2422;ld.shared.f32 nvf2426,[nvrd381+4096];ld.shared.v4.f32{nvf2427,nvf2428,nvf2429,nvf2430},[global_smem+4128];neg.ftz.f32 nvf2431,nvf2426;fma.rn.ftz.f32 nvf2432,nvf2431,nvf2427,nvf2425;ld.shared.f32 nvf2433,[nvrd383+4096];neg.ftz.f32 nvf2434,nvf2433;fma.rn.ftz.f32 nvf2435,nvf2434,nvf2428,nvf2432;ld.shared.f32 nvf2436,[nvrd385+4096];neg.ftz.f32 nvf2437,nvf2436;fma.rn.ftz.f32 nvf2438,nvf2437,nvf2429,nvf2435;ld.shared.f32 nvf2439,[nvrd387+4096];neg.ftz.f32 nvf2440,nvf2439;fma.rn.ftz.f32 nvf2441,nvf2440,nvf2430,nvf2438;ld.shared.f32 nvf2442,[nvrd389+4096];ld.shared.v4.f32{nvf2443,nvf2444,nvf2445,nvf2446},[global_smem+4144];neg.ftz.f32 nvf2447,nvf2442;fma.rn.ftz.f32 nvf2448,nvf2447,nvf2443,nvf2441;ld.shared.f32 nvf2449,[nvrd391+4096];neg.ftz.f32 nvf2450,nvf2449;fma.rn.ftz.f32 nvf2451,nvf2450,nvf2444,nvf2448;ld.shared.f32 nvf2452,[nvrd393+4096];neg.ftz.f32 nvf2453,nvf2452;fma.rn.ftz.f32 nvf2454,nvf2453,nvf2445,nvf2451;ld.shared.f32 nvf2455,[nvrd395+4096];neg.ftz.f32 nvf2456,nvf2455;fma.rn.ftz.f32 nvf2457,nvf2456,nvf2446,nvf2454;ld.shared.f32 nvf2458,[nvrd397+4096];ld.shared.v4.f32{nvf2459,nvf2460,nvf2461,nvf2462},[global_smem+4160];neg.ftz.f32 nvf2463,nvf2458;fma.rn.ftz.f32 nvf2464,nvf2463,nvf2459,nvf2457;ld.shared.f32 nvf2465,[nvrd399+4096];neg.ftz.f32 nvf2466,nvf2465;fma.rn.ftz.f32 nvf2467,nvf2466,nvf2460,nvf2464;ld.shared.f32 nvf2468,[nvrd401+4096];neg.ftz.f32 nvf2469,nvf2468;fma.rn.ftz.f32 nvf2470,nvf2469,nvf2461,nvf2467;ld.shared.f32 nvf2471,[nvrd403+4096];neg.ftz.f32 nvf2472,nvf2471;fma.rn.ftz.f32 nvf2473,nvf2472,nvf2462,nvf2470;ld.shared.f32 nvf2474,[nvrd405+4096];ld.shared.v4.f32{nvf2475,nvf2476,nvf2477,nvf2478},[global_smem+4176];neg.ftz.f32 nvf2479,nvf2474;fma.rn.ftz.f32 nvf2480,nvf2479,nvf2475,nvf2473;ld.shared.f32 nvf2481,[nvrd407+4096];neg.ftz.f32 nvf2482,nvf2481;fma.rn.ftz.f32 nvf2483,nvf2482,nvf2476,nvf2480;ld.shared.f32 nvf2484,[nvrd409+4096];neg.ftz.f32 nvf2485,nvf2484;fma.rn.ftz.f32 nvf2486,nvf2485,nvf2477,nvf2483;ld.shared.f32 nvf2487,[nvrd411+4096];neg.ftz.f32 nvf2488,nvf2487;fma.rn.ftz.f32 nvf2489,nvf2488,nvf2478,nvf2486;ld.shared.f32 nvf2490,[nvrd413+4096];ld.shared.v4.f32{nvf2491,nvf2492,nvf2493,nvf2494},[global_smem+4192];neg.ftz.f32 nvf2495,nvf2490;fma.rn.ftz.f32 nvf2496,nvf2495,nvf2491,nvf2489;ld.shared.f32 nvf2497,[nvrd415+4096];neg.ftz.f32 nvf2498,nvf2497;fma.rn.ftz.f32 nvf2499,nvf2498,nvf2492,nvf2496;ld.shared.f32 nvf2500,[nvrd417+4096];neg.ftz.f32 nvf2501,nvf2500;fma.rn.ftz.f32 nvf2502,nvf2501,nvf2493,nvf2499;ld.shared.f32 nvf2503,[nvrd419+4096];neg.ftz.f32 nvf2504,nvf2503;fma.rn.ftz.f32 nvf2505,nvf2504,nvf2494,nvf2502;ld.shared.f32 nvf2506,[nvrd421+4096];ld.shared.v4.f32{nvf2507,nvf2508,nvf2509,nvf2510},[global_smem+4208];neg.ftz.f32 nvf2511,nvf2506;fma.rn.ftz.f32 nvf2512,nvf2511,nvf2507,nvf2505;ld.shared.f32 nvf2513,[nvrd423+4096];neg.ftz.f32 nvf2514,nvf2513;fma.rn.ftz.f32 nvf2515,nvf2514,nvf2508,nvf2512;ld.shared.f32 nvf2516,[nvrd425+4096];neg.ftz.f32 nvf2517,nvf2516;fma.rn.ftz.f32 nvf2518,nvf2517,nvf2509,nvf2515;ld.shared.f32 nvf2519,[nvrd427+4096];neg.ftz.f32 nvf2520,nvf2519;fma.rn.ftz.f32 nvf2521,nvf2520,nvf2510,nvf2518;ld.shared.v4.f32{nvf2522,nvf2523,nvf2524,nvf2525},[global_smem+4224];fma.rn.ftz.f32 nvf2526,nvf2399,nvf2523,nvf2363;fma.rn.ftz.f32 nvf2527,nvf2402,nvf2522,nvf2526;fma.rn.ftz.f32 nvf2528,nvf2405,nvf2525,nvf2527;fma.rn.ftz.f32 nvf2529,nvf2408,nvf2524,nvf2528;ld.shared.v4.f32{nvf2530,nvf2531,nvf2532,nvf2533},[global_smem+4240];fma.rn.ftz.f32 nvf2534,nvf2415,nvf2531,nvf2529;fma.rn.ftz.f32 nvf2535,nvf2418,nvf2530,nvf2534;fma.rn.ftz.f32 nvf2536,nvf2421,nvf2533,nvf2535;fma.rn.ftz.f32 nvf2537,nvf2424,nvf2532,nvf2536;ld.shared.v4.f32{nvf2538,nvf2539,nvf2540,nvf2541},[global_smem+4256];fma.rn.ftz.f32 nvf2542,nvf2431,nvf2539,nvf2537;fma.rn.ftz.f32 nvf2543,nvf2434,nvf2538,nvf2542;fma.rn.ftz.f32 nvf2544,nvf2437,nvf2541,nvf2543;fma.rn.ftz.f32 nvf2545,nvf2440,nvf2540,nvf2544;ld.shared.v4.f32{nvf2546,nvf2547,nvf2548,nvf2549},[global_smem+4272];fma.rn.ftz.f32 nvf2550,nvf2447,nvf2547,nvf2545;fma.rn.ftz.f32 nvf2551,nvf2450,nvf2546,nvf2550;fma.rn.ftz.f32 nvf2552,nvf2453,nvf2549,nvf2551;fma.rn.ftz.f32 nvf2553,nvf2456,nvf2548,nvf2552;ld.shared.v4.f32{nvf2554,nvf2555,nvf2556,nvf2557},[global_smem+4288];fma.rn.ftz.f32 nvf2558,nvf2463,nvf2555,nvf2553;fma.rn.ftz.f32 nvf2559,nvf2466,nvf2554,nvf2558;fma.rn.ftz.f32 nvf2560,nvf2469,nvf2557,nvf2559;fma.rn.ftz.f32 nvf2561,nvf2472,nvf2556,nvf2560;ld.shared.v4.f32{nvf2562,nvf2563,nvf2564,nvf2565},[global_smem+4304];fma.rn.ftz.f32 nvf2566,nvf2479,nvf2563,nvf2561;fma.rn.ftz.f32 nvf2567,nvf2482,nvf2562,nvf2566;fma.rn.ftz.f32 nvf2568,nvf2485,nvf2565,nvf2567;fma.rn.ftz.f32 nvf2569,nvf2488,nvf2564,nvf2568;ld.shared.v4.f32{nvf2570,nvf2571,nvf2572,nvf2573},[global_smem+4320];fma.rn.ftz.f32 nvf2574,nvf2495,nvf2571,nvf2569;fma.rn.ftz.f32 nvf2575,nvf2498,nvf2570,nvf2574;fma.rn.ftz.f32 nvf2576,nvf2501,nvf2573,nvf2575;fma.rn.ftz.f32 nvf2577,nvf2504,nvf2572,nvf2576;ld.shared.v4.f32{nvf2578,nvf2579,nvf2580,nvf2581},[global_smem+4336];fma.rn.ftz.f32 nvf2582,nvf2511,nvf2579,nvf2577;fma.rn.ftz.f32 nvf2583,nvf2514,nvf2578,nvf2582;fma.rn.ftz.f32 nvf2584,nvf2517,nvf2581,nvf2583;fma.rn.ftz.f32 nvf2585,nvf2520,nvf2580,nvf2584;ld.shared.v4.f32{nvf2586,nvf2587,nvf2588,nvf2589},[global_smem+4352];fma.rn.ftz.f32 nvf2590,nvf2399,nvf2588,nvf2364;fma.rn.ftz.f32 nvf2591,nvf2402,nvf2589,nvf2590;fma.rn.ftz.f32 nvf2592,nvf2405,nvf2586,nvf2591;fma.rn.ftz.f32 nvf2593,nvf2408,nvf2587,nvf2592;ld.shared.v4.f32{nvf2594,nvf2595,nvf2596,nvf2597},[global_smem+4368];fma.rn.ftz.f32 nvf2598,nvf2415,nvf2596,nvf2593;fma.rn.ftz.f32 nvf2599,nvf2418,nvf2597,nvf2598;fma.rn.ftz.f32 nvf2600,nvf2421,nvf2594,nvf2599;fma.rn.ftz.f32 nvf2601,nvf2424,nvf2595,nvf2600;ld.shared.v4.f32{nvf2602,nvf2603,nvf2604,nvf2605},[global_smem+4384];fma.rn.ftz.f32 nvf2606,nvf2431,nvf2604,nvf2601;fma.rn.ftz.f32 nvf2607,nvf2434,nvf2605,nvf2606;fma.rn.ftz.f32 nvf2608,nvf2437,nvf2602,nvf2607;fma.rn.ftz.f32 nvf2609,nvf2440,nvf2603,nvf2608;ld.shared.v4.f32{nvf2610,nvf2611,nvf2612,nvf2613},[global_smem+4400];fma.rn.ftz.f32 nvf2614,nvf2447,nvf2612,nvf2609;fma.rn.ftz.f32 nvf2615,nvf2450,nvf2613,nvf2614;fma.rn.ftz.f32 nvf2616,nvf2453,nvf2610,nvf2615;fma.rn.ftz.f32 nvf2617,nvf2456,nvf2611,nvf2616;ld.shared.v4.f32{nvf2618,nvf2619,nvf2620,nvf2621},[global_smem+4416];fma.rn.ftz.f32 nvf2622,nvf2463,nvf2620,nvf2617;fma.rn.ftz.f32 nvf2623,nvf2466,nvf2621,nvf2622;fma.rn.ftz.f32 nvf2624,nvf2469,nvf2618,nvf2623;fma.rn.ftz.f32 nvf2625,nvf2472,nvf2619,nvf2624;ld.shared.v4.f32{nvf2626,nvf2627,nvf2628,nvf2629},[global_smem+4432];fma.rn.ftz.f32 nvf2630,nvf2479,nvf2628,nvf2625;fma.rn.ftz.f32 nvf2631,nvf2482,nvf2629,nvf2630;fma.rn.ftz.f32 nvf2632,nvf2485,nvf2626,nvf2631;fma.rn.ftz.f32 nvf2633,nvf2488,nvf2627,nvf2632;ld.shared.v4.f32{nvf2634,nvf2635,nvf2636,nvf2637},[global_smem+4448];fma.rn.ftz.f32 nvf2638,nvf2495,nvf2636,nvf2633;fma.rn.ftz.f32 nvf2639,nvf2498,nvf2637,nvf2638;fma.rn.ftz.f32 nvf2640,nvf2501,nvf2634,nvf2639;fma.rn.ftz.f32 nvf2641,nvf2504,nvf2635,nvf2640;ld.shared.v4.f32{nvf2642,nvf2643,nvf2644,nvf2645},[global_smem+4464];fma.rn.ftz.f32 nvf2646,nvf2511,nvf2644,nvf2641;fma.rn.ftz.f32 nvf2647,nvf2514,nvf2645,nvf2646;fma.rn.ftz.f32 nvf2648,nvf2517,nvf2642,nvf2647;fma.rn.ftz.f32 nvf2649,nvf2520,nvf2643,nvf2648;ld.shared.v4.f32{nvf2650,nvf2651,nvf2652,nvf2653},[global_smem+4480];fma.rn.ftz.f32 nvf2654,nvf2399,nvf2653,nvf2365;fma.rn.ftz.f32 nvf2655,nvf2402,nvf2652,nvf2654;fma.rn.ftz.f32 nvf2656,nvf2405,nvf2651,nvf2655;fma.rn.ftz.f32 nvf2657,nvf2408,nvf2650,nvf2656;ld.shared.v4.f32{nvf2658,nvf2659,nvf2660,nvf2661},[global_smem+4496];fma.rn.ftz.f32 nvf2662,nvf2415,nvf2661,nvf2657;fma.rn.ftz.f32 nvf2663,nvf2418,nvf2660,nvf2662;fma.rn.ftz.f32 nvf2664,nvf2421,nvf2659,nvf2663;fma.rn.ftz.f32 nvf2665,nvf2424,nvf2658,nvf2664;ld.shared.v4.f32{nvf2666,nvf2667,nvf2668,nvf2669},[global_smem+4512];fma.rn.ftz.f32 nvf2670,nvf2431,nvf2669,nvf2665;fma.rn.ftz.f32 nvf2671,nvf2434,nvf2668,nvf2670;fma.rn.ftz.f32 nvf2672,nvf2437,nvf2667,nvf2671;fma.rn.ftz.f32 nvf2673,nvf2440,nvf2666,nvf2672;ld.shared.v4.f32{nvf2674,nvf2675,nvf2676,nvf2677},[global_smem+4528];fma.rn.ftz.f32 nvf2678,nvf2447,nvf2677,nvf2673;fma.rn.ftz.f32 nvf2679,nvf2450,nvf2676,nvf2678;fma.rn.ftz.f32 nvf2680,nvf2453,nvf2675,nvf2679;fma.rn.ftz.f32 nvf2681,nvf2456,nvf2674,nvf2680;ld.shared.v4.f32{nvf2682,nvf2683,nvf2684,nvf2685},[global_smem+4544];fma.rn.ftz.f32 nvf2686,nvf2463,nvf2685,nvf2681;fma.rn.ftz.f32 nvf2687,nvf2466,nvf2684,nvf2686;fma.rn.ftz.f32 nvf2688,nvf2469,nvf2683,nvf2687;fma.rn.ftz.f32 nvf2689,nvf2472,nvf2682,nvf2688;ld.shared.v4.f32{nvf2690,nvf2691,nvf2692,nvf2693},[global_smem+4560];fma.rn.ftz.f32 nvf2694,nvf2479,nvf2693,nvf2689;fma.rn.ftz.f32 nvf2695,nvf2482,nvf2692,nvf2694;fma.rn.ftz.f32 nvf2696,nvf2485,nvf2691,nvf2695;fma.rn.ftz.f32 nvf2697,nvf2488,nvf2690,nvf2696;ld.shared.v4.f32{nvf2698,nvf2699,nvf2700,nvf2701},[global_smem+4576];fma.rn.ftz.f32 nvf2702,nvf2495,nvf2701,nvf2697;fma.rn.ftz.f32 nvf2703,nvf2498,nvf2700,nvf2702;fma.rn.ftz.f32 nvf2704,nvf2501,nvf2699,nvf2703;fma.rn.ftz.f32 nvf2705,nvf2504,nvf2698,nvf2704;ld.shared.v4.f32{nvf2706,nvf2707,nvf2708,nvf2709},[global_smem+4592];fma.rn.ftz.f32 nvf2710,nvf2511,nvf2709,nvf2705;fma.rn.ftz.f32 nvf2711,nvf2514,nvf2708,nvf2710;fma.rn.ftz.f32 nvf2712,nvf2517,nvf2707,nvf2711;fma.rn.ftz.f32 nvf2713,nvf2520,nvf2706,nvf2712;ld.shared.v4.f32{nvf2714,nvf2715,nvf2716,nvf2717},[global_smem+4624];fma.rn.ftz.f32 nvf2718,nvf2399,nvf2714,nvf2366;fma.rn.ftz.f32 nvf2719,nvf2402,nvf2715,nvf2718;fma.rn.ftz.f32 nvf2720,nvf2405,nvf2716,nvf2719;fma.rn.ftz.f32 nvf2721,nvf2408,nvf2717,nvf2720;ld.shared.v4.f32{nvf2722,nvf2723,nvf2724,nvf2725},[global_smem+4608];fma.rn.ftz.f32 nvf2726,nvf2415,nvf2722,nvf2721;fma.rn.ftz.f32 nvf2727,nvf2418,nvf2723,nvf2726;fma.rn.ftz.f32 nvf2728,nvf2421,nvf2724,nvf2727;fma.rn.ftz.f32 nvf2729,nvf2424,nvf2725,nvf2728;ld.shared.v4.f32{nvf2730,nvf2731,nvf2732,nvf2733},[global_smem+4656];fma.rn.ftz.f32 nvf2734,nvf2431,nvf2730,nvf2729;fma.rn.ftz.f32 nvf2735,nvf2434,nvf2731,nvf2734;fma.rn.ftz.f32 nvf2736,nvf2437,nvf2732,nvf2735;fma.rn.ftz.f32 nvf2737,nvf2440,nvf2733,nvf2736;ld.shared.v4.f32{nvf2738,nvf2739,nvf2740,nvf2741},[global_smem+4640];fma.rn.ftz.f32 nvf2742,nvf2447,nvf2738,nvf2737;fma.rn.ftz.f32 nvf2743,nvf2450,nvf2739,nvf2742;fma.rn.ftz.f32 nvf2744,nvf2453,nvf2740,nvf2743;fma.rn.ftz.f32 nvf2745,nvf2456,nvf2741,nvf2744;ld.shared.v4.f32{nvf2746,nvf2747,nvf2748,nvf2749},[global_smem+4688];fma.rn.ftz.f32 nvf2750,nvf2463,nvf2746,nvf2745;fma.rn.ftz.f32 nvf2751,nvf2466,nvf2747,nvf2750;fma.rn.ftz.f32 nvf2752,nvf2469,nvf2748,nvf2751;fma.rn.ftz.f32 nvf2753,nvf2472,nvf2749,nvf2752;ld.shared.v4.f32{nvf2754,nvf2755,nvf2756,nvf2757},[global_smem+4672];fma.rn.ftz.f32 nvf2758,nvf2479,nvf2754,nvf2753;fma.rn.ftz.f32 nvf2759,nvf2482,nvf2755,nvf2758;fma.rn.ftz.f32 nvf2760,nvf2485,nvf2756,nvf2759;fma.rn.ftz.f32 nvf2761,nvf2488,nvf2757,nvf2760;ld.shared.v4.f32{nvf2762,nvf2763,nvf2764,nvf2765},[global_smem+4720];fma.rn.ftz.f32 nvf2766,nvf2495,nvf2762,nvf2761;fma.rn.ftz.f32 nvf2767,nvf2498,nvf2763,nvf2766;fma.rn.ftz.f32 nvf2768,nvf2501,nvf2764,nvf2767;fma.rn.ftz.f32 nvf2769,nvf2504,nvf2765,nvf2768;ld.shared.v4.f32{nvf2770,nvf2771,nvf2772,nvf2773},[global_smem+4704];fma.rn.ftz.f32 nvf2774,nvf2511,nvf2770,nvf2769;fma.rn.ftz.f32 nvf2775,nvf2514,nvf2771,nvf2774;fma.rn.ftz.f32 nvf2776,nvf2517,nvf2772,nvf2775;fma.rn.ftz.f32 nvf2777,nvf2520,nvf2773,nvf2776;ld.shared.v4.f32{nvf2778,nvf2779,nvf2780,nvf2781},[global_smem+4752];fma.rn.ftz.f32 nvf2782,nvf2399,nvf2779,nvf2367;fma.rn.ftz.f32 nvf2783,nvf2402,nvf2778,nvf2782;fma.rn.ftz.f32 nvf2784,nvf2405,nvf2781,nvf2783;fma.rn.ftz.f32 nvf2785,nvf2408,nvf2780,nvf2784;ld.shared.v4.f32{nvf2786,nvf2787,nvf2788,nvf2789},[global_smem+4736];fma.rn.ftz.f32 nvf2790,nvf2415,nvf2787,nvf2785;fma.rn.ftz.f32 nvf2791,nvf2418,nvf2786,nvf2790;fma.rn.ftz.f32 nvf2792,nvf2421,nvf2789,nvf2791;fma.rn.ftz.f32 nvf2793,nvf2424,nvf2788,nvf2792;ld.shared.v4.f32{nvf2794,nvf2795,nvf2796,nvf2797},[global_smem+4784];fma.rn.ftz.f32 nvf2798,nvf2431,nvf2795,nvf2793;fma.rn.ftz.f32 nvf2799,nvf2434,nvf2794,nvf2798;fma.rn.ftz.f32 nvf2800,nvf2437,nvf2797,nvf2799;fma.rn.ftz.f32 nvf2801,nvf2440,nvf2796,nvf2800;ld.shared.v4.f32{nvf2802,nvf2803,nvf2804,nvf2805},[global_smem+4768];fma.rn.ftz.f32 nvf2806,nvf2447,nvf2803,nvf2801;fma.rn.ftz.f32 nvf2807,nvf2450,nvf2802,nvf2806;fma.rn.ftz.f32 nvf2808,nvf2453,nvf2805,nvf2807;fma.rn.ftz.f32 nvf2809,nvf2456,nvf2804,nvf2808;ld.shared.v4.f32{nvf2810,nvf2811,nvf2812,nvf2813},[global_smem+4816];fma.rn.ftz.f32 nvf2814,nvf2463,nvf2811,nvf2809;fma.rn.ftz.f32 nvf2815,nvf2466,nvf2810,nvf2814;fma.rn.ftz.f32 nvf2816,nvf2469,nvf2813,nvf2815;fma.rn.ftz.f32 nvf2817,nvf2472,nvf2812,nvf2816;ld.shared.v4.f32{nvf2818,nvf2819,nvf2820,nvf2821},[global_smem+4800];fma.rn.ftz.f32 nvf2822,nvf2479,nvf2819,nvf2817;fma.rn.ftz.f32 nvf2823,nvf2482,nvf2818,nvf2822;fma.rn.ftz.f32 nvf2824,nvf2485,nvf2821,nvf2823;fma.rn.ftz.f32 nvf2825,nvf2488,nvf2820,nvf2824;ld.shared.v4.f32{nvf2826,nvf2827,nvf2828,nvf2829},[global_smem+4848];fma.rn.ftz.f32 nvf2830,nvf2495,nvf2827,nvf2825;fma.rn.ftz.f32 nvf2831,nvf2498,nvf2826,nvf2830;fma.rn.ftz.f32 nvf2832,nvf2501,nvf2829,nvf2831;fma.rn.ftz.f32 nvf2833,nvf2504,nvf2828,nvf2832;ld.shared.v4.f32{nvf2834,nvf2835,nvf2836,nvf2837},[global_smem+4832];fma.rn.ftz.f32 nvf2838,nvf2511,nvf2835,nvf2833;fma.rn.ftz.f32 nvf2839,nvf2514,nvf2834,nvf2838;fma.rn.ftz.f32 nvf2840,nvf2517,nvf2837,nvf2839;fma.rn.ftz.f32 nvf2841,nvf2520,nvf2836,nvf2840;ld.shared.v4.f32{nvf2842,nvf2843,nvf2844,nvf2845},[global_smem+4880];fma.rn.ftz.f32 nvf2846,nvf2399,nvf2844,nvf2368;fma.rn.ftz.f32 nvf2847,nvf2402,nvf2845,nvf2846;fma.rn.ftz.f32 nvf2848,nvf2405,nvf2842,nvf2847;fma.rn.ftz.f32 nvf2849,nvf2408,nvf2843,nvf2848;ld.shared.v4.f32{nvf2850,nvf2851,nvf2852,nvf2853},[global_smem+4864];fma.rn.ftz.f32 nvf2854,nvf2415,nvf2852,nvf2849;fma.rn.ftz.f32 nvf2855,nvf2418,nvf2853,nvf2854;fma.rn.ftz.f32 nvf2856,nvf2421,nvf2850,nvf2855;fma.rn.ftz.f32 nvf2857,nvf2424,nvf2851,nvf2856;ld.shared.v4.f32{nvf2858,nvf2859,nvf2860,nvf2861},[global_smem+4912];fma.rn.ftz.f32 nvf2862,nvf2431,nvf2860,nvf2857;fma.rn.ftz.f32 nvf2863,nvf2434,nvf2861,nvf2862;fma.rn.ftz.f32 nvf2864,nvf2437,nvf2858,nvf2863;fma.rn.ftz.f32 nvf2865,nvf2440,nvf2859,nvf2864;ld.shared.v4.f32{nvf2866,nvf2867,nvf2868,nvf2869},[global_smem+4896];fma.rn.ftz.f32 nvf2870,nvf2447,nvf2868,nvf2865;fma.rn.ftz.f32 nvf2871,nvf2450,nvf2869,nvf2870;fma.rn.ftz.f32 nvf2872,nvf2453,nvf2866,nvf2871;fma.rn.ftz.f32 nvf2873,nvf2456,nvf2867,nvf2872;ld.shared.v4.f32{nvf2874,nvf2875,nvf2876,nvf2877},[global_smem+4944];fma.rn.ftz.f32 nvf2878,nvf2463,nvf2876,nvf2873;fma.rn.ftz.f32 nvf2879,nvf2466,nvf2877,nvf2878;fma.rn.ftz.f32 nvf2880,nvf2469,nvf2874,nvf2879;fma.rn.ftz.f32 nvf2881,nvf2472,nvf2875,nvf2880;ld.shared.v4.f32{nvf2882,nvf2883,nvf2884,nvf2885},[global_smem+4928];fma.rn.ftz.f32 nvf2886,nvf2479,nvf2884,nvf2881;fma.rn.ftz.f32 nvf2887,nvf2482,nvf2885,nvf2886;fma.rn.ftz.f32 nvf2888,nvf2485,nvf2882,nvf2887;fma.rn.ftz.f32 nvf2889,nvf2488,nvf2883,nvf2888;ld.shared.v4.f32{nvf2890,nvf2891,nvf2892,nvf2893},[global_smem+4976];fma.rn.ftz.f32 nvf2894,nvf2495,nvf2892,nvf2889;fma.rn.ftz.f32 nvf2895,nvf2498,nvf2893,nvf2894;fma.rn.ftz.f32 nvf2896,nvf2501,nvf2890,nvf2895;fma.rn.ftz.f32 nvf2897,nvf2504,nvf2891,nvf2896;ld.shared.v4.f32{nvf2898,nvf2899,nvf2900,nvf2901},[global_smem+4960];fma.rn.ftz.f32 nvf2902,nvf2511,nvf2900,nvf2897;fma.rn.ftz.f32 nvf2903,nvf2514,nvf2901,nvf2902;fma.rn.ftz.f32 nvf2904,nvf2517,nvf2898,nvf2903;fma.rn.ftz.f32 nvf2905,nvf2520,nvf2899,nvf2904;ld.shared.v4.f32{nvf2906,nvf2907,nvf2908,nvf2909},[global_smem+5008];fma.rn.ftz.f32 nvf2910,nvf2399,nvf2909,nvf2369;fma.rn.ftz.f32 nvf2911,nvf2402,nvf2908,nvf2910;fma.rn.ftz.f32 nvf2912,nvf2405,nvf2907,nvf2911;fma.rn.ftz.f32 nvf2913,nvf2408,nvf2906,nvf2912;ld.shared.v4.f32{nvf2914,nvf2915,nvf2916,nvf2917},[global_smem+4992];fma.rn.ftz.f32 nvf2918,nvf2415,nvf2917,nvf2913;fma.rn.ftz.f32 nvf2919,nvf2418,nvf2916,nvf2918;fma.rn.ftz.f32 nvf2920,nvf2421,nvf2915,nvf2919;fma.rn.ftz.f32 nvf2921,nvf2424,nvf2914,nvf2920;ld.shared.v4.f32{nvf2922,nvf2923,nvf2924,nvf2925},[global_smem+5040];fma.rn.ftz.f32 nvf2926,nvf2431,nvf2925,nvf2921;fma.rn.ftz.f32 nvf2927,nvf2434,nvf2924,nvf2926;fma.rn.ftz.f32 nvf2928,nvf2437,nvf2923,nvf2927;fma.rn.ftz.f32 nvf2929,nvf2440,nvf2922,nvf2928;ld.shared.v4.f32{nvf2930,nvf2931,nvf2932,nvf2933},[global_smem+5024];fma.rn.ftz.f32 nvf2934,nvf2447,nvf2933,nvf2929;fma.rn.ftz.f32 nvf2935,nvf2450,nvf2932,nvf2934;fma.rn.ftz.f32 nvf2936,nvf2453,nvf2931,nvf2935;fma.rn.ftz.f32 nvf2937,nvf2456,nvf2930,nvf2936;ld.shared.v4.f32{nvf2938,nvf2939,nvf2940,nvf2941},[global_smem+5072];fma.rn.ftz.f32 nvf2942,nvf2463,nvf2941,nvf2937;fma.rn.ftz.f32 nvf2943,nvf2466,nvf2940,nvf2942;fma.rn.ftz.f32 nvf2944,nvf2469,nvf2939,nvf2943;fma.rn.ftz.f32 nvf2945,nvf2472,nvf2938,nvf2944;ld.shared.v4.f32{nvf2946,nvf2947,nvf2948,nvf2949},[global_smem+5056];fma.rn.ftz.f32 nvf2950,nvf2479,nvf2949,nvf2945;fma.rn.ftz.f32 nvf2951,nvf2482,nvf2948,nvf2950;fma.rn.ftz.f32 nvf2952,nvf2485,nvf2947,nvf2951;fma.rn.ftz.f32 nvf2953,nvf2488,nvf2946,nvf2952;ld.shared.v4.f32{nvf2954,nvf2955,nvf2956,nvf2957},[global_smem+5104];fma.rn.ftz.f32 nvf2958,nvf2495,nvf2957,nvf2953;fma.rn.ftz.f32 nvf2959,nvf2498,nvf2956,nvf2958;fma.rn.ftz.f32 nvf2960,nvf2501,nvf2955,nvf2959;fma.rn.ftz.f32 nvf2961,nvf2504,nvf2954,nvf2960;ld.shared.v4.f32{nvf2962,nvf2963,nvf2964,nvf2965},[global_smem+5088];fma.rn.ftz.f32 nvf2966,nvf2511,nvf2965,nvf2961;fma.rn.ftz.f32 nvf2967,nvf2514,nvf2964,nvf2966;fma.rn.ftz.f32 nvf2968,nvf2517,nvf2963,nvf2967;fma.rn.ftz.f32 nvf2969,nvf2520,nvf2962,nvf2968;ld.shared.v4.f32{nvf2970,nvf2971,nvf2972,nvf2973},[global_smem+5152];fma.rn.ftz.f32 nvf2974,nvf2399,nvf2970,nvf2370;fma.rn.ftz.f32 nvf2975,nvf2402,nvf2971,nvf2974;fma.rn.ftz.f32 nvf2976,nvf2405,nvf2972,nvf2975;fma.rn.ftz.f32 nvf2977,nvf2408,nvf2973,nvf2976;ld.shared.v4.f32{nvf2978,nvf2979,nvf2980,nvf2981},[global_smem+5168];fma.rn.ftz.f32 nvf2982,nvf2415,nvf2978,nvf2977;fma.rn.ftz.f32 nvf2983,nvf2418,nvf2979,nvf2982;fma.rn.ftz.f32 nvf2984,nvf2421,nvf2980,nvf2983;fma.rn.ftz.f32 nvf2985,nvf2424,nvf2981,nvf2984;ld.shared.v4.f32{nvf2986,nvf2987,nvf2988,nvf2989},[global_smem+5120];fma.rn.ftz.f32 nvf2990,nvf2431,nvf2986,nvf2985;fma.rn.ftz.f32 nvf2991,nvf2434,nvf2987,nvf2990;fma.rn.ftz.f32 nvf2992,nvf2437,nvf2988,nvf2991;fma.rn.ftz.f32 nvf2993,nvf2440,nvf2989,nvf2992;ld.shared.v4.f32{nvf2994,nvf2995,nvf2996,nvf2997},[global_smem+5136];fma.rn.ftz.f32 nvf2998,nvf2447,nvf2994,nvf2993;fma.rn.ftz.f32 nvf2999,nvf2450,nvf2995,nvf2998;fma.rn.ftz.f32 nvf3000,nvf2453,nvf2996,nvf2999;fma.rn.ftz.f32 nvf3001,nvf2456,nvf2997,nvf3000;ld.shared.v4.f32{nvf3002,nvf3003,nvf3004,nvf3005},[global_smem+5216];fma.rn.ftz.f32 nvf3006,nvf2463,nvf3002,nvf3001;fma.rn.ftz.f32 nvf3007,nvf2466,nvf3003,nvf3006;fma.rn.ftz.f32 nvf3008,nvf2469,nvf3004,nvf3007;fma.rn.ftz.f32 nvf3009,nvf2472,nvf3005,nvf3008;ld.shared.v4.f32{nvf3010,nvf3011,nvf3012,nvf3013},[global_smem+5232];fma.rn.ftz.f32 nvf3014,nvf2479,nvf3010,nvf3009;fma.rn.ftz.f32 nvf3015,nvf2482,nvf3011,nvf3014;fma.rn.ftz.f32 nvf3016,nvf2485,nvf3012,nvf3015;fma.rn.ftz.f32 nvf3017,nvf2488,nvf3013,nvf3016;ld.shared.v4.f32{nvf3018,nvf3019,nvf3020,nvf3021},[global_smem+5184];fma.rn.ftz.f32 nvf3022,nvf2495,nvf3018,nvf3017;fma.rn.ftz.f32 nvf3023,nvf2498,nvf3019,nvf3022;fma.rn.ftz.f32 nvf3024,nvf2501,nvf3020,nvf3023;fma.rn.ftz.f32 nvf3025,nvf2504,nvf3021,nvf3024;ld.shared.v4.f32{nvf3026,nvf3027,nvf3028,nvf3029},[global_smem+5200];fma.rn.ftz.f32 nvf3030,nvf2511,nvf3026,nvf3025;fma.rn.ftz.f32 nvf3031,nvf2514,nvf3027,nvf3030;fma.rn.ftz.f32 nvf3032,nvf2517,nvf3028,nvf3031;fma.rn.ftz.f32 nvf3033,nvf2520,nvf3029,nvf3032;ld.shared.v4.f32{nvf3034,nvf3035,nvf3036,nvf3037},[global_smem+5280];fma.rn.ftz.f32 nvf3038,nvf2399,nvf3035,nvf2371;fma.rn.ftz.f32 nvf3039,nvf2402,nvf3034,nvf3038;fma.rn.ftz.f32 nvf3040,nvf2405,nvf3037,nvf3039;fma.rn.ftz.f32 nvf3041,nvf2408,nvf3036,nvf3040;ld.shared.v4.f32{nvf3042,nvf3043,nvf3044,nvf3045},[global_smem+5296];fma.rn.ftz.f32 nvf3046,nvf2415,nvf3043,nvf3041;fma.rn.ftz.f32 nvf3047,nvf2418,nvf3042,nvf3046;fma.rn.ftz.f32 nvf3048,nvf2421,nvf3045,nvf3047;fma.rn.ftz.f32 nvf3049,nvf2424,nvf3044,nvf3048;ld.shared.v4.f32{nvf3050,nvf3051,nvf3052,nvf3053},[global_smem+5248];fma.rn.ftz.f32 nvf3054,nvf2431,nvf3051,nvf3049;fma.rn.ftz.f32 nvf3055,nvf2434,nvf3050,nvf3054;fma.rn.ftz.f32 nvf3056,nvf2437,nvf3053,nvf3055;fma.rn.ftz.f32 nvf3057,nvf2440,nvf3052,nvf3056;ld.shared.v4.f32{nvf3058,nvf3059,nvf3060,nvf3061},[global_smem+5264];fma.rn.ftz.f32 nvf3062,nvf2447,nvf3059,nvf3057;fma.rn.ftz.f32 nvf3063,nvf2450,nvf3058,nvf3062;fma.rn.ftz.f32 nvf3064,nvf2453,nvf3061,nvf3063;fma.rn.ftz.f32 nvf3065,nvf2456,nvf3060,nvf3064;ld.shared.v4.f32{nvf3066,nvf3067,nvf3068,nvf3069},[global_smem+5344];fma.rn.ftz.f32 nvf3070,nvf2463,nvf3067,nvf3065;fma.rn.ftz.f32 nvf3071,nvf2466,nvf3066,nvf3070;fma.rn.ftz.f32 nvf3072,nvf2469,nvf3069,nvf3071;fma.rn.ftz.f32 nvf3073,nvf2472,nvf3068,nvf3072;ld.shared.v4.f32{nvf3074,nvf3075,nvf3076,nvf3077},[global_smem+5360];fma.rn.ftz.f32 nvf3078,nvf2479,nvf3075,nvf3073;fma.rn.ftz.f32 nvf3079,nvf2482,nvf3074,nvf3078;fma.rn.ftz.f32 nvf3080,nvf2485,nvf3077,nvf3079;fma.rn.ftz.f32 nvf3081,nvf2488,nvf3076,nvf3080;ld.shared.v4.f32{nvf3082,nvf3083,nvf3084,nvf3085},[global_smem+5312];fma.rn.ftz.f32 nvf3086,nvf2495,nvf3083,nvf3081;fma.rn.ftz.f32 nvf3087,nvf2498,nvf3082,nvf3086;fma.rn.ftz.f32 nvf3088,nvf2501,nvf3085,nvf3087;fma.rn.ftz.f32 nvf3089,nvf2504,nvf3084,nvf3088;ld.shared.v4.f32{nvf3090,nvf3091,nvf3092,nvf3093},[global_smem+5328];fma.rn.ftz.f32 nvf3094,nvf2511,nvf3091,nvf3089;fma.rn.ftz.f32 nvf3095,nvf2514,nvf3090,nvf3094;fma.rn.ftz.f32 nvf3096,nvf2517,nvf3093,nvf3095;fma.rn.ftz.f32 nvf3097,nvf2520,nvf3092,nvf3096;ld.shared.v4.f32{nvf3098,nvf3099,nvf3100,nvf3101},[global_smem+5408];fma.rn.ftz.f32 nvf3102,nvf2399,nvf3100,nvf2372;fma.rn.ftz.f32 nvf3103,nvf2402,nvf3101,nvf3102;fma.rn.ftz.f32 nvf3104,nvf2405,nvf3098,nvf3103;fma.rn.ftz.f32 nvf3105,nvf2408,nvf3099,nvf3104;ld.shared.v4.f32{nvf3106,nvf3107,nvf3108,nvf3109},[global_smem+5424];fma.rn.ftz.f32 nvf3110,nvf2415,nvf3108,nvf3105;fma.rn.ftz.f32 nvf3111,nvf2418,nvf3109,nvf3110;fma.rn.ftz.f32 nvf3112,nvf2421,nvf3106,nvf3111;fma.rn.ftz.f32 nvf3113,nvf2424,nvf3107,nvf3112;ld.shared.v4.f32{nvf3114,nvf3115,nvf3116,nvf3117},[global_smem+5376];fma.rn.ftz.f32 nvf3118,nvf2431,nvf3116,nvf3113;fma.rn.ftz.f32 nvf3119,nvf2434,nvf3117,nvf3118;fma.rn.ftz.f32 nvf3120,nvf2437,nvf3114,nvf3119;fma.rn.ftz.f32 nvf3121,nvf2440,nvf3115,nvf3120;ld.shared.v4.f32{nvf3122,nvf3123,nvf3124,nvf3125},[global_smem+5392];fma.rn.ftz.f32 nvf3126,nvf2447,nvf3124,nvf3121;fma.rn.ftz.f32 nvf3127,nvf2450,nvf3125,nvf3126;fma.rn.ftz.f32 nvf3128,nvf2453,nvf3122,nvf3127;fma.rn.ftz.f32 nvf3129,nvf2456,nvf3123,nvf3128;ld.shared.v4.f32{nvf3130,nvf3131,nvf3132,nvf3133},[global_smem+5472];fma.rn.ftz.f32 nvf3134,nvf2463,nvf3132,nvf3129;fma.rn.ftz.f32 nvf3135,nvf2466,nvf3133,nvf3134;fma.rn.ftz.f32 nvf3136,nvf2469,nvf3130,nvf3135;fma.rn.ftz.f32 nvf3137,nvf2472,nvf3131,nvf3136;ld.shared.v4.f32{nvf3138,nvf3139,nvf3140,nvf3141},[global_smem+5488];fma.rn.ftz.f32 nvf3142,nvf2479,nvf3140,nvf3137;fma.rn.ftz.f32 nvf3143,nvf2482,nvf3141,nvf3142;fma.rn.ftz.f32 nvf3144,nvf2485,nvf3138,nvf3143;fma.rn.ftz.f32 nvf3145,nvf2488,nvf3139,nvf3144;ld.shared.v4.f32{nvf3146,nvf3147,nvf3148,nvf3149},[global_smem+5440];fma.rn.ftz.f32 nvf3150,nvf2495,nvf3148,nvf3145;fma.rn.ftz.f32 nvf3151,nvf2498,nvf3149,nvf3150;fma.rn.ftz.f32 nvf3152,nvf2501,nvf3146,nvf3151;fma.rn.ftz.f32 nvf3153,nvf2504,nvf3147,nvf3152;ld.shared.v4.f32{nvf3154,nvf3155,nvf3156,nvf3157},[global_smem+5456];fma.rn.ftz.f32 nvf3158,nvf2511,nvf3156,nvf3153;fma.rn.ftz.f32 nvf3159,nvf2514,nvf3157,nvf3158;fma.rn.ftz.f32 nvf3160,nvf2517,nvf3154,nvf3159;fma.rn.ftz.f32 nvf3161,nvf2520,nvf3155,nvf3160;ld.shared.v4.f32{nvf3162,nvf3163,nvf3164,nvf3165},[global_smem+5536];fma.rn.ftz.f32 nvf3166,nvf2399,nvf3165,nvf2373;fma.rn.ftz.f32 nvf3167,nvf2402,nvf3164,nvf3166;fma.rn.ftz.f32 nvf3168,nvf2405,nvf3163,nvf3167;fma.rn.ftz.f32 nvf3169,nvf2408,nvf3162,nvf3168;ld.shared.v4.f32{nvf3170,nvf3171,nvf3172,nvf3173},[global_smem+5552];fma.rn.ftz.f32 nvf3174,nvf2415,nvf3173,nvf3169;fma.rn.ftz.f32 nvf3175,nvf2418,nvf3172,nvf3174;fma.rn.ftz.f32 nvf3176,nvf2421,nvf3171,nvf3175;fma.rn.ftz.f32 nvf3177,nvf2424,nvf3170,nvf3176;ld.shared.v4.f32{nvf3178,nvf3179,nvf3180,nvf3181},[global_smem+5504];fma.rn.ftz.f32 nvf3182,nvf2431,nvf3181,nvf3177;fma.rn.ftz.f32 nvf3183,nvf2434,nvf3180,nvf3182;fma.rn.ftz.f32 nvf3184,nvf2437,nvf3179,nvf3183;fma.rn.ftz.f32 nvf3185,nvf2440,nvf3178,nvf3184;ld.shared.v4.f32{nvf3186,nvf3187,nvf3188,nvf3189},[global_smem+5520];fma.rn.ftz.f32 nvf3190,nvf2447,nvf3189,nvf3185;fma.rn.ftz.f32 nvf3191,nvf2450,nvf3188,nvf3190;fma.rn.ftz.f32 nvf3192,nvf2453,nvf3187,nvf3191;fma.rn.ftz.f32 nvf3193,nvf2456,nvf3186,nvf3192;ld.shared.v4.f32{nvf3194,nvf3195,nvf3196,nvf3197},[global_smem+5600];fma.rn.ftz.f32 nvf3198,nvf2463,nvf3197,nvf3193;fma.rn.ftz.f32 nvf3199,nvf2466,nvf3196,nvf3198;fma.rn.ftz.f32 nvf3200,nvf2469,nvf3195,nvf3199;fma.rn.ftz.f32 nvf3201,nvf2472,nvf3194,nvf3200;ld.shared.v4.f32{nvf3202,nvf3203,nvf3204,nvf3205},[global_smem+5616];fma.rn.ftz.f32 nvf3206,nvf2479,nvf3205,nvf3201;fma.rn.ftz.f32 nvf3207,nvf2482,nvf3204,nvf3206;fma.rn.ftz.f32 nvf3208,nvf2485,nvf3203,nvf3207;fma.rn.ftz.f32 nvf3209,nvf2488,nvf3202,nvf3208;ld.shared.v4.f32{nvf3210,nvf3211,nvf3212,nvf3213},[global_smem+5568];fma.rn.ftz.f32 nvf3214,nvf2495,nvf3213,nvf3209;fma.rn.ftz.f32 nvf3215,nvf2498,nvf3212,nvf3214;fma.rn.ftz.f32 nvf3216,nvf2501,nvf3211,nvf3215;fma.rn.ftz.f32 nvf3217,nvf2504,nvf3210,nvf3216;ld.shared.v4.f32{nvf3218,nvf3219,nvf3220,nvf3221},[global_smem+5584];fma.rn.ftz.f32 nvf3222,nvf2511,nvf3221,nvf3217;fma.rn.ftz.f32 nvf3223,nvf2514,nvf3220,nvf3222;fma.rn.ftz.f32 nvf3224,nvf2517,nvf3219,nvf3223;fma.rn.ftz.f32 nvf3225,nvf2520,nvf3218,nvf3224;ld.shared.v4.f32{nvf3226,nvf3227,nvf3228,nvf3229},[global_smem+5680];fma.rn.ftz.f32 nvf3230,nvf2399,nvf3226,nvf2374;fma.rn.ftz.f32 nvf3231,nvf2402,nvf3227,nvf3230;fma.rn.ftz.f32 nvf3232,nvf2405,nvf3228,nvf3231;fma.rn.ftz.f32 nvf3233,nvf2408,nvf3229,nvf3232;ld.shared.v4.f32{nvf3234,nvf3235,nvf3236,nvf3237},[global_smem+5664];fma.rn.ftz.f32 nvf3238,nvf2415,nvf3234,nvf3233;fma.rn.ftz.f32 nvf3239,nvf2418,nvf3235,nvf3238;fma.rn.ftz.f32 nvf3240,nvf2421,nvf3236,nvf3239;fma.rn.ftz.f32 nvf3241,nvf2424,nvf3237,nvf3240;ld.shared.v4.f32{nvf3242,nvf3243,nvf3244,nvf3245},[global_smem+5648];fma.rn.ftz.f32 nvf3246,nvf2431,nvf3242,nvf3241;fma.rn.ftz.f32 nvf3247,nvf2434,nvf3243,nvf3246;fma.rn.ftz.f32 nvf3248,nvf2437,nvf3244,nvf3247;fma.rn.ftz.f32 nvf3249,nvf2440,nvf3245,nvf3248;ld.shared.v4.f32{nvf3250,nvf3251,nvf3252,nvf3253},[global_smem+5632];fma.rn.ftz.f32 nvf3254,nvf2447,nvf3250,nvf3249;fma.rn.ftz.f32 nvf3255,nvf2450,nvf3251,nvf3254;fma.rn.ftz.f32 nvf3256,nvf2453,nvf3252,nvf3255;fma.rn.ftz.f32 nvf3257,nvf2456,nvf3253,nvf3256;ld.shared.v4.f32{nvf3258,nvf3259,nvf3260,nvf3261},[global_smem+5744];fma.rn.ftz.f32 nvf3262,nvf2463,nvf3258,nvf3257;fma.rn.ftz.f32 nvf3263,nvf2466,nvf3259,nvf3262;fma.rn.ftz.f32 nvf3264,nvf2469,nvf3260,nvf3263;fma.rn.ftz.f32 nvf3265,nvf2472,nvf3261,nvf3264;ld.shared.v4.f32{nvf3266,nvf3267,nvf3268,nvf3269},[global_smem+5728];fma.rn.ftz.f32 nvf3270,nvf2479,nvf3266,nvf3265;fma.rn.ftz.f32 nvf3271,nvf2482,nvf3267,nvf3270;fma.rn.ftz.f32 nvf3272,nvf2485,nvf3268,nvf3271;fma.rn.ftz.f32 nvf3273,nvf2488,nvf3269,nvf3272;ld.shared.v4.f32{nvf3274,nvf3275,nvf3276,nvf3277},[global_smem+5712];fma.rn.ftz.f32 nvf3278,nvf2495,nvf3274,nvf3273;fma.rn.ftz.f32 nvf3279,nvf2498,nvf3275,nvf3278;fma.rn.ftz.f32 nvf3280,nvf2501,nvf3276,nvf3279;fma.rn.ftz.f32 nvf3281,nvf2504,nvf3277,nvf3280;ld.shared.v4.f32{nvf3282,nvf3283,nvf3284,nvf3285},[global_smem+5696];fma.rn.ftz.f32 nvf3286,nvf2511,nvf3282,nvf3281;fma.rn.ftz.f32 nvf3287,nvf2514,nvf3283,nvf3286;fma.rn.ftz.f32 nvf3288,nvf2517,nvf3284,nvf3287;fma.rn.ftz.f32 nvf3289,nvf2520,nvf3285,nvf3288;ld.shared.v4.f32{nvf3290,nvf3291,nvf3292,nvf3293},[global_smem+5808];fma.rn.ftz.f32 nvf3294,nvf2399,nvf3291,nvf2375;fma.rn.ftz.f32 nvf3295,nvf2402,nvf3290,nvf3294;fma.rn.ftz.f32 nvf3296,nvf2405,nvf3293,nvf3295;fma.rn.ftz.f32 nvf3297,nvf2408,nvf3292,nvf3296;ld.shared.v4.f32{nvf3298,nvf3299,nvf3300,nvf3301},[global_smem+5792];fma.rn.ftz.f32 nvf3302,nvf2415,nvf3299,nvf3297;fma.rn.ftz.f32 nvf3303,nvf2418,nvf3298,nvf3302;fma.rn.ftz.f32 nvf3304,nvf2421,nvf3301,nvf3303;fma.rn.ftz.f32 nvf3305,nvf2424,nvf3300,nvf3304;ld.shared.v4.f32{nvf3306,nvf3307,nvf3308,nvf3309},[global_smem+5776];fma.rn.ftz.f32 nvf3310,nvf2431,nvf3307,nvf3305;fma.rn.ftz.f32 nvf3311,nvf2434,nvf3306,nvf3310;fma.rn.ftz.f32 nvf3312,nvf2437,nvf3309,nvf3311;fma.rn.ftz.f32 nvf3313,nvf2440,nvf3308,nvf3312;ld.shared.v4.f32{nvf3314,nvf3315,nvf3316,nvf3317},[global_smem+5760];fma.rn.ftz.f32 nvf3318,nvf2447,nvf3315,nvf3313;fma.rn.ftz.f32 nvf3319,nvf2450,nvf3314,nvf3318;fma.rn.ftz.f32 nvf3320,nvf2453,nvf3317,nvf3319;fma.rn.ftz.f32 nvf3321,nvf2456,nvf3316,nvf3320;ld.shared.v4.f32{nvf3322,nvf3323,nvf3324,nvf3325},[global_smem+5872];fma.rn.ftz.f32 nvf3326,nvf2463,nvf3323,nvf3321;fma.rn.ftz.f32 nvf3327,nvf2466,nvf3322,nvf3326;fma.rn.ftz.f32 nvf3328,nvf2469,nvf3325,nvf3327;fma.rn.ftz.f32 nvf3329,nvf2472,nvf3324,nvf3328;ld.shared.v4.f32{nvf3330,nvf3331,nvf3332,nvf3333},[global_smem+5856];fma.rn.ftz.f32 nvf3334,nvf2479,nvf3331,nvf3329;fma.rn.ftz.f32 nvf3335,nvf2482,nvf3330,nvf3334;fma.rn.ftz.f32 nvf3336,nvf2485,nvf3333,nvf3335;fma.rn.ftz.f32 nvf3337,nvf2488,nvf3332,nvf3336;ld.shared.v4.f32{nvf3338,nvf3339,nvf3340,nvf3341},[global_smem+5840];fma.rn.ftz.f32 nvf3342,nvf2495,nvf3339,nvf3337;fma.rn.ftz.f32 nvf3343,nvf2498,nvf3338,nvf3342;fma.rn.ftz.f32 nvf3344,nvf2501,nvf3341,nvf3343;fma.rn.ftz.f32 nvf3345,nvf2504,nvf3340,nvf3344;ld.shared.v4.f32{nvf3346,nvf3347,nvf3348,nvf3349},[global_smem+5824];fma.rn.ftz.f32 nvf3350,nvf2511,nvf3347,nvf3345;fma.rn.ftz.f32 nvf3351,nvf2514,nvf3346,nvf3350;fma.rn.ftz.f32 nvf3352,nvf2517,nvf3349,nvf3351;fma.rn.ftz.f32 nvf3353,nvf2520,nvf3348,nvf3352;ld.shared.v4.f32{nvf3354,nvf3355,nvf3356,nvf3357},[global_smem+5936];fma.rn.ftz.f32 nvf3358,nvf2399,nvf3356,nvf2376;fma.rn.ftz.f32 nvf3359,nvf2402,nvf3357,nvf3358;fma.rn.ftz.f32 nvf3360,nvf2405,nvf3354,nvf3359;fma.rn.ftz.f32 nvf3361,nvf2408,nvf3355,nvf3360;ld.shared.v4.f32{nvf3362,nvf3363,nvf3364,nvf3365},[global_smem+5920];fma.rn.ftz.f32 nvf3366,nvf2415,nvf3364,nvf3361;fma.rn.ftz.f32 nvf3367,nvf2418,nvf3365,nvf3366;fma.rn.ftz.f32 nvf3368,nvf2421,nvf3362,nvf3367;fma.rn.ftz.f32 nvf3369,nvf2424,nvf3363,nvf3368;ld.shared.v4.f32{nvf3370,nvf3371,nvf3372,nvf3373},[global_smem+5904];fma.rn.ftz.f32 nvf3374,nvf2431,nvf3372,nvf3369;fma.rn.ftz.f32 nvf3375,nvf2434,nvf3373,nvf3374;fma.rn.ftz.f32 nvf3376,nvf2437,nvf3370,nvf3375;fma.rn.ftz.f32 nvf3377,nvf2440,nvf3371,nvf3376;ld.shared.v4.f32{nvf3378,nvf3379,nvf3380,nvf3381},[global_smem+5888];fma.rn.ftz.f32 nvf3382,nvf2447,nvf3380,nvf3377;fma.rn.ftz.f32 nvf3383,nvf2450,nvf3381,nvf3382;fma.rn.ftz.f32 nvf3384,nvf2453,nvf3378,nvf3383;fma.rn.ftz.f32 nvf3385,nvf2456,nvf3379,nvf3384;ld.shared.v4.f32{nvf3386,nvf3387,nvf3388,nvf3389},[global_smem+6000];fma.rn.ftz.f32 nvf3390,nvf2463,nvf3388,nvf3385;fma.rn.ftz.f32 nvf3391,nvf2466,nvf3389,nvf3390;fma.rn.ftz.f32 nvf3392,nvf2469,nvf3386,nvf3391;fma.rn.ftz.f32 nvf3393,nvf2472,nvf3387,nvf3392;ld.shared.v4.f32{nvf3394,nvf3395,nvf3396,nvf3397},[global_smem+5984];fma.rn.ftz.f32 nvf3398,nvf2479,nvf3396,nvf3393;fma.rn.ftz.f32 nvf3399,nvf2482,nvf3397,nvf3398;fma.rn.ftz.f32 nvf3400,nvf2485,nvf3394,nvf3399;fma.rn.ftz.f32 nvf3401,nvf2488,nvf3395,nvf3400;ld.shared.v4.f32{nvf3402,nvf3403,nvf3404,nvf3405},[global_smem+5968];fma.rn.ftz.f32 nvf3406,nvf2495,nvf3404,nvf3401;fma.rn.ftz.f32 nvf3407,nvf2498,nvf3405,nvf3406;fma.rn.ftz.f32 nvf3408,nvf2501,nvf3402,nvf3407;fma.rn.ftz.f32 nvf3409,nvf2504,nvf3403,nvf3408;ld.shared.v4.f32{nvf3410,nvf3411,nvf3412,nvf3413},[global_smem+5952];fma.rn.ftz.f32 nvf3414,nvf2511,nvf3412,nvf3409;fma.rn.ftz.f32 nvf3415,nvf2514,nvf3413,nvf3414;fma.rn.ftz.f32 nvf3416,nvf2517,nvf3410,nvf3415;fma.rn.ftz.f32 nvf3417,nvf2520,nvf3411,nvf3416;ld.shared.v4.f32{nvf3418,nvf3419,nvf3420,nvf3421},[global_smem+6064];fma.rn.ftz.f32 nvf3422,nvf2399,nvf3421,nvf2377;fma.rn.ftz.f32 nvf3423,nvf2402,nvf3420,nvf3422;fma.rn.ftz.f32 nvf3424,nvf2405,nvf3419,nvf3423;fma.rn.ftz.f32 nvf3425,nvf2408,nvf3418,nvf3424;ld.shared.v4.f32{nvf3426,nvf3427,nvf3428,nvf3429},[global_smem+6048];fma.rn.ftz.f32 nvf3430,nvf2415,nvf3429,nvf3425;fma.rn.ftz.f32 nvf3431,nvf2418,nvf3428,nvf3430;fma.rn.ftz.f32 nvf3432,nvf2421,nvf3427,nvf3431;fma.rn.ftz.f32 nvf3433,nvf2424,nvf3426,nvf3432;ld.shared.v4.f32{nvf3434,nvf3435,nvf3436,nvf3437},[global_smem+6032];fma.rn.ftz.f32 nvf3438,nvf2431,nvf3437,nvf3433;fma.rn.ftz.f32 nvf3439,nvf2434,nvf3436,nvf3438;fma.rn.ftz.f32 nvf3440,nvf2437,nvf3435,nvf3439;fma.rn.ftz.f32 nvf3441,nvf2440,nvf3434,nvf3440;ld.shared.v4.f32{nvf3442,nvf3443,nvf3444,nvf3445},[global_smem+6016];fma.rn.ftz.f32 nvf3446,nvf2447,nvf3445,nvf3441;fma.rn.ftz.f32 nvf3447,nvf2450,nvf3444,nvf3446;fma.rn.ftz.f32 nvf3448,nvf2453,nvf3443,nvf3447;fma.rn.ftz.f32 nvf3449,nvf2456,nvf3442,nvf3448;ld.shared.v4.f32{nvf3450,nvf3451,nvf3452,nvf3453},[global_smem+6128];fma.rn.ftz.f32 nvf3454,nvf2463,nvf3453,nvf3449;fma.rn.ftz.f32 nvf3455,nvf2466,nvf3452,nvf3454;fma.rn.ftz.f32 nvf3456,nvf2469,nvf3451,nvf3455;fma.rn.ftz.f32 nvf3457,nvf2472,nvf3450,nvf3456;ld.shared.v4.f32{nvf3458,nvf3459,nvf3460,nvf3461},[global_smem+6112];fma.rn.ftz.f32 nvf3462,nvf2479,nvf3461,nvf3457;fma.rn.ftz.f32 nvf3463,nvf2482,nvf3460,nvf3462;fma.rn.ftz.f32 nvf3464,nvf2485,nvf3459,nvf3463;fma.rn.ftz.f32 nvf3465,nvf2488,nvf3458,nvf3464;ld.shared.v4.f32{nvf3466,nvf3467,nvf3468,nvf3469},[global_smem+6096];fma.rn.ftz.f32 nvf3470,nvf2495,nvf3469,nvf3465;fma.rn.ftz.f32 nvf3471,nvf2498,nvf3468,nvf3470;fma.rn.ftz.f32 nvf3472,nvf2501,nvf3467,nvf3471;fma.rn.ftz.f32 nvf3473,nvf2504,nvf3466,nvf3472;ld.shared.v4.f32{nvf3474,nvf3475,nvf3476,nvf3477},[global_smem+6080];fma.rn.ftz.f32 nvf3478,nvf2511,nvf3477,nvf3473;fma.rn.ftz.f32 nvf3479,nvf2514,nvf3476,nvf3478;fma.rn.ftz.f32 nvf3480,nvf2517,nvf3475,nvf3479;fma.rn.ftz.f32 nvf3481,nvf2520,nvf3474,nvf3480;ld.shared.v4.f32{nvf3482,nvf3483,nvf3484,nvf3485},[global_smem+6208];fma.rn.ftz.f32 nvf3486,nvf2399,nvf3482,nvf2378;fma.rn.ftz.f32 nvf3487,nvf2402,nvf3483,nvf3486;fma.rn.ftz.f32 nvf3488,nvf2405,nvf3484,nvf3487;fma.rn.ftz.f32 nvf3489,nvf2408,nvf3485,nvf3488;ld.shared.v4.f32{nvf3490,nvf3491,nvf3492,nvf3493},[global_smem+6224];fma.rn.ftz.f32 nvf3494,nvf2415,nvf3490,nvf3489;fma.rn.ftz.f32 nvf3495,nvf2418,nvf3491,nvf3494;fma.rn.ftz.f32 nvf3496,nvf2421,nvf3492,nvf3495;fma.rn.ftz.f32 nvf3497,nvf2424,nvf3493,nvf3496;ld.shared.v4.f32{nvf3498,nvf3499,nvf3500,nvf3501},[global_smem+6240];fma.rn.ftz.f32 nvf3502,nvf2431,nvf3498,nvf3497;fma.rn.ftz.f32 nvf3503,nvf2434,nvf3499,nvf3502;fma.rn.ftz.f32 nvf3504,nvf2437,nvf3500,nvf3503;fma.rn.ftz.f32 nvf3505,nvf2440,nvf3501,nvf3504;ld.shared.v4.f32{nvf3506,nvf3507,nvf3508,nvf3509},[global_smem+6256];fma.rn.ftz.f32 nvf3510,nvf2447,nvf3506,nvf3505;fma.rn.ftz.f32 nvf3511,nvf2450,nvf3507,nvf3510;fma.rn.ftz.f32 nvf3512,nvf2453,nvf3508,nvf3511;fma.rn.ftz.f32 nvf3513,nvf2456,nvf3509,nvf3512;ld.shared.v4.f32{nvf3514,nvf3515,nvf3516,nvf3517},[global_smem+6144];fma.rn.ftz.f32 nvf3518,nvf2463,nvf3514,nvf3513;fma.rn.ftz.f32 nvf3519,nvf2466,nvf3515,nvf3518;fma.rn.ftz.f32 nvf3520,nvf2469,nvf3516,nvf3519;fma.rn.ftz.f32 nvf3521,nvf2472,nvf3517,nvf3520;ld.shared.v4.f32{nvf3522,nvf3523,nvf3524,nvf3525},[global_smem+6160];fma.rn.ftz.f32 nvf3526,nvf2479,nvf3522,nvf3521;fma.rn.ftz.f32 nvf3527,nvf2482,nvf3523,nvf3526;fma.rn.ftz.f32 nvf3528,nvf2485,nvf3524,nvf3527;fma.rn.ftz.f32 nvf3529,nvf2488,nvf3525,nvf3528;ld.shared.v4.f32{nvf3530,nvf3531,nvf3532,nvf3533},[global_smem+6176];fma.rn.ftz.f32 nvf3534,nvf2495,nvf3530,nvf3529;fma.rn.ftz.f32 nvf3535,nvf2498,nvf3531,nvf3534;fma.rn.ftz.f32 nvf3536,nvf2501,nvf3532,nvf3535;fma.rn.ftz.f32 nvf3537,nvf2504,nvf3533,nvf3536;ld.shared.v4.f32{nvf3538,nvf3539,nvf3540,nvf3541},[global_smem+6192];fma.rn.ftz.f32 nvf3542,nvf2511,nvf3538,nvf3537;fma.rn.ftz.f32 nvf3543,nvf2514,nvf3539,nvf3542;fma.rn.ftz.f32 nvf3544,nvf2517,nvf3540,nvf3543;fma.rn.ftz.f32 nvf3545,nvf2520,nvf3541,nvf3544;ld.shared.v4.f32{nvf3546,nvf3547,nvf3548,nvf3549},[global_smem+6336];fma.rn.ftz.f32 nvf3550,nvf2399,nvf3547,nvf2379;fma.rn.ftz.f32 nvf3551,nvf2402,nvf3546,nvf3550;fma.rn.ftz.f32 nvf3552,nvf2405,nvf3549,nvf3551;fma.rn.ftz.f32 nvf3553,nvf2408,nvf3548,nvf3552;ld.shared.v4.f32{nvf3554,nvf3555,nvf3556,nvf3557},[global_smem+6352];fma.rn.ftz.f32 nvf3558,nvf2415,nvf3555,nvf3553;fma.rn.ftz.f32 nvf3559,nvf2418,nvf3554,nvf3558;fma.rn.ftz.f32 nvf3560,nvf2421,nvf3557,nvf3559;fma.rn.ftz.f32 nvf3561,nvf2424,nvf3556,nvf3560;ld.shared.v4.f32{nvf3562,nvf3563,nvf3564,nvf3565},[global_smem+6368];fma.rn.ftz.f32 nvf3566,nvf2431,nvf3563,nvf3561;fma.rn.ftz.f32 nvf3567,nvf2434,nvf3562,nvf3566;fma.rn.ftz.f32 nvf3568,nvf2437,nvf3565,nvf3567;fma.rn.ftz.f32 nvf3569,nvf2440,nvf3564,nvf3568;ld.shared.v4.f32{nvf3570,nvf3571,nvf3572,nvf3573},[global_smem+6384];fma.rn.ftz.f32 nvf3574,nvf2447,nvf3571,nvf3569;fma.rn.ftz.f32 nvf3575,nvf2450,nvf3570,nvf3574;fma.rn.ftz.f32 nvf3576,nvf2453,nvf3573,nvf3575;fma.rn.ftz.f32 nvf3577,nvf2456,nvf3572,nvf3576;ld.shared.v4.f32{nvf3578,nvf3579,nvf3580,nvf3581},[global_smem+6272];fma.rn.ftz.f32 nvf3582,nvf2463,nvf3579,nvf3577;fma.rn.ftz.f32 nvf3583,nvf2466,nvf3578,nvf3582;fma.rn.ftz.f32 nvf3584,nvf2469,nvf3581,nvf3583;fma.rn.ftz.f32 nvf3585,nvf2472,nvf3580,nvf3584;ld.shared.v4.f32{nvf3586,nvf3587,nvf3588,nvf3589},[global_smem+6288];fma.rn.ftz.f32 nvf3590,nvf2479,nvf3587,nvf3585;fma.rn.ftz.f32 nvf3591,nvf2482,nvf3586,nvf3590;fma.rn.ftz.f32 nvf3592,nvf2485,nvf3589,nvf3591;fma.rn.ftz.f32 nvf3593,nvf2488,nvf3588,nvf3592;ld.shared.v4.f32{nvf3594,nvf3595,nvf3596,nvf3597},[global_smem+6304];fma.rn.ftz.f32 nvf3598,nvf2495,nvf3595,nvf3593;fma.rn.ftz.f32 nvf3599,nvf2498,nvf3594,nvf3598;fma.rn.ftz.f32 nvf3600,nvf2501,nvf3597,nvf3599;fma.rn.ftz.f32 nvf3601,nvf2504,nvf3596,nvf3600;ld.shared.v4.f32{nvf3602,nvf3603,nvf3604,nvf3605},[global_smem+6320];fma.rn.ftz.f32 nvf3606,nvf2511,nvf3603,nvf3601;fma.rn.ftz.f32 nvf3607,nvf2514,nvf3602,nvf3606;fma.rn.ftz.f32 nvf3608,nvf2517,nvf3605,nvf3607;fma.rn.ftz.f32 nvf3609,nvf2520,nvf3604,nvf3608;ld.shared.v4.f32{nvf3610,nvf3611,nvf3612,nvf3613},[global_smem+6464];fma.rn.ftz.f32 nvf3614,nvf2399,nvf3612,nvf2380;fma.rn.ftz.f32 nvf3615,nvf2402,nvf3613,nvf3614;fma.rn.ftz.f32 nvf3616,nvf2405,nvf3610,nvf3615;fma.rn.ftz.f32 nvf3617,nvf2408,nvf3611,nvf3616;ld.shared.v4.f32{nvf3618,nvf3619,nvf3620,nvf3621},[global_smem+6480];fma.rn.ftz.f32 nvf3622,nvf2415,nvf3620,nvf3617;fma.rn.ftz.f32 nvf3623,nvf2418,nvf3621,nvf3622;fma.rn.ftz.f32 nvf3624,nvf2421,nvf3618,nvf3623;fma.rn.ftz.f32 nvf3625,nvf2424,nvf3619,nvf3624;ld.shared.v4.f32{nvf3626,nvf3627,nvf3628,nvf3629},[global_smem+6496];fma.rn.ftz.f32 nvf3630,nvf2431,nvf3628,nvf3625;fma.rn.ftz.f32 nvf3631,nvf2434,nvf3629,nvf3630;fma.rn.ftz.f32 nvf3632,nvf2437,nvf3626,nvf3631;fma.rn.ftz.f32 nvf3633,nvf2440,nvf3627,nvf3632;ld.shared.v4.f32{nvf3634,nvf3635,nvf3636,nvf3637},[global_smem+6512];fma.rn.ftz.f32 nvf3638,nvf2447,nvf3636,nvf3633;fma.rn.ftz.f32 nvf3639,nvf2450,nvf3637,nvf3638;fma.rn.ftz.f32 nvf3640,nvf2453,nvf3634,nvf3639;fma.rn.ftz.f32 nvf3641,nvf2456,nvf3635,nvf3640;ld.shared.v4.f32{nvf3642,nvf3643,nvf3644,nvf3645},[global_smem+6400];fma.rn.ftz.f32 nvf3646,nvf2463,nvf3644,nvf3641;fma.rn.ftz.f32 nvf3647,nvf2466,nvf3645,nvf3646;fma.rn.ftz.f32 nvf3648,nvf2469,nvf3642,nvf3647;fma.rn.ftz.f32 nvf3649,nvf2472,nvf3643,nvf3648;ld.shared.v4.f32{nvf3650,nvf3651,nvf3652,nvf3653},[global_smem+6416];fma.rn.ftz.f32 nvf3654,nvf2479,nvf3652,nvf3649;fma.rn.ftz.f32 nvf3655,nvf2482,nvf3653,nvf3654;fma.rn.ftz.f32 nvf3656,nvf2485,nvf3650,nvf3655;fma.rn.ftz.f32 nvf3657,nvf2488,nvf3651,nvf3656;ld.shared.v4.f32{nvf3658,nvf3659,nvf3660,nvf3661},[global_smem+6432];fma.rn.ftz.f32 nvf3662,nvf2495,nvf3660,nvf3657;fma.rn.ftz.f32 nvf3663,nvf2498,nvf3661,nvf3662;fma.rn.ftz.f32 nvf3664,nvf2501,nvf3658,nvf3663;fma.rn.ftz.f32 nvf3665,nvf2504,nvf3659,nvf3664;ld.shared.v4.f32{nvf3666,nvf3667,nvf3668,nvf3669},[global_smem+6448];fma.rn.ftz.f32 nvf3670,nvf2511,nvf3668,nvf3665;fma.rn.ftz.f32 nvf3671,nvf2514,nvf3669,nvf3670;fma.rn.ftz.f32 nvf3672,nvf2517,nvf3666,nvf3671;fma.rn.ftz.f32 nvf3673,nvf2520,nvf3667,nvf3672;ld.shared.v4.f32{nvf3674,nvf3675,nvf3676,nvf3677},[global_smem+6592];fma.rn.ftz.f32 nvf3678,nvf2399,nvf3677,nvf2381;fma.rn.ftz.f32 nvf3679,nvf2402,nvf3676,nvf3678;fma.rn.ftz.f32 nvf3680,nvf2405,nvf3675,nvf3679;fma.rn.ftz.f32 nvf3681,nvf2408,nvf3674,nvf3680;ld.shared.v4.f32{nvf3682,nvf3683,nvf3684,nvf3685},[global_smem+6608];fma.rn.ftz.f32 nvf3686,nvf2415,nvf3685,nvf3681;fma.rn.ftz.f32 nvf3687,nvf2418,nvf3684,nvf3686;fma.rn.ftz.f32 nvf3688,nvf2421,nvf3683,nvf3687;fma.rn.ftz.f32 nvf3689,nvf2424,nvf3682,nvf3688;ld.shared.v4.f32{nvf3690,nvf3691,nvf3692,nvf3693},[global_smem+6624];fma.rn.ftz.f32 nvf3694,nvf2431,nvf3693,nvf3689;fma.rn.ftz.f32 nvf3695,nvf2434,nvf3692,nvf3694;fma.rn.ftz.f32 nvf3696,nvf2437,nvf3691,nvf3695;fma.rn.ftz.f32 nvf3697,nvf2440,nvf3690,nvf3696;ld.shared.v4.f32{nvf3698,nvf3699,nvf3700,nvf3701},[global_smem+6640];fma.rn.ftz.f32 nvf3702,nvf2447,nvf3701,nvf3697;fma.rn.ftz.f32 nvf3703,nvf2450,nvf3700,nvf3702;fma.rn.ftz.f32 nvf3704,nvf2453,nvf3699,nvf3703;fma.rn.ftz.f32 nvf3705,nvf2456,nvf3698,nvf3704;ld.shared.v4.f32{nvf3706,nvf3707,nvf3708,nvf3709},[global_smem+6528];fma.rn.ftz.f32 nvf3710,nvf2463,nvf3709,nvf3705;fma.rn.ftz.f32 nvf3711,nvf2466,nvf3708,nvf3710;fma.rn.ftz.f32 nvf3712,nvf2469,nvf3707,nvf3711;fma.rn.ftz.f32 nvf3713,nvf2472,nvf3706,nvf3712;ld.shared.v4.f32{nvf3714,nvf3715,nvf3716,nvf3717},[global_smem+6544];fma.rn.ftz.f32 nvf3718,nvf2479,nvf3717,nvf3713;fma.rn.ftz.f32 nvf3719,nvf2482,nvf3716,nvf3718;fma.rn.ftz.f32 nvf3720,nvf2485,nvf3715,nvf3719;fma.rn.ftz.f32 nvf3721,nvf2488,nvf3714,nvf3720;ld.shared.v4.f32{nvf3722,nvf3723,nvf3724,nvf3725},[global_smem+6560];fma.rn.ftz.f32 nvf3726,nvf2495,nvf3725,nvf3721;fma.rn.ftz.f32 nvf3727,nvf2498,nvf3724,nvf3726;fma.rn.ftz.f32 nvf3728,nvf2501,nvf3723,nvf3727;fma.rn.ftz.f32 nvf3729,nvf2504,nvf3722,nvf3728;ld.shared.v4.f32{nvf3730,nvf3731,nvf3732,nvf3733},[global_smem+6576];fma.rn.ftz.f32 nvf3734,nvf2511,nvf3733,nvf3729;fma.rn.ftz.f32 nvf3735,nvf2514,nvf3732,nvf3734;fma.rn.ftz.f32 nvf3736,nvf2517,nvf3731,nvf3735;fma.rn.ftz.f32 nvf3737,nvf2520,nvf3730,nvf3736;ld.shared.v4.f32{nvf3738,nvf3739,nvf3740,nvf3741},[global_smem+6736];fma.rn.ftz.f32 nvf3742,nvf2399,nvf3738,nvf2382;fma.rn.ftz.f32 nvf3743,nvf2402,nvf3739,nvf3742;fma.rn.ftz.f32 nvf3744,nvf2405,nvf3740,nvf3743;fma.rn.ftz.f32 nvf3745,nvf2408,nvf3741,nvf3744;ld.shared.v4.f32{nvf3746,nvf3747,nvf3748,nvf3749},[global_smem+6720];fma.rn.ftz.f32 nvf3750,nvf2415,nvf3746,nvf3745;fma.rn.ftz.f32 nvf3751,nvf2418,nvf3747,nvf3750;fma.rn.ftz.f32 nvf3752,nvf2421,nvf3748,nvf3751;fma.rn.ftz.f32 nvf3753,nvf2424,nvf3749,nvf3752;ld.shared.v4.f32{nvf3754,nvf3755,nvf3756,nvf3757},[global_smem+6768];fma.rn.ftz.f32 nvf3758,nvf2431,nvf3754,nvf3753;fma.rn.ftz.f32 nvf3759,nvf2434,nvf3755,nvf3758;fma.rn.ftz.f32 nvf3760,nvf2437,nvf3756,nvf3759;fma.rn.ftz.f32 nvf3761,nvf2440,nvf3757,nvf3760;ld.shared.v4.f32{nvf3762,nvf3763,nvf3764,nvf3765},[global_smem+6752];fma.rn.ftz.f32 nvf3766,nvf2447,nvf3762,nvf3761;fma.rn.ftz.f32 nvf3767,nvf2450,nvf3763,nvf3766;fma.rn.ftz.f32 nvf3768,nvf2453,nvf3764,nvf3767;fma.rn.ftz.f32 nvf3769,nvf2456,nvf3765,nvf3768;ld.shared.v4.f32{nvf3770,nvf3771,nvf3772,nvf3773},[global_smem+6672];fma.rn.ftz.f32 nvf3774,nvf2463,nvf3770,nvf3769;fma.rn.ftz.f32 nvf3775,nvf2466,nvf3771,nvf3774;fma.rn.ftz.f32 nvf3776,nvf2469,nvf3772,nvf3775;fma.rn.ftz.f32 nvf3777,nvf2472,nvf3773,nvf3776;ld.shared.v4.f32{nvf3778,nvf3779,nvf3780,nvf3781},[global_smem+6656];fma.rn.ftz.f32 nvf3782,nvf2479,nvf3778,nvf3777;fma.rn.ftz.f32 nvf3783,nvf2482,nvf3779,nvf3782;fma.rn.ftz.f32 nvf3784,nvf2485,nvf3780,nvf3783;fma.rn.ftz.f32 nvf3785,nvf2488,nvf3781,nvf3784;ld.shared.v4.f32{nvf3786,nvf3787,nvf3788,nvf3789},[global_smem+6704];fma.rn.ftz.f32 nvf3790,nvf2495,nvf3786,nvf3785;fma.rn.ftz.f32 nvf3791,nvf2498,nvf3787,nvf3790;fma.rn.ftz.f32 nvf3792,nvf2501,nvf3788,nvf3791;fma.rn.ftz.f32 nvf3793,nvf2504,nvf3789,nvf3792;ld.shared.v4.f32{nvf3794,nvf3795,nvf3796,nvf3797},[global_smem+6688];fma.rn.ftz.f32 nvf3798,nvf2511,nvf3794,nvf3793;fma.rn.ftz.f32 nvf3799,nvf2514,nvf3795,nvf3798;fma.rn.ftz.f32 nvf3800,nvf2517,nvf3796,nvf3799;fma.rn.ftz.f32 nvf3801,nvf2520,nvf3797,nvf3800;ld.shared.v4.f32{nvf3802,nvf3803,nvf3804,nvf3805},[global_smem+6864];fma.rn.ftz.f32 nvf3806,nvf2399,nvf3803,nvf2383;fma.rn.ftz.f32 nvf3807,nvf2402,nvf3802,nvf3806;fma.rn.ftz.f32 nvf3808,nvf2405,nvf3805,nvf3807;fma.rn.ftz.f32 nvf3809,nvf2408,nvf3804,nvf3808;ld.shared.v4.f32{nvf3810,nvf3811,nvf3812,nvf3813},[global_smem+6848];fma.rn.ftz.f32 nvf3814,nvf2415,nvf3811,nvf3809;fma.rn.ftz.f32 nvf3815,nvf2418,nvf3810,nvf3814;fma.rn.ftz.f32 nvf3816,nvf2421,nvf3813,nvf3815;fma.rn.ftz.f32 nvf3817,nvf2424,nvf3812,nvf3816;ld.shared.v4.f32{nvf3818,nvf3819,nvf3820,nvf3821},[global_smem+6896];fma.rn.ftz.f32 nvf3822,nvf2431,nvf3819,nvf3817;fma.rn.ftz.f32 nvf3823,nvf2434,nvf3818,nvf3822;fma.rn.ftz.f32 nvf3824,nvf2437,nvf3821,nvf3823;fma.rn.ftz.f32 nvf3825,nvf2440,nvf3820,nvf3824;ld.shared.v4.f32{nvf3826,nvf3827,nvf3828,nvf3829},[global_smem+6880];fma.rn.ftz.f32 nvf3830,nvf2447,nvf3827,nvf3825;fma.rn.ftz.f32 nvf3831,nvf2450,nvf3826,nvf3830;fma.rn.ftz.f32 nvf3832,nvf2453,nvf3829,nvf3831;fma.rn.ftz.f32 nvf3833,nvf2456,nvf3828,nvf3832;ld.shared.v4.f32{nvf3834,nvf3835,nvf3836,nvf3837},[global_smem+6800];fma.rn.ftz.f32 nvf3838,nvf2463,nvf3835,nvf3833;fma.rn.ftz.f32 nvf3839,nvf2466,nvf3834,nvf3838;fma.rn.ftz.f32 nvf3840,nvf2469,nvf3837,nvf3839;fma.rn.ftz.f32 nvf3841,nvf2472,nvf3836,nvf3840;ld.shared.v4.f32{nvf3842,nvf3843,nvf3844,nvf3845},[global_smem+6784];fma.rn.ftz.f32 nvf3846,nvf2479,nvf3843,nvf3841;fma.rn.ftz.f32 nvf3847,nvf2482,nvf3842,nvf3846;fma.rn.ftz.f32 nvf3848,nvf2485,nvf3845,nvf3847;fma.rn.ftz.f32 nvf3849,nvf2488,nvf3844,nvf3848;ld.shared.v4.f32{nvf3850,nvf3851,nvf3852,nvf3853},[global_smem+6832];fma.rn.ftz.f32 nvf3854,nvf2495,nvf3851,nvf3849;fma.rn.ftz.f32 nvf3855,nvf2498,nvf3850,nvf3854;fma.rn.ftz.f32 nvf3856,nvf2501,nvf3853,nvf3855;fma.rn.ftz.f32 nvf3857,nvf2504,nvf3852,nvf3856;ld.shared.v4.f32{nvf3858,nvf3859,nvf3860,nvf3861},[global_smem+6816];fma.rn.ftz.f32 nvf3862,nvf2511,nvf3859,nvf3857;fma.rn.ftz.f32 nvf3863,nvf2514,nvf3858,nvf3862;fma.rn.ftz.f32 nvf3864,nvf2517,nvf3861,nvf3863;fma.rn.ftz.f32 nvf3865,nvf2520,nvf3860,nvf3864;ld.shared.v4.f32{nvf3866,nvf3867,nvf3868,nvf3869},[global_smem+6992];fma.rn.ftz.f32 nvf3870,nvf2399,nvf3868,nvf2384;fma.rn.ftz.f32 nvf3871,nvf2402,nvf3869,nvf3870;fma.rn.ftz.f32 nvf3872,nvf2405,nvf3866,nvf3871;fma.rn.ftz.f32 nvf3873,nvf2408,nvf3867,nvf3872;ld.shared.v4.f32{nvf3874,nvf3875,nvf3876,nvf3877},[global_smem+6976];fma.rn.ftz.f32 nvf3878,nvf2415,nvf3876,nvf3873;fma.rn.ftz.f32 nvf3879,nvf2418,nvf3877,nvf3878;fma.rn.ftz.f32 nvf3880,nvf2421,nvf3874,nvf3879;fma.rn.ftz.f32 nvf3881,nvf2424,nvf3875,nvf3880;ld.shared.v4.f32{nvf3882,nvf3883,nvf3884,nvf3885},[global_smem+7024];fma.rn.ftz.f32 nvf3886,nvf2431,nvf3884,nvf3881;fma.rn.ftz.f32 nvf3887,nvf2434,nvf3885,nvf3886;fma.rn.ftz.f32 nvf3888,nvf2437,nvf3882,nvf3887;fma.rn.ftz.f32 nvf3889,nvf2440,nvf3883,nvf3888;ld.shared.v4.f32{nvf3890,nvf3891,nvf3892,nvf3893},[global_smem+7008];fma.rn.ftz.f32 nvf3894,nvf2447,nvf3892,nvf3889;fma.rn.ftz.f32 nvf3895,nvf2450,nvf3893,nvf3894;fma.rn.ftz.f32 nvf3896,nvf2453,nvf3890,nvf3895;fma.rn.ftz.f32 nvf3897,nvf2456,nvf3891,nvf3896;ld.shared.v4.f32{nvf3898,nvf3899,nvf3900,nvf3901},[global_smem+6928];fma.rn.ftz.f32 nvf3902,nvf2463,nvf3900,nvf3897;fma.rn.ftz.f32 nvf3903,nvf2466,nvf3901,nvf3902;fma.rn.ftz.f32 nvf3904,nvf2469,nvf3898,nvf3903;fma.rn.ftz.f32 nvf3905,nvf2472,nvf3899,nvf3904;ld.shared.v4.f32{nvf3906,nvf3907,nvf3908,nvf3909},[global_smem+6912];fma.rn.ftz.f32 nvf3910,nvf2479,nvf3908,nvf3905;fma.rn.ftz.f32 nvf3911,nvf2482,nvf3909,nvf3910;fma.rn.ftz.f32 nvf3912,nvf2485,nvf3906,nvf3911;fma.rn.ftz.f32 nvf3913,nvf2488,nvf3907,nvf3912;ld.shared.v4.f32{nvf3914,nvf3915,nvf3916,nvf3917},[global_smem+6960];fma.rn.ftz.f32 nvf3918,nvf2495,nvf3916,nvf3913;fma.rn.ftz.f32 nvf3919,nvf2498,nvf3917,nvf3918;fma.rn.ftz.f32 nvf3920,nvf2501,nvf3914,nvf3919;fma.rn.ftz.f32 nvf3921,nvf2504,nvf3915,nvf3920;ld.shared.v4.f32{nvf3922,nvf3923,nvf3924,nvf3925},[global_smem+6944];fma.rn.ftz.f32 nvf3926,nvf2511,nvf3924,nvf3921;fma.rn.ftz.f32 nvf3927,nvf2514,nvf3925,nvf3926;fma.rn.ftz.f32 nvf3928,nvf2517,nvf3922,nvf3927;fma.rn.ftz.f32 nvf3929,nvf2520,nvf3923,nvf3928;ld.shared.v4.f32{nvf3930,nvf3931,nvf3932,nvf3933},[global_smem+7120];fma.rn.ftz.f32 nvf3934,nvf2399,nvf3933,nvf2385;fma.rn.ftz.f32 nvf3935,nvf2402,nvf3932,nvf3934;fma.rn.ftz.f32 nvf3936,nvf2405,nvf3931,nvf3935;fma.rn.ftz.f32 nvf3937,nvf2408,nvf3930,nvf3936;ld.shared.v4.f32{nvf3938,nvf3939,nvf3940,nvf3941},[global_smem+7104];fma.rn.ftz.f32 nvf3942,nvf2415,nvf3941,nvf3937;fma.rn.ftz.f32 nvf3943,nvf2418,nvf3940,nvf3942;fma.rn.ftz.f32 nvf3944,nvf2421,nvf3939,nvf3943;fma.rn.ftz.f32 nvf3945,nvf2424,nvf3938,nvf3944;ld.shared.v4.f32{nvf3946,nvf3947,nvf3948,nvf3949},[global_smem+7152];fma.rn.ftz.f32 nvf3950,nvf2431,nvf3949,nvf3945;fma.rn.ftz.f32 nvf3951,nvf2434,nvf3948,nvf3950;fma.rn.ftz.f32 nvf3952,nvf2437,nvf3947,nvf3951;fma.rn.ftz.f32 nvf3953,nvf2440,nvf3946,nvf3952;ld.shared.v4.f32{nvf3954,nvf3955,nvf3956,nvf3957},[global_smem+7136];fma.rn.ftz.f32 nvf3958,nvf2447,nvf3957,nvf3953;fma.rn.ftz.f32 nvf3959,nvf2450,nvf3956,nvf3958;fma.rn.ftz.f32 nvf3960,nvf2453,nvf3955,nvf3959;fma.rn.ftz.f32 nvf3961,nvf2456,nvf3954,nvf3960;ld.shared.v4.f32{nvf3962,nvf3963,nvf3964,nvf3965},[global_smem+7056];fma.rn.ftz.f32 nvf3966,nvf2463,nvf3965,nvf3961;fma.rn.ftz.f32 nvf3967,nvf2466,nvf3964,nvf3966;fma.rn.ftz.f32 nvf3968,nvf2469,nvf3963,nvf3967;fma.rn.ftz.f32 nvf3969,nvf2472,nvf3962,nvf3968;ld.shared.v4.f32{nvf3970,nvf3971,nvf3972,nvf3973},[global_smem+7040];fma.rn.ftz.f32 nvf3974,nvf2479,nvf3973,nvf3969;fma.rn.ftz.f32 nvf3975,nvf2482,nvf3972,nvf3974;fma.rn.ftz.f32 nvf3976,nvf2485,nvf3971,nvf3975;fma.rn.ftz.f32 nvf3977,nvf2488,nvf3970,nvf3976;ld.shared.v4.f32{nvf3978,nvf3979,nvf3980,nvf3981},[global_smem+7088];fma.rn.ftz.f32 nvf3982,nvf2495,nvf3981,nvf3977;fma.rn.ftz.f32 nvf3983,nvf2498,nvf3980,nvf3982;fma.rn.ftz.f32 nvf3984,nvf2501,nvf3979,nvf3983;fma.rn.ftz.f32 nvf3985,nvf2504,nvf3978,nvf3984;ld.shared.v4.f32{nvf3986,nvf3987,nvf3988,nvf3989},[global_smem+7072];fma.rn.ftz.f32 nvf3990,nvf2511,nvf3989,nvf3985;fma.rn.ftz.f32 nvf3991,nvf2514,nvf3988,nvf3990;fma.rn.ftz.f32 nvf3992,nvf2517,nvf3987,nvf3991;fma.rn.ftz.f32 nvf3993,nvf2520,nvf3986,nvf3992;ld.shared.v4.f32{nvf3994,nvf3995,nvf3996,nvf3997},[global_smem+7264];fma.rn.ftz.f32 nvf3998,nvf2399,nvf3994,nvf2386;fma.rn.ftz.f32 nvf3999,nvf2402,nvf3995,nvf3998;fma.rn.ftz.f32 nvf4000,nvf2405,nvf3996,nvf3999;fma.rn.ftz.f32 nvf4001,nvf2408,nvf3997,nvf4000;ld.shared.v4.f32{nvf4002,nvf4003,nvf4004,nvf4005},[global_smem+7280];fma.rn.ftz.f32 nvf4006,nvf2415,nvf4002,nvf4001;fma.rn.ftz.f32 nvf4007,nvf2418,nvf4003,nvf4006;fma.rn.ftz.f32 nvf4008,nvf2421,nvf4004,nvf4007;fma.rn.ftz.f32 nvf4009,nvf2424,nvf4005,nvf4008;ld.shared.v4.f32{nvf4010,nvf4011,nvf4012,nvf4013},[global_smem+7232];fma.rn.ftz.f32 nvf4014,nvf2431,nvf4010,nvf4009;fma.rn.ftz.f32 nvf4015,nvf2434,nvf4011,nvf4014;fma.rn.ftz.f32 nvf4016,nvf2437,nvf4012,nvf4015;fma.rn.ftz.f32 nvf4017,nvf2440,nvf4013,nvf4016;ld.shared.v4.f32{nvf4018,nvf4019,nvf4020,nvf4021},[global_smem+7248];fma.rn.ftz.f32 nvf4022,nvf2447,nvf4018,nvf4017;fma.rn.ftz.f32 nvf4023,nvf2450,nvf4019,nvf4022;fma.rn.ftz.f32 nvf4024,nvf2453,nvf4020,nvf4023;fma.rn.ftz.f32 nvf4025,nvf2456,nvf4021,nvf4024;ld.shared.v4.f32{nvf4026,nvf4027,nvf4028,nvf4029},[global_smem+7200];fma.rn.ftz.f32 nvf4030,nvf2463,nvf4026,nvf4025;fma.rn.ftz.f32 nvf4031,nvf2466,nvf4027,nvf4030;fma.rn.ftz.f32 nvf4032,nvf2469,nvf4028,nvf4031;fma.rn.ftz.f32 nvf4033,nvf2472,nvf4029,nvf4032;ld.shared.v4.f32{nvf4034,nvf4035,nvf4036,nvf4037},[global_smem+7216];fma.rn.ftz.f32 nvf4038,nvf2479,nvf4034,nvf4033;fma.rn.ftz.f32 nvf4039,nvf2482,nvf4035,nvf4038;fma.rn.ftz.f32 nvf4040,nvf2485,nvf4036,nvf4039;fma.rn.ftz.f32 nvf4041,nvf2488,nvf4037,nvf4040;ld.shared.v4.f32{nvf4042,nvf4043,nvf4044,nvf4045},[global_smem+7168];fma.rn.ftz.f32 nvf4046,nvf2495,nvf4042,nvf4041;fma.rn.ftz.f32 nvf4047,nvf2498,nvf4043,nvf4046;fma.rn.ftz.f32 nvf4048,nvf2501,nvf4044,nvf4047;fma.rn.ftz.f32 nvf4049,nvf2504,nvf4045,nvf4048;ld.shared.v4.f32{nvf4050,nvf4051,nvf4052,nvf4053},[global_smem+7184];fma.rn.ftz.f32 nvf4054,nvf2511,nvf4050,nvf4049;fma.rn.ftz.f32 nvf4055,nvf2514,nvf4051,nvf4054;fma.rn.ftz.f32 nvf4056,nvf2517,nvf4052,nvf4055;fma.rn.ftz.f32 nvf4057,nvf2520,nvf4053,nvf4056;ld.shared.v4.f32{nvf4058,nvf4059,nvf4060,nvf4061},[global_smem+7392];fma.rn.ftz.f32 nvf4062,nvf2399,nvf4059,nvf2387;fma.rn.ftz.f32 nvf4063,nvf2402,nvf4058,nvf4062;fma.rn.ftz.f32 nvf4064,nvf2405,nvf4061,nvf4063;fma.rn.ftz.f32 nvf4065,nvf2408,nvf4060,nvf4064;ld.shared.v4.f32{nvf4066,nvf4067,nvf4068,nvf4069},[global_smem+7408];fma.rn.ftz.f32 nvf4070,nvf2415,nvf4067,nvf4065;fma.rn.ftz.f32 nvf4071,nvf2418,nvf4066,nvf4070;fma.rn.ftz.f32 nvf4072,nvf2421,nvf4069,nvf4071;fma.rn.ftz.f32 nvf4073,nvf2424,nvf4068,nvf4072;ld.shared.v4.f32{nvf4074,nvf4075,nvf4076,nvf4077},[global_smem+7360];fma.rn.ftz.f32 nvf4078,nvf2431,nvf4075,nvf4073;fma.rn.ftz.f32 nvf4079,nvf2434,nvf4074,nvf4078;fma.rn.ftz.f32 nvf4080,nvf2437,nvf4077,nvf4079;fma.rn.ftz.f32 nvf4081,nvf2440,nvf4076,nvf4080;ld.shared.v4.f32{nvf4082,nvf4083,nvf4084,nvf4085},[global_smem+7376];fma.rn.ftz.f32 nvf4086,nvf2447,nvf4083,nvf4081;fma.rn.ftz.f32 nvf4087,nvf2450,nvf4082,nvf4086;fma.rn.ftz.f32 nvf4088,nvf2453,nvf4085,nvf4087;fma.rn.ftz.f32 nvf4089,nvf2456,nvf4084,nvf4088;ld.shared.v4.f32{nvf4090,nvf4091,nvf4092,nvf4093},[global_smem+7328];fma.rn.ftz.f32 nvf4094,nvf2463,nvf4091,nvf4089;fma.rn.ftz.f32 nvf4095,nvf2466,nvf4090,nvf4094;fma.rn.ftz.f32 nvf4096,nvf2469,nvf4093,nvf4095;fma.rn.ftz.f32 nvf4097,nvf2472,nvf4092,nvf4096;ld.shared.v4.f32{nvf4098,nvf4099,nvf4100,nvf4101},[global_smem+7344];fma.rn.ftz.f32 nvf4102,nvf2479,nvf4099,nvf4097;fma.rn.ftz.f32 nvf4103,nvf2482,nvf4098,nvf4102;fma.rn.ftz.f32 nvf4104,nvf2485,nvf4101,nvf4103;fma.rn.ftz.f32 nvf4105,nvf2488,nvf4100,nvf4104;ld.shared.v4.f32{nvf4106,nvf4107,nvf4108,nvf4109},[global_smem+7296];fma.rn.ftz.f32 nvf4110,nvf2495,nvf4107,nvf4105;fma.rn.ftz.f32 nvf4111,nvf2498,nvf4106,nvf4110;fma.rn.ftz.f32 nvf4112,nvf2501,nvf4109,nvf4111;fma.rn.ftz.f32 nvf4113,nvf2504,nvf4108,nvf4112;ld.shared.v4.f32{nvf4114,nvf4115,nvf4116,nvf4117},[global_smem+7312];fma.rn.ftz.f32 nvf4118,nvf2511,nvf4115,nvf4113;fma.rn.ftz.f32 nvf4119,nvf2514,nvf4114,nvf4118;fma.rn.ftz.f32 nvf4120,nvf2517,nvf4117,nvf4119;fma.rn.ftz.f32 nvf4121,nvf2520,nvf4116,nvf4120;ld.shared.v4.f32{nvf4122,nvf4123,nvf4124,nvf4125},[global_smem+7520];fma.rn.ftz.f32 nvf4126,nvf2399,nvf4124,nvf2388;fma.rn.ftz.f32 nvf4127,nvf2402,nvf4125,nvf4126;fma.rn.ftz.f32 nvf4128,nvf2405,nvf4122,nvf4127;fma.rn.ftz.f32 nvf4129,nvf2408,nvf4123,nvf4128;ld.shared.v4.f32{nvf4130,nvf4131,nvf4132,nvf4133},[global_smem+7536];fma.rn.ftz.f32 nvf4134,nvf2415,nvf4132,nvf4129;fma.rn.ftz.f32 nvf4135,nvf2418,nvf4133,nvf4134;fma.rn.ftz.f32 nvf4136,nvf2421,nvf4130,nvf4135;fma.rn.ftz.f32 nvf4137,nvf2424,nvf4131,nvf4136;ld.shared.v4.f32{nvf4138,nvf4139,nvf4140,nvf4141},[global_smem+7488];fma.rn.ftz.f32 nvf4142,nvf2431,nvf4140,nvf4137;fma.rn.ftz.f32 nvf4143,nvf2434,nvf4141,nvf4142;fma.rn.ftz.f32 nvf4144,nvf2437,nvf4138,nvf4143;fma.rn.ftz.f32 nvf4145,nvf2440,nvf4139,nvf4144;ld.shared.v4.f32{nvf4146,nvf4147,nvf4148,nvf4149},[global_smem+7504];fma.rn.ftz.f32 nvf4150,nvf2447,nvf4148,nvf4145;fma.rn.ftz.f32 nvf4151,nvf2450,nvf4149,nvf4150;fma.rn.ftz.f32 nvf4152,nvf2453,nvf4146,nvf4151;fma.rn.ftz.f32 nvf4153,nvf2456,nvf4147,nvf4152;ld.shared.v4.f32{nvf4154,nvf4155,nvf4156,nvf4157},[global_smem+7456];fma.rn.ftz.f32 nvf4158,nvf2463,nvf4156,nvf4153;fma.rn.ftz.f32 nvf4159,nvf2466,nvf4157,nvf4158;fma.rn.ftz.f32 nvf4160,nvf2469,nvf4154,nvf4159;fma.rn.ftz.f32 nvf4161,nvf2472,nvf4155,nvf4160;ld.shared.v4.f32{nvf4162,nvf4163,nvf4164,nvf4165},[global_smem+7472];fma.rn.ftz.f32 nvf4166,nvf2479,nvf4164,nvf4161;fma.rn.ftz.f32 nvf4167,nvf2482,nvf4165,nvf4166;fma.rn.ftz.f32 nvf4168,nvf2485,nvf4162,nvf4167;fma.rn.ftz.f32 nvf4169,nvf2488,nvf4163,nvf4168;ld.shared.v4.f32{nvf4170,nvf4171,nvf4172,nvf4173},[global_smem+7424];fma.rn.ftz.f32 nvf4174,nvf2495,nvf4172,nvf4169;fma.rn.ftz.f32 nvf4175,nvf2498,nvf4173,nvf4174;fma.rn.ftz.f32 nvf4176,nvf2501,nvf4170,nvf4175;fma.rn.ftz.f32 nvf4177,nvf2504,nvf4171,nvf4176;ld.shared.v4.f32{nvf4178,nvf4179,nvf4180,nvf4181},[global_smem+7440];fma.rn.ftz.f32 nvf4182,nvf2511,nvf4180,nvf4177;fma.rn.ftz.f32 nvf4183,nvf2514,nvf4181,nvf4182;fma.rn.ftz.f32 nvf4184,nvf2517,nvf4178,nvf4183;fma.rn.ftz.f32 nvf4185,nvf2520,nvf4179,nvf4184;ld.shared.v4.f32{nvf4186,nvf4187,nvf4188,nvf4189},[global_smem+7648];fma.rn.ftz.f32 nvf4190,nvf2399,nvf4189,nvf2389;fma.rn.ftz.f32 nvf4191,nvf2402,nvf4188,nvf4190;fma.rn.ftz.f32 nvf4192,nvf2405,nvf4187,nvf4191;fma.rn.ftz.f32 nvf4193,nvf2408,nvf4186,nvf4192;ld.shared.v4.f32{nvf4194,nvf4195,nvf4196,nvf4197},[global_smem+7664];fma.rn.ftz.f32 nvf4198,nvf2415,nvf4197,nvf4193;fma.rn.ftz.f32 nvf4199,nvf2418,nvf4196,nvf4198;fma.rn.ftz.f32 nvf4200,nvf2421,nvf4195,nvf4199;fma.rn.ftz.f32 nvf4201,nvf2424,nvf4194,nvf4200;ld.shared.v4.f32{nvf4202,nvf4203,nvf4204,nvf4205},[global_smem+7616];fma.rn.ftz.f32 nvf4206,nvf2431,nvf4205,nvf4201;fma.rn.ftz.f32 nvf4207,nvf2434,nvf4204,nvf4206;fma.rn.ftz.f32 nvf4208,nvf2437,nvf4203,nvf4207;fma.rn.ftz.f32 nvf4209,nvf2440,nvf4202,nvf4208;ld.shared.v4.f32{nvf4210,nvf4211,nvf4212,nvf4213},[global_smem+7632];fma.rn.ftz.f32 nvf4214,nvf2447,nvf4213,nvf4209;fma.rn.ftz.f32 nvf4215,nvf2450,nvf4212,nvf4214;fma.rn.ftz.f32 nvf4216,nvf2453,nvf4211,nvf4215;fma.rn.ftz.f32 nvf4217,nvf2456,nvf4210,nvf4216;ld.shared.v4.f32{nvf4218,nvf4219,nvf4220,nvf4221},[global_smem+7584];fma.rn.ftz.f32 nvf4222,nvf2463,nvf4221,nvf4217;fma.rn.ftz.f32 nvf4223,nvf2466,nvf4220,nvf4222;fma.rn.ftz.f32 nvf4224,nvf2469,nvf4219,nvf4223;fma.rn.ftz.f32 nvf4225,nvf2472,nvf4218,nvf4224;ld.shared.v4.f32{nvf4226,nvf4227,nvf4228,nvf4229},[global_smem+7600];fma.rn.ftz.f32 nvf4230,nvf2479,nvf4229,nvf4225;fma.rn.ftz.f32 nvf4231,nvf2482,nvf4228,nvf4230;fma.rn.ftz.f32 nvf4232,nvf2485,nvf4227,nvf4231;fma.rn.ftz.f32 nvf4233,nvf2488,nvf4226,nvf4232;ld.shared.v4.f32{nvf4234,nvf4235,nvf4236,nvf4237},[global_smem+7552];fma.rn.ftz.f32 nvf4238,nvf2495,nvf4237,nvf4233;fma.rn.ftz.f32 nvf4239,nvf2498,nvf4236,nvf4238;fma.rn.ftz.f32 nvf4240,nvf2501,nvf4235,nvf4239;fma.rn.ftz.f32 nvf4241,nvf2504,nvf4234,nvf4240;ld.shared.v4.f32{nvf4242,nvf4243,nvf4244,nvf4245},[global_smem+7568];fma.rn.ftz.f32 nvf4246,nvf2511,nvf4245,nvf4241;fma.rn.ftz.f32 nvf4247,nvf2514,nvf4244,nvf4246;fma.rn.ftz.f32 nvf4248,nvf2517,nvf4243,nvf4247;fma.rn.ftz.f32 nvf4249,nvf2520,nvf4242,nvf4248;ld.shared.v4.f32{nvf4250,nvf4251,nvf4252,nvf4253},[global_smem+7792];fma.rn.ftz.f32 nvf4254,nvf2399,nvf4250,nvf2390;fma.rn.ftz.f32 nvf4255,nvf2402,nvf4251,nvf4254;fma.rn.ftz.f32 nvf4256,nvf2405,nvf4252,nvf4255;fma.rn.ftz.f32 nvf4257,nvf2408,nvf4253,nvf4256;ld.shared.v4.f32{nvf4258,nvf4259,nvf4260,nvf4261},[global_smem+7776];fma.rn.ftz.f32 nvf4262,nvf2415,nvf4258,nvf4257;fma.rn.ftz.f32 nvf4263,nvf2418,nvf4259,nvf4262;fma.rn.ftz.f32 nvf4264,nvf2421,nvf4260,nvf4263;fma.rn.ftz.f32 nvf4265,nvf2424,nvf4261,nvf4264;ld.shared.v4.f32{nvf4266,nvf4267,nvf4268,nvf4269},[global_smem+7760];fma.rn.ftz.f32 nvf4270,nvf2431,nvf4266,nvf4265;fma.rn.ftz.f32 nvf4271,nvf2434,nvf4267,nvf4270;fma.rn.ftz.f32 nvf4272,nvf2437,nvf4268,nvf4271;fma.rn.ftz.f32 nvf4273,nvf2440,nvf4269,nvf4272;ld.shared.v4.f32{nvf4274,nvf4275,nvf4276,nvf4277},[global_smem+7744];fma.rn.ftz.f32 nvf4278,nvf2447,nvf4274,nvf4273;fma.rn.ftz.f32 nvf4279,nvf2450,nvf4275,nvf4278;fma.rn.ftz.f32 nvf4280,nvf2453,nvf4276,nvf4279;fma.rn.ftz.f32 nvf4281,nvf2456,nvf4277,nvf4280;ld.shared.v4.f32{nvf4282,nvf4283,nvf4284,nvf4285},[global_smem+7728];fma.rn.ftz.f32 nvf4286,nvf2463,nvf4282,nvf4281;fma.rn.ftz.f32 nvf4287,nvf2466,nvf4283,nvf4286;fma.rn.ftz.f32 nvf4288,nvf2469,nvf4284,nvf4287;fma.rn.ftz.f32 nvf4289,nvf2472,nvf4285,nvf4288;ld.shared.v4.f32{nvf4290,nvf4291,nvf4292,nvf4293},[global_smem+7712];fma.rn.ftz.f32 nvf4294,nvf2479,nvf4290,nvf4289;fma.rn.ftz.f32 nvf4295,nvf2482,nvf4291,nvf4294;fma.rn.ftz.f32 nvf4296,nvf2485,nvf4292,nvf4295;fma.rn.ftz.f32 nvf4297,nvf2488,nvf4293,nvf4296;ld.shared.v4.f32{nvf4298,nvf4299,nvf4300,nvf4301},[global_smem+7696];fma.rn.ftz.f32 nvf4302,nvf2495,nvf4298,nvf4297;fma.rn.ftz.f32 nvf4303,nvf2498,nvf4299,nvf4302;fma.rn.ftz.f32 nvf4304,nvf2501,nvf4300,nvf4303;fma.rn.ftz.f32 nvf4305,nvf2504,nvf4301,nvf4304;ld.shared.v4.f32{nvf4306,nvf4307,nvf4308,nvf4309},[global_smem+7680];fma.rn.ftz.f32 nvf4310,nvf2511,nvf4306,nvf4305;fma.rn.ftz.f32 nvf4311,nvf2514,nvf4307,nvf4310;fma.rn.ftz.f32 nvf4312,nvf2517,nvf4308,nvf4311;fma.rn.ftz.f32 nvf4313,nvf2520,nvf4309,nvf4312;ld.shared.v4.f32{nvf4314,nvf4315,nvf4316,nvf4317},[global_smem+7920];fma.rn.ftz.f32 nvf4318,nvf2399,nvf4315,nvf2391;fma.rn.ftz.f32 nvf4319,nvf2402,nvf4314,nvf4318;fma.rn.ftz.f32 nvf4320,nvf2405,nvf4317,nvf4319;fma.rn.ftz.f32 nvf4321,nvf2408,nvf4316,nvf4320;ld.shared.v4.f32{nvf4322,nvf4323,nvf4324,nvf4325},[global_smem+7904];fma.rn.ftz.f32 nvf4326,nvf2415,nvf4323,nvf4321;fma.rn.ftz.f32 nvf4327,nvf2418,nvf4322,nvf4326;fma.rn.ftz.f32 nvf4328,nvf2421,nvf4325,nvf4327;fma.rn.ftz.f32 nvf4329,nvf2424,nvf4324,nvf4328;ld.shared.v4.f32{nvf4330,nvf4331,nvf4332,nvf4333},[global_smem+7888];fma.rn.ftz.f32 nvf4334,nvf2431,nvf4331,nvf4329;fma.rn.ftz.f32 nvf4335,nvf2434,nvf4330,nvf4334;fma.rn.ftz.f32 nvf4336,nvf2437,nvf4333,nvf4335;fma.rn.ftz.f32 nvf4337,nvf2440,nvf4332,nvf4336;ld.shared.v4.f32{nvf4338,nvf4339,nvf4340,nvf4341},[global_smem+7872];fma.rn.ftz.f32 nvf4342,nvf2447,nvf4339,nvf4337;fma.rn.ftz.f32 nvf4343,nvf2450,nvf4338,nvf4342;fma.rn.ftz.f32 nvf4344,nvf2453,nvf4341,nvf4343;fma.rn.ftz.f32 nvf4345,nvf2456,nvf4340,nvf4344;ld.shared.v4.f32{nvf4346,nvf4347,nvf4348,nvf4349},[global_smem+7856];fma.rn.ftz.f32 nvf4350,nvf2463,nvf4347,nvf4345;fma.rn.ftz.f32 nvf4351,nvf2466,nvf4346,nvf4350;fma.rn.ftz.f32 nvf4352,nvf2469,nvf4349,nvf4351;fma.rn.ftz.f32 nvf4353,nvf2472,nvf4348,nvf4352;ld.shared.v4.f32{nvf4354,nvf4355,nvf4356,nvf4357},[global_smem+7840];fma.rn.ftz.f32 nvf4358,nvf2479,nvf4355,nvf4353;fma.rn.ftz.f32 nvf4359,nvf2482,nvf4354,nvf4358;fma.rn.ftz.f32 nvf4360,nvf2485,nvf4357,nvf4359;fma.rn.ftz.f32 nvf4361,nvf2488,nvf4356,nvf4360;ld.shared.v4.f32{nvf4362,nvf4363,nvf4364,nvf4365},[global_smem+7824];fma.rn.ftz.f32 nvf4366,nvf2495,nvf4363,nvf4361;fma.rn.ftz.f32 nvf4367,nvf2498,nvf4362,nvf4366;fma.rn.ftz.f32 nvf4368,nvf2501,nvf4365,nvf4367;fma.rn.ftz.f32 nvf4369,nvf2504,nvf4364,nvf4368;ld.shared.v4.f32{nvf4370,nvf4371,nvf4372,nvf4373},[global_smem+7808];fma.rn.ftz.f32 nvf4374,nvf2511,nvf4371,nvf4369;fma.rn.ftz.f32 nvf4375,nvf2514,nvf4370,nvf4374;fma.rn.ftz.f32 nvf4376,nvf2517,nvf4373,nvf4375;fma.rn.ftz.f32 nvf4377,nvf2520,nvf4372,nvf4376;ld.shared.v4.f32{nvf4378,nvf4379,nvf4380,nvf4381},[global_smem+8048];fma.rn.ftz.f32 nvf4382,nvf2399,nvf4380,nvf2392;fma.rn.ftz.f32 nvf4383,nvf2402,nvf4381,nvf4382;fma.rn.ftz.f32 nvf4384,nvf2405,nvf4378,nvf4383;fma.rn.ftz.f32 nvf4385,nvf2408,nvf4379,nvf4384;ld.shared.v4.f32{nvf4386,nvf4387,nvf4388,nvf4389},[global_smem+8032];fma.rn.ftz.f32 nvf4390,nvf2415,nvf4388,nvf4385;fma.rn.ftz.f32 nvf4391,nvf2418,nvf4389,nvf4390;fma.rn.ftz.f32 nvf4392,nvf2421,nvf4386,nvf4391;fma.rn.ftz.f32 nvf4393,nvf2424,nvf4387,nvf4392;ld.shared.v4.f32{nvf4394,nvf4395,nvf4396,nvf4397},[global_smem+8016];fma.rn.ftz.f32 nvf4398,nvf2431,nvf4396,nvf4393;fma.rn.ftz.f32 nvf4399,nvf2434,nvf4397,nvf4398;fma.rn.ftz.f32 nvf4400,nvf2437,nvf4394,nvf4399;fma.rn.ftz.f32 nvf4401,nvf2440,nvf4395,nvf4400;ld.shared.v4.f32{nvf4402,nvf4403,nvf4404,nvf4405},[global_smem+8000];fma.rn.ftz.f32 nvf4406,nvf2447,nvf4404,nvf4401;fma.rn.ftz.f32 nvf4407,nvf2450,nvf4405,nvf4406;fma.rn.ftz.f32 nvf4408,nvf2453,nvf4402,nvf4407;fma.rn.ftz.f32 nvf4409,nvf2456,nvf4403,nvf4408;ld.shared.v4.f32{nvf4410,nvf4411,nvf4412,nvf4413},[global_smem+7984];fma.rn.ftz.f32 nvf4414,nvf2463,nvf4412,nvf4409;fma.rn.ftz.f32 nvf4415,nvf2466,nvf4413,nvf4414;fma.rn.ftz.f32 nvf4416,nvf2469,nvf4410,nvf4415;fma.rn.ftz.f32 nvf4417,nvf2472,nvf4411,nvf4416;ld.shared.v4.f32{nvf4418,nvf4419,nvf4420,nvf4421},[global_smem+7968];fma.rn.ftz.f32 nvf4422,nvf2479,nvf4420,nvf4417;fma.rn.ftz.f32 nvf4423,nvf2482,nvf4421,nvf4422;fma.rn.ftz.f32 nvf4424,nvf2485,nvf4418,nvf4423;fma.rn.ftz.f32 nvf4425,nvf2488,nvf4419,nvf4424;ld.shared.v4.f32{nvf4426,nvf4427,nvf4428,nvf4429},[global_smem+7952];fma.rn.ftz.f32 nvf4430,nvf2495,nvf4428,nvf4425;fma.rn.ftz.f32 nvf4431,nvf2498,nvf4429,nvf4430;fma.rn.ftz.f32 nvf4432,nvf2501,nvf4426,nvf4431;fma.rn.ftz.f32 nvf4433,nvf2504,nvf4427,nvf4432;ld.shared.v4.f32{nvf4434,nvf4435,nvf4436,nvf4437},[global_smem+7936];fma.rn.ftz.f32 nvf4438,nvf2511,nvf4436,nvf4433;fma.rn.ftz.f32 nvf4439,nvf2514,nvf4437,nvf4438;fma.rn.ftz.f32 nvf4440,nvf2517,nvf4434,nvf4439;fma.rn.ftz.f32 nvf4441,nvf2520,nvf4435,nvf4440;ld.shared.v4.f32{nvf4442,nvf4443,nvf4444,nvf4445},[global_smem+8176];fma.rn.ftz.f32 nvf4446,nvf2399,nvf4445,nvf2393;fma.rn.ftz.f32 nvf4447,nvf2402,nvf4444,nvf4446;fma.rn.ftz.f32 nvf4448,nvf2405,nvf4443,nvf4447;fma.rn.ftz.f32 nvf4449,nvf2408,nvf4442,nvf4448;ld.shared.v4.f32{nvf4450,nvf4451,nvf4452,nvf4453},[global_smem+8160];fma.rn.ftz.f32 nvf4454,nvf2415,nvf4453,nvf4449;fma.rn.ftz.f32 nvf4455,nvf2418,nvf4452,nvf4454;fma.rn.ftz.f32 nvf4456,nvf2421,nvf4451,nvf4455;fma.rn.ftz.f32 nvf4457,nvf2424,nvf4450,nvf4456;ld.shared.v4.f32{nvf4458,nvf4459,nvf4460,nvf4461},[global_smem+8144];fma.rn.ftz.f32 nvf4462,nvf2431,nvf4461,nvf4457;fma.rn.ftz.f32 nvf4463,nvf2434,nvf4460,nvf4462;fma.rn.ftz.f32 nvf4464,nvf2437,nvf4459,nvf4463;fma.rn.ftz.f32 nvf4465,nvf2440,nvf4458,nvf4464;ld.shared.v4.f32{nvf4466,nvf4467,nvf4468,nvf4469},[global_smem+8128];fma.rn.ftz.f32 nvf4470,nvf2447,nvf4469,nvf4465;fma.rn.ftz.f32 nvf4471,nvf2450,nvf4468,nvf4470;fma.rn.ftz.f32 nvf4472,nvf2453,nvf4467,nvf4471;fma.rn.ftz.f32 nvf4473,nvf2456,nvf4466,nvf4472;ld.shared.v4.f32{nvf4474,nvf4475,nvf4476,nvf4477},[global_smem+8112];fma.rn.ftz.f32 nvf4478,nvf2463,nvf4477,nvf4473;fma.rn.ftz.f32 nvf4479,nvf2466,nvf4476,nvf4478;fma.rn.ftz.f32 nvf4480,nvf2469,nvf4475,nvf4479;fma.rn.ftz.f32 nvf4481,nvf2472,nvf4474,nvf4480;ld.shared.v4.f32{nvf4482,nvf4483,nvf4484,nvf4485},[global_smem+8096];fma.rn.ftz.f32 nvf4486,nvf2479,nvf4485,nvf4481;fma.rn.ftz.f32 nvf4487,nvf2482,nvf4484,nvf4486;fma.rn.ftz.f32 nvf4488,nvf2485,nvf4483,nvf4487;fma.rn.ftz.f32 nvf4489,nvf2488,nvf4482,nvf4488;ld.shared.v4.f32{nvf4490,nvf4491,nvf4492,nvf4493},[global_smem+8080];fma.rn.ftz.f32 nvf4494,nvf2495,nvf4493,nvf4489;fma.rn.ftz.f32 nvf4495,nvf2498,nvf4492,nvf4494;fma.rn.ftz.f32 nvf4496,nvf2501,nvf4491,nvf4495;fma.rn.ftz.f32 nvf4497,nvf2504,nvf4490,nvf4496;ld.shared.v4.f32{nvf4498,nvf4499,nvf4500,nvf4501},[global_smem+8064];fma.rn.ftz.f32 nvf4502,nvf2511,nvf4501,nvf4497;fma.rn.ftz.f32 nvf4503,nvf2514,nvf4500,nvf4502;fma.rn.ftz.f32 nvf4504,nvf2517,nvf4499,nvf4503;fma.rn.ftz.f32 nvf4505,nvf2520,nvf4498,nvf4504;max.ftz.f32 nvf4506,nvf2521,0f00800000;sqrt.approx.ftz.f32 nvf4507,nvf4506;mov.b32 nvr918,nvf4507;shfl.sync.idx.b32 nvr919|nvp606,nvr918,0,31,-1;mov.b32 nvf4508,nvr919;div.approx.ftz.f32 nvf4509,nvf2521,nvf4508;mov.b32 nvr920,nvf4509;shfl.sync.idx.b32 nvr921|nvp607,nvr920,1,31,-1;mov.b32 nvf4510,nvr921;neg.ftz.f32 nvf4511,nvf4509;fma.rn.ftz.f32 nvf4512,nvf4511,nvf4510,nvf2585;shfl.sync.idx.b32 nvr922|nvp608,nvr920,2,31,-1;mov.b32 nvf4513,nvr922;fma.rn.ftz.f32 nvf4514,nvf4511,nvf4513,nvf2649;shfl.sync.idx.b32 nvr923|nvp609,nvr920,3,31,-1;mov.b32 nvf4515,nvr923;fma.rn.ftz.f32 nvf4516,nvf4511,nvf4515,nvf2713;shfl.sync.idx.b32 nvr924|nvp610,nvr920,4,31,-1;mov.b32 nvf4517,nvr924;fma.rn.ftz.f32 nvf4518,nvf4511,nvf4517,nvf2777;shfl.sync.idx.b32 nvr925|nvp611,nvr920,5,31,-1;mov.b32 nvf4519,nvr925;fma.rn.ftz.f32 nvf4520,nvf4511,nvf4519,nvf2841;shfl.sync.idx.b32 nvr926|nvp612,nvr920,6,31,-1;mov.b32 nvf4521,nvr926;fma.rn.ftz.f32 nvf4522,nvf4511,nvf4521,nvf2905;shfl.sync.idx.b32 nvr927|nvp613,nvr920,7,31,-1;mov.b32 nvf4523,nvr927;fma.rn.ftz.f32 nvf4524,nvf4511,nvf4523,nvf2969;shfl.sync.idx.b32 nvr928|nvp614,nvr920,8,31,-1;mov.b32 nvf4525,nvr928;fma.rn.ftz.f32 nvf4526,nvf4511,nvf4525,nvf3033;shfl.sync.idx.b32 nvr929|nvp615,nvr920,9,31,-1;mov.b32 nvf4527,nvr929;fma.rn.ftz.f32 nvf4528,nvf4511,nvf4527,nvf3097;shfl.sync.idx.b32 nvr930|nvp616,nvr920,10,31,-1;mov.b32 nvf4529,nvr930;fma.rn.ftz.f32 nvf4530,nvf4511,nvf4529,nvf3161;shfl.sync.idx.b32 nvr931|nvp617,nvr920,11,31,-1;mov.b32 nvf4531,nvr931;fma.rn.ftz.f32 nvf4532,nvf4511,nvf4531,nvf3225;shfl.sync.idx.b32 nvr932|nvp618,nvr920,12,31,-1;mov.b32 nvf4533,nvr932;fma.rn.ftz.f32 nvf4534,nvf4511,nvf4533,nvf3289;shfl.sync.idx.b32 nvr933|nvp619,nvr920,13,31,-1;mov.b32 nvf4535,nvr933;fma.rn.ftz.f32 nvf4536,nvf4511,nvf4535,nvf3353;shfl.sync.idx.b32 nvr934|nvp620,nvr920,14,31,-1;mov.b32 nvf4537,nvr934;fma.rn.ftz.f32 nvf4538,nvf4511,nvf4537,nvf3417;shfl.sync.idx.b32 nvr935|nvp621,nvr920,15,31,-1;mov.b32 nvf4539,nvr935;fma.rn.ftz.f32 nvf4540,nvf4511,nvf4539,nvf3481;shfl.sync.idx.b32 nvr936|nvp622,nvr920,16,31,-1;mov.b32 nvf4541,nvr936;fma.rn.ftz.f32 nvf4542,nvf4511,nvf4541,nvf3545;shfl.sync.idx.b32 nvr937|nvp623,nvr920,17,31,-1;mov.b32 nvf4543,nvr937;fma.rn.ftz.f32 nvf4544,nvf4511,nvf4543,nvf3609;shfl.sync.idx.b32 nvr938|nvp624,nvr920,18,31,-1;mov.b32 nvf4545,nvr938;fma.rn.ftz.f32 nvf4546,nvf4511,nvf4545,nvf3673;shfl.sync.idx.b32 nvr939|nvp625,nvr920,19,31,-1;mov.b32 nvf4547,nvr939;fma.rn.ftz.f32 nvf4548,nvf4511,nvf4547,nvf3737;shfl.sync.idx.b32 nvr940|nvp626,nvr920,20,31,-1;mov.b32 nvf4549,nvr940;fma.rn.ftz.f32 nvf4550,nvf4511,nvf4549,nvf3801;shfl.sync.idx.b32 nvr941|nvp627,nvr920,21,31,-1;mov.b32 nvf4551,nvr941;fma.rn.ftz.f32 nvf4552,nvf4511,nvf4551,nvf3865;shfl.sync.idx.b32 nvr942|nvp628,nvr920,22,31,-1;mov.b32 nvf4553,nvr942;fma.rn.ftz.f32 nvf4554,nvf4511,nvf4553,nvf3929;shfl.sync.idx.b32 nvr943|nvp629,nvr920,23,31,-1;mov.b32 nvf4555,nvr943;fma.rn.ftz.f32 nvf4556,nvf4511,nvf4555,nvf3993;shfl.sync.idx.b32 nvr944|nvp630,nvr920,24,31,-1;mov.b32 nvf4557,nvr944;fma.rn.ftz.f32 nvf4558,nvf4511,nvf4557,nvf4057;shfl.sync.idx.b32 nvr945|nvp631,nvr920,25,31,-1;mov.b32 nvf4559,nvr945;fma.rn.ftz.f32 nvf4560,nvf4511,nvf4559,nvf4121;shfl.sync.idx.b32 nvr946|nvp632,nvr920,26,31,-1;mov.b32 nvf4561,nvr946;fma.rn.ftz.f32 nvf4562,nvf4511,nvf4561,nvf4185;shfl.sync.idx.b32 nvr947|nvp633,nvr920,27,31,-1;mov.b32 nvf4563,nvr947;fma.rn.ftz.f32 nvf4564,nvf4511,nvf4563,nvf4249;shfl.sync.idx.b32 nvr948|nvp634,nvr920,28,31,-1;mov.b32 nvf4565,nvr948;fma.rn.ftz.f32 nvf4566,nvf4511,nvf4565,nvf4313;shfl.sync.idx.b32 nvr949|nvp635,nvr920,29,31,-1;mov.b32 nvf4567,nvr949;fma.rn.ftz.f32 nvf4568,nvf4511,nvf4567,nvf4377;shfl.sync.idx.b32 nvr950|nvp636,nvr920,30,31,-1;mov.b32 nvf4569,nvr950;fma.rn.ftz.f32 nvf4570,nvf4511,nvf4569,nvf4441;shfl.sync.idx.b32 nvr951|nvp637,nvr920,31,31,-1;mov.b32 nvf4571,nvr951;fma.rn.ftz.f32 nvf4572,nvf4511,nvf4571,nvf4505;max.ftz.f32 nvf4573,nvf4512,0f00800000;sqrt.approx.ftz.f32 nvf4574,nvf4573;mov.b32 nvr952,nvf4574;shfl.sync.idx.b32 nvr953|nvp638,nvr952,1,31,-1;mov.b32 nvf4575,nvr953;div.approx.ftz.f32 nvf4576,nvf4512,nvf4575;mov.b32 nvr954,nvf4576;shfl.sync.idx.b32 nvr955|nvp639,nvr954,2,31,-1;mov.b32 nvf4577,nvr955;neg.ftz.f32 nvf4578,nvf4576;fma.rn.ftz.f32 nvf4579,nvf4578,nvf4577,nvf4514;shfl.sync.idx.b32 nvr956|nvp640,nvr954,3,31,-1;mov.b32 nvf4580,nvr956;fma.rn.ftz.f32 nvf4581,nvf4578,nvf4580,nvf4516;shfl.sync.idx.b32 nvr957|nvp641,nvr954,4,31,-1;mov.b32 nvf4582,nvr957;fma.rn.ftz.f32 nvf4583,nvf4578,nvf4582,nvf4518;shfl.sync.idx.b32 nvr958|nvp642,nvr954,5,31,-1;mov.b32 nvf4584,nvr958;fma.rn.ftz.f32 nvf4585,nvf4578,nvf4584,nvf4520;shfl.sync.idx.b32 nvr959|nvp643,nvr954,6,31,-1;mov.b32 nvf4586,nvr959;fma.rn.ftz.f32 nvf4587,nvf4578,nvf4586,nvf4522;shfl.sync.idx.b32 nvr960|nvp644,nvr954,7,31,-1;mov.b32 nvf4588,nvr960;fma.rn.ftz.f32 nvf4589,nvf4578,nvf4588,nvf4524;shfl.sync.idx.b32 nvr961|nvp645,nvr954,8,31,-1;mov.b32 nvf4590,nvr961;fma.rn.ftz.f32 nvf4591,nvf4578,nvf4590,nvf4526;shfl.sync.idx.b32 nvr962|nvp646,nvr954,9,31,-1;mov.b32 nvf4592,nvr962;fma.rn.ftz.f32 nvf4593,nvf4578,nvf4592,nvf4528;shfl.sync.idx.b32 nvr963|nvp647,nvr954,10,31,-1;mov.b32 nvf4594,nvr963;fma.rn.ftz.f32 nvf4595,nvf4578,nvf4594,nvf4530;shfl.sync.idx.b32 nvr964|nvp648,nvr954,11,31,-1;mov.b32 nvf4596,nvr964;fma.rn.ftz.f32 nvf4597,nvf4578,nvf4596,nvf4532;shfl.sync.idx.b32 nvr965|nvp649,nvr954,12,31,-1;mov.b32 nvf4598,nvr965;fma.rn.ftz.f32 nvf4599,nvf4578,nvf4598,nvf4534;shfl.sync.idx.b32 nvr966|nvp650,nvr954,13,31,-1;mov.b32 nvf4600,nvr966;fma.rn.ftz.f32 nvf4601,nvf4578,nvf4600,nvf4536;shfl.sync.idx.b32 nvr967|nvp651,nvr954,14,31,-1;mov.b32 nvf4602,nvr967;fma.rn.ftz.f32 nvf4603,nvf4578,nvf4602,nvf4538;shfl.sync.idx.b32 nvr968|nvp652,nvr954,15,31,-1;mov.b32 nvf4604,nvr968;fma.rn.ftz.f32 nvf4605,nvf4578,nvf4604,nvf4540;shfl.sync.idx.b32 nvr969|nvp653,nvr954,16,31,-1;mov.b32 nvf4606,nvr969;fma.rn.ftz.f32 nvf4607,nvf4578,nvf4606,nvf4542;shfl.sync.idx.b32 nvr970|nvp654,nvr954,17,31,-1;mov.b32 nvf4608,nvr970;fma.rn.ftz.f32 nvf4609,nvf4578,nvf4608,nvf4544;shfl.sync.idx.b32 nvr971|nvp655,nvr954,18,31,-1;mov.b32 nvf4610,nvr971;fma.rn.ftz.f32 nvf4611,nvf4578,nvf4610,nvf4546;shfl.sync.idx.b32 nvr972|nvp656,nvr954,19,31,-1;mov.b32 nvf4612,nvr972;fma.rn.ftz.f32 nvf4613,nvf4578,nvf4612,nvf4548;shfl.sync.idx.b32 nvr973|nvp657,nvr954,20,31,-1;mov.b32 nvf4614,nvr973;fma.rn.ftz.f32 nvf4615,nvf4578,nvf4614,nvf4550;shfl.sync.idx.b32 nvr974|nvp658,nvr954,21,31,-1;mov.b32 nvf4616,nvr974;fma.rn.ftz.f32 nvf4617,nvf4578,nvf4616,nvf4552;shfl.sync.idx.b32 nvr975|nvp659,nvr954,22,31,-1;mov.b32 nvf4618,nvr975;fma.rn.ftz.f32 nvf4619,nvf4578,nvf4618,nvf4554;shfl.sync.idx.b32 nvr976|nvp660,nvr954,23,31,-1;mov.b32 nvf4620,nvr976;fma.rn.ftz.f32 nvf4621,nvf4578,nvf4620,nvf4556;shfl.sync.idx.b32 nvr977|nvp661,nvr954,24,31,-1;mov.b32 nvf4622,nvr977;fma.rn.ftz.f32 nvf4623,nvf4578,nvf4622,nvf4558;shfl.sync.idx.b32 nvr978|nvp662,nvr954,25,31,-1;mov.b32 nvf4624,nvr978;fma.rn.ftz.f32 nvf4625,nvf4578,nvf4624,nvf4560;shfl.sync.idx.b32 nvr979|nvp663,nvr954,26,31,-1;mov.b32 nvf4626,nvr979;fma.rn.ftz.f32 nvf4627,nvf4578,nvf4626,nvf4562;shfl.sync.idx.b32 nvr980|nvp664,nvr954,27,31,-1;mov.b32 nvf4628,nvr980;fma.rn.ftz.f32 nvf4629,nvf4578,nvf4628,nvf4564;shfl.sync.idx.b32 nvr981|nvp665,nvr954,28,31,-1;mov.b32 nvf4630,nvr981;fma.rn.ftz.f32 nvf4631,nvf4578,nvf4630,nvf4566;shfl.sync.idx.b32 nvr982|nvp666,nvr954,29,31,-1;mov.b32 nvf4632,nvr982;fma.rn.ftz.f32 nvf4633,nvf4578,nvf4632,nvf4568;shfl.sync.idx.b32 nvr983|nvp667,nvr954,30,31,-1;mov.b32 nvf4634,nvr983;fma.rn.ftz.f32 nvf4635,nvf4578,nvf4634,nvf4570;shfl.sync.idx.b32 nvr984|nvp668,nvr954,31,31,-1;mov.b32 nvf4636,nvr984;fma.rn.ftz.f32 nvf4637,nvf4578,nvf4636,nvf4572;max.ftz.f32 nvf4638,nvf4579,0f00800000;sqrt.approx.ftz.f32 nvf4639,nvf4638;mov.b32 nvr985,nvf4639;shfl.sync.idx.b32 nvr986|nvp669,nvr985,2,31,-1;mov.b32 nvf4640,nvr986;div.approx.ftz.f32 nvf4641,nvf4579,nvf4640;mov.b32 nvr987,nvf4641;shfl.sync.idx.b32 nvr988|nvp670,nvr987,3,31,-1;mov.b32 nvf4642,nvr988;neg.ftz.f32 nvf4643,nvf4641;fma.rn.ftz.f32 nvf4644,nvf4643,nvf4642,nvf4581;shfl.sync.idx.b32 nvr989|nvp671,nvr987,4,31,-1;mov.b32 nvf4645,nvr989;fma.rn.ftz.f32 nvf4646,nvf4643,nvf4645,nvf4583;shfl.sync.idx.b32 nvr990|nvp672,nvr987,5,31,-1;mov.b32 nvf4647,nvr990;fma.rn.ftz.f32 nvf4648,nvf4643,nvf4647,nvf4585;shfl.sync.idx.b32 nvr991|nvp673,nvr987,6,31,-1;mov.b32 nvf4649,nvr991;fma.rn.ftz.f32 nvf4650,nvf4643,nvf4649,nvf4587;shfl.sync.idx.b32 nvr992|nvp674,nvr987,7,31,-1;mov.b32 nvf4651,nvr992;fma.rn.ftz.f32 nvf4652,nvf4643,nvf4651,nvf4589;shfl.sync.idx.b32 nvr993|nvp675,nvr987,8,31,-1;mov.b32 nvf4653,nvr993;fma.rn.ftz.f32 nvf4654,nvf4643,nvf4653,nvf4591;shfl.sync.idx.b32 nvr994|nvp676,nvr987,9,31,-1;mov.b32 nvf4655,nvr994;fma.rn.ftz.f32 nvf4656,nvf4643,nvf4655,nvf4593;shfl.sync.idx.b32 nvr995|nvp677,nvr987,10,31,-1;mov.b32 nvf4657,nvr995;fma.rn.ftz.f32 nvf4658,nvf4643,nvf4657,nvf4595;shfl.sync.idx.b32 nvr996|nvp678,nvr987,11,31,-1;mov.b32 nvf4659,nvr996;fma.rn.ftz.f32 nvf4660,nvf4643,nvf4659,nvf4597;shfl.sync.idx.b32 nvr997|nvp679,nvr987,12,31,-1;mov.b32 nvf4661,nvr997;fma.rn.ftz.f32 nvf4662,nvf4643,nvf4661,nvf4599;shfl.sync.idx.b32 nvr998|nvp680,nvr987,13,31,-1;mov.b32 nvf4663,nvr998;fma.rn.ftz.f32 nvf4664,nvf4643,nvf4663,nvf4601;shfl.sync.idx.b32 nvr999|nvp681,nvr987,14,31,-1;mov.b32 nvf4665,nvr999;fma.rn.ftz.f32 nvf4666,nvf4643,nvf4665,nvf4603;shfl.sync.idx.b32 nvr1000|nvp682,nvr987,15,31,-1;mov.b32 nvf4667,nvr1000;fma.rn.ftz.f32 nvf4668,nvf4643,nvf4667,nvf4605;shfl.sync.idx.b32 nvr1001|nvp683,nvr987,16,31,-1;mov.b32 nvf4669,nvr1001;fma.rn.ftz.f32 nvf4670,nvf4643,nvf4669,nvf4607;shfl.sync.idx.b32 nvr1002|nvp684,nvr987,17,31,-1;mov.b32 nvf4671,nvr1002;fma.rn.ftz.f32 nvf4672,nvf4643,nvf4671,nvf4609;shfl.sync.idx.b32 nvr1003|nvp685,nvr987,18,31,-1;mov.b32 nvf4673,nvr1003;fma.rn.ftz.f32 nvf4674,nvf4643,nvf4673,nvf4611;shfl.sync.idx.b32 nvr1004|nvp686,nvr987,19,31,-1;mov.b32 nvf4675,nvr1004;fma.rn.ftz.f32 nvf4676,nvf4643,nvf4675,nvf4613;shfl.sync.idx.b32 nvr1005|nvp687,nvr987,20,31,-1;mov.b32 nvf4677,nvr1005;fma.rn.ftz.f32 nvf4678,nvf4643,nvf4677,nvf4615;shfl.sync.idx.b32 nvr1006|nvp688,nvr987,21,31,-1;mov.b32 nvf4679,nvr1006;fma.rn.ftz.f32 nvf4680,nvf4643,nvf4679,nvf4617;shfl.sync.idx.b32 nvr1007|nvp689,nvr987,22,31,-1;mov.b32 nvf4681,nvr1007;fma.rn.ftz.f32 nvf4682,nvf4643,nvf4681,nvf4619;shfl.sync.idx.b32 nvr1008|nvp690,nvr987,23,31,-1;mov.b32 nvf4683,nvr1008;fma.rn.ftz.f32 nvf4684,nvf4643,nvf4683,nvf4621;shfl.sync.idx.b32 nvr1009|nvp691,nvr987,24,31,-1;mov.b32 nvf4685,nvr1009;fma.rn.ftz.f32 nvf4686,nvf4643,nvf4685,nvf4623;shfl.sync.idx.b32 nvr1010|nvp692,nvr987,25,31,-1;mov.b32 nvf4687,nvr1010;fma.rn.ftz.f32 nvf4688,nvf4643,nvf4687,nvf4625;shfl.sync.idx.b32 nvr1011|nvp693,nvr987,26,31,-1;mov.b32 nvf4689,nvr1011;fma.rn.ftz.f32 nvf4690,nvf4643,nvf4689,nvf4627;shfl.sync.idx.b32 nvr1012|nvp694,nvr987,27,31,-1;mov.b32 nvf4691,nvr1012;fma.rn.ftz.f32 nvf4692,nvf4643,nvf4691,nvf4629;shfl.sync.idx.b32 nvr1013|nvp695,nvr987,28,31,-1;mov.b32 nvf4693,nvr1013;fma.rn.ftz.f32 nvf4694,nvf4643,nvf4693,nvf4631;shfl.sync.idx.b32 nvr1014|nvp696,nvr987,29,31,-1;mov.b32 nvf4695,nvr1014;fma.rn.ftz.f32 nvf4696,nvf4643,nvf4695,nvf4633;shfl.sync.idx.b32 nvr1015|nvp697,nvr987,30,31,-1;mov.b32 nvf4697,nvr1015;fma.rn.ftz.f32 nvf4698,nvf4643,nvf4697,nvf4635;shfl.sync.idx.b32 nvr1016|nvp698,nvr987,31,31,-1;mov.b32 nvf4699,nvr1016;fma.rn.ftz.f32 nvf4700,nvf4643,nvf4699,nvf4637;max.ftz.f32 nvf4701,nvf4644,0f00800000;sqrt.approx.ftz.f32 nvf4702,nvf4701;mov.b32 nvr1017,nvf4702;shfl.sync.idx.b32 nvr1018|nvp699,nvr1017,3,31,-1;mov.b32 nvf4703,nvr1018;div.approx.ftz.f32 nvf4704,nvf4644,nvf4703;mov.b32 nvr1019,nvf4704;shfl.sync.idx.b32 nvr1020|nvp700,nvr1019,4,31,-1;mov.b32 nvf4705,nvr1020;neg.ftz.f32 nvf4706,nvf4704;fma.rn.ftz.f32 nvf4707,nvf4706,nvf4705,nvf4646;shfl.sync.idx.b32 nvr1021|nvp701,nvr1019,5,31,-1;mov.b32 nvf4708,nvr1021;fma.rn.ftz.f32 nvf4709,nvf4706,nvf4708,nvf4648;shfl.sync.idx.b32 nvr1022|nvp702,nvr1019,6,31,-1;mov.b32 nvf4710,nvr1022;fma.rn.ftz.f32 nvf4711,nvf4706,nvf4710,nvf4650;shfl.sync.idx.b32 nvr1023|nvp703,nvr1019,7,31,-1;mov.b32 nvf4712,nvr1023;fma.rn.ftz.f32 nvf4713,nvf4706,nvf4712,nvf4652;shfl.sync.idx.b32 nvr1024|nvp704,nvr1019,8,31,-1;mov.b32 nvf4714,nvr1024;fma.rn.ftz.f32 nvf4715,nvf4706,nvf4714,nvf4654;shfl.sync.idx.b32 nvr1025|nvp705,nvr1019,9,31,-1;mov.b32 nvf4716,nvr1025;fma.rn.ftz.f32 nvf4717,nvf4706,nvf4716,nvf4656;shfl.sync.idx.b32 nvr1026|nvp706,nvr1019,10,31,-1;mov.b32 nvf4718,nvr1026;fma.rn.ftz.f32 nvf4719,nvf4706,nvf4718,nvf4658;shfl.sync.idx.b32 nvr1027|nvp707,nvr1019,11,31,-1;mov.b32 nvf4720,nvr1027;fma.rn.ftz.f32 nvf4721,nvf4706,nvf4720,nvf4660;shfl.sync.idx.b32 nvr1028|nvp708,nvr1019,12,31,-1;mov.b32 nvf4722,nvr1028;fma.rn.ftz.f32 nvf4723,nvf4706,nvf4722,nvf4662;shfl.sync.idx.b32 nvr1029|nvp709,nvr1019,13,31,-1;mov.b32 nvf4724,nvr1029;fma.rn.ftz.f32 nvf4725,nvf4706,nvf4724,nvf4664;shfl.sync.idx.b32 nvr1030|nvp710,nvr1019,14,31,-1;mov.b32 nvf4726,nvr1030;fma.rn.ftz.f32 nvf4727,nvf4706,nvf4726,nvf4666;shfl.sync.idx.b32 nvr1031|nvp711,nvr1019,15,31,-1;mov.b32 nvf4728,nvr1031;fma.rn.ftz.f32 nvf4729,nvf4706,nvf4728,nvf4668;shfl.sync.idx.b32 nvr1032|nvp712,nvr1019,16,31,-1;mov.b32 nvf4730,nvr1032;fma.rn.ftz.f32 nvf4731,nvf4706,nvf4730,nvf4670;shfl.sync.idx.b32 nvr1033|nvp713,nvr1019,17,31,-1;mov.b32 nvf4732,nvr1033;fma.rn.ftz.f32 nvf4733,nvf4706,nvf4732,nvf4672;shfl.sync.idx.b32 nvr1034|nvp714,nvr1019,18,31,-1;mov.b32 nvf4734,nvr1034;fma.rn.ftz.f32 nvf4735,nvf4706,nvf4734,nvf4674;shfl.sync.idx.b32 nvr1035|nvp715,nvr1019,19,31,-1;mov.b32 nvf4736,nvr1035;fma.rn.ftz.f32 nvf4737,nvf4706,nvf4736,nvf4676;shfl.sync.idx.b32 nvr1036|nvp716,nvr1019,20,31,-1;mov.b32 nvf4738,nvr1036;fma.rn.ftz.f32 nvf4739,nvf4706,nvf4738,nvf4678;shfl.sync.idx.b32 nvr1037|nvp717,nvr1019,21,31,-1;mov.b32 nvf4740,nvr1037;fma.rn.ftz.f32 nvf4741,nvf4706,nvf4740,nvf4680;shfl.sync.idx.b32 nvr1038|nvp718,nvr1019,22,31,-1;mov.b32 nvf4742,nvr1038;fma.rn.ftz.f32 nvf4743,nvf4706,nvf4742,nvf4682;shfl.sync.idx.b32 nvr1039|nvp719,nvr1019,23,31,-1;mov.b32 nvf4744,nvr1039;fma.rn.ftz.f32 nvf4745,nvf4706,nvf4744,nvf4684;shfl.sync.idx.b32 nvr1040|nvp720,nvr1019,24,31,-1;mov.b32 nvf4746,nvr1040;fma.rn.ftz.f32 nvf4747,nvf4706,nvf4746,nvf4686;shfl.sync.idx.b32 nvr1041|nvp721,nvr1019,25,31,-1;mov.b32 nvf4748,nvr1041;fma.rn.ftz.f32 nvf4749,nvf4706,nvf4748,nvf4688;shfl.sync.idx.b32 nvr1042|nvp722,nvr1019,26,31,-1;mov.b32 nvf4750,nvr1042;fma.rn.ftz.f32 nvf4751,nvf4706,nvf4750,nvf4690;shfl.sync.idx.b32 nvr1043|nvp723,nvr1019,27,31,-1;mov.b32 nvf4752,nvr1043;fma.rn.ftz.f32 nvf4753,nvf4706,nvf4752,nvf4692;shfl.sync.idx.b32 nvr1044|nvp724,nvr1019,28,31,-1;mov.b32 nvf4754,nvr1044;fma.rn.ftz.f32 nvf4755,nvf4706,nvf4754,nvf4694;shfl.sync.idx.b32 nvr1045|nvp725,nvr1019,29,31,-1;mov.b32 nvf4756,nvr1045;fma.rn.ftz.f32 nvf4757,nvf4706,nvf4756,nvf4696;shfl.sync.idx.b32 nvr1046|nvp726,nvr1019,30,31,-1;mov.b32 nvf4758,nvr1046;fma.rn.ftz.f32 nvf4759,nvf4706,nvf4758,nvf4698;shfl.sync.idx.b32 nvr1047|nvp727,nvr1019,31,31,-1;mov.b32 nvf4760,nvr1047;fma.rn.ftz.f32 nvf4761,nvf4706,nvf4760,nvf4700;max.ftz.f32 nvf4762,nvf4707,0f00800000;sqrt.approx.ftz.f32 nvf4763,nvf4762;mov.b32 nvr1048,nvf4763;shfl.sync.idx.b32 nvr1049|nvp728,nvr1048,4,31,-1;mov.b32 nvf4764,nvr1049;div.approx.ftz.f32 nvf4765,nvf4707,nvf4764;mov.b32 nvr1050,nvf4765;shfl.sync.idx.b32 nvr1051|nvp729,nvr1050,5,31,-1;mov.b32 nvf4766,nvr1051;neg.ftz.f32 nvf4767,nvf4765;fma.rn.ftz.f32 nvf4768,nvf4767,nvf4766,nvf4709;shfl.sync.idx.b32 nvr1052|nvp730,nvr1050,6,31,-1;mov.b32 nvf4769,nvr1052;fma.rn.ftz.f32 nvf4770,nvf4767,nvf4769,nvf4711;shfl.sync.idx.b32 nvr1053|nvp731,nvr1050,7,31,-1;mov.b32 nvf4771,nvr1053;fma.rn.ftz.f32 nvf4772,nvf4767,nvf4771,nvf4713;shfl.sync.idx.b32 nvr1054|nvp732,nvr1050,8,31,-1;mov.b32 nvf4773,nvr1054;fma.rn.ftz.f32 nvf4774,nvf4767,nvf4773,nvf4715;shfl.sync.idx.b32 nvr1055|nvp733,nvr1050,9,31,-1;mov.b32 nvf4775,nvr1055;fma.rn.ftz.f32 nvf4776,nvf4767,nvf4775,nvf4717;shfl.sync.idx.b32 nvr1056|nvp734,nvr1050,10,31,-1;mov.b32 nvf4777,nvr1056;fma.rn.ftz.f32 nvf4778,nvf4767,nvf4777,nvf4719;shfl.sync.idx.b32 nvr1057|nvp735,nvr1050,11,31,-1;mov.b32 nvf4779,nvr1057;fma.rn.ftz.f32 nvf4780,nvf4767,nvf4779,nvf4721;shfl.sync.idx.b32 nvr1058|nvp736,nvr1050,12,31,-1;mov.b32 nvf4781,nvr1058;fma.rn.ftz.f32 nvf4782,nvf4767,nvf4781,nvf4723;shfl.sync.idx.b32 nvr1059|nvp737,nvr1050,13,31,-1;mov.b32 nvf4783,nvr1059;fma.rn.ftz.f32 nvf4784,nvf4767,nvf4783,nvf4725;shfl.sync.idx.b32 nvr1060|nvp738,nvr1050,14,31,-1;mov.b32 nvf4785,nvr1060;fma.rn.ftz.f32 nvf4786,nvf4767,nvf4785,nvf4727;shfl.sync.idx.b32 nvr1061|nvp739,nvr1050,15,31,-1;mov.b32 nvf4787,nvr1061;fma.rn.ftz.f32 nvf4788,nvf4767,nvf4787,nvf4729;shfl.sync.idx.b32 nvr1062|nvp740,nvr1050,16,31,-1;mov.b32 nvf4789,nvr1062;fma.rn.ftz.f32 nvf4790,nvf4767,nvf4789,nvf4731;shfl.sync.idx.b32 nvr1063|nvp741,nvr1050,17,31,-1;mov.b32 nvf4791,nvr1063;fma.rn.ftz.f32 nvf4792,nvf4767,nvf4791,nvf4733;shfl.sync.idx.b32 nvr1064|nvp742,nvr1050,18,31,-1;mov.b32 nvf4793,nvr1064;fma.rn.ftz.f32 nvf4794,nvf4767,nvf4793,nvf4735;shfl.sync.idx.b32 nvr1065|nvp743,nvr1050,19,31,-1;mov.b32 nvf4795,nvr1065;fma.rn.ftz.f32 nvf4796,nvf4767,nvf4795,nvf4737;shfl.sync.idx.b32 nvr1066|nvp744,nvr1050,20,31,-1;mov.b32 nvf4797,nvr1066;fma.rn.ftz.f32 nvf4798,nvf4767,nvf4797,nvf4739;shfl.sync.idx.b32 nvr1067|nvp745,nvr1050,21,31,-1;mov.b32 nvf4799,nvr1067;fma.rn.ftz.f32 nvf4800,nvf4767,nvf4799,nvf4741;shfl.sync.idx.b32 nvr1068|nvp746,nvr1050,22,31,-1;mov.b32 nvf4801,nvr1068;fma.rn.ftz.f32 nvf4802,nvf4767,nvf4801,nvf4743;shfl.sync.idx.b32 nvr1069|nvp747,nvr1050,23,31,-1;mov.b32 nvf4803,nvr1069;fma.rn.ftz.f32 nvf4804,nvf4767,nvf4803,nvf4745;shfl.sync.idx.b32 nvr1070|nvp748,nvr1050,24,31,-1;mov.b32 nvf4805,nvr1070;fma.rn.ftz.f32 nvf4806,nvf4767,nvf4805,nvf4747;shfl.sync.idx.b32 nvr1071|nvp749,nvr1050,25,31,-1;mov.b32 nvf4807,nvr1071;fma.rn.ftz.f32 nvf4808,nvf4767,nvf4807,nvf4749;shfl.sync.idx.b32 nvr1072|nvp750,nvr1050,26,31,-1;mov.b32 nvf4809,nvr1072;fma.rn.ftz.f32 nvf4810,nvf4767,nvf4809,nvf4751;shfl.sync.idx.b32 nvr1073|nvp751,nvr1050,27,31,-1;mov.b32 nvf4811,nvr1073;fma.rn.ftz.f32 nvf4812,nvf4767,nvf4811,nvf4753;shfl.sync.idx.b32 nvr1074|nvp752,nvr1050,28,31,-1;mov.b32 nvf4813,nvr1074;fma.rn.ftz.f32 nvf4814,nvf4767,nvf4813,nvf4755;shfl.sync.idx.b32 nvr1075|nvp753,nvr1050,29,31,-1;mov.b32 nvf4815,nvr1075;fma.rn.ftz.f32 nvf4816,nvf4767,nvf4815,nvf4757;shfl.sync.idx.b32 nvr1076|nvp754,nvr1050,30,31,-1;mov.b32 nvf4817,nvr1076;fma.rn.ftz.f32 nvf4818,nvf4767,nvf4817,nvf4759;shfl.sync.idx.b32 nvr1077|nvp755,nvr1050,31,31,-1;mov.b32 nvf4819,nvr1077;fma.rn.ftz.f32 nvf4820,nvf4767,nvf4819,nvf4761;max.ftz.f32 nvf4821,nvf4768,0f00800000;sqrt.approx.ftz.f32 nvf4822,nvf4821;mov.b32 nvr1078,nvf4822;shfl.sync.idx.b32 nvr1079|nvp756,nvr1078,5,31,-1;mov.b32 nvf4823,nvr1079;div.approx.ftz.f32 nvf4824,nvf4768,nvf4823;mov.b32 nvr1080,nvf4824;shfl.sync.idx.b32 nvr1081|nvp757,nvr1080,6,31,-1;mov.b32 nvf4825,nvr1081;neg.ftz.f32 nvf4826,nvf4824;fma.rn.ftz.f32 nvf4827,nvf4826,nvf4825,nvf4770;shfl.sync.idx.b32 nvr1082|nvp758,nvr1080,7,31,-1;mov.b32 nvf4828,nvr1082;fma.rn.ftz.f32 nvf4829,nvf4826,nvf4828,nvf4772;shfl.sync.idx.b32 nvr1083|nvp759,nvr1080,8,31,-1;mov.b32 nvf4830,nvr1083;fma.rn.ftz.f32 nvf4831,nvf4826,nvf4830,nvf4774;shfl.sync.idx.b32 nvr1084|nvp760,nvr1080,9,31,-1;mov.b32 nvf4832,nvr1084;fma.rn.ftz.f32 nvf4833,nvf4826,nvf4832,nvf4776;shfl.sync.idx.b32 nvr1085|nvp761,nvr1080,10,31,-1;mov.b32 nvf4834,nvr1085;fma.rn.ftz.f32 nvf4835,nvf4826,nvf4834,nvf4778;shfl.sync.idx.b32 nvr1086|nvp762,nvr1080,11,31,-1;mov.b32 nvf4836,nvr1086;fma.rn.ftz.f32 nvf4837,nvf4826,nvf4836,nvf4780;shfl.sync.idx.b32 nvr1087|nvp763,nvr1080,12,31,-1;mov.b32 nvf4838,nvr1087;fma.rn.ftz.f32 nvf4839,nvf4826,nvf4838,nvf4782;shfl.sync.idx.b32 nvr1088|nvp764,nvr1080,13,31,-1;mov.b32 nvf4840,nvr1088;fma.rn.ftz.f32 nvf4841,nvf4826,nvf4840,nvf4784;shfl.sync.idx.b32 nvr1089|nvp765,nvr1080,14,31,-1;mov.b32 nvf4842,nvr1089;fma.rn.ftz.f32 nvf4843,nvf4826,nvf4842,nvf4786;shfl.sync.idx.b32 nvr1090|nvp766,nvr1080,15,31,-1;mov.b32 nvf4844,nvr1090;fma.rn.ftz.f32 nvf4845,nvf4826,nvf4844,nvf4788;shfl.sync.idx.b32 nvr1091|nvp767,nvr1080,16,31,-1;mov.b32 nvf4846,nvr1091;fma.rn.ftz.f32 nvf4847,nvf4826,nvf4846,nvf4790;shfl.sync.idx.b32 nvr1092|nvp768,nvr1080,17,31,-1;mov.b32 nvf4848,nvr1092;fma.rn.ftz.f32 nvf4849,nvf4826,nvf4848,nvf4792;shfl.sync.idx.b32 nvr1093|nvp769,nvr1080,18,31,-1;mov.b32 nvf4850,nvr1093;fma.rn.ftz.f32 nvf4851,nvf4826,nvf4850,nvf4794;shfl.sync.idx.b32 nvr1094|nvp770,nvr1080,19,31,-1;mov.b32 nvf4852,nvr1094;fma.rn.ftz.f32 nvf4853,nvf4826,nvf4852,nvf4796;shfl.sync.idx.b32 nvr1095|nvp771,nvr1080,20,31,-1;mov.b32 nvf4854,nvr1095;fma.rn.ftz.f32 nvf4855,nvf4826,nvf4854,nvf4798;shfl.sync.idx.b32 nvr1096|nvp772,nvr1080,21,31,-1;mov.b32 nvf4856,nvr1096;fma.rn.ftz.f32 nvf4857,nvf4826,nvf4856,nvf4800;shfl.sync.idx.b32 nvr1097|nvp773,nvr1080,22,31,-1;mov.b32 nvf4858,nvr1097;fma.rn.ftz.f32 nvf4859,nvf4826,nvf4858,nvf4802;shfl.sync.idx.b32 nvr1098|nvp774,nvr1080,23,31,-1;mov.b32 nvf4860,nvr1098;fma.rn.ftz.f32 nvf4861,nvf4826,nvf4860,nvf4804;shfl.sync.idx.b32 nvr1099|nvp775,nvr1080,24,31,-1;mov.b32 nvf4862,nvr1099;fma.rn.ftz.f32 nvf4863,nvf4826,nvf4862,nvf4806;shfl.sync.idx.b32 nvr1100|nvp776,nvr1080,25,31,-1;mov.b32 nvf4864,nvr1100;fma.rn.ftz.f32 nvf4865,nvf4826,nvf4864,nvf4808;shfl.sync.idx.b32 nvr1101|nvp777,nvr1080,26,31,-1;mov.b32 nvf4866,nvr1101;fma.rn.ftz.f32 nvf4867,nvf4826,nvf4866,nvf4810;shfl.sync.idx.b32 nvr1102|nvp778,nvr1080,27,31,-1;mov.b32 nvf4868,nvr1102;fma.rn.ftz.f32 nvf4869,nvf4826,nvf4868,nvf4812;shfl.sync.idx.b32 nvr1103|nvp779,nvr1080,28,31,-1;mov.b32 nvf4870,nvr1103;fma.rn.ftz.f32 nvf4871,nvf4826,nvf4870,nvf4814;shfl.sync.idx.b32 nvr1104|nvp780,nvr1080,29,31,-1;mov.b32 nvf4872,nvr1104;fma.rn.ftz.f32 nvf4873,nvf4826,nvf4872,nvf4816;shfl.sync.idx.b32 nvr1105|nvp781,nvr1080,30,31,-1;mov.b32 nvf4874,nvr1105;fma.rn.ftz.f32 nvf4875,nvf4826,nvf4874,nvf4818;shfl.sync.idx.b32 nvr1106|nvp782,nvr1080,31,31,-1;mov.b32 nvf4876,nvr1106;fma.rn.ftz.f32 nvf4877,nvf4826,nvf4876,nvf4820;max.ftz.f32 nvf4878,nvf4827,0f00800000;sqrt.approx.ftz.f32 nvf4879,nvf4878;mov.b32 nvr1107,nvf4879;shfl.sync.idx.b32 nvr1108|nvp783,nvr1107,6,31,-1;mov.b32 nvf4880,nvr1108;div.approx.ftz.f32 nvf4881,nvf4827,nvf4880;mov.b32 nvr1109,nvf4881;shfl.sync.idx.b32 nvr1110|nvp784,nvr1109,7,31,-1;mov.b32 nvf4882,nvr1110;neg.ftz.f32 nvf4883,nvf4881;fma.rn.ftz.f32 nvf4884,nvf4883,nvf4882,nvf4829;shfl.sync.idx.b32 nvr1111|nvp785,nvr1109,8,31,-1;mov.b32 nvf4885,nvr1111;fma.rn.ftz.f32 nvf4886,nvf4883,nvf4885,nvf4831;shfl.sync.idx.b32 nvr1112|nvp786,nvr1109,9,31,-1;mov.b32 nvf4887,nvr1112;fma.rn.ftz.f32 nvf4888,nvf4883,nvf4887,nvf4833;shfl.sync.idx.b32 nvr1113|nvp787,nvr1109,10,31,-1;mov.b32 nvf4889,nvr1113;fma.rn.ftz.f32 nvf4890,nvf4883,nvf4889,nvf4835;shfl.sync.idx.b32 nvr1114|nvp788,nvr1109,11,31,-1;mov.b32 nvf4891,nvr1114;fma.rn.ftz.f32 nvf4892,nvf4883,nvf4891,nvf4837;shfl.sync.idx.b32 nvr1115|nvp789,nvr1109,12,31,-1;mov.b32 nvf4893,nvr1115;fma.rn.ftz.f32 nvf4894,nvf4883,nvf4893,nvf4839;shfl.sync.idx.b32 nvr1116|nvp790,nvr1109,13,31,-1;mov.b32 nvf4895,nvr1116;fma.rn.ftz.f32 nvf4896,nvf4883,nvf4895,nvf4841;shfl.sync.idx.b32 nvr1117|nvp791,nvr1109,14,31,-1;mov.b32 nvf4897,nvr1117;fma.rn.ftz.f32 nvf4898,nvf4883,nvf4897,nvf4843;shfl.sync.idx.b32 nvr1118|nvp792,nvr1109,15,31,-1;mov.b32 nvf4899,nvr1118;fma.rn.ftz.f32 nvf4900,nvf4883,nvf4899,nvf4845;shfl.sync.idx.b32 nvr1119|nvp793,nvr1109,16,31,-1;mov.b32 nvf4901,nvr1119;fma.rn.ftz.f32 nvf4902,nvf4883,nvf4901,nvf4847;shfl.sync.idx.b32 nvr1120|nvp794,nvr1109,17,31,-1;mov.b32 nvf4903,nvr1120;fma.rn.ftz.f32 nvf4904,nvf4883,nvf4903,nvf4849;shfl.sync.idx.b32 nvr1121|nvp795,nvr1109,18,31,-1;mov.b32 nvf4905,nvr1121;fma.rn.ftz.f32 nvf4906,nvf4883,nvf4905,nvf4851;shfl.sync.idx.b32 nvr1122|nvp796,nvr1109,19,31,-1;mov.b32 nvf4907,nvr1122;fma.rn.ftz.f32 nvf4908,nvf4883,nvf4907,nvf4853;shfl.sync.idx.b32 nvr1123|nvp797,nvr1109,20,31,-1;mov.b32 nvf4909,nvr1123;fma.rn.ftz.f32 nvf4910,nvf4883,nvf4909,nvf4855;shfl.sync.idx.b32 nvr1124|nvp798,nvr1109,21,31,-1;mov.b32 nvf4911,nvr1124;fma.rn.ftz.f32 nvf4912,nvf4883,nvf4911,nvf4857;shfl.sync.idx.b32 nvr1125|nvp799,nvr1109,22,31,-1;mov.b32 nvf4913,nvr1125;fma.rn.ftz.f32 nvf4914,nvf4883,nvf4913,nvf4859;shfl.sync.idx.b32 nvr1126|nvp800,nvr1109,23,31,-1;mov.b32 nvf4915,nvr1126;fma.rn.ftz.f32 nvf4916,nvf4883,nvf4915,nvf4861;shfl.sync.idx.b32 nvr1127|nvp801,nvr1109,24,31,-1;mov.b32 nvf4917,nvr1127;fma.rn.ftz.f32 nvf4918,nvf4883,nvf4917,nvf4863;shfl.sync.idx.b32 nvr1128|nvp802,nvr1109,25,31,-1;mov.b32 nvf4919,nvr1128;fma.rn.ftz.f32 nvf4920,nvf4883,nvf4919,nvf4865;shfl.sync.idx.b32 nvr1129|nvp803,nvr1109,26,31,-1;mov.b32 nvf4921,nvr1129;fma.rn.ftz.f32 nvf4922,nvf4883,nvf4921,nvf4867;shfl.sync.idx.b32 nvr1130|nvp804,nvr1109,27,31,-1;mov.b32 nvf4923,nvr1130;fma.rn.ftz.f32 nvf4924,nvf4883,nvf4923,nvf4869;shfl.sync.idx.b32 nvr1131|nvp805,nvr1109,28,31,-1;mov.b32 nvf4925,nvr1131;fma.rn.ftz.f32 nvf4926,nvf4883,nvf4925,nvf4871;shfl.sync.idx.b32 nvr1132|nvp806,nvr1109,29,31,-1;mov.b32 nvf4927,nvr1132;fma.rn.ftz.f32 nvf4928,nvf4883,nvf4927,nvf4873;shfl.sync.idx.b32 nvr1133|nvp807,nvr1109,30,31,-1;mov.b32 nvf4929,nvr1133;fma.rn.ftz.f32 nvf4930,nvf4883,nvf4929,nvf4875;shfl.sync.idx.b32 nvr1134|nvp808,nvr1109,31,31,-1;mov.b32 nvf4931,nvr1134;fma.rn.ftz.f32 nvf4932,nvf4883,nvf4931,nvf4877;max.ftz.f32 nvf4933,nvf4884,0f00800000;sqrt.approx.ftz.f32 nvf4934,nvf4933;mov.b32 nvr1135,nvf4934;shfl.sync.idx.b32 nvr1136|nvp809,nvr1135,7,31,-1;mov.b32 nvf4935,nvr1136;div.approx.ftz.f32 nvf4936,nvf4884,nvf4935;mov.b32 nvr1137,nvf4936;shfl.sync.idx.b32 nvr1138|nvp810,nvr1137,8,31,-1;mov.b32 nvf4937,nvr1138;neg.ftz.f32 nvf4938,nvf4936;fma.rn.ftz.f32 nvf4939,nvf4938,nvf4937,nvf4886;shfl.sync.idx.b32 nvr1139|nvp811,nvr1137,9,31,-1;mov.b32 nvf4940,nvr1139;fma.rn.ftz.f32 nvf4941,nvf4938,nvf4940,nvf4888;shfl.sync.idx.b32 nvr1140|nvp812,nvr1137,10,31,-1;mov.b32 nvf4942,nvr1140;fma.rn.ftz.f32 nvf4943,nvf4938,nvf4942,nvf4890;shfl.sync.idx.b32 nvr1141|nvp813,nvr1137,11,31,-1;mov.b32 nvf4944,nvr1141;fma.rn.ftz.f32 nvf4945,nvf4938,nvf4944,nvf4892;shfl.sync.idx.b32 nvr1142|nvp814,nvr1137,12,31,-1;mov.b32 nvf4946,nvr1142;fma.rn.ftz.f32 nvf4947,nvf4938,nvf4946,nvf4894;shfl.sync.idx.b32 nvr1143|nvp815,nvr1137,13,31,-1;mov.b32 nvf4948,nvr1143;fma.rn.ftz.f32 nvf4949,nvf4938,nvf4948,nvf4896;shfl.sync.idx.b32 nvr1144|nvp816,nvr1137,14,31,-1;mov.b32 nvf4950,nvr1144;fma.rn.ftz.f32 nvf4951,nvf4938,nvf4950,nvf4898;shfl.sync.idx.b32 nvr1145|nvp817,nvr1137,15,31,-1;mov.b32 nvf4952,nvr1145;fma.rn.ftz.f32 nvf4953,nvf4938,nvf4952,nvf4900;shfl.sync.idx.b32 nvr1146|nvp818,nvr1137,16,31,-1;mov.b32 nvf4954,nvr1146;fma.rn.ftz.f32 nvf4955,nvf4938,nvf4954,nvf4902;shfl.sync.idx.b32 nvr1147|nvp819,nvr1137,17,31,-1;mov.b32 nvf4956,nvr1147;fma.rn.ftz.f32 nvf4957,nvf4938,nvf4956,nvf4904;shfl.sync.idx.b32 nvr1148|nvp820,nvr1137,18,31,-1;mov.b32 nvf4958,nvr1148;fma.rn.ftz.f32 nvf4959,nvf4938,nvf4958,nvf4906;shfl.sync.idx.b32 nvr1149|nvp821,nvr1137,19,31,-1;mov.b32 nvf4960,nvr1149;fma.rn.ftz.f32 nvf4961,nvf4938,nvf4960,nvf4908;shfl.sync.idx.b32 nvr1150|nvp822,nvr1137,20,31,-1;mov.b32 nvf4962,nvr1150;fma.rn.ftz.f32 nvf4963,nvf4938,nvf4962,nvf4910;shfl.sync.idx.b32 nvr1151|nvp823,nvr1137,21,31,-1;mov.b32 nvf4964,nvr1151;fma.rn.ftz.f32 nvf4965,nvf4938,nvf4964,nvf4912;shfl.sync.idx.b32 nvr1152|nvp824,nvr1137,22,31,-1;mov.b32 nvf4966,nvr1152;fma.rn.ftz.f32 nvf4967,nvf4938,nvf4966,nvf4914;shfl.sync.idx.b32 nvr1153|nvp825,nvr1137,23,31,-1;mov.b32 nvf4968,nvr1153;fma.rn.ftz.f32 nvf4969,nvf4938,nvf4968,nvf4916;shfl.sync.idx.b32 nvr1154|nvp826,nvr1137,24,31,-1;mov.b32 nvf4970,nvr1154;fma.rn.ftz.f32 nvf4971,nvf4938,nvf4970,nvf4918;shfl.sync.idx.b32 nvr1155|nvp827,nvr1137,25,31,-1;mov.b32 nvf4972,nvr1155;fma.rn.ftz.f32 nvf4973,nvf4938,nvf4972,nvf4920;shfl.sync.idx.b32 nvr1156|nvp828,nvr1137,26,31,-1;mov.b32 nvf4974,nvr1156;fma.rn.ftz.f32 nvf4975,nvf4938,nvf4974,nvf4922;shfl.sync.idx.b32 nvr1157|nvp829,nvr1137,27,31,-1;mov.b32 nvf4976,nvr1157;fma.rn.ftz.f32 nvf4977,nvf4938,nvf4976,nvf4924;shfl.sync.idx.b32 nvr1158|nvp830,nvr1137,28,31,-1;mov.b32 nvf4978,nvr1158;fma.rn.ftz.f32 nvf4979,nvf4938,nvf4978,nvf4926;shfl.sync.idx.b32 nvr1159|nvp831,nvr1137,29,31,-1;mov.b32 nvf4980,nvr1159;fma.rn.ftz.f32 nvf4981,nvf4938,nvf4980,nvf4928;shfl.sync.idx.b32 nvr1160|nvp832,nvr1137,30,31,-1;mov.b32 nvf4982,nvr1160;fma.rn.ftz.f32 nvf4983,nvf4938,nvf4982,nvf4930;shfl.sync.idx.b32 nvr1161|nvp833,nvr1137,31,31,-1;mov.b32 nvf4984,nvr1161;fma.rn.ftz.f32 nvf4985,nvf4938,nvf4984,nvf4932;max.ftz.f32 nvf4986,nvf4939,0f00800000;sqrt.approx.ftz.f32 nvf4987,nvf4986;mov.b32 nvr1162,nvf4987;shfl.sync.idx.b32 nvr1163|nvp834,nvr1162,8,31,-1;mov.b32 nvf4988,nvr1163;div.approx.ftz.f32 nvf4989,nvf4939,nvf4988;mov.b32 nvr1164,nvf4989;shfl.sync.idx.b32 nvr1165|nvp835,nvr1164,9,31,-1;mov.b32 nvf4990,nvr1165;neg.ftz.f32 nvf4991,nvf4989;fma.rn.ftz.f32 nvf4992,nvf4991,nvf4990,nvf4941;shfl.sync.idx.b32 nvr1166|nvp836,nvr1164,10,31,-1;mov.b32 nvf4993,nvr1166;fma.rn.ftz.f32 nvf4994,nvf4991,nvf4993,nvf4943;shfl.sync.idx.b32 nvr1167|nvp837,nvr1164,11,31,-1;mov.b32 nvf4995,nvr1167;fma.rn.ftz.f32 nvf4996,nvf4991,nvf4995,nvf4945;shfl.sync.idx.b32 nvr1168|nvp838,nvr1164,12,31,-1;mov.b32 nvf4997,nvr1168;fma.rn.ftz.f32 nvf4998,nvf4991,nvf4997,nvf4947;shfl.sync.idx.b32 nvr1169|nvp839,nvr1164,13,31,-1;mov.b32 nvf4999,nvr1169;fma.rn.ftz.f32 nvf5000,nvf4991,nvf4999,nvf4949;shfl.sync.idx.b32 nvr1170|nvp840,nvr1164,14,31,-1;mov.b32 nvf5001,nvr1170;fma.rn.ftz.f32 nvf5002,nvf4991,nvf5001,nvf4951;shfl.sync.idx.b32 nvr1171|nvp841,nvr1164,15,31,-1;mov.b32 nvf5003,nvr1171;fma.rn.ftz.f32 nvf5004,nvf4991,nvf5003,nvf4953;shfl.sync.idx.b32 nvr1172|nvp842,nvr1164,16,31,-1;mov.b32 nvf5005,nvr1172;fma.rn.ftz.f32 nvf5006,nvf4991,nvf5005,nvf4955;shfl.sync.idx.b32 nvr1173|nvp843,nvr1164,17,31,-1;mov.b32 nvf5007,nvr1173;fma.rn.ftz.f32 nvf5008,nvf4991,nvf5007,nvf4957;shfl.sync.idx.b32 nvr1174|nvp844,nvr1164,18,31,-1;mov.b32 nvf5009,nvr1174;fma.rn.ftz.f32 nvf5010,nvf4991,nvf5009,nvf4959;shfl.sync.idx.b32 nvr1175|nvp845,nvr1164,19,31,-1;mov.b32 nvf5011,nvr1175;fma.rn.ftz.f32 nvf5012,nvf4991,nvf5011,nvf4961;shfl.sync.idx.b32 nvr1176|nvp846,nvr1164,20,31,-1;mov.b32 nvf5013,nvr1176;fma.rn.ftz.f32 nvf5014,nvf4991,nvf5013,nvf4963;shfl.sync.idx.b32 nvr1177|nvp847,nvr1164,21,31,-1;mov.b32 nvf5015,nvr1177;fma.rn.ftz.f32 nvf5016,nvf4991,nvf5015,nvf4965;shfl.sync.idx.b32 nvr1178|nvp848,nvr1164,22,31,-1;mov.b32 nvf5017,nvr1178;fma.rn.ftz.f32 nvf5018,nvf4991,nvf5017,nvf4967;shfl.sync.idx.b32 nvr1179|nvp849,nvr1164,23,31,-1;mov.b32 nvf5019,nvr1179;fma.rn.ftz.f32 nvf5020,nvf4991,nvf5019,nvf4969;shfl.sync.idx.b32 nvr1180|nvp850,nvr1164,24,31,-1;mov.b32 nvf5021,nvr1180;fma.rn.ftz.f32 nvf5022,nvf4991,nvf5021,nvf4971;shfl.sync.idx.b32 nvr1181|nvp851,nvr1164,25,31,-1;mov.b32 nvf5023,nvr1181;fma.rn.ftz.f32 nvf5024,nvf4991,nvf5023,nvf4973;shfl.sync.idx.b32 nvr1182|nvp852,nvr1164,26,31,-1;mov.b32 nvf5025,nvr1182;fma.rn.ftz.f32 nvf5026,nvf4991,nvf5025,nvf4975;shfl.sync.idx.b32 nvr1183|nvp853,nvr1164,27,31,-1;mov.b32 nvf5027,nvr1183;fma.rn.ftz.f32 nvf5028,nvf4991,nvf5027,nvf4977;shfl.sync.idx.b32 nvr1184|nvp854,nvr1164,28,31,-1;mov.b32 nvf5029,nvr1184;fma.rn.ftz.f32 nvf5030,nvf4991,nvf5029,nvf4979;shfl.sync.idx.b32 nvr1185|nvp855,nvr1164,29,31,-1;mov.b32 nvf5031,nvr1185;fma.rn.ftz.f32 nvf5032,nvf4991,nvf5031,nvf4981;shfl.sync.idx.b32 nvr1186|nvp856,nvr1164,30,31,-1;mov.b32 nvf5033,nvr1186;fma.rn.ftz.f32 nvf5034,nvf4991,nvf5033,nvf4983;shfl.sync.idx.b32 nvr1187|nvp857,nvr1164,31,31,-1;mov.b32 nvf5035,nvr1187;fma.rn.ftz.f32 nvf5036,nvf4991,nvf5035,nvf4985;max.ftz.f32 nvf5037,nvf4992,0f00800000;sqrt.approx.ftz.f32 nvf5038,nvf5037;mov.b32 nvr1188,nvf5038;shfl.sync.idx.b32 nvr1189|nvp858,nvr1188,9,31,-1;mov.b32 nvf5039,nvr1189;div.approx.ftz.f32 nvf5040,nvf4992,nvf5039;mov.b32 nvr1190,nvf5040;shfl.sync.idx.b32 nvr1191|nvp859,nvr1190,10,31,-1;mov.b32 nvf5041,nvr1191;neg.ftz.f32 nvf5042,nvf5040;fma.rn.ftz.f32 nvf5043,nvf5042,nvf5041,nvf4994;shfl.sync.idx.b32 nvr1192|nvp860,nvr1190,11,31,-1;mov.b32 nvf5044,nvr1192;fma.rn.ftz.f32 nvf5045,nvf5042,nvf5044,nvf4996;shfl.sync.idx.b32 nvr1193|nvp861,nvr1190,12,31,-1;mov.b32 nvf5046,nvr1193;fma.rn.ftz.f32 nvf5047,nvf5042,nvf5046,nvf4998;shfl.sync.idx.b32 nvr1194|nvp862,nvr1190,13,31,-1;mov.b32 nvf5048,nvr1194;fma.rn.ftz.f32 nvf5049,nvf5042,nvf5048,nvf5000;shfl.sync.idx.b32 nvr1195|nvp863,nvr1190,14,31,-1;mov.b32 nvf5050,nvr1195;fma.rn.ftz.f32 nvf5051,nvf5042,nvf5050,nvf5002;shfl.sync.idx.b32 nvr1196|nvp864,nvr1190,15,31,-1;mov.b32 nvf5052,nvr1196;fma.rn.ftz.f32 nvf5053,nvf5042,nvf5052,nvf5004;shfl.sync.idx.b32 nvr1197|nvp865,nvr1190,16,31,-1;mov.b32 nvf5054,nvr1197;fma.rn.ftz.f32 nvf5055,nvf5042,nvf5054,nvf5006;shfl.sync.idx.b32 nvr1198|nvp866,nvr1190,17,31,-1;mov.b32 nvf5056,nvr1198;fma.rn.ftz.f32 nvf5057,nvf5042,nvf5056,nvf5008;shfl.sync.idx.b32 nvr1199|nvp867,nvr1190,18,31,-1;mov.b32 nvf5058,nvr1199;fma.rn.ftz.f32 nvf5059,nvf5042,nvf5058,nvf5010;shfl.sync.idx.b32 nvr1200|nvp868,nvr1190,19,31,-1;mov.b32 nvf5060,nvr1200;fma.rn.ftz.f32 nvf5061,nvf5042,nvf5060,nvf5012;shfl.sync.idx.b32 nvr1201|nvp869,nvr1190,20,31,-1;mov.b32 nvf5062,nvr1201;fma.rn.ftz.f32 nvf5063,nvf5042,nvf5062,nvf5014;shfl.sync.idx.b32 nvr1202|nvp870,nvr1190,21,31,-1;mov.b32 nvf5064,nvr1202;fma.rn.ftz.f32 nvf5065,nvf5042,nvf5064,nvf5016;shfl.sync.idx.b32 nvr1203|nvp871,nvr1190,22,31,-1;mov.b32 nvf5066,nvr1203;fma.rn.ftz.f32 nvf5067,nvf5042,nvf5066,nvf5018;shfl.sync.idx.b32 nvr1204|nvp872,nvr1190,23,31,-1;mov.b32 nvf5068,nvr1204;fma.rn.ftz.f32 nvf5069,nvf5042,nvf5068,nvf5020;shfl.sync.idx.b32 nvr1205|nvp873,nvr1190,24,31,-1;mov.b32 nvf5070,nvr1205;fma.rn.ftz.f32 nvf5071,nvf5042,nvf5070,nvf5022;shfl.sync.idx.b32 nvr1206|nvp874,nvr1190,25,31,-1;mov.b32 nvf5072,nvr1206;fma.rn.ftz.f32 nvf5073,nvf5042,nvf5072,nvf5024;shfl.sync.idx.b32 nvr1207|nvp875,nvr1190,26,31,-1;mov.b32 nvf5074,nvr1207;fma.rn.ftz.f32 nvf5075,nvf5042,nvf5074,nvf5026;shfl.sync.idx.b32 nvr1208|nvp876,nvr1190,27,31,-1;mov.b32 nvf5076,nvr1208;fma.rn.ftz.f32 nvf5077,nvf5042,nvf5076,nvf5028;shfl.sync.idx.b32 nvr1209|nvp877,nvr1190,28,31,-1;mov.b32 nvf5078,nvr1209;fma.rn.ftz.f32 nvf5079,nvf5042,nvf5078,nvf5030;shfl.sync.idx.b32 nvr1210|nvp878,nvr1190,29,31,-1;mov.b32 nvf5080,nvr1210;fma.rn.ftz.f32 nvf5081,nvf5042,nvf5080,nvf5032;shfl.sync.idx.b32 nvr1211|nvp879,nvr1190,30,31,-1;mov.b32 nvf5082,nvr1211;fma.rn.ftz.f32 nvf5083,nvf5042,nvf5082,nvf5034;shfl.sync.idx.b32 nvr1212|nvp880,nvr1190,31,31,-1;mov.b32 nvf5084,nvr1212;fma.rn.ftz.f32 nvf5085,nvf5042,nvf5084,nvf5036;max.ftz.f32 nvf5086,nvf5043,0f00800000;sqrt.approx.ftz.f32 nvf5087,nvf5086;mov.b32 nvr1213,nvf5087;shfl.sync.idx.b32 nvr1214|nvp881,nvr1213,10,31,-1;mov.b32 nvf5088,nvr1214;div.approx.ftz.f32 nvf5089,nvf5043,nvf5088;mov.b32 nvr1215,nvf5089;shfl.sync.idx.b32 nvr1216|nvp882,nvr1215,11,31,-1;mov.b32 nvf5090,nvr1216;neg.ftz.f32 nvf5091,nvf5089;fma.rn.ftz.f32 nvf5092,nvf5091,nvf5090,nvf5045;shfl.sync.idx.b32 nvr1217|nvp883,nvr1215,12,31,-1;mov.b32 nvf5093,nvr1217;fma.rn.ftz.f32 nvf5094,nvf5091,nvf5093,nvf5047;shfl.sync.idx.b32 nvr1218|nvp884,nvr1215,13,31,-1;mov.b32 nvf5095,nvr1218;fma.rn.ftz.f32 nvf5096,nvf5091,nvf5095,nvf5049;shfl.sync.idx.b32 nvr1219|nvp885,nvr1215,14,31,-1;mov.b32 nvf5097,nvr1219;fma.rn.ftz.f32 nvf5098,nvf5091,nvf5097,nvf5051;shfl.sync.idx.b32 nvr1220|nvp886,nvr1215,15,31,-1;mov.b32 nvf5099,nvr1220;fma.rn.ftz.f32 nvf5100,nvf5091,nvf5099,nvf5053;shfl.sync.idx.b32 nvr1221|nvp887,nvr1215,16,31,-1;mov.b32 nvf5101,nvr1221;fma.rn.ftz.f32 nvf5102,nvf5091,nvf5101,nvf5055;shfl.sync.idx.b32 nvr1222|nvp888,nvr1215,17,31,-1;mov.b32 nvf5103,nvr1222;fma.rn.ftz.f32 nvf5104,nvf5091,nvf5103,nvf5057;shfl.sync.idx.b32 nvr1223|nvp889,nvr1215,18,31,-1;mov.b32 nvf5105,nvr1223;fma.rn.ftz.f32 nvf5106,nvf5091,nvf5105,nvf5059;shfl.sync.idx.b32 nvr1224|nvp890,nvr1215,19,31,-1;mov.b32 nvf5107,nvr1224;fma.rn.ftz.f32 nvf5108,nvf5091,nvf5107,nvf5061;shfl.sync.idx.b32 nvr1225|nvp891,nvr1215,20,31,-1;mov.b32 nvf5109,nvr1225;fma.rn.ftz.f32 nvf5110,nvf5091,nvf5109,nvf5063;shfl.sync.idx.b32 nvr1226|nvp892,nvr1215,21,31,-1;mov.b32 nvf5111,nvr1226;fma.rn.ftz.f32 nvf5112,nvf5091,nvf5111,nvf5065;shfl.sync.idx.b32 nvr1227|nvp893,nvr1215,22,31,-1;mov.b32 nvf5113,nvr1227;fma.rn.ftz.f32 nvf5114,nvf5091,nvf5113,nvf5067;shfl.sync.idx.b32 nvr1228|nvp894,nvr1215,23,31,-1;mov.b32 nvf5115,nvr1228;fma.rn.ftz.f32 nvf5116,nvf5091,nvf5115,nvf5069;shfl.sync.idx.b32 nvr1229|nvp895,nvr1215,24,31,-1;mov.b32 nvf5117,nvr1229;fma.rn.ftz.f32 nvf5118,nvf5091,nvf5117,nvf5071;shfl.sync.idx.b32 nvr1230|nvp896,nvr1215,25,31,-1;mov.b32 nvf5119,nvr1230;fma.rn.ftz.f32 nvf5120,nvf5091,nvf5119,nvf5073;shfl.sync.idx.b32 nvr1231|nvp897,nvr1215,26,31,-1;mov.b32 nvf5121,nvr1231;fma.rn.ftz.f32 nvf5122,nvf5091,nvf5121,nvf5075;shfl.sync.idx.b32 nvr1232|nvp898,nvr1215,27,31,-1;mov.b32 nvf5123,nvr1232;fma.rn.ftz.f32 nvf5124,nvf5091,nvf5123,nvf5077;shfl.sync.idx.b32 nvr1233|nvp899,nvr1215,28,31,-1;mov.b32 nvf5125,nvr1233;fma.rn.ftz.f32 nvf5126,nvf5091,nvf5125,nvf5079;shfl.sync.idx.b32 nvr1234|nvp900,nvr1215,29,31,-1;mov.b32 nvf5127,nvr1234;fma.rn.ftz.f32 nvf5128,nvf5091,nvf5127,nvf5081;shfl.sync.idx.b32 nvr1235|nvp901,nvr1215,30,31,-1;mov.b32 nvf5129,nvr1235;fma.rn.ftz.f32 nvf5130,nvf5091,nvf5129,nvf5083;shfl.sync.idx.b32 nvr1236|nvp902,nvr1215,31,31,-1;mov.b32 nvf5131,nvr1236;fma.rn.ftz.f32 nvf5132,nvf5091,nvf5131,nvf5085;max.ftz.f32 nvf5133,nvf5092,0f00800000;sqrt.approx.ftz.f32 nvf5134,nvf5133;mov.b32 nvr1237,nvf5134;shfl.sync.idx.b32 nvr1238|nvp903,nvr1237,11,31,-1;mov.b32 nvf5135,nvr1238;div.approx.ftz.f32 nvf5136,nvf5092,nvf5135;mov.b32 nvr1239,nvf5136;shfl.sync.idx.b32 nvr1240|nvp904,nvr1239,12,31,-1;mov.b32 nvf5137,nvr1240;neg.ftz.f32 nvf5138,nvf5136;fma.rn.ftz.f32 nvf5139,nvf5138,nvf5137,nvf5094;shfl.sync.idx.b32 nvr1241|nvp905,nvr1239,13,31,-1;mov.b32 nvf5140,nvr1241;fma.rn.ftz.f32 nvf5141,nvf5138,nvf5140,nvf5096;shfl.sync.idx.b32 nvr1242|nvp906,nvr1239,14,31,-1;mov.b32 nvf5142,nvr1242;fma.rn.ftz.f32 nvf5143,nvf5138,nvf5142,nvf5098;shfl.sync.idx.b32 nvr1243|nvp907,nvr1239,15,31,-1;mov.b32 nvf5144,nvr1243;fma.rn.ftz.f32 nvf5145,nvf5138,nvf5144,nvf5100;shfl.sync.idx.b32 nvr1244|nvp908,nvr1239,16,31,-1;mov.b32 nvf5146,nvr1244;fma.rn.ftz.f32 nvf5147,nvf5138,nvf5146,nvf5102;shfl.sync.idx.b32 nvr1245|nvp909,nvr1239,17,31,-1;mov.b32 nvf5148,nvr1245;fma.rn.ftz.f32 nvf5149,nvf5138,nvf5148,nvf5104;shfl.sync.idx.b32 nvr1246|nvp910,nvr1239,18,31,-1;mov.b32 nvf5150,nvr1246;fma.rn.ftz.f32 nvf5151,nvf5138,nvf5150,nvf5106;shfl.sync.idx.b32 nvr1247|nvp911,nvr1239,19,31,-1;mov.b32 nvf5152,nvr1247;fma.rn.ftz.f32 nvf5153,nvf5138,nvf5152,nvf5108;shfl.sync.idx.b32 nvr1248|nvp912,nvr1239,20,31,-1;mov.b32 nvf5154,nvr1248;fma.rn.ftz.f32 nvf5155,nvf5138,nvf5154,nvf5110;shfl.sync.idx.b32 nvr1249|nvp913,nvr1239,21,31,-1;mov.b32 nvf5156,nvr1249;fma.rn.ftz.f32 nvf5157,nvf5138,nvf5156,nvf5112;shfl.sync.idx.b32 nvr1250|nvp914,nvr1239,22,31,-1;mov.b32 nvf5158,nvr1250;fma.rn.ftz.f32 nvf5159,nvf5138,nvf5158,nvf5114;shfl.sync.idx.b32 nvr1251|nvp915,nvr1239,23,31,-1;mov.b32 nvf5160,nvr1251;fma.rn.ftz.f32 nvf5161,nvf5138,nvf5160,nvf5116;shfl.sync.idx.b32 nvr1252|nvp916,nvr1239,24,31,-1;mov.b32 nvf5162,nvr1252;fma.rn.ftz.f32 nvf5163,nvf5138,nvf5162,nvf5118;shfl.sync.idx.b32 nvr1253|nvp917,nvr1239,25,31,-1;mov.b32 nvf5164,nvr1253;fma.rn.ftz.f32 nvf5165,nvf5138,nvf5164,nvf5120;shfl.sync.idx.b32 nvr1254|nvp918,nvr1239,26,31,-1;mov.b32 nvf5166,nvr1254;fma.rn.ftz.f32 nvf5167,nvf5138,nvf5166,nvf5122;shfl.sync.idx.b32 nvr1255|nvp919,nvr1239,27,31,-1;mov.b32 nvf5168,nvr1255;fma.rn.ftz.f32 nvf5169,nvf5138,nvf5168,nvf5124;shfl.sync.idx.b32 nvr1256|nvp920,nvr1239,28,31,-1;mov.b32 nvf5170,nvr1256;fma.rn.ftz.f32 nvf5171,nvf5138,nvf5170,nvf5126;shfl.sync.idx.b32 nvr1257|nvp921,nvr1239,29,31,-1;mov.b32 nvf5172,nvr1257;fma.rn.ftz.f32 nvf5173,nvf5138,nvf5172,nvf5128;shfl.sync.idx.b32 nvr1258|nvp922,nvr1239,30,31,-1;mov.b32 nvf5174,nvr1258;fma.rn.ftz.f32 nvf5175,nvf5138,nvf5174,nvf5130;shfl.sync.idx.b32 nvr1259|nvp923,nvr1239,31,31,-1;mov.b32 nvf5176,nvr1259;fma.rn.ftz.f32 nvf5177,nvf5138,nvf5176,nvf5132;max.ftz.f32 nvf5178,nvf5139,0f00800000;sqrt.approx.ftz.f32 nvf5179,nvf5178;mov.b32 nvr1260,nvf5179;shfl.sync.idx.b32 nvr1261|nvp924,nvr1260,12,31,-1;mov.b32 nvf5180,nvr1261;div.approx.ftz.f32 nvf5181,nvf5139,nvf5180;mov.b32 nvr1262,nvf5181;shfl.sync.idx.b32 nvr1263|nvp925,nvr1262,13,31,-1;mov.b32 nvf5182,nvr1263;neg.ftz.f32 nvf5183,nvf5181;fma.rn.ftz.f32 nvf5184,nvf5183,nvf5182,nvf5141;shfl.sync.idx.b32 nvr1264|nvp926,nvr1262,14,31,-1;mov.b32 nvf5185,nvr1264;fma.rn.ftz.f32 nvf5186,nvf5183,nvf5185,nvf5143;shfl.sync.idx.b32 nvr1265|nvp927,nvr1262,15,31,-1;mov.b32 nvf5187,nvr1265;fma.rn.ftz.f32 nvf5188,nvf5183,nvf5187,nvf5145;shfl.sync.idx.b32 nvr1266|nvp928,nvr1262,16,31,-1;mov.b32 nvf5189,nvr1266;fma.rn.ftz.f32 nvf5190,nvf5183,nvf5189,nvf5147;shfl.sync.idx.b32 nvr1267|nvp929,nvr1262,17,31,-1;mov.b32 nvf5191,nvr1267;fma.rn.ftz.f32 nvf5192,nvf5183,nvf5191,nvf5149;shfl.sync.idx.b32 nvr1268|nvp930,nvr1262,18,31,-1;mov.b32 nvf5193,nvr1268;fma.rn.ftz.f32 nvf5194,nvf5183,nvf5193,nvf5151;shfl.sync.idx.b32 nvr1269|nvp931,nvr1262,19,31,-1;mov.b32 nvf5195,nvr1269;fma.rn.ftz.f32 nvf5196,nvf5183,nvf5195,nvf5153;shfl.sync.idx.b32 nvr1270|nvp932,nvr1262,20,31,-1;mov.b32 nvf5197,nvr1270;fma.rn.ftz.f32 nvf5198,nvf5183,nvf5197,nvf5155;shfl.sync.idx.b32 nvr1271|nvp933,nvr1262,21,31,-1;mov.b32 nvf5199,nvr1271;fma.rn.ftz.f32 nvf5200,nvf5183,nvf5199,nvf5157;shfl.sync.idx.b32 nvr1272|nvp934,nvr1262,22,31,-1;mov.b32 nvf5201,nvr1272;fma.rn.ftz.f32 nvf5202,nvf5183,nvf5201,nvf5159;shfl.sync.idx.b32 nvr1273|nvp935,nvr1262,23,31,-1;mov.b32 nvf5203,nvr1273;fma.rn.ftz.f32 nvf5204,nvf5183,nvf5203,nvf5161;shfl.sync.idx.b32 nvr1274|nvp936,nvr1262,24,31,-1;mov.b32 nvf5205,nvr1274;fma.rn.ftz.f32 nvf5206,nvf5183,nvf5205,nvf5163;shfl.sync.idx.b32 nvr1275|nvp937,nvr1262,25,31,-1;mov.b32 nvf5207,nvr1275;fma.rn.ftz.f32 nvf5208,nvf5183,nvf5207,nvf5165;shfl.sync.idx.b32 nvr1276|nvp938,nvr1262,26,31,-1;mov.b32 nvf5209,nvr1276;fma.rn.ftz.f32 nvf5210,nvf5183,nvf5209,nvf5167;shfl.sync.idx.b32 nvr1277|nvp939,nvr1262,27,31,-1;mov.b32 nvf5211,nvr1277;fma.rn.ftz.f32 nvf5212,nvf5183,nvf5211,nvf5169;shfl.sync.idx.b32 nvr1278|nvp940,nvr1262,28,31,-1;mov.b32 nvf5213,nvr1278;fma.rn.ftz.f32 nvf5214,nvf5183,nvf5213,nvf5171;shfl.sync.idx.b32 nvr1279|nvp941,nvr1262,29,31,-1;mov.b32 nvf5215,nvr1279;fma.rn.ftz.f32 nvf5216,nvf5183,nvf5215,nvf5173;shfl.sync.idx.b32 nvr1280|nvp942,nvr1262,30,31,-1;mov.b32 nvf5217,nvr1280;fma.rn.ftz.f32 nvf5218,nvf5183,nvf5217,nvf5175;shfl.sync.idx.b32 nvr1281|nvp943,nvr1262,31,31,-1;mov.b32 nvf5219,nvr1281;fma.rn.ftz.f32 nvf5220,nvf5183,nvf5219,nvf5177;max.ftz.f32 nvf5221,nvf5184,0f00800000;sqrt.approx.ftz.f32 nvf5222,nvf5221;mov.b32 nvr1282,nvf5222;shfl.sync.idx.b32 nvr1283|nvp944,nvr1282,13,31,-1;mov.b32 nvf5223,nvr1283;div.approx.ftz.f32 nvf5224,nvf5184,nvf5223;mov.b32 nvr1284,nvf5224;shfl.sync.idx.b32 nvr1285|nvp945,nvr1284,14,31,-1;mov.b32 nvf5225,nvr1285;neg.ftz.f32 nvf5226,nvf5224;fma.rn.ftz.f32 nvf5227,nvf5226,nvf5225,nvf5186;shfl.sync.idx.b32 nvr1286|nvp946,nvr1284,15,31,-1;mov.b32 nvf5228,nvr1286;fma.rn.ftz.f32 nvf5229,nvf5226,nvf5228,nvf5188;shfl.sync.idx.b32 nvr1287|nvp947,nvr1284,16,31,-1;mov.b32 nvf5230,nvr1287;fma.rn.ftz.f32 nvf5231,nvf5226,nvf5230,nvf5190;shfl.sync.idx.b32 nvr1288|nvp948,nvr1284,17,31,-1;mov.b32 nvf5232,nvr1288;fma.rn.ftz.f32 nvf5233,nvf5226,nvf5232,nvf5192;shfl.sync.idx.b32 nvr1289|nvp949,nvr1284,18,31,-1;mov.b32 nvf5234,nvr1289;fma.rn.ftz.f32 nvf5235,nvf5226,nvf5234,nvf5194;shfl.sync.idx.b32 nvr1290|nvp950,nvr1284,19,31,-1;mov.b32 nvf5236,nvr1290;fma.rn.ftz.f32 nvf5237,nvf5226,nvf5236,nvf5196;shfl.sync.idx.b32 nvr1291|nvp951,nvr1284,20,31,-1;mov.b32 nvf5238,nvr1291;fma.rn.ftz.f32 nvf5239,nvf5226,nvf5238,nvf5198;shfl.sync.idx.b32 nvr1292|nvp952,nvr1284,21,31,-1;mov.b32 nvf5240,nvr1292;fma.rn.ftz.f32 nvf5241,nvf5226,nvf5240,nvf5200;shfl.sync.idx.b32 nvr1293|nvp953,nvr1284,22,31,-1;mov.b32 nvf5242,nvr1293;fma.rn.ftz.f32 nvf5243,nvf5226,nvf5242,nvf5202;shfl.sync.idx.b32 nvr1294|nvp954,nvr1284,23,31,-1;mov.b32 nvf5244,nvr1294;fma.rn.ftz.f32 nvf5245,nvf5226,nvf5244,nvf5204;shfl.sync.idx.b32 nvr1295|nvp955,nvr1284,24,31,-1;mov.b32 nvf5246,nvr1295;fma.rn.ftz.f32 nvf5247,nvf5226,nvf5246,nvf5206;shfl.sync.idx.b32 nvr1296|nvp956,nvr1284,25,31,-1;mov.b32 nvf5248,nvr1296;fma.rn.ftz.f32 nvf5249,nvf5226,nvf5248,nvf5208;shfl.sync.idx.b32 nvr1297|nvp957,nvr1284,26,31,-1;mov.b32 nvf5250,nvr1297;fma.rn.ftz.f32 nvf5251,nvf5226,nvf5250,nvf5210;shfl.sync.idx.b32 nvr1298|nvp958,nvr1284,27,31,-1;mov.b32 nvf5252,nvr1298;fma.rn.ftz.f32 nvf5253,nvf5226,nvf5252,nvf5212;shfl.sync.idx.b32 nvr1299|nvp959,nvr1284,28,31,-1;mov.b32 nvf5254,nvr1299;fma.rn.ftz.f32 nvf5255,nvf5226,nvf5254,nvf5214;shfl.sync.idx.b32 nvr1300|nvp960,nvr1284,29,31,-1;mov.b32 nvf5256,nvr1300;fma.rn.ftz.f32 nvf5257,nvf5226,nvf5256,nvf5216;shfl.sync.idx.b32 nvr1301|nvp961,nvr1284,30,31,-1;mov.b32 nvf5258,nvr1301;fma.rn.ftz.f32 nvf5259,nvf5226,nvf5258,nvf5218;shfl.sync.idx.b32 nvr1302|nvp962,nvr1284,31,31,-1;mov.b32 nvf5260,nvr1302;fma.rn.ftz.f32 nvf5261,nvf5226,nvf5260,nvf5220;max.ftz.f32 nvf5262,nvf5227,0f00800000;sqrt.approx.ftz.f32 nvf5263,nvf5262;mov.b32 nvr1303,nvf5263;shfl.sync.idx.b32 nvr1304|nvp963,nvr1303,14,31,-1;mov.b32 nvf5264,nvr1304;div.approx.ftz.f32 nvf5265,nvf5227,nvf5264;mov.b32 nvr1305,nvf5265;shfl.sync.idx.b32 nvr1306|nvp964,nvr1305,15,31,-1;mov.b32 nvf5266,nvr1306;neg.ftz.f32 nvf5267,nvf5265;fma.rn.ftz.f32 nvf5268,nvf5267,nvf5266,nvf5229;shfl.sync.idx.b32 nvr1307|nvp965,nvr1305,16,31,-1;mov.b32 nvf5269,nvr1307;fma.rn.ftz.f32 nvf5270,nvf5267,nvf5269,nvf5231;shfl.sync.idx.b32 nvr1308|nvp966,nvr1305,17,31,-1;mov.b32 nvf5271,nvr1308;fma.rn.ftz.f32 nvf5272,nvf5267,nvf5271,nvf5233;shfl.sync.idx.b32 nvr1309|nvp967,nvr1305,18,31,-1;mov.b32 nvf5273,nvr1309;fma.rn.ftz.f32 nvf5274,nvf5267,nvf5273,nvf5235;shfl.sync.idx.b32 nvr1310|nvp968,nvr1305,19,31,-1;mov.b32 nvf5275,nvr1310;fma.rn.ftz.f32 nvf5276,nvf5267,nvf5275,nvf5237;shfl.sync.idx.b32 nvr1311|nvp969,nvr1305,20,31,-1;mov.b32 nvf5277,nvr1311;fma.rn.ftz.f32 nvf5278,nvf5267,nvf5277,nvf5239;shfl.sync.idx.b32 nvr1312|nvp970,nvr1305,21,31,-1;mov.b32 nvf5279,nvr1312;fma.rn.ftz.f32 nvf5280,nvf5267,nvf5279,nvf5241;shfl.sync.idx.b32 nvr1313|nvp971,nvr1305,22,31,-1;mov.b32 nvf5281,nvr1313;fma.rn.ftz.f32 nvf5282,nvf5267,nvf5281,nvf5243;shfl.sync.idx.b32 nvr1314|nvp972,nvr1305,23,31,-1;mov.b32 nvf5283,nvr1314;fma.rn.ftz.f32 nvf5284,nvf5267,nvf5283,nvf5245;shfl.sync.idx.b32 nvr1315|nvp973,nvr1305,24,31,-1;mov.b32 nvf5285,nvr1315;fma.rn.ftz.f32 nvf5286,nvf5267,nvf5285,nvf5247;shfl.sync.idx.b32 nvr1316|nvp974,nvr1305,25,31,-1;mov.b32 nvf5287,nvr1316;fma.rn.ftz.f32 nvf5288,nvf5267,nvf5287,nvf5249;shfl.sync.idx.b32 nvr1317|nvp975,nvr1305,26,31,-1;mov.b32 nvf5289,nvr1317;fma.rn.ftz.f32 nvf5290,nvf5267,nvf5289,nvf5251;shfl.sync.idx.b32 nvr1318|nvp976,nvr1305,27,31,-1;mov.b32 nvf5291,nvr1318;fma.rn.ftz.f32 nvf5292,nvf5267,nvf5291,nvf5253;shfl.sync.idx.b32 nvr1319|nvp977,nvr1305,28,31,-1;mov.b32 nvf5293,nvr1319;fma.rn.ftz.f32 nvf5294,nvf5267,nvf5293,nvf5255;shfl.sync.idx.b32 nvr1320|nvp978,nvr1305,29,31,-1;mov.b32 nvf5295,nvr1320;fma.rn.ftz.f32 nvf5296,nvf5267,nvf5295,nvf5257;shfl.sync.idx.b32 nvr1321|nvp979,nvr1305,30,31,-1;mov.b32 nvf5297,nvr1321;fma.rn.ftz.f32 nvf5298,nvf5267,nvf5297,nvf5259;shfl.sync.idx.b32 nvr1322|nvp980,nvr1305,31,31,-1;mov.b32 nvf5299,nvr1322;fma.rn.ftz.f32 nvf5300,nvf5267,nvf5299,nvf5261;max.ftz.f32 nvf5301,nvf5268,0f00800000;sqrt.approx.ftz.f32 nvf5302,nvf5301;mov.b32 nvr1323,nvf5302;shfl.sync.idx.b32 nvr1324|nvp981,nvr1323,15,31,-1;mov.b32 nvf5303,nvr1324;div.approx.ftz.f32 nvf5304,nvf5268,nvf5303;mov.b32 nvr1325,nvf5304;shfl.sync.idx.b32 nvr1326|nvp982,nvr1325,16,31,-1;mov.b32 nvf5305,nvr1326;neg.ftz.f32 nvf5306,nvf5304;fma.rn.ftz.f32 nvf5307,nvf5306,nvf5305,nvf5270;shfl.sync.idx.b32 nvr1327|nvp983,nvr1325,17,31,-1;mov.b32 nvf5308,nvr1327;fma.rn.ftz.f32 nvf5309,nvf5306,nvf5308,nvf5272;shfl.sync.idx.b32 nvr1328|nvp984,nvr1325,18,31,-1;mov.b32 nvf5310,nvr1328;fma.rn.ftz.f32 nvf5311,nvf5306,nvf5310,nvf5274;shfl.sync.idx.b32 nvr1329|nvp985,nvr1325,19,31,-1;mov.b32 nvf5312,nvr1329;fma.rn.ftz.f32 nvf5313,nvf5306,nvf5312,nvf5276;shfl.sync.idx.b32 nvr1330|nvp986,nvr1325,20,31,-1;mov.b32 nvf5314,nvr1330;fma.rn.ftz.f32 nvf5315,nvf5306,nvf5314,nvf5278;shfl.sync.idx.b32 nvr1331|nvp987,nvr1325,21,31,-1;mov.b32 nvf5316,nvr1331;fma.rn.ftz.f32 nvf5317,nvf5306,nvf5316,nvf5280;shfl.sync.idx.b32 nvr1332|nvp988,nvr1325,22,31,-1;mov.b32 nvf5318,nvr1332;fma.rn.ftz.f32 nvf5319,nvf5306,nvf5318,nvf5282;shfl.sync.idx.b32 nvr1333|nvp989,nvr1325,23,31,-1;mov.b32 nvf5320,nvr1333;fma.rn.ftz.f32 nvf5321,nvf5306,nvf5320,nvf5284;shfl.sync.idx.b32 nvr1334|nvp990,nvr1325,24,31,-1;mov.b32 nvf5322,nvr1334;fma.rn.ftz.f32 nvf5323,nvf5306,nvf5322,nvf5286;shfl.sync.idx.b32 nvr1335|nvp991,nvr1325,25,31,-1;mov.b32 nvf5324,nvr1335;fma.rn.ftz.f32 nvf5325,nvf5306,nvf5324,nvf5288;shfl.sync.idx.b32 nvr1336|nvp992,nvr1325,26,31,-1;mov.b32 nvf5326,nvr1336;fma.rn.ftz.f32 nvf5327,nvf5306,nvf5326,nvf5290;shfl.sync.idx.b32 nvr1337|nvp993,nvr1325,27,31,-1;mov.b32 nvf5328,nvr1337;fma.rn.ftz.f32 nvf5329,nvf5306,nvf5328,nvf5292;shfl.sync.idx.b32 nvr1338|nvp994,nvr1325,28,31,-1;mov.b32 nvf5330,nvr1338;fma.rn.ftz.f32 nvf5331,nvf5306,nvf5330,nvf5294;shfl.sync.idx.b32 nvr1339|nvp995,nvr1325,29,31,-1;mov.b32 nvf5332,nvr1339;fma.rn.ftz.f32 nvf5333,nvf5306,nvf5332,nvf5296;shfl.sync.idx.b32 nvr1340|nvp996,nvr1325,30,31,-1;mov.b32 nvf5334,nvr1340;fma.rn.ftz.f32 nvf5335,nvf5306,nvf5334,nvf5298;shfl.sync.idx.b32 nvr1341|nvp997,nvr1325,31,31,-1;mov.b32 nvf5336,nvr1341;fma.rn.ftz.f32 nvf5337,nvf5306,nvf5336,nvf5300;max.ftz.f32 nvf5338,nvf5307,0f00800000;sqrt.approx.ftz.f32 nvf5339,nvf5338;mov.b32 nvr1342,nvf5339;shfl.sync.idx.b32 nvr1343|nvp998,nvr1342,16,31,-1;mov.b32 nvf5340,nvr1343;div.approx.ftz.f32 nvf5341,nvf5307,nvf5340;mov.b32 nvr1344,nvf5341;shfl.sync.idx.b32 nvr1345|nvp999,nvr1344,17,31,-1;mov.b32 nvf5342,nvr1345;neg.ftz.f32 nvf5343,nvf5341;fma.rn.ftz.f32 nvf5344,nvf5343,nvf5342,nvf5309;shfl.sync.idx.b32 nvr1346|nvp1000,nvr1344,18,31,-1;mov.b32 nvf5345,nvr1346;fma.rn.ftz.f32 nvf5346,nvf5343,nvf5345,nvf5311;shfl.sync.idx.b32 nvr1347|nvp1001,nvr1344,19,31,-1;mov.b32 nvf5347,nvr1347;fma.rn.ftz.f32 nvf5348,nvf5343,nvf5347,nvf5313;shfl.sync.idx.b32 nvr1348|nvp1002,nvr1344,20,31,-1;mov.b32 nvf5349,nvr1348;fma.rn.ftz.f32 nvf5350,nvf5343,nvf5349,nvf5315;shfl.sync.idx.b32 nvr1349|nvp1003,nvr1344,21,31,-1;mov.b32 nvf5351,nvr1349;fma.rn.ftz.f32 nvf5352,nvf5343,nvf5351,nvf5317;shfl.sync.idx.b32 nvr1350|nvp1004,nvr1344,22,31,-1;mov.b32 nvf5353,nvr1350;fma.rn.ftz.f32 nvf5354,nvf5343,nvf5353,nvf5319;shfl.sync.idx.b32 nvr1351|nvp1005,nvr1344,23,31,-1;mov.b32 nvf5355,nvr1351;fma.rn.ftz.f32 nvf5356,nvf5343,nvf5355,nvf5321;shfl.sync.idx.b32 nvr1352|nvp1006,nvr1344,24,31,-1;mov.b32 nvf5357,nvr1352;fma.rn.ftz.f32 nvf5358,nvf5343,nvf5357,nvf5323;shfl.sync.idx.b32 nvr1353|nvp1007,nvr1344,25,31,-1;mov.b32 nvf5359,nvr1353;fma.rn.ftz.f32 nvf5360,nvf5343,nvf5359,nvf5325;shfl.sync.idx.b32 nvr1354|nvp1008,nvr1344,26,31,-1;mov.b32 nvf5361,nvr1354;fma.rn.ftz.f32 nvf5362,nvf5343,nvf5361,nvf5327;shfl.sync.idx.b32 nvr1355|nvp1009,nvr1344,27,31,-1;mov.b32 nvf5363,nvr1355;fma.rn.ftz.f32 nvf5364,nvf5343,nvf5363,nvf5329;shfl.sync.idx.b32 nvr1356|nvp1010,nvr1344,28,31,-1;mov.b32 nvf5365,nvr1356;fma.rn.ftz.f32 nvf5366,nvf5343,nvf5365,nvf5331;shfl.sync.idx.b32 nvr1357|nvp1011,nvr1344,29,31,-1;mov.b32 nvf5367,nvr1357;fma.rn.ftz.f32 nvf5368,nvf5343,nvf5367,nvf5333;shfl.sync.idx.b32 nvr1358|nvp1012,nvr1344,30,31,-1;mov.b32 nvf5369,nvr1358;fma.rn.ftz.f32 nvf5370,nvf5343,nvf5369,nvf5335;shfl.sync.idx.b32 nvr1359|nvp1013,nvr1344,31,31,-1;mov.b32 nvf5371,nvr1359;fma.rn.ftz.f32 nvf5372,nvf5343,nvf5371,nvf5337;max.ftz.f32 nvf5373,nvf5344,0f00800000;sqrt.approx.ftz.f32 nvf5374,nvf5373;mov.b32 nvr1360,nvf5374;shfl.sync.idx.b32 nvr1361|nvp1014,nvr1360,17,31,-1;mov.b32 nvf5375,nvr1361;div.approx.ftz.f32 nvf5376,nvf5344,nvf5375;mov.b32 nvr1362,nvf5376;shfl.sync.idx.b32 nvr1363|nvp1015,nvr1362,18,31,-1;mov.b32 nvf5377,nvr1363;neg.ftz.f32 nvf5378,nvf5376;fma.rn.ftz.f32 nvf5379,nvf5378,nvf5377,nvf5346;shfl.sync.idx.b32 nvr1364|nvp1016,nvr1362,19,31,-1;mov.b32 nvf5380,nvr1364;fma.rn.ftz.f32 nvf5381,nvf5378,nvf5380,nvf5348;shfl.sync.idx.b32 nvr1365|nvp1017,nvr1362,20,31,-1;mov.b32 nvf5382,nvr1365;fma.rn.ftz.f32 nvf5383,nvf5378,nvf5382,nvf5350;shfl.sync.idx.b32 nvr1366|nvp1018,nvr1362,21,31,-1;mov.b32 nvf5384,nvr1366;fma.rn.ftz.f32 nvf5385,nvf5378,nvf5384,nvf5352;shfl.sync.idx.b32 nvr1367|nvp1019,nvr1362,22,31,-1;mov.b32 nvf5386,nvr1367;fma.rn.ftz.f32 nvf5387,nvf5378,nvf5386,nvf5354;shfl.sync.idx.b32 nvr1368|nvp1020,nvr1362,23,31,-1;mov.b32 nvf5388,nvr1368;fma.rn.ftz.f32 nvf5389,nvf5378,nvf5388,nvf5356;shfl.sync.idx.b32 nvr1369|nvp1021,nvr1362,24,31,-1;mov.b32 nvf5390,nvr1369;fma.rn.ftz.f32 nvf5391,nvf5378,nvf5390,nvf5358;shfl.sync.idx.b32 nvr1370|nvp1022,nvr1362,25,31,-1;mov.b32 nvf5392,nvr1370;fma.rn.ftz.f32 nvf5393,nvf5378,nvf5392,nvf5360;shfl.sync.idx.b32 nvr1371|nvp1023,nvr1362,26,31,-1;mov.b32 nvf5394,nvr1371;fma.rn.ftz.f32 nvf5395,nvf5378,nvf5394,nvf5362;shfl.sync.idx.b32 nvr1372|nvp1024,nvr1362,27,31,-1;mov.b32 nvf5396,nvr1372;fma.rn.ftz.f32 nvf5397,nvf5378,nvf5396,nvf5364;shfl.sync.idx.b32 nvr1373|nvp1025,nvr1362,28,31,-1;mov.b32 nvf5398,nvr1373;fma.rn.ftz.f32 nvf5399,nvf5378,nvf5398,nvf5366;shfl.sync.idx.b32 nvr1374|nvp1026,nvr1362,29,31,-1;mov.b32 nvf5400,nvr1374;fma.rn.ftz.f32 nvf5401,nvf5378,nvf5400,nvf5368;shfl.sync.idx.b32 nvr1375|nvp1027,nvr1362,30,31,-1;mov.b32 nvf5402,nvr1375;fma.rn.ftz.f32 nvf5403,nvf5378,nvf5402,nvf5370;shfl.sync.idx.b32 nvr1376|nvp1028,nvr1362,31,31,-1;mov.b32 nvf5404,nvr1376;fma.rn.ftz.f32 nvf5405,nvf5378,nvf5404,nvf5372;max.ftz.f32 nvf5406,nvf5379,0f00800000;sqrt.approx.ftz.f32 nvf5407,nvf5406;mov.b32 nvr1377,nvf5407;shfl.sync.idx.b32 nvr1378|nvp1029,nvr1377,18,31,-1;mov.b32 nvf5408,nvr1378;div.approx.ftz.f32 nvf5409,nvf5379,nvf5408;mov.b32 nvr1379,nvf5409;shfl.sync.idx.b32 nvr1380|nvp1030,nvr1379,19,31,-1;mov.b32 nvf5410,nvr1380;neg.ftz.f32 nvf5411,nvf5409;fma.rn.ftz.f32 nvf5412,nvf5411,nvf5410,nvf5381;shfl.sync.idx.b32 nvr1381|nvp1031,nvr1379,20,31,-1;mov.b32 nvf5413,nvr1381;fma.rn.ftz.f32 nvf5414,nvf5411,nvf5413,nvf5383;shfl.sync.idx.b32 nvr1382|nvp1032,nvr1379,21,31,-1;mov.b32 nvf5415,nvr1382;fma.rn.ftz.f32 nvf5416,nvf5411,nvf5415,nvf5385;shfl.sync.idx.b32 nvr1383|nvp1033,nvr1379,22,31,-1;mov.b32 nvf5417,nvr1383;fma.rn.ftz.f32 nvf5418,nvf5411,nvf5417,nvf5387;shfl.sync.idx.b32 nvr1384|nvp1034,nvr1379,23,31,-1;mov.b32 nvf5419,nvr1384;fma.rn.ftz.f32 nvf5420,nvf5411,nvf5419,nvf5389;shfl.sync.idx.b32 nvr1385|nvp1035,nvr1379,24,31,-1;mov.b32 nvf5421,nvr1385;fma.rn.ftz.f32 nvf5422,nvf5411,nvf5421,nvf5391;shfl.sync.idx.b32 nvr1386|nvp1036,nvr1379,25,31,-1;mov.b32 nvf5423,nvr1386;fma.rn.ftz.f32 nvf5424,nvf5411,nvf5423,nvf5393;shfl.sync.idx.b32 nvr1387|nvp1037,nvr1379,26,31,-1;mov.b32 nvf5425,nvr1387;fma.rn.ftz.f32 nvf5426,nvf5411,nvf5425,nvf5395;shfl.sync.idx.b32 nvr1388|nvp1038,nvr1379,27,31,-1;mov.b32 nvf5427,nvr1388;fma.rn.ftz.f32 nvf5428,nvf5411,nvf5427,nvf5397;shfl.sync.idx.b32 nvr1389|nvp1039,nvr1379,28,31,-1;mov.b32 nvf5429,nvr1389;fma.rn.ftz.f32 nvf5430,nvf5411,nvf5429,nvf5399;shfl.sync.idx.b32 nvr1390|nvp1040,nvr1379,29,31,-1;mov.b32 nvf5431,nvr1390;fma.rn.ftz.f32 nvf5432,nvf5411,nvf5431,nvf5401;shfl.sync.idx.b32 nvr1391|nvp1041,nvr1379,30,31,-1;mov.b32 nvf5433,nvr1391;fma.rn.ftz.f32 nvf5434,nvf5411,nvf5433,nvf5403;shfl.sync.idx.b32 nvr1392|nvp1042,nvr1379,31,31,-1;mov.b32 nvf5435,nvr1392;fma.rn.ftz.f32 nvf5436,nvf5411,nvf5435,nvf5405;max.ftz.f32 nvf5437,nvf5412,0f00800000;sqrt.approx.ftz.f32 nvf5438,nvf5437;mov.b32 nvr1393,nvf5438;shfl.sync.idx.b32 nvr1394|nvp1043,nvr1393,19,31,-1;mov.b32 nvf5439,nvr1394;div.approx.ftz.f32 nvf5440,nvf5412,nvf5439;mov.b32 nvr1395,nvf5440;shfl.sync.idx.b32 nvr1396|nvp1044,nvr1395,20,31,-1;mov.b32 nvf5441,nvr1396;neg.ftz.f32 nvf5442,nvf5440;fma.rn.ftz.f32 nvf5443,nvf5442,nvf5441,nvf5414;shfl.sync.idx.b32 nvr1397|nvp1045,nvr1395,21,31,-1;mov.b32 nvf5444,nvr1397;fma.rn.ftz.f32 nvf5445,nvf5442,nvf5444,nvf5416;shfl.sync.idx.b32 nvr1398|nvp1046,nvr1395,22,31,-1;mov.b32 nvf5446,nvr1398;fma.rn.ftz.f32 nvf5447,nvf5442,nvf5446,nvf5418;shfl.sync.idx.b32 nvr1399|nvp1047,nvr1395,23,31,-1;mov.b32 nvf5448,nvr1399;fma.rn.ftz.f32 nvf5449,nvf5442,nvf5448,nvf5420;shfl.sync.idx.b32 nvr1400|nvp1048,nvr1395,24,31,-1;mov.b32 nvf5450,nvr1400;fma.rn.ftz.f32 nvf5451,nvf5442,nvf5450,nvf5422;shfl.sync.idx.b32 nvr1401|nvp1049,nvr1395,25,31,-1;mov.b32 nvf5452,nvr1401;fma.rn.ftz.f32 nvf5453,nvf5442,nvf5452,nvf5424;shfl.sync.idx.b32 nvr1402|nvp1050,nvr1395,26,31,-1;mov.b32 nvf5454,nvr1402;fma.rn.ftz.f32 nvf5455,nvf5442,nvf5454,nvf5426;shfl.sync.idx.b32 nvr1403|nvp1051,nvr1395,27,31,-1;mov.b32 nvf5456,nvr1403;fma.rn.ftz.f32 nvf5457,nvf5442,nvf5456,nvf5428;shfl.sync.idx.b32 nvr1404|nvp1052,nvr1395,28,31,-1;mov.b32 nvf5458,nvr1404;fma.rn.ftz.f32 nvf5459,nvf5442,nvf5458,nvf5430;shfl.sync.idx.b32 nvr1405|nvp1053,nvr1395,29,31,-1;mov.b32 nvf5460,nvr1405;fma.rn.ftz.f32 nvf5461,nvf5442,nvf5460,nvf5432;shfl.sync.idx.b32 nvr1406|nvp1054,nvr1395,30,31,-1;mov.b32 nvf5462,nvr1406;fma.rn.ftz.f32 nvf5463,nvf5442,nvf5462,nvf5434;shfl.sync.idx.b32 nvr1407|nvp1055,nvr1395,31,31,-1;mov.b32 nvf5464,nvr1407;fma.rn.ftz.f32 nvf5465,nvf5442,nvf5464,nvf5436;max.ftz.f32 nvf5466,nvf5443,0f00800000;sqrt.approx.ftz.f32 nvf5467,nvf5466;mov.b32 nvr1408,nvf5467;shfl.sync.idx.b32 nvr1409|nvp1056,nvr1408,20,31,-1;mov.b32 nvf5468,nvr1409;div.approx.ftz.f32 nvf5469,nvf5443,nvf5468;mov.b32 nvr1410,nvf5469;shfl.sync.idx.b32 nvr1411|nvp1057,nvr1410,21,31,-1;mov.b32 nvf5470,nvr1411;neg.ftz.f32 nvf5471,nvf5469;fma.rn.ftz.f32 nvf5472,nvf5471,nvf5470,nvf5445;shfl.sync.idx.b32 nvr1412|nvp1058,nvr1410,22,31,-1;mov.b32 nvf5473,nvr1412;fma.rn.ftz.f32 nvf5474,nvf5471,nvf5473,nvf5447;shfl.sync.idx.b32 nvr1413|nvp1059,nvr1410,23,31,-1;mov.b32 nvf5475,nvr1413;fma.rn.ftz.f32 nvf5476,nvf5471,nvf5475,nvf5449;shfl.sync.idx.b32 nvr1414|nvp1060,nvr1410,24,31,-1;mov.b32 nvf5477,nvr1414;fma.rn.ftz.f32 nvf5478,nvf5471,nvf5477,nvf5451;shfl.sync.idx.b32 nvr1415|nvp1061,nvr1410,25,31,-1;mov.b32 nvf5479,nvr1415;fma.rn.ftz.f32 nvf5480,nvf5471,nvf5479,nvf5453;shfl.sync.idx.b32 nvr1416|nvp1062,nvr1410,26,31,-1;mov.b32 nvf5481,nvr1416;fma.rn.ftz.f32 nvf5482,nvf5471,nvf5481,nvf5455;shfl.sync.idx.b32 nvr1417|nvp1063,nvr1410,27,31,-1;mov.b32 nvf5483,nvr1417;fma.rn.ftz.f32 nvf5484,nvf5471,nvf5483,nvf5457;shfl.sync.idx.b32 nvr1418|nvp1064,nvr1410,28,31,-1;mov.b32 nvf5485,nvr1418;fma.rn.ftz.f32 nvf5486,nvf5471,nvf5485,nvf5459;shfl.sync.idx.b32 nvr1419|nvp1065,nvr1410,29,31,-1;mov.b32 nvf5487,nvr1419;fma.rn.ftz.f32 nvf5488,nvf5471,nvf5487,nvf5461;shfl.sync.idx.b32 nvr1420|nvp1066,nvr1410,30,31,-1;mov.b32 nvf5489,nvr1420;fma.rn.ftz.f32 nvf5490,nvf5471,nvf5489,nvf5463;shfl.sync.idx.b32 nvr1421|nvp1067,nvr1410,31,31,-1;mov.b32 nvf5491,nvr1421;fma.rn.ftz.f32 nvf5492,nvf5471,nvf5491,nvf5465;max.ftz.f32 nvf5493,nvf5472,0f00800000;sqrt.approx.ftz.f32 nvf5494,nvf5493;mov.b32 nvr1422,nvf5494;shfl.sync.idx.b32 nvr1423|nvp1068,nvr1422,21,31,-1;mov.b32 nvf5495,nvr1423;div.approx.ftz.f32 nvf5496,nvf5472,nvf5495;mov.b32 nvr1424,nvf5496;shfl.sync.idx.b32 nvr1425|nvp1069,nvr1424,22,31,-1;mov.b32 nvf5497,nvr1425;neg.ftz.f32 nvf5498,nvf5496;fma.rn.ftz.f32 nvf5499,nvf5498,nvf5497,nvf5474;shfl.sync.idx.b32 nvr1426|nvp1070,nvr1424,23,31,-1;mov.b32 nvf5500,nvr1426;fma.rn.ftz.f32 nvf5501,nvf5498,nvf5500,nvf5476;shfl.sync.idx.b32 nvr1427|nvp1071,nvr1424,24,31,-1;mov.b32 nvf5502,nvr1427;fma.rn.ftz.f32 nvf5503,nvf5498,nvf5502,nvf5478;shfl.sync.idx.b32 nvr1428|nvp1072,nvr1424,25,31,-1;mov.b32 nvf5504,nvr1428;fma.rn.ftz.f32 nvf5505,nvf5498,nvf5504,nvf5480;shfl.sync.idx.b32 nvr1429|nvp1073,nvr1424,26,31,-1;mov.b32 nvf5506,nvr1429;fma.rn.ftz.f32 nvf5507,nvf5498,nvf5506,nvf5482;shfl.sync.idx.b32 nvr1430|nvp1074,nvr1424,27,31,-1;mov.b32 nvf5508,nvr1430;fma.rn.ftz.f32 nvf5509,nvf5498,nvf5508,nvf5484;shfl.sync.idx.b32 nvr1431|nvp1075,nvr1424,28,31,-1;mov.b32 nvf5510,nvr1431;fma.rn.ftz.f32 nvf5511,nvf5498,nvf5510,nvf5486;shfl.sync.idx.b32 nvr1432|nvp1076,nvr1424,29,31,-1;mov.b32 nvf5512,nvr1432;fma.rn.ftz.f32 nvf5513,nvf5498,nvf5512,nvf5488;shfl.sync.idx.b32 nvr1433|nvp1077,nvr1424,30,31,-1;mov.b32 nvf5514,nvr1433;fma.rn.ftz.f32 nvf5515,nvf5498,nvf5514,nvf5490;shfl.sync.idx.b32 nvr1434|nvp1078,nvr1424,31,31,-1;mov.b32 nvf5516,nvr1434;fma.rn.ftz.f32 nvf5517,nvf5498,nvf5516,nvf5492;max.ftz.f32 nvf5518,nvf5499,0f00800000;sqrt.approx.ftz.f32 nvf5519,nvf5518;mov.b32 nvr1435,nvf5519;shfl.sync.idx.b32 nvr1436|nvp1079,nvr1435,22,31,-1;mov.b32 nvf5520,nvr1436;div.approx.ftz.f32 nvf5521,nvf5499,nvf5520;mov.b32 nvr1437,nvf5521;shfl.sync.idx.b32 nvr1438|nvp1080,nvr1437,23,31,-1;mov.b32 nvf5522,nvr1438;neg.ftz.f32 nvf5523,nvf5521;fma.rn.ftz.f32 nvf5524,nvf5523,nvf5522,nvf5501;shfl.sync.idx.b32 nvr1439|nvp1081,nvr1437,24,31,-1;mov.b32 nvf5525,nvr1439;fma.rn.ftz.f32 nvf5526,nvf5523,nvf5525,nvf5503;shfl.sync.idx.b32 nvr1440|nvp1082,nvr1437,25,31,-1;mov.b32 nvf5527,nvr1440;fma.rn.ftz.f32 nvf5528,nvf5523,nvf5527,nvf5505;shfl.sync.idx.b32 nvr1441|nvp1083,nvr1437,26,31,-1;mov.b32 nvf5529,nvr1441;fma.rn.ftz.f32 nvf5530,nvf5523,nvf5529,nvf5507;shfl.sync.idx.b32 nvr1442|nvp1084,nvr1437,27,31,-1;mov.b32 nvf5531,nvr1442;fma.rn.ftz.f32 nvf5532,nvf5523,nvf5531,nvf5509;shfl.sync.idx.b32 nvr1443|nvp1085,nvr1437,28,31,-1;mov.b32 nvf5533,nvr1443;fma.rn.ftz.f32 nvf5534,nvf5523,nvf5533,nvf5511;shfl.sync.idx.b32 nvr1444|nvp1086,nvr1437,29,31,-1;mov.b32 nvf5535,nvr1444;fma.rn.ftz.f32 nvf5536,nvf5523,nvf5535,nvf5513;shfl.sync.idx.b32 nvr1445|nvp1087,nvr1437,30,31,-1;mov.b32 nvf5537,nvr1445;fma.rn.ftz.f32 nvf5538,nvf5523,nvf5537,nvf5515;shfl.sync.idx.b32 nvr1446|nvp1088,nvr1437,31,31,-1;mov.b32 nvf5539,nvr1446;fma.rn.ftz.f32 nvf5540,nvf5523,nvf5539,nvf5517;max.ftz.f32 nvf5541,nvf5524,0f00800000;sqrt.approx.ftz.f32 nvf5542,nvf5541;mov.b32 nvr1447,nvf5542;shfl.sync.idx.b32 nvr1448|nvp1089,nvr1447,23,31,-1;mov.b32 nvf5543,nvr1448;div.approx.ftz.f32 nvf5544,nvf5524,nvf5543;mov.b32 nvr1449,nvf5544;shfl.sync.idx.b32 nvr1450|nvp1090,nvr1449,24,31,-1;mov.b32 nvf5545,nvr1450;neg.ftz.f32 nvf5546,nvf5544;fma.rn.ftz.f32 nvf5547,nvf5546,nvf5545,nvf5526;shfl.sync.idx.b32 nvr1451|nvp1091,nvr1449,25,31,-1;mov.b32 nvf5548,nvr1451;fma.rn.ftz.f32 nvf5549,nvf5546,nvf5548,nvf5528;shfl.sync.idx.b32 nvr1452|nvp1092,nvr1449,26,31,-1;mov.b32 nvf5550,nvr1452;fma.rn.ftz.f32 nvf5551,nvf5546,nvf5550,nvf5530;shfl.sync.idx.b32 nvr1453|nvp1093,nvr1449,27,31,-1;mov.b32 nvf5552,nvr1453;fma.rn.ftz.f32 nvf5553,nvf5546,nvf5552,nvf5532;shfl.sync.idx.b32 nvr1454|nvp1094,nvr1449,28,31,-1;mov.b32 nvf5554,nvr1454;fma.rn.ftz.f32 nvf5555,nvf5546,nvf5554,nvf5534;shfl.sync.idx.b32 nvr1455|nvp1095,nvr1449,29,31,-1;mov.b32 nvf5556,nvr1455;fma.rn.ftz.f32 nvf5557,nvf5546,nvf5556,nvf5536;shfl.sync.idx.b32 nvr1456|nvp1096,nvr1449,30,31,-1;mov.b32 nvf5558,nvr1456;fma.rn.ftz.f32 nvf5559,nvf5546,nvf5558,nvf5538;shfl.sync.idx.b32 nvr1457|nvp1097,nvr1449,31,31,-1;mov.b32 nvf5560,nvr1457;fma.rn.ftz.f32 nvf5561,nvf5546,nvf5560,nvf5540;max.ftz.f32 nvf5562,nvf5547,0f00800000;sqrt.approx.ftz.f32 nvf5563,nvf5562;mov.b32 nvr1458,nvf5563;shfl.sync.idx.b32 nvr1459|nvp1098,nvr1458,24,31,-1;mov.b32 nvf5564,nvr1459;div.approx.ftz.f32 nvf5565,nvf5547,nvf5564;mov.b32 nvr1460,nvf5565;shfl.sync.idx.b32 nvr1461|nvp1099,nvr1460,25,31,-1;mov.b32 nvf5566,nvr1461;neg.ftz.f32 nvf5567,nvf5565;fma.rn.ftz.f32 nvf5568,nvf5567,nvf5566,nvf5549;shfl.sync.idx.b32 nvr1462|nvp1100,nvr1460,26,31,-1;mov.b32 nvf5569,nvr1462;fma.rn.ftz.f32 nvf5570,nvf5567,nvf5569,nvf5551;shfl.sync.idx.b32 nvr1463|nvp1101,nvr1460,27,31,-1;mov.b32 nvf5571,nvr1463;fma.rn.ftz.f32 nvf5572,nvf5567,nvf5571,nvf5553;shfl.sync.idx.b32 nvr1464|nvp1102,nvr1460,28,31,-1;mov.b32 nvf5573,nvr1464;fma.rn.ftz.f32 nvf5574,nvf5567,nvf5573,nvf5555;shfl.sync.idx.b32 nvr1465|nvp1103,nvr1460,29,31,-1;mov.b32 nvf5575,nvr1465;fma.rn.ftz.f32 nvf5576,nvf5567,nvf5575,nvf5557;shfl.sync.idx.b32 nvr1466|nvp1104,nvr1460,30,31,-1;mov.b32 nvf5577,nvr1466;fma.rn.ftz.f32 nvf5578,nvf5567,nvf5577,nvf5559;shfl.sync.idx.b32 nvr1467|nvp1105,nvr1460,31,31,-1;mov.b32 nvf5579,nvr1467;fma.rn.ftz.f32 nvf5580,nvf5567,nvf5579,nvf5561;max.ftz.f32 nvf5581,nvf5568,0f00800000;sqrt.approx.ftz.f32 nvf5582,nvf5581;mov.b32 nvr1468,nvf5582;shfl.sync.idx.b32 nvr1469|nvp1106,nvr1468,25,31,-1;mov.b32 nvf5583,nvr1469;div.approx.ftz.f32 nvf5584,nvf5568,nvf5583;mov.b32 nvr1470,nvf5584;shfl.sync.idx.b32 nvr1471|nvp1107,nvr1470,26,31,-1;mov.b32 nvf5585,nvr1471;neg.ftz.f32 nvf5586,nvf5584;fma.rn.ftz.f32 nvf5587,nvf5586,nvf5585,nvf5570;shfl.sync.idx.b32 nvr1472|nvp1108,nvr1470,27,31,-1;mov.b32 nvf5588,nvr1472;fma.rn.ftz.f32 nvf5589,nvf5586,nvf5588,nvf5572;shfl.sync.idx.b32 nvr1473|nvp1109,nvr1470,28,31,-1;mov.b32 nvf5590,nvr1473;fma.rn.ftz.f32 nvf5591,nvf5586,nvf5590,nvf5574;shfl.sync.idx.b32 nvr1474|nvp1110,nvr1470,29,31,-1;mov.b32 nvf5592,nvr1474;fma.rn.ftz.f32 nvf5593,nvf5586,nvf5592,nvf5576;shfl.sync.idx.b32 nvr1475|nvp1111,nvr1470,30,31,-1;mov.b32 nvf5594,nvr1475;fma.rn.ftz.f32 nvf5595,nvf5586,nvf5594,nvf5578;shfl.sync.idx.b32 nvr1476|nvp1112,nvr1470,31,31,-1;mov.b32 nvf5596,nvr1476;fma.rn.ftz.f32 nvf5597,nvf5586,nvf5596,nvf5580;max.ftz.f32 nvf5598,nvf5587,0f00800000;sqrt.approx.ftz.f32 nvf5599,nvf5598;mov.b32 nvr1477,nvf5599;shfl.sync.idx.b32 nvr1478|nvp1113,nvr1477,26,31,-1;mov.b32 nvf5600,nvr1478;div.approx.ftz.f32 nvf5601,nvf5587,nvf5600;mov.b32 nvr1479,nvf5601;shfl.sync.idx.b32 nvr1480|nvp1114,nvr1479,27,31,-1;mov.b32 nvf5602,nvr1480;neg.ftz.f32 nvf5603,nvf5601;fma.rn.ftz.f32 nvf5604,nvf5603,nvf5602,nvf5589;shfl.sync.idx.b32 nvr1481|nvp1115,nvr1479,28,31,-1;mov.b32 nvf5605,nvr1481;fma.rn.ftz.f32 nvf5606,nvf5603,nvf5605,nvf5591;shfl.sync.idx.b32 nvr1482|nvp1116,nvr1479,29,31,-1;mov.b32 nvf5607,nvr1482;fma.rn.ftz.f32 nvf5608,nvf5603,nvf5607,nvf5593;shfl.sync.idx.b32 nvr1483|nvp1117,nvr1479,30,31,-1;mov.b32 nvf5609,nvr1483;fma.rn.ftz.f32 nvf5610,nvf5603,nvf5609,nvf5595;shfl.sync.idx.b32 nvr1484|nvp1118,nvr1479,31,31,-1;mov.b32 nvf5611,nvr1484;fma.rn.ftz.f32 nvf5612,nvf5603,nvf5611,nvf5597;max.ftz.f32 nvf5613,nvf5604,0f00800000;sqrt.approx.ftz.f32 nvf5614,nvf5613;mov.b32 nvr1485,nvf5614;shfl.sync.idx.b32 nvr1486|nvp1119,nvr1485,27,31,-1;mov.b32 nvf5615,nvr1486;div.approx.ftz.f32 nvf5616,nvf5604,nvf5615;mov.b32 nvr1487,nvf5616;shfl.sync.idx.b32 nvr1488|nvp1120,nvr1487,28,31,-1;mov.b32 nvf5617,nvr1488;neg.ftz.f32 nvf5618,nvf5616;fma.rn.ftz.f32 nvf5619,nvf5618,nvf5617,nvf5606;shfl.sync.idx.b32 nvr1489|nvp1121,nvr1487,29,31,-1;mov.b32 nvf5620,nvr1489;fma.rn.ftz.f32 nvf5621,nvf5618,nvf5620,nvf5608;shfl.sync.idx.b32 nvr1490|nvp1122,nvr1487,30,31,-1;mov.b32 nvf5622,nvr1490;fma.rn.ftz.f32 nvf5623,nvf5618,nvf5622,nvf5610;shfl.sync.idx.b32 nvr1491|nvp1123,nvr1487,31,31,-1;mov.b32 nvf5624,nvr1491;fma.rn.ftz.f32 nvf5625,nvf5618,nvf5624,nvf5612;max.ftz.f32 nvf5626,nvf5619,0f00800000;sqrt.approx.ftz.f32 nvf5627,nvf5626;mov.b32 nvr1492,nvf5627;shfl.sync.idx.b32 nvr1493|nvp1124,nvr1492,28,31,-1;mov.b32 nvf5628,nvr1493;div.approx.ftz.f32 nvf5629,nvf5619,nvf5628;mov.b32 nvr1494,nvf5629;shfl.sync.idx.b32 nvr1495|nvp1125,nvr1494,29,31,-1;mov.b32 nvf5630,nvr1495;neg.ftz.f32 nvf5631,nvf5629;fma.rn.ftz.f32 nvf5632,nvf5631,nvf5630,nvf5621;shfl.sync.idx.b32 nvr1496|nvp1126,nvr1494,30,31,-1;mov.b32 nvf5633,nvr1496;fma.rn.ftz.f32 nvf5634,nvf5631,nvf5633,nvf5623;shfl.sync.idx.b32 nvr1497|nvp1127,nvr1494,31,31,-1;mov.b32 nvf5635,nvr1497;fma.rn.ftz.f32 nvf5636,nvf5631,nvf5635,nvf5625;max.ftz.f32 nvf5637,nvf5632,0f00800000;sqrt.approx.ftz.f32 nvf5638,nvf5637;mov.b32 nvr1498,nvf5638;shfl.sync.idx.b32 nvr1499|nvp1128,nvr1498,29,31,-1;mov.b32 nvf5639,nvr1499;div.approx.ftz.f32 nvf5640,nvf5632,nvf5639;mov.b32 nvr1500,nvf5640;shfl.sync.idx.b32 nvr1501|nvp1129,nvr1500,30,31,-1;mov.b32 nvf5641,nvr1501;neg.ftz.f32 nvf5642,nvf5640;fma.rn.ftz.f32 nvf5643,nvf5642,nvf5641,nvf5634;shfl.sync.idx.b32 nvr1502|nvp1130,nvr1500,31,31,-1;mov.b32 nvf5644,nvr1502;fma.rn.ftz.f32 nvf5645,nvf5642,nvf5644,nvf5636;max.ftz.f32 nvf5646,nvf5643,0f00800000;sqrt.approx.ftz.f32 nvf5647,nvf5646;mov.b32 nvr1503,nvf5647;shfl.sync.idx.b32 nvr1504|nvp1131,nvr1503,30,31,-1;mov.b32 nvf5648,nvr1504;div.approx.ftz.f32 nvf5649,nvf5643,nvf5648;mov.b32 nvr1505,nvf5649;shfl.sync.idx.b32 nvr1506|nvp1132,nvr1505,31,31,-1;mov.b32 nvf5650,nvr1506;neg.ftz.f32 nvf5651,nvf5649;fma.rn.ftz.f32 nvf5652,nvf5651,nvf5650,nvf5645;max.ftz.f32 nvf5653,nvf5652,0f00800000;sqrt.approx.ftz.f32 nvf5654,nvf5653;mov.b32 nvr1507,nvf5654;shfl.sync.idx.b32 nvr1508|nvp1133,nvr1507,31,31,-1;mov.b32 nvf5655,nvr1508;div.approx.ftz.f32 nvf5656,nvf5652,nvf5655;st.shared.f32[nvrd365],nvf4509;selp.f32 nvf5657,0f00000000,nvf4576,nvp605;st.shared.f32[nvrd367],nvf5657;selp.f32 nvf5658,nvf4641,0f00000000,nvp604;st.shared.f32[nvrd369],nvf5658;selp.f32 nvf5659,nvf4704,0f00000000,nvp603;st.shared.f32[nvrd371],nvf5659;selp.f32 nvf5660,nvf4765,0f00000000,nvp602;st.shared.f32[nvrd373],nvf5660;selp.f32 nvf5661,nvf4824,0f00000000,nvp601;st.shared.f32[nvrd375],nvf5661;selp.f32 nvf5662,nvf4881,0f00000000,nvp600;st.shared.f32[nvrd377],nvf5662;selp.f32 nvf5663,nvf4936,0f00000000,nvp599;st.shared.f32[nvrd379],nvf5663;selp.f32 nvf5664,nvf4989,0f00000000,nvp598;st.shared.f32[nvrd381],nvf5664;selp.f32 nvf5665,nvf5040,0f00000000,nvp597;st.shared.f32[nvrd383],nvf5665;selp.f32 nvf5666,nvf5089,0f00000000,nvp596;st.shared.f32[nvrd385],nvf5666;selp.f32 nvf5667,nvf5136,0f00000000,nvp595;st.shared.f32[nvrd387],nvf5667;selp.f32 nvf5668,nvf5181,0f00000000,nvp594;st.shared.f32[nvrd389],nvf5668;selp.f32 nvf5669,nvf5224,0f00000000,nvp593;st.shared.f32[nvrd391],nvf5669;selp.f32 nvf5670,nvf5265,0f00000000,nvp592;st.shared.f32[nvrd393],nvf5670;selp.f32 nvf5671,nvf5304,0f00000000,nvp591;st.shared.f32[nvrd395],nvf5671;selp.f32 nvf5672,nvf5341,0f00000000,nvp590;st.shared.f32[nvrd397],nvf5672;selp.f32 nvf5673,nvf5376,0f00000000,nvp589;st.shared.f32[nvrd399],nvf5673;selp.f32 nvf5674,nvf5409,0f00000000,nvp588;st.shared.f32[nvrd401],nvf5674;selp.f32 nvf5675,nvf5440,0f00000000,nvp587;st.shared.f32[nvrd403],nvf5675;selp.f32 nvf5676,nvf5469,0f00000000,nvp586;st.shared.f32[nvrd405],nvf5676;selp.f32 nvf5677,nvf5496,0f00000000,nvp585;st.shared.f32[nvrd407],nvf5677;selp.f32 nvf5678,nvf5521,0f00000000,nvp584;st.shared.f32[nvrd409],nvf5678;selp.f32 nvf5679,nvf5544,0f00000000,nvp583;st.shared.f32[nvrd411],nvf5679;selp.f32 nvf5680,nvf5565,0f00000000,nvp582;st.shared.f32[nvrd413],nvf5680;selp.f32 nvf5681,nvf5584,0f00000000,nvp581;st.shared.f32[nvrd415],nvf5681;selp.f32 nvf5682,nvf5601,0f00000000,nvp580;st.shared.f32[nvrd417],nvf5682;selp.f32 nvf5683,nvf5616,0f00000000,nvp579;st.shared.f32[nvrd419],nvf5683;selp.f32 nvf5684,nvf5629,0f00000000,nvp578;st.shared.f32[nvrd421],nvf5684;selp.f32 nvf5685,nvf5640,0f00000000,nvp577;st.shared.f32[nvrd423],nvf5685;selp.f32 nvf5686,nvf5649,0f00000000,nvp576;st.shared.f32[nvrd425],nvf5686;selp.f32 nvf5687,nvf5656,0f00000000,nvp575;st.shared.f32[nvrd427],nvf5687;bar.warp.sync -1;mov.u64 nvrd428,nvrd457;add.s64 nvrd497,nvrd428,8320;mov.u32 nvr1547,nvr52;L__BB0_30:\nsetp.lt.u32 nvp1134,nvr1543,nvr52;@nvp1134 bra L__BB0_32;xor.b32 nvr1509,nvr52,nvr1543;cvt.u64.u32 nvrd429,nvr1509;add.s64 nvrd430,nvrd494,nvrd429;shl.b64 nvrd431,nvrd430,2;add.s64 nvrd433,nvrd364,nvrd431;ld.shared.f32 nvf5688,[nvrd433];add.s64 nvrd434,nvrd497,nvrd92;st.global.f32[nvrd434],nvf5688;L__BB0_32:\nsetp.lt.u32 nvp1135,nvr1544,nvr52;@nvp1135 bra L__BB0_34;xor.b32 nvr1510,nvr52,nvr1544;cvt.u64.u32 nvrd435,nvr1510;add.s64 nvrd436,nvrd493,nvrd435;shl.b64 nvrd437,nvrd436,2;add.s64 nvrd439,nvrd364,nvrd437;ld.shared.f32 nvf5689,[nvrd439];add.s64 nvrd440,nvrd497,nvrd93;st.global.f32[nvrd440],nvf5689;L__BB0_34:\nsetp.lt.u32 nvp1136,nvr1545,nvr52;@nvp1136 bra L__BB0_36;xor.b32 nvr1511,nvr52,nvr1545;cvt.u64.u32 nvrd441,nvr1511;add.s64 nvrd442,nvrd495,nvrd441;shl.b64 nvrd443,nvrd442,2;add.s64 nvrd445,nvrd364,nvrd443;ld.shared.f32 nvf5690,[nvrd445];add.s64 nvrd446,nvrd497,nvrd94;st.global.f32[nvrd446],nvf5690;L__BB0_36:\nsetp.lt.u32 nvp1137,nvr1546,nvr52;@nvp1137 bra L__BB0_38;xor.b32 nvr1512,nvr52,nvr1546;cvt.u64.u32 nvrd447,nvr1512;add.s64 nvrd448,nvrd496,nvrd447;shl.b64 nvrd449,nvrd448,2;add.s64 nvrd451,nvrd364,nvrd449;ld.shared.f32 nvf5691,[nvrd451];add.s64 nvrd452,nvrd497,nvrd95;st.global.f32[nvrd452],nvf5691;L__BB0_38:\nadd.s32 nvr83,nvr1547,128;add.s64 nvrd497,nvrd497,1024;add.s32 nvr1546,nvr1546,4;add.s64 nvrd496,nvrd496,128;add.s32 nvr1545,nvr1545,4;add.s64 nvrd495,nvrd495,128;add.s64 nvrd494,nvrd494,128;add.s64 nvrd493,nvrd493,128;add.s32 nvr1544,nvr1544,4;add.s32 nvr1543,nvr1543,4;setp.lt.u32 nvp1138,nvr1547,896;mov.u32 nvr1547,nvr83;@nvp1138 bra L__BB0_30;L__BB0_39:\nret;}'

@gluon.jit
def _potrf64_nvvm_kernel(source, output, ASM: gl.constexpr, num_warps: gl.constexpr):
 gl.inline_asm_elementwise('griddepcontrol.launch_dependents; mov.u32 $0, 0;', '=r', [], dtype=gl.int32, is_pure=False, pack=1)
 scratch = gl.allocate_shared_memory(gl.int64, [1024], mbarrier.MBarrierLayout())
 sentinel = scratch.slice(1023, 1)
 gl.inline_asm_elementwise(ASM, '=r,l,l', [source, output], dtype=gl.int32, is_pure=False, pack=1)
 mbarrier.init(sentinel, count=1)
 mbarrier.invalidate(sentinel)

@gluon.jit
def _potrf32_scheduled_kernel(source, output, batch: gl.constexpr, ASM: gl.constexpr, num_warps: gl.constexpr):
 gl.inline_asm_elementwise('griddepcontrol.launch_dependents; mov.u32 $0, 0;', '=r', [], dtype=gl.int32, is_pure=False, pack=1)
 scratch = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 sentinel = scratch.slice(2047, 1)
 gl.inline_asm_elementwise(ASM, '=r,l,l,r', [source, output, batch], dtype=gl.int32, is_pure=False, pack=1)
 mbarrier.init(sentinel, count=1)
 mbarrier.invalidate(sentinel)

@triton.jit
def _full_potrf64_blocked_kernel(source, output, UPDATE_PRECISION: tl.constexpr):
 matrix = tl.program_id(0)
 axis = tl.arange(0, 32)
 rows = axis[:, None]
 columns = axis[None, :]
 base = matrix * 64 * 64
 lower = rows >= columns
 values = tl.load(source + base + rows * 64 + columns, mask=lower, other=0.0).to(tl.float32)
 tl.store(output + base + rows * 64 + columns, 0.0, mask=columns > rows)
 tl.store(output + base + rows * 64 + 32 + columns, 0.0)
 for group_start in tl.static_range(0, 32, 2):
  column0 = tl.sum(tl.where(columns == group_start, values, 0.0), axis=1)
  diagonal0 = tl.sum(tl.where(axis == group_start, column0, 0.0), axis=0)
  root0 = tl.sqrt(tl.maximum(diagonal0, 1.1754943508222875e-38))
  factor0 = tl.where(axis >= group_start, column0 / root0, 0.0)
  cross = tl.sum(tl.where(axis == group_start + 1, factor0, 0.0), axis=0)
  column1 = tl.sum(tl.where(columns == group_start + 1, values, 0.0), axis=1)
  diagonal1 = tl.sum(tl.where(axis == group_start + 1, column1, 0.0), axis=0)
  root1 = tl.sqrt(tl.maximum(diagonal1 - cross * cross, 1.1754943508222875e-38))
  factor1 = tl.where(axis >= group_start + 1, (column1 - factor0 * cross) / root1, 0.0)
  tl.store(output + base + axis * 64 + group_start, factor0, mask=axis >= group_start)
  tl.store(output + base + axis * 64 + group_start + 1, factor1, mask=axis >= group_start + 1)
  trailing = (rows >= group_start + 2) & (columns >= group_start + 2) & lower
  values = tl.where(trailing, values - factor0[:, None] * factor0[None, :] - factor1[:, None] * factor1[None, :], values)
 factor00 = tl.load(output + base + rows * 64 + columns, mask=lower, other=0.0)
 panel = tl.load(source + base + (32 + rows) * 64 + columns).to(tl.float32)
 for group_start in tl.static_range(0, 32, 2):
  rhs0 = tl.sum(tl.where(columns == group_start, panel, 0.0), axis=1)
  factor0 = tl.sum(tl.where(columns == group_start, factor00, 0.0), axis=1)
  diagonal0 = tl.sum(tl.where(axis == group_start, factor0, 0.0), axis=0)
  solved0 = rhs0 / diagonal0
  cross = tl.sum(tl.where(axis == group_start + 1, factor0, 0.0), axis=0)
  rhs1 = tl.sum(tl.where(columns == group_start + 1, panel, 0.0), axis=1) - solved0 * cross
  factor1 = tl.sum(tl.where(columns == group_start + 1, factor00, 0.0), axis=1)
  diagonal1 = tl.sum(tl.where(axis == group_start + 1, factor1, 0.0), axis=0)
  solved1 = rhs1 / diagonal1
  panel = tl.where(columns == group_start, solved0[:, None], tl.where(columns == group_start + 1, solved1[:, None], tl.where(columns >= group_start + 2, panel - solved0[:, None] * factor0[None, :] - solved1[:, None] * factor1[None, :], panel)))
  tl.store(output + base + (32 + axis) * 64 + group_start, solved0)
  tl.store(output + base + (32 + axis) * 64 + group_start + 1, solved1)
 values = tl.load(source + base + (32 + rows) * 64 + 32 + columns, mask=lower, other=0.0).to(tl.float32)
 values -= tl.where(lower, _factor_dot(panel, tl.trans(panel), UPDATE_PRECISION), 0.0)
 tl.store(output + base + (32 + rows) * 64 + 32 + columns, 0.0, mask=columns > rows)
 for group_start in tl.static_range(0, 32, 2):
  column0 = tl.sum(tl.where(columns == group_start, values, 0.0), axis=1)
  diagonal0 = tl.sum(tl.where(axis == group_start, column0, 0.0), axis=0)
  root0 = tl.sqrt(tl.maximum(diagonal0, 1.1754943508222875e-38))
  factor0 = tl.where(axis >= group_start, column0 / root0, 0.0)
  cross = tl.sum(tl.where(axis == group_start + 1, factor0, 0.0), axis=0)
  column1 = tl.sum(tl.where(columns == group_start + 1, values, 0.0), axis=1)
  diagonal1 = tl.sum(tl.where(axis == group_start + 1, column1, 0.0), axis=0)
  root1 = tl.sqrt(tl.maximum(diagonal1 - cross * cross, 1.1754943508222875e-38))
  factor1 = tl.where(axis >= group_start + 1, (column1 - factor0 * cross) / root1, 0.0)
  tl.store(output + base + (32 + axis) * 64 + 32 + group_start, factor0, mask=axis >= group_start)
  tl.store(output + base + (32 + axis) * 64 + 32 + group_start + 1, factor1, mask=axis >= group_start + 1)
  trailing = (rows >= group_start + 2) & (columns >= group_start + 2) & lower
  values = tl.where(trailing, values - factor0[:, None] * factor0[None, :] - factor1[:, None] * factor1[None, :], values)

@triton.jit
def _tile_potrf_kernel(output, n: tl.constexpr, panel, TILE: tl.constexpr, PIVOT_FLOOR: tl.constexpr):
 """Factor an already-updated diagonal tile in place."""
 matrix = tl.program_id(0)
 axis = tl.arange(0, TILE)
 rows = axis[:, None]
 columns = axis[None, :]
 global_rows = panel + rows
 global_columns = panel + columns
 valid = (global_rows < n) & (global_columns < n)
 base = matrix * n * n
 values = tl.load(output + base + global_rows * n + global_columns, mask=valid & (rows >= columns), other=0.0).to(tl.float32)
 for column_index in tl.range(0, TILE, 1):
  column_valid = panel + column_index < n
  column = tl.sum(tl.where(columns == column_index, values, 0.0), axis=1)
  diagonal = tl.sum(tl.where(axis == column_index, column, 0.0), axis=0)
  root = tl.sqrt(tl.maximum(diagonal, PIVOT_FLOOR))
  factor_column = tl.where((axis >= column_index) & column_valid, column / root, 0.0)
  tl.store(output + base + (panel + axis) * n + panel + column_index, factor_column, mask=(panel + axis < n) & (axis >= column_index) & column_valid)
  trailing = (rows > column_index) & (columns > column_index) & (rows >= columns) & valid & column_valid
  values = tl.where(trailing, values - factor_column[:, None] * factor_column[None, :], values)

@triton.jit
def _panel_trsm_kernel(output, n: tl.constexpr, panel, TILE: tl.constexpr, ROWS: tl.constexpr):
 """Solve L_ik = A_ik L_kk^-T for one TILE x TILE row block."""
 row_block = tl.program_id(0)
 matrix = tl.program_id(1)
 row_axis = tl.arange(0, ROWS)
 columns = tl.arange(0, TILE)
 rows = panel + TILE + row_block * ROWS + row_axis
 base = matrix * n * n
 values = tl.load(output + base + rows[:, None] * n + panel + columns[None, :], mask=(rows[:, None] < n) & (panel + columns[None, :] < n), other=0.0).to(tl.float32)
 for column_index in tl.range(0, TILE, 1):
  column_valid = panel + column_index < n
  rhs = tl.sum(tl.where(columns[None, :] == column_index, values, 0.0), axis=1)
  diagonal_row = tl.load(output + base + (panel + column_index) * n + panel + columns, mask=(columns < column_index) & column_valid, other=0.0)
  product = tl.sum(tl.where(columns[None, :] < column_index, values * diagonal_row[None, :], 0.0), axis=1)
  diagonal = tl.load(output + base + (panel + column_index) * n + panel + column_index, mask=column_valid, other=1.0)
  solution = (rhs - product) / diagonal
  values = tl.where((columns[None, :] == column_index) & column_valid, solution[:, None], values)
 tl.store(output + base + rows[:, None] * n + panel + columns[None, :], values, mask=(rows[:, None] < n) & (panel + columns[None, :] < n))

@triton.jit
def _zero_flags_kernel(flags, count: tl.constexpr, BLOCK: tl.constexpr):
 offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
 tl.store(flags + offsets, 0.0, mask=offsets < count)

@triton.jit
def _zero_flags_signal_n2048_backing_kernel(flags, count: tl.constexpr, BLOCK: tl.constexpr):
 offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
 tl.store(flags + offsets, 0.0, mask=offsets < count)
 tl_cuda.gdc_launch_dependents()

@triton.jit
def _factor_dot(left, right, HIGH_PRECISION: tl.constexpr):
 if HIGH_PRECISION == 3:
  return tl.dot(left.to(tl.float16), right.to(tl.float16))
 if HIGH_PRECISION:
  left_high = left.to(tl.float16)
  right_high = right.to(tl.float16)
  right_low = (right - right_high.to(tl.float32)).to(tl.float16)
  if HIGH_PRECISION == 2:
   return tl.dot(left_high, right_high) + tl.dot(left_high, right_low)
  left_low = (left - left_high.to(tl.float32)).to(tl.float16)
  return tl.dot(left_high, right_high) + tl.dot(left_low, right_high) + tl.dot(left_high, right_low)
 return tl.dot(left, right, input_precision='tf32')

@triton.jit
def _register_potrf16(values, PIVOT_FLOOR: tl.constexpr, HIGH_PRECISION: tl.constexpr, KEEP_STRICT4: tl.constexpr, EXACT_INVERSE: tl.constexpr, PAIR_POTRF: tl.constexpr, PAIR_DOT: tl.constexpr, QUAD_POTRF: tl.constexpr, LINEAR_INVERSE: tl.constexpr, APPROX_RSQRT: tl.constexpr):
 axis = tl.arange(0, 16)
 rows = axis[:, None]
 columns = axis[None, :]
 lower = tl.zeros((16, 16), tl.float32)
 if QUAD_POTRF:
  for group_start in tl.static_range(0, 16, 4):
   column0 = tl.sum(tl.where(columns == group_start, values, 0.0), axis=1)
   diagonal0 = tl.sum(tl.where(axis == group_start, column0, 0.0), axis=0)
   root0 = tl.sqrt(tl.maximum(diagonal0, PIVOT_FLOOR))
   factor0 = tl.where(axis >= group_start, column0 / root0, 0.0)
   cross10 = tl.sum(tl.where(axis == group_start + 1, factor0, 0.0), axis=0)
   column1 = tl.sum(tl.where(columns == group_start + 1, values, 0.0), axis=1) - factor0 * cross10
   diagonal1 = tl.sum(tl.where(axis == group_start + 1, column1, 0.0), axis=0)
   root1 = tl.sqrt(tl.maximum(diagonal1, PIVOT_FLOOR))
   factor1 = tl.where(axis >= group_start + 1, column1 / root1, 0.0)
   cross20 = tl.sum(tl.where(axis == group_start + 2, factor0, 0.0), axis=0)
   cross21 = tl.sum(tl.where(axis == group_start + 2, factor1, 0.0), axis=0)
   column2 = tl.sum(tl.where(columns == group_start + 2, values, 0.0), axis=1) - factor0 * cross20 - factor1 * cross21
   diagonal2 = tl.sum(tl.where(axis == group_start + 2, column2, 0.0), axis=0)
   root2 = tl.sqrt(tl.maximum(diagonal2, PIVOT_FLOOR))
   factor2 = tl.where(axis >= group_start + 2, column2 / root2, 0.0)
   cross30 = tl.sum(tl.where(axis == group_start + 3, factor0, 0.0), axis=0)
   cross31 = tl.sum(tl.where(axis == group_start + 3, factor1, 0.0), axis=0)
   cross32 = tl.sum(tl.where(axis == group_start + 3, factor2, 0.0), axis=0)
   column3 = tl.sum(tl.where(columns == group_start + 3, values, 0.0), axis=1) - factor0 * cross30 - factor1 * cross31 - factor2 * cross32
   diagonal3 = tl.sum(tl.where(axis == group_start + 3, column3, 0.0), axis=0)
   root3 = tl.sqrt(tl.maximum(diagonal3, PIVOT_FLOOR))
   factor3 = tl.where(axis >= group_start + 3, column3 / root3, 0.0)
   factor_group = tl.where(columns == group_start, factor0[:, None], tl.where(columns == group_start + 1, factor1[:, None], tl.where(columns == group_start + 2, factor2[:, None], tl.where(columns == group_start + 3, factor3[:, None], 0.0))))
   group_columns = (columns >= group_start) & (columns < group_start + 4)
   lower = tl.where(group_columns, factor_group, lower)
   product = _factor_dot(factor_group, tl.trans(factor_group), HIGH_PRECISION)
   trailing = (rows >= group_start + 4) & (columns >= group_start + 4) & (rows >= columns)
   values = tl.where(trailing, values - product, values)
 elif PAIR_POTRF:
  for group_start in tl.static_range(0, 16, 2):
   column0 = tl.sum(tl.where(columns == group_start, values, 0.0), axis=1)
   diagonal0 = tl.sum(tl.where(axis == group_start, column0, 0.0), axis=0)
   root0 = tl.sqrt(tl.maximum(diagonal0, PIVOT_FLOOR))
   factor0 = tl.where(axis >= group_start, column0 / root0, 0.0)
   cross = tl.sum(tl.where(axis == group_start + 1, factor0, 0.0), axis=0)
   column1 = tl.sum(tl.where(columns == group_start + 1, values, 0.0), axis=1)
   diagonal1 = tl.sum(tl.where(axis == group_start + 1, column1, 0.0), axis=0)
   root1 = tl.sqrt(tl.maximum(diagonal1 - cross * cross, PIVOT_FLOOR))
   factor1 = tl.where(axis >= group_start + 1, (column1 - factor0 * cross) / root1, 0.0)
   factor_group = tl.where(columns == group_start, factor0[:, None], tl.where(columns == group_start + 1, factor1[:, None], 0.0))
   group_columns = (columns >= group_start) & (columns < group_start + 2)
   lower = tl.where(group_columns, factor_group, lower)
   if PAIR_DOT:
    product = _factor_dot(factor_group, tl.trans(factor_group), HIGH_PRECISION)
   else:
    product = factor0[:, None] * factor0[None, :] + factor1[:, None] * factor1[None, :]
   trailing = (rows >= group_start + 2) & (columns >= group_start + 2) & (rows >= columns)
   values = tl.where(trailing, values - product, values)
 else:
  for column_index in tl.range(0, 16, 1):
   column = tl.sum(tl.where(columns == column_index, values, 0.0), axis=1)
   diagonal = tl.sum(tl.where(axis == column_index, column, 0.0), axis=0)
   pivot = tl.maximum(diagonal, PIVOT_FLOOR)
   if APPROX_RSQRT:
    inverse_root = tl.rsqrt(pivot)
    factor_column = tl.where(axis >= column_index, column * inverse_root, 0.0)
   else:
    root = tl.sqrt(pivot)
    factor_column = tl.where(axis >= column_index, column / root, 0.0)
   lower = tl.where(columns == column_index, factor_column[:, None], lower)
   trailing = (rows > column_index) & (columns > column_index) & (rows >= columns)
   values = tl.where(trailing, values - factor_column[:, None] * factor_column[None, :], values)
 diagonal = tl.sum(tl.where(rows == columns, lower, 0.0), axis=1)
 identity = tl.where(rows == columns, 1.0, 0.0)
 strict = tl.where(rows > columns, lower / diagonal[:, None], 0.0)
 inverse = identity - strict
 if LINEAR_INVERSE:
  inverse += _factor_dot(strict, strict, HIGH_PRECISION)
 else:
  strict2 = _factor_dot(strict, strict, HIGH_PRECISION)
  inverse = _factor_dot(inverse, identity + strict2, HIGH_PRECISION)
  if HIGH_PRECISION or KEEP_STRICT4 or EXACT_INVERSE:
   strict4 = _factor_dot(strict2, strict2, HIGH_PRECISION)
   if HIGH_PRECISION or EXACT_INVERSE:
    inverse = _factor_dot(inverse, identity + strict4, HIGH_PRECISION)
   else:
    inverse += strict4
    if PAIR_POTRF:
     inverse -= _factor_dot(strict4, strict, HIGH_PRECISION)
  if HIGH_PRECISION or EXACT_INVERSE:
   strict8 = _factor_dot(strict4, strict4, HIGH_PRECISION)
   inverse = _factor_dot(inverse, identity + strict8, HIGH_PRECISION)
 inverse /= diagonal[None, :]
 return (lower, inverse)

@triton.jit
def _potrf32_recursive16_global(output, base, n: tl.constexpr, _b, PIVOT_FLOOR: tl.constexpr, HIGH_PRECISION: tl.constexpr, DIAGONAL_PRECISION: tl.constexpr, FIRST_LEAF_PRECISION: tl.constexpr=-1, APPROX_RSQRT_MASK: tl.constexpr=-1):
 axis = tl.arange(0, 16)
 rows = axis[:, None]
 columns = axis[None, :]
 first = tl.load(output + base + (_b + rows) * n + _b + columns, mask=rows >= columns, other=0.0).to(tl.float32)
 first_leaf_precision = DIAGONAL_PRECISION
 if FIRST_LEAF_PRECISION >= 0:
  first_leaf_precision = FIRST_LEAF_PRECISION
 if APPROX_RSQRT_MASK < 0:
  lower0, inverse0 = _register_potrf16(first, PIVOT_FLOOR, first_leaf_precision, n.bit_length() == 10, n <= 256, n.bit_length() == 9 or n.bit_length() == 11 or n.bit_length() == 12, n >= 1024, False, False, n == 64)
 else:
  lower0, inverse0 = _register_potrf16(first, PIVOT_FLOOR, first_leaf_precision, n.bit_length() == 10, n <= 256, n.bit_length() == 9 or n.bit_length() == 11 or n.bit_length() == 12, n >= 1024, False, False, APPROX_RSQRT_MASK & 1 != 0)
 tl.store(output + base + (_b + rows) * n + _b + columns, lower0, mask=rows >= columns)
 tl.store(output + base + (_b + columns) * n + _b + rows, inverse0, mask=rows > columns)
 cross = tl.load(output + base + (_b + 16 + rows) * n + _b + columns).to(tl.float32)
 lower10 = _factor_dot(cross, tl.trans(inverse0), DIAGONAL_PRECISION)
 tl.store(output + base + (_b + 16 + rows) * n + _b + columns, lower10)
 second = tl.load(output + base + (_b + 16 + rows) * n + _b + 16 + columns, mask=rows >= columns, other=0.0).to(tl.float32)
 second -= _factor_dot(lower10, tl.trans(lower10), DIAGONAL_PRECISION)
 if APPROX_RSQRT_MASK < 0:
  lower1, inverse1 = _register_potrf16(second, PIVOT_FLOOR, DIAGONAL_PRECISION, n.bit_length() == 9, n <= 128, n.bit_length() == 9 or n.bit_length() == 11 or n.bit_length() == 12, n >= 1024, False, False, n == 64)
 else:
  lower1, inverse1 = _register_potrf16(second, PIVOT_FLOOR, DIAGONAL_PRECISION, n.bit_length() == 9, n <= 128, n.bit_length() == 9 or n.bit_length() == 11 or n.bit_length() == 12, n >= 1024, False, False, APPROX_RSQRT_MASK & 2 != 0)
 tl.store(output + base + (_b + 16 + rows) * n + _b + 16 + columns, lower1, mask=rows >= columns)
 tl.store(output + base + (_b + 16 + columns) * n + _b + 16 + rows, inverse1, mask=rows > columns)
 tl.debug_barrier()

@triton.jit
def _trsm32_recursive16_global(output, base, n: tl.constexpr, solve_rows, _b, valid_rows, HIGH_PRECISION: tl.constexpr):
 axis = tl.arange(0, 16)
 rows = axis[:, None]
 columns = axis[None, :]
 rhs0 = tl.load(output + base + solve_rows[:, None] * n + _b + axis[None, :], mask=valid_rows[:, None], other=0.0).to(tl.float32)
 inverse0 = tl.load(output + base + (_b + columns) * n + _b + rows, mask=rows > columns, other=0.0)
 diagonal0 = tl.load(output + base + (_b + axis) * n + _b + axis)
 inverse0 += tl.where(rows == columns, 1.0 / diagonal0[:, None], 0.0)
 solved0 = _factor_dot(rhs0, tl.trans(inverse0), HIGH_PRECISION)
 rhs1 = tl.load(output + base + solve_rows[:, None] * n + _b + 16 + axis[None, :], mask=valid_rows[:, None], other=0.0).to(tl.float32)
 lower10 = tl.load(output + base + (_b + 16 + rows) * n + _b + columns)
 rhs1 -= _factor_dot(solved0, tl.trans(lower10), n <= 128)
 inverse1 = tl.load(output + base + (_b + 16 + columns) * n + _b + 16 + rows, mask=rows > columns, other=0.0)
 diagonal1 = tl.load(output + base + (_b + 16 + axis) * n + _b + 16 + axis)
 inverse1 += tl.where(rows == columns, 1.0 / diagonal1[:, None], 0.0)
 solved1 = _factor_dot(rhs1, tl.trans(inverse1), HIGH_PRECISION)
 tl.store(output + base + solve_rows[:, None] * n + _b + axis[None, :], solved0, mask=valid_rows[:, None])
 tl.store(output + base + solve_rows[:, None] * n + _b + 16 + axis[None, :], solved1, mask=valid_rows[:, None])
 tl.debug_barrier()

@triton.jit
def _register_potrf8(values, PIVOT_FLOOR: tl.constexpr):
 axis = tl.arange(0, 8)
 rows = axis[:, None]
 columns = axis[None, :]
 lower = tl.zeros((8, 8), tl.float32)
 for column_index in tl.range(0, 8, 1):
  column = tl.sum(tl.where(columns == column_index, values, 0.0), axis=1)
  diagonal = tl.sum(tl.where(axis == column_index, column, 0.0), axis=0)
  root = tl.sqrt(tl.maximum(diagonal, PIVOT_FLOOR))
  factor_column = tl.where(axis >= column_index, column / root, 0.0)
  lower = tl.where(columns == column_index, factor_column[:, None], lower)
  trailing = (rows > column_index) & (columns > column_index) & (rows >= columns)
  values = tl.where(trailing, values - factor_column[:, None] * factor_column[None, :], values)
 diagonal = tl.sum(tl.where(rows == columns, lower, 0.0), axis=1)
 identity = tl.where(rows == columns, 1.0, 0.0)
 strict = tl.where(rows > columns, lower / diagonal[:, None], 0.0)
 strict2 = tl.sum(strict[:, :, None] * strict[None, :, :], axis=1)
 strict4 = tl.sum(strict2[:, :, None] * strict2[None, :, :], axis=1)
 inverse = identity - strict
 inverse = tl.sum(inverse[:, :, None] * (identity + strict2)[None, :, :], axis=1)
 inverse = tl.sum(inverse[:, :, None] * (identity + strict4)[None, :, :], axis=1)
 inverse /= diagonal[None, :]
 return (lower, inverse)

@triton.jit
def _register_incomplete_potrf8(values, PIVOT_FLOOR: tl.constexpr, MICROBLOCK: tl.constexpr):
 axis = tl.arange(0, 8)
 rows = axis[:, None]
 columns = axis[None, :]
 lower = tl.zeros((8, 8), tl.float32)
 for group_start in tl.static_range(0, 8, MICROBLOCK):
  in_group = (axis >= group_start) & (axis < group_start + MICROBLOCK)
  diagonal = tl.sum(tl.where(rows == columns, values, 0.0), axis=1)
  roots = tl.where(in_group, tl.sqrt(tl.maximum(diagonal, PIVOT_FLOOR)), 1.0)
  group_columns = in_group[None, :]
  factor_group = tl.where(rows == columns, roots[:, None], tl.where((rows >= group_start + MICROBLOCK) & group_columns, values / roots[None, :], 0.0))
  factor_group = tl.where(group_columns, factor_group, 0.0)
  lower = tl.where(group_columns, factor_group, lower)
  product = tl.sum(factor_group[:, None, :] * factor_group[None, :, :], axis=2)
  trailing = (rows >= group_start + MICROBLOCK) & (columns >= group_start + MICROBLOCK) & (rows >= columns)
  values = tl.where(trailing, values - product, values)
 diagonal = tl.sum(tl.where(rows == columns, lower, 0.0), axis=1)
 identity = tl.where(rows == columns, 1.0, 0.0)
 strict = tl.where(rows > columns, lower / diagonal[:, None], 0.0)
 strict2 = tl.sum(strict[:, :, None] * strict[None, :, :], axis=1)
 strict4 = tl.sum(strict2[:, :, None] * strict2[None, :, :], axis=1)
 inverse = identity - strict
 inverse = tl.sum(inverse[:, :, None] * (identity + strict2)[None, :, :], axis=1)
 inverse = tl.sum(inverse[:, :, None] * (identity + strict4)[None, :, :], axis=1)
 inverse /= diagonal[None, :]
 return (lower, inverse)

@triton.jit
def _register_potrf4(values, PIVOT_FLOOR: tl.constexpr):
 axis = tl.arange(0, 4)
 rows = axis[:, None]
 columns = axis[None, :]
 lower = tl.zeros((4, 4), tl.float32)
 for column_index in tl.range(0, 4, 1):
  column = tl.sum(tl.where(columns == column_index, values, 0.0), axis=1)
  diagonal = tl.sum(tl.where(axis == column_index, column, 0.0), axis=0)
  root = tl.sqrt(tl.maximum(diagonal, PIVOT_FLOOR))
  factor_column = tl.where(axis >= column_index, column / root, 0.0)
  lower = tl.where(columns == column_index, factor_column[:, None], lower)
  trailing = (rows > column_index) & (columns > column_index) & (rows >= columns)
  values = tl.where(trailing, values - factor_column[:, None] * factor_column[None, :], values)
 diagonal = tl.sum(tl.where(rows == columns, lower, 0.0), axis=1)
 identity = tl.where(rows == columns, 1.0, 0.0)
 strict = tl.where(rows > columns, lower / diagonal[:, None], 0.0)
 strict2 = tl.sum(strict[:, :, None] * strict[None, :, :], axis=1)
 inverse = tl.sum((identity - strict)[:, :, None] * (identity + strict2)[None, :, :], axis=1)
 inverse /= diagonal[None, :]
 return (lower, inverse)

@triton.jit
def _potrf8_recursive4_global(output, base, n: tl.constexpr, _b, PIVOT_FLOOR: tl.constexpr):
 axis = tl.arange(0, 4)
 rows = axis[:, None]
 columns = axis[None, :]
 first = tl.load(output + base + (_b + rows) * n + _b + columns, mask=rows >= columns, other=0.0).to(tl.float32)
 lower0, inverse0 = _register_potrf4(first, PIVOT_FLOOR)
 tl.store(output + base + (_b + rows) * n + _b + columns, lower0, mask=rows >= columns)
 tl.store(output + base + (_b + columns) * n + _b + rows, inverse0, mask=rows > columns)
 cross = tl.load(output + base + (_b + 4 + rows) * n + _b + columns).to(tl.float32)
 lower10 = tl.sum(cross[:, None, :] * inverse0[None, :, :], axis=2)
 tl.store(output + base + (_b + 4 + rows) * n + _b + columns, lower10)
 second = tl.load(output + base + (_b + 4 + rows) * n + _b + 4 + columns, mask=rows >= columns, other=0.0).to(tl.float32)
 second -= tl.sum(lower10[:, None, :] * lower10[None, :, :], axis=2)
 lower1, inverse1 = _register_potrf4(second, PIVOT_FLOOR)
 tl.store(output + base + (_b + 4 + rows) * n + _b + 4 + columns, lower1, mask=rows >= columns)
 tl.store(output + base + (_b + 4 + columns) * n + _b + 4 + rows, inverse1, mask=rows > columns)
 tl.debug_barrier()

@triton.jit
def _potrf8_hybrid4_global(output, base, n: tl.constexpr, _b, PIVOT_FLOOR: tl.constexpr):
 _potrf8_recursive4_global(output, base, n, _b, PIVOT_FLOOR)
 axis = tl.arange(0, 4)
 rows = axis[:, None]
 columns = axis[None, :]
 inverse0 = tl.load(output + base + (_b + columns) * n + _b + rows, mask=rows > columns, other=0.0)
 diagonal0 = tl.load(output + base + (_b + axis) * n + _b + axis)
 inverse0 += tl.where(rows == columns, 1.0 / diagonal0[:, None], 0.0)
 inverse1 = tl.load(output + base + (_b + 4 + columns) * n + _b + 4 + rows, mask=rows > columns, other=0.0)
 diagonal1 = tl.load(output + base + (_b + 4 + axis) * n + _b + 4 + axis)
 inverse1 += tl.where(rows == columns, 1.0 / diagonal1[:, None], 0.0)
 lower10 = tl.load(output + base + (_b + 4 + rows) * n + _b + columns)
 right = tl.sum(lower10[:, :, None] * inverse0[None, :, :], axis=1)
 inverse10 = -tl.sum(inverse1[:, :, None] * right[None, :, :], axis=1)
 tl.store(output + base + (_b + columns) * n + _b + 4 + rows, inverse10)
 tl.debug_barrier()

@triton.jit
def _potrf16_recursive4_global(output, base, n: tl.constexpr, _b, PIVOT_FLOOR: tl.constexpr):
 _potrf8_hybrid4_global(output, base, n, _b, PIVOT_FLOOR)
 axis = tl.arange(0, 8)
 rows = axis[:, None]
 columns = axis[None, :]
 inverse0 = tl.load(output + base + (_b + columns) * n + _b + rows, mask=rows > columns, other=0.0)
 diagonal0 = tl.load(output + base + (_b + axis) * n + _b + axis)
 inverse0 += tl.where(rows == columns, 1.0 / diagonal0[:, None], 0.0)
 cross = tl.load(output + base + (_b + 8 + rows) * n + _b + columns).to(tl.float32)
 lower10 = tl.sum(cross[:, None, :] * inverse0[None, :, :], axis=2)
 tl.store(output + base + (_b + 8 + rows) * n + _b + columns, lower10)
 second = tl.load(output + base + (_b + 8 + rows) * n + _b + 8 + columns, mask=rows >= columns, other=0.0).to(tl.float32)
 second -= tl.sum(lower10[:, None, :] * lower10[None, :, :], axis=2)
 tl.store(output + base + (_b + 8 + rows) * n + _b + 8 + columns, second, mask=rows >= columns)
 tl.debug_barrier()
 _potrf8_hybrid4_global(output, base, n, _b + 8, PIVOT_FLOOR)

@triton.jit
def _potrf16_recursive8_global(output, base, n: tl.constexpr, _b, PIVOT_FLOOR: tl.constexpr, INCOMPLETE_MICROBLOCK: tl.constexpr):
 axis = tl.arange(0, 8)
 rows = axis[:, None]
 columns = axis[None, :]
 first = tl.load(output + base + (_b + rows) * n + _b + columns, mask=rows >= columns, other=0.0).to(tl.float32)
 if INCOMPLETE_MICROBLOCK > 0:
  lower0, inverse0 = _register_incomplete_potrf8(first, PIVOT_FLOOR, INCOMPLETE_MICROBLOCK)
 else:
  lower0, inverse0 = _register_potrf8(first, PIVOT_FLOOR)
 tl.store(output + base + (_b + rows) * n + _b + columns, lower0, mask=rows >= columns)
 tl.store(output + base + (_b + columns) * n + _b + rows, inverse0, mask=rows > columns)
 cross = tl.load(output + base + (_b + 8 + rows) * n + _b + columns).to(tl.float32)
 lower10 = tl.sum(cross[:, None, :] * inverse0[None, :, :], axis=2)
 tl.store(output + base + (_b + 8 + rows) * n + _b + columns, lower10)
 second = tl.load(output + base + (_b + 8 + rows) * n + _b + 8 + columns, mask=rows >= columns, other=0.0).to(tl.float32)
 second -= tl.sum(lower10[:, None, :] * lower10[None, :, :], axis=2)
 if INCOMPLETE_MICROBLOCK > 0:
  lower1, inverse1 = _register_incomplete_potrf8(second, PIVOT_FLOOR, INCOMPLETE_MICROBLOCK)
 else:
  lower1, inverse1 = _register_potrf8(second, PIVOT_FLOOR)
 tl.store(output + base + (_b + 8 + rows) * n + _b + 8 + columns, lower1, mask=rows >= columns)
 tl.store(output + base + (_b + 8 + columns) * n + _b + 8 + rows, inverse1, mask=rows > columns)
 tl.debug_barrier()

@triton.jit
def _trsm16_recursive8_global(output, base, n: tl.constexpr, solve_rows, _b, valid_rows):
 axis = tl.arange(0, 8)
 rows = axis[:, None]
 columns = axis[None, :]
 rhs0 = tl.load(output + base + solve_rows[:, None] * n + _b + axis[None, :], mask=valid_rows[:, None], other=0.0).to(tl.float32)
 inverse0 = tl.load(output + base + (_b + columns) * n + _b + rows, mask=rows > columns, other=0.0)
 diagonal0 = tl.load(output + base + (_b + axis) * n + _b + axis)
 inverse0 += tl.where(rows == columns, 1.0 / diagonal0[:, None], 0.0)
 solved0 = tl.sum(rhs0[:, None, :] * inverse0[None, :, :], axis=2)
 rhs1 = tl.load(output + base + solve_rows[:, None] * n + _b + 8 + axis[None, :], mask=valid_rows[:, None], other=0.0).to(tl.float32)
 lower10 = tl.load(output + base + (_b + 8 + rows) * n + _b + columns)
 rhs1 -= tl.sum(solved0[:, None, :] * lower10[None, :, :], axis=2)
 inverse1 = tl.load(output + base + (_b + 8 + columns) * n + _b + 8 + rows, mask=rows > columns, other=0.0)
 diagonal1 = tl.load(output + base + (_b + 8 + axis) * n + _b + 8 + axis)
 inverse1 += tl.where(rows == columns, 1.0 / diagonal1[:, None], 0.0)
 solved1 = tl.sum(rhs1[:, None, :] * inverse1[None, :, :], axis=2)
 tl.store(output + base + solve_rows[:, None] * n + _b + axis[None, :], solved0, mask=valid_rows[:, None])
 tl.store(output + base + solve_rows[:, None] * n + _b + 8 + axis[None, :], solved1, mask=valid_rows[:, None])
 tl.debug_barrier()

@triton.jit
def _potrf32_recursive8_global(output, base, n: tl.constexpr, _b, PIVOT_FLOOR: tl.constexpr):
 _potrf16_recursive8_global(output, base, n, _b, PIVOT_FLOOR, 0)
 internal_rows = _b + 16 + tl.arange(0, 16)
 _trsm16_recursive8_global(output, base, n, internal_rows, _b, internal_rows < _b + 32)
 axis = tl.arange(0, 16)
 rows = axis[:, None]
 columns = axis[None, :]
 reduction = tl.arange(0, 16)
 lower10 = tl.load(output + base + (_b + 16 + rows) * n + _b + reduction[None, :])
 second = tl.load(output + base + (_b + 16 + rows) * n + _b + 16 + columns, mask=rows >= columns, other=0.0)
 second -= tl.dot(lower10, tl.trans(lower10), input_precision='tf32')
 tl.store(output + base + (_b + 16 + rows) * n + _b + 16 + columns, second, mask=rows >= columns)
 tl.debug_barrier()
 _potrf16_recursive8_global(output, base, n, _b + 16, PIVOT_FLOOR, 0)

@triton.jit
def _trsm32_recursive8_global(output, base, n: tl.constexpr, solve_rows, _b, valid_rows):
 _trsm16_recursive8_global(output, base, n, solve_rows, _b, valid_rows)
 axis = tl.arange(0, 16)
 reduction = tl.arange(0, 16)
 previous_solution = tl.load(output + base + solve_rows[:, None] * n + _b + reduction[None, :], mask=valid_rows[:, None], other=0.0)
 lower10 = tl.load(output + base + (_b + 16 + axis[:, None]) * n + _b + reduction[None, :])
 rhs1 = tl.load(output + base + solve_rows[:, None] * n + _b + 16 + axis[None, :], mask=valid_rows[:, None], other=0.0)
 rhs1 -= tl.dot(previous_solution, tl.trans(lower10), input_precision='tf32')
 tl.store(output + base + solve_rows[:, None] * n + _b + 16 + axis[None, :], rhs1, mask=valid_rows[:, None])
 tl.debug_barrier()
 _trsm16_recursive8_global(output, base, n, solve_rows, _b + 16, valid_rows)

@triton.jit
def _potrf16_hybrid8_global(output, base, n: tl.constexpr, _b, PIVOT_FLOOR: tl.constexpr, LEAF4: tl.constexpr, INCOMPLETE_MICROBLOCK: tl.constexpr):
 if INCOMPLETE_MICROBLOCK > 0:
  _potrf16_recursive8_global(output, base, n, _b, PIVOT_FLOOR, INCOMPLETE_MICROBLOCK)
 elif LEAF4:
  _potrf16_recursive4_global(output, base, n, _b, PIVOT_FLOOR)
 else:
  _potrf16_recursive8_global(output, base, n, _b, PIVOT_FLOOR, 0)
 axis = tl.arange(0, 8)
 rows = axis[:, None]
 columns = axis[None, :]
 inverse0 = tl.load(output + base + (_b + columns) * n + _b + rows, mask=rows > columns, other=0.0)
 diagonal0 = tl.load(output + base + (_b + axis) * n + _b + axis)
 inverse0 += tl.where(rows == columns, 1.0 / diagonal0[:, None], 0.0)
 inverse1 = tl.load(output + base + (_b + 8 + columns) * n + _b + 8 + rows, mask=rows > columns, other=0.0)
 diagonal1 = tl.load(output + base + (_b + 8 + axis) * n + _b + 8 + axis)
 inverse1 += tl.where(rows == columns, 1.0 / diagonal1[:, None], 0.0)
 lower10 = tl.load(output + base + (_b + 8 + rows) * n + _b + columns)
 right = tl.sum(lower10[:, :, None] * inverse0[None, :, :], axis=1)
 inverse10 = -tl.sum(inverse1[:, :, None] * right[None, :, :], axis=1)
 tl.store(output + base + (_b + columns) * n + _b + 8 + rows, inverse10)
 tl.debug_barrier()

@triton.jit
def _trsm16_full_inverse_global(output, base, n: tl.constexpr, solve_rows, _b, valid_rows, HIGH_PRECISION: tl.constexpr):
 axis = tl.arange(0, 16)
 rows = axis[:, None]
 columns = axis[None, :]
 rhs = tl.load(output + base + solve_rows[:, None] * n + _b + axis[None, :], mask=valid_rows[:, None], other=0.0).to(tl.float32)
 inverse = tl.load(output + base + (_b + columns) * n + _b + rows, mask=rows > columns, other=0.0)
 factor_diagonal = tl.load(output + base + (_b + axis) * n + _b + axis)
 inverse += tl.where(rows == columns, 1.0 / factor_diagonal[:, None], 0.0)
 solution = _factor_dot(rhs, tl.trans(inverse), HIGH_PRECISION)
 tl.store(output + base + solve_rows[:, None] * n + _b + axis[None, :], solution, mask=valid_rows[:, None])
 tl.debug_barrier()

@triton.jit
def _potrf32_hybrid8_global(output, base, n: tl.constexpr, _b, PIVOT_FLOOR: tl.constexpr, HIGH_PRECISION: tl.constexpr, LEAF4: tl.constexpr, INCOMPLETE_MICROBLOCK: tl.constexpr):
 _potrf16_hybrid8_global(output, base, n, _b, PIVOT_FLOOR, LEAF4, INCOMPLETE_MICROBLOCK)
 internal_rows = _b + 16 + tl.arange(0, 16)
 _trsm16_full_inverse_global(output, base, n, internal_rows, _b, internal_rows < _b + 32, HIGH_PRECISION)
 axis = tl.arange(0, 16)
 rows = axis[:, None]
 columns = axis[None, :]
 reduction = tl.arange(0, 16)
 lower10 = tl.load(output + base + (_b + 16 + rows) * n + _b + reduction[None, :])
 second = tl.load(output + base + (_b + 16 + rows) * n + _b + 16 + columns, mask=rows >= columns, other=0.0)
 second -= tl.dot(lower10, tl.trans(lower10), input_precision='tf32')
 tl.store(output + base + (_b + 16 + rows) * n + _b + 16 + columns, second, mask=rows >= columns)
 tl.debug_barrier()
 _potrf16_hybrid8_global(output, base, n, _b + 16, PIVOT_FLOOR, LEAF4, INCOMPLETE_MICROBLOCK)

@triton.jit
def _complete_inverse32_global(output, base, n: tl.constexpr, _b, HIGH_PRECISION: tl.constexpr):
 axis = tl.arange(0, 16)
 rows = axis[:, None]
 columns = axis[None, :]
 inverse0 = tl.load(output + base + (_b + columns) * n + _b + rows, mask=rows > columns, other=0.0)
 diagonal0 = tl.load(output + base + (_b + axis) * n + _b + axis)
 inverse0 += tl.where(rows == columns, 1.0 / diagonal0[:, None], 0.0)
 inverse1 = tl.load(output + base + (_b + 16 + columns) * n + _b + 16 + rows, mask=rows > columns, other=0.0)
 diagonal1 = tl.load(output + base + (_b + 16 + axis) * n + _b + 16 + axis)
 inverse1 += tl.where(rows == columns, 1.0 / diagonal1[:, None], 0.0)
 lower10 = tl.load(output + base + (_b + 16 + rows) * n + _b + columns)
 right = _factor_dot(lower10, inverse0, HIGH_PRECISION)
 inverse10 = -_factor_dot(inverse1, right, HIGH_PRECISION)
 tl.store(output + base + (_b + columns) * n + _b + 16 + rows, inverse10)
 tl.debug_barrier()

@triton.jit
def _trsm32_full_inverse_global(output, base, n: tl.constexpr, solve_rows, _b, valid_rows, HIGH_PRECISION: tl.constexpr):
 axis = tl.arange(0, 32)
 rows = axis[:, None]
 columns = axis[None, :]
 rhs = tl.load(output + base + solve_rows[:, None] * n + _b + axis[None, :], mask=valid_rows[:, None], other=0.0).to(tl.float32)
 inverse = tl.load(output + base + (_b + columns) * n + _b + rows, mask=rows > columns, other=0.0)
 factor_diagonal = tl.load(output + base + (_b + axis) * n + _b + axis)
 inverse += tl.where(rows == columns, 1.0 / factor_diagonal[:, None], 0.0)
 solution = _factor_dot(rhs, tl.trans(inverse), HIGH_PRECISION)
 tl.store(output + base + solve_rows[:, None] * n + _b + axis[None, :], solution, mask=valid_rows[:, None])
 tl.debug_barrier()

@triton.jit
def _factor32_dispatch_global(output, base, n: tl.constexpr, _b, PIVOT_FLOOR: tl.constexpr, LEAF8: tl.constexpr, HYBRID8: tl.constexpr, HYBRID4: tl.constexpr, FULL_INVERSE32: tl.constexpr, HIGH_PRECISION: tl.constexpr, DIAGONAL_PRECISION: tl.constexpr, INCOMPLETE_MICROBLOCK: tl.constexpr, FIRST_LEAF_PRECISION: tl.constexpr=-1, APPROX_RSQRT_MASK: tl.constexpr=-1):
 if HYBRID8:
  _potrf32_hybrid8_global(output, base, n, _b, PIVOT_FLOOR, HIGH_PRECISION, HYBRID4, INCOMPLETE_MICROBLOCK)
 elif LEAF8:
  _potrf32_recursive8_global(output, base, n, _b, PIVOT_FLOOR)
 else:
  _potrf32_recursive16_global(output, base, n, _b, PIVOT_FLOOR, HIGH_PRECISION, DIAGONAL_PRECISION, FIRST_LEAF_PRECISION, APPROX_RSQRT_MASK)
 if not LEAF8 and FULL_INVERSE32:
  _complete_inverse32_global(output, base, n, _b, HIGH_PRECISION)

@triton.jit
def _solve32_dispatch_global(output, base, n: tl.constexpr, solve_rows, _b, valid_rows, LEAF8: tl.constexpr, FULL_INVERSE32: tl.constexpr, HIGH_PRECISION: tl.constexpr):
 if LEAF8:
  _trsm32_recursive8_global(output, base, n, solve_rows, _b, valid_rows)
 elif FULL_INVERSE32:
  _trsm32_full_inverse_global(output, base, n, solve_rows, _b, valid_rows, HIGH_PRECISION)
 else:
  _trsm32_recursive16_global(output, base, n, solve_rows, _b, valid_rows, HIGH_PRECISION)

@triton.jit
def _factor64_block_global(output, base, n: tl.constexpr, _b, PIVOT_FLOOR: tl.constexpr, LEAF8: tl.constexpr, HYBRID8: tl.constexpr, HYBRID4: tl.constexpr, FULL_INVERSE32: tl.constexpr, HIGH_PRECISION: tl.constexpr, DIAGONAL_PRECISION: tl.constexpr, INCOMPLETE_MICROBLOCK: tl.constexpr, FIRST_LEAF_PRECISION: tl.constexpr=-1, APPROX_RSQRT_MASK: tl.constexpr=-1):
 first_half_rsqrt_mask: tl.constexpr = -1
 second_half_rsqrt_mask: tl.constexpr = -1
 if APPROX_RSQRT_MASK >= 0:
  first_half_rsqrt_mask = APPROX_RSQRT_MASK & 3
  second_half_rsqrt_mask = APPROX_RSQRT_MASK >> 2 & 3
 _factor32_dispatch_global(output, base, n, _b, PIVOT_FLOOR, LEAF8, HYBRID8, HYBRID4, FULL_INVERSE32, HIGH_PRECISION, DIAGONAL_PRECISION, INCOMPLETE_MICROBLOCK, FIRST_LEAF_PRECISION, first_half_rsqrt_mask)
 internal_rows = _b + 32 + tl.arange(0, 32)
 _solve32_dispatch_global(output, base, n, internal_rows, _b, internal_rows < _b + 64, LEAF8, FULL_INVERSE32, HIGH_PRECISION)
 axis = tl.arange(0, 32)
 rows = axis[:, None]
 columns = axis[None, :]
 lower10 = tl.load(output + base + (_b + 32 + rows) * n + _b + columns)
 second = tl.load(output + base + (_b + 32 + rows) * n + _b + 32 + columns, mask=rows >= columns, other=0.0)
 second -= _factor_dot(lower10, tl.trans(lower10), DIAGONAL_PRECISION)
 tl.store(output + base + (_b + 32 + rows) * n + _b + 32 + columns, second, mask=rows >= columns)
 tl.debug_barrier()
 _factor32_dispatch_global(output, base, n, _b + 32, PIVOT_FLOOR, LEAF8, HYBRID8, HYBRID4, FULL_INVERSE32, HIGH_PRECISION, DIAGONAL_PRECISION, INCOMPLETE_MICROBLOCK, -1, second_half_rsqrt_mask)

@triton.jit
def _complete_inverse64_global(output, base, n: tl.constexpr, _b, HIGH_PRECISION: tl.constexpr):
 axis = tl.arange(0, 32)
 rows = axis[:, None]
 columns = axis[None, :]
 inverse0 = tl.load(output + base + (_b + columns) * n + _b + rows, mask=rows > columns, other=0.0)
 diagonal0 = tl.load(output + base + (_b + axis) * n + _b + axis)
 inverse0 += tl.where(rows == columns, 1.0 / diagonal0[:, None], 0.0)
 inverse1 = tl.load(output + base + (_b + 32 + columns) * n + _b + 32 + rows, mask=rows > columns, other=0.0)
 diagonal1 = tl.load(output + base + (_b + 32 + axis) * n + _b + 32 + axis)
 inverse1 += tl.where(rows == columns, 1.0 / diagonal1[:, None], 0.0)
 lower10 = tl.load(output + base + (_b + 32 + rows) * n + _b + columns)
 right = _factor_dot(lower10, inverse0, HIGH_PRECISION)
 inverse10 = -_factor_dot(inverse1, right, HIGH_PRECISION)
 tl.store(output + base + (_b + columns) * n + _b + 32 + rows, inverse10)
 tl.debug_barrier()

@triton.jit
def _complete_inverse64_diagonal_global(output, base, n: tl.constexpr, _b):
 """Zeroth-order off-block inverse for weakly correlated dense leaves."""
 axis = tl.arange(0, 32)
 rows = axis[:, None]
 columns = axis[None, :]
 diagonal0 = tl.load(output + base + (_b + axis) * n + _b + axis)
 diagonal1 = tl.load(output + base + (_b + 32 + axis) * n + _b + 32 + axis)
 lower10 = tl.load(output + base + (_b + 32 + rows) * n + _b + columns)
 inverse10 = -lower10 / (diagonal1[:, None] * diagonal0[None, :])
 tl.store(output + base + (_b + columns) * n + _b + 32 + rows, inverse10)
 tl.debug_barrier()

@triton.jit
def _trsm64_full_inverse_global(output, base, n: tl.constexpr, solve_rows, _b, valid_rows, HIGH_PRECISION: tl.constexpr, USE_BF16: tl.constexpr):
 axis = tl.arange(0, 64)
 rows = axis[:, None]
 columns = axis[None, :]
 rhs = tl.load(output + base + solve_rows[:, None] * n + _b + columns, mask=valid_rows[:, None], other=0.0).to(tl.float32)
 inverse = tl.load(output + base + (_b + columns) * n + _b + rows, mask=rows > columns, other=0.0)
 factor_diagonal = tl.load(output + base + (_b + axis) * n + _b + axis)
 inverse += tl.where(rows == columns, 1.0 / factor_diagonal[:, None], 0.0)
 if USE_BF16:
  solution = tl.dot(rhs.to(tl.bfloat16), tl.trans(inverse.to(tl.bfloat16)))
 else:
  solution = _factor_dot(rhs, tl.trans(inverse), HIGH_PRECISION)
 tl.store(output + base + solve_rows[:, None] * n + _b + columns, solution, mask=valid_rows[:, None])
 tl.debug_barrier()

@triton.jit
def _solve64_block_global(output, base, n: tl.constexpr, solve_rows, _b, valid_rows, LEAF8: tl.constexpr, FULL_INVERSE32: tl.constexpr, HIGH_PRECISION: tl.constexpr, DIAGONAL_PRECISION: tl.constexpr):
 _solve32_dispatch_global(output, base, n, solve_rows, _b, valid_rows, LEAF8, FULL_INVERSE32, HIGH_PRECISION)
 axis = tl.arange(0, 32)
 rows = axis[:, None]
 columns = axis[None, :]
 previous_solution = tl.load(output + base + solve_rows[:, None] * n + _b + columns, mask=valid_rows[:, None], other=0.0)
 lower10 = tl.load(output + base + (_b + 32 + rows) * n + _b + columns)
 rhs1 = tl.load(output + base + solve_rows[:, None] * n + _b + 32 + columns, mask=valid_rows[:, None], other=0.0)
 rhs1 -= _factor_dot(previous_solution, tl.trans(lower10), DIAGONAL_PRECISION)
 tl.store(output + base + solve_rows[:, None] * n + _b + 32 + columns, rhs1, mask=valid_rows[:, None])
 tl.debug_barrier()
 _solve32_dispatch_global(output, base, n, solve_rows, _b + 32, valid_rows, LEAF8, FULL_INVERSE32, HIGH_PRECISION)

@triton.jit
def _potrf64_trsm_full_inverse_kernel(output, flags, n: tl.constexpr, batch: tl.constexpr, panel, generation, row_blocks, ROWS: tl.constexpr, PIVOT_FLOOR: tl.constexpr, LEAF8: tl.constexpr, HYBRID8: tl.constexpr, HYBRID4: tl.constexpr, FULL_INVERSE32: tl.constexpr, HIGH_PRECISION: tl.constexpr, DIAGONAL_PRECISION: tl.constexpr, USE_BF16_TRSM: tl.constexpr, INCOMPLETE_MICROBLOCK: tl.constexpr):
 program = tl.program_id(0)
 if program < batch:
  matrix = program
  base = matrix * n * n
  _factor64_block_global(output, base, n, panel, PIVOT_FLOOR, LEAF8, HYBRID8, HYBRID4, FULL_INVERSE32, HIGH_PRECISION, DIAGONAL_PRECISION, INCOMPLETE_MICROBLOCK)
  _complete_inverse64_global(output, base, n, panel, HIGH_PRECISION)
  tl.atomic_xchg(flags + matrix, generation * 1.0)
 else:
  linear = program - batch
  matrix = linear // row_blocks
  row_block = linear - matrix * row_blocks
  ready = tl.atomic_add(flags + matrix, 0.0)
  while ready < generation:
   ready = tl.atomic_add(flags + matrix, 0.0)
  row_axis = tl.arange(0, ROWS)
  solve_rows = panel + 64 + row_block * ROWS + row_axis
  valid_rows = solve_rows < n
  base = matrix * n * n
  _trsm64_full_inverse_global(output, base, n, solve_rows, panel, valid_rows, HIGH_PRECISION, USE_BF16_TRSM)

@triton.jit
def _factor64_only_kernel(output, n: tl.constexpr, panel, PIVOT_FLOOR: tl.constexpr, LEAF8: tl.constexpr, HYBRID8: tl.constexpr, HYBRID4: tl.constexpr, FULL_INVERSE32: tl.constexpr, BUILD_INVERSE64: tl.constexpr, APPROX_INVERSE64: tl.constexpr, HIGH_PRECISION: tl.constexpr, DIAGONAL_PRECISION: tl.constexpr, INCOMPLETE_MICROBLOCK: tl.constexpr):
 matrix = tl.program_id(0)
 base = matrix * n * n
 _factor64_block_global(output, base, n, panel, PIVOT_FLOOR, LEAF8, HYBRID8, HYBRID4, FULL_INVERSE32, HIGH_PRECISION, DIAGONAL_PRECISION, INCOMPLETE_MICROBLOCK)
 if BUILD_INVERSE64:
  if APPROX_INVERSE64:
   _complete_inverse64_diagonal_global(output, base, n, panel)
  else:
   _complete_inverse64_global(output, base, n, panel, HIGH_PRECISION)

@triton.jit
def _full_potrf128_recursive16_kernel(source, output, FACTOR_PRECISION: tl.constexpr, SECOND_FACTOR_PRECISION: tl.constexpr, SECOND_DIAGONAL_PRECISION: tl.constexpr, SOLVE_PRECISION: tl.constexpr, UPDATE_PRECISION: tl.constexpr, FIRST_DIAGONAL_PRECISION: tl.constexpr=-1):
 """Initialize and factor one complete 128x128 matrix in one launch."""
 matrix = tl.program_id(0)
 base = matrix * 128 * 128
 axis = tl.arange(0, 64)
 rows = axis[:, None]
 columns = axis[None, :]
 lower = rows >= columns
 block00 = tl.load(source + base + rows * 128 + columns, mask=lower, other=0.0)
 tl.store(output + base + rows * 128 + columns, block00, mask=lower)
 tl.store(output + base + rows * 128 + columns, 0.0, mask=columns > rows)
 block10 = tl.load(source + base + (64 + rows) * 128 + columns)
 tl.store(output + base + (64 + rows) * 128 + columns, block10)
 tl.store(output + base + rows * 128 + 64 + columns, 0.0)
 block11 = tl.load(source + base + (64 + rows) * 128 + 64 + columns, mask=lower, other=0.0)
 tl.store(output + base + (64 + rows) * 128 + 64 + columns, block11, mask=lower)
 tl.store(output + base + (64 + rows) * 128 + 64 + columns, 0.0, mask=columns > rows)
 tl.debug_barrier()
 _factor64_block_global(output, base, 128, 0, 1.1754943508222875e-38, False, False, False, False, FACTOR_PRECISION, FACTOR_PRECISION, 0, FIRST_DIAGONAL_PRECISION, 14)
 internal_rows = 64 + axis
 _solve64_block_global(output, base, 128, internal_rows, 0, internal_rows < 128, False, False, SOLVE_PRECISION, SOLVE_PRECISION)
 lower10 = tl.load(output + base + (64 + rows) * 128 + columns)
 second = tl.load(output + base + (64 + rows) * 128 + 64 + columns, mask=lower, other=0.0)
 second -= _factor_dot(lower10, tl.trans(lower10), UPDATE_PRECISION)
 tl.store(output + base + (64 + rows) * 128 + 64 + columns, second, mask=lower)
 tl.debug_barrier()
 _factor64_block_global(output, base, 128, 64, 1.1754943508222875e-38, False, False, False, False, SECOND_FACTOR_PRECISION, SECOND_DIAGONAL_PRECISION, 0, -1, 15)
 scratch_axis = tl.arange(0, 32)
 scratch_rows = scratch_axis[:, None]
 scratch_columns = scratch_axis[None, :]
 for scratch_block in tl.static_range(0, 4):
  scratch_start = scratch_block * 32
  tl.store(output + base + (scratch_start + scratch_rows) * 128 + scratch_start + scratch_columns, 0.0, mask=scratch_columns > scratch_rows)

@triton.jit
def _resident_block64_cholesky_kernel(source, output, n: tl.constexpr, BLOCKS: tl.constexpr, FACTOR_PRECISION: tl.constexpr, SOLVE_PRECISION: tl.constexpr, UPDATE_PRECISION: tl.constexpr, FULL_INVERSE32: tl.constexpr):
 """One matrix per CTA, with the complete block wavefront resident."""
 matrix = tl.program_id(0)
 base = matrix * n * n
 axis = tl.arange(0, 64)
 rows = axis[:, None]
 columns = axis[None, :]
 for panel_block in tl.range(0, BLOCKS, 1):
  panel_start = panel_block * 64
  diagonal = tl.load(source + base + (panel_start + rows) * n + panel_start + columns).to(tl.float32)
  for previous_block in tl.range(0, panel_block, 1):
   previous_start = previous_block * 64
   history = tl.load(output + base + (panel_start + rows) * n + previous_start + columns)
   diagonal -= _factor_dot(history, tl.trans(history), UPDATE_PRECISION)
  tl.store(output + base + (panel_start + rows) * n + panel_start + columns, diagonal, mask=rows >= columns)
  tl.debug_barrier()
  _factor64_block_global(output, base, n, panel_start, 1.1754943508222875e-38, False, False, False, FULL_INVERSE32, FACTOR_PRECISION, FACTOR_PRECISION, 0)
  for row_block in tl.range(panel_block + 1, BLOCKS, 1):
   row_start = row_block * 64
   rhs = tl.load(source + base + (row_start + rows) * n + panel_start + columns).to(tl.float32)
   for previous_block in tl.range(0, panel_block, 1):
    previous_start = previous_block * 64
    left = tl.load(output + base + (row_start + rows) * n + previous_start + columns)
    right = tl.load(output + base + (panel_start + rows) * n + previous_start + columns)
    rhs -= _factor_dot(left, tl.trans(right), UPDATE_PRECISION)
   tl.store(output + base + (row_start + rows) * n + panel_start + columns, rhs)
   tl.debug_barrier()
   solve_rows = row_start + axis
   _solve64_block_global(output, base, n, solve_rows, panel_start, solve_rows < row_start + 64, False, FULL_INVERSE32, SOLVE_PRECISION, SOLVE_PRECISION)
   tl.store(output + base + (panel_start + rows) * n + row_start + columns, 0.0)
  tl.store(output + base + (panel_start + rows) * n + panel_start + columns, 0.0, mask=columns > rows)
  tl.debug_barrier()

@triton.jit
def _jacobi_dot(left, right, USE_FP16: tl.constexpr):
 if USE_FP16:
  return tl.dot(left.to(tl.float16), right.to(tl.float16))
 return tl.dot(left, right, input_precision='tf32')

@triton.jit
def _jacobi_solve_dot(left, right, PRECISION: tl.constexpr):
 if PRECISION == 2:
  return tl.dot(left.to(tl.float8e4nv), right.to(tl.float8e4nv))
 if PRECISION == 1:
  return tl.dot(left.to(tl.float16), right.to(tl.float16))
 return tl.dot(left, right, input_precision='tf32')

@triton.jit
def _jacobi_factor_inverse_from_lower(lower, REFINEMENT_STEPS: tl.constexpr, LAST_REFINEMENT_GAIN: tl.constexpr, INVERSE_ORDER: tl.constexpr, REFINEMENT_FP16: tl.constexpr=False, INVERSE_FP16: tl.constexpr=False, FP8_REFINEMENT_STEPS: tl.constexpr=0, USE_RSQRT: tl.constexpr=False):
 axis = tl.arange(0, 64)
 rows = axis[:, None]
 columns = axis[None, :]
 diagonal = tl.sum(tl.where(rows == columns, lower, 0.0), axis=0)
 safe_diagonal = tl.maximum(diagonal, 1.1754943508222875e-38)
 if USE_RSQRT:
  inverse_roots = tl.rsqrt(safe_diagonal)
  roots = safe_diagonal * inverse_roots
 else:
  roots = tl.sqrt(safe_diagonal)
 symmetric = lower + tl.trans(lower) - tl.where(rows == columns, diagonal[None, :], 0.0)
 if USE_RSQRT:
  normalized = symmetric * inverse_roots[:, None] * inverse_roots[None, :]
 else:
  normalized = symmetric / (roots[:, None] * roots[None, :])
 factor = tl.where(rows == columns, 1.0, tl.where(rows > columns, normalized, 0.0))
 for refinement in tl.static_range(0, REFINEMENT_STEPS):
  if refinement < FP8_REFINEMENT_STEPS:
   product = tl.dot(factor.to(tl.float8e4nv), tl.trans(factor.to(tl.float8e4nv)))
  elif REFINEMENT_FP16:
   product = _jacobi_dot(factor, tl.trans(factor), True)
  elif REFINEMENT_STEPS >= 6 and refinement + 1 == REFINEMENT_STEPS:
   product = tl.dot(factor, tl.trans(factor), input_precision='tf32x3')
  else:
   product = tl.dot(factor, tl.trans(factor), input_precision='tf32')
  residual = normalized - product
  gain = 1.0
  if refinement + 1 == REFINEMENT_STEPS:
   gain = LAST_REFINEMENT_GAIN
  factor += gain * tl.where(rows == columns, 0.5 * residual, tl.where(rows > columns, residual, 0.0))
 scaled_factor = roots[:, None] * factor
 identity = tl.where(rows == columns, 1.0, 0.0)
 nilpotent = factor - identity
 inverse = identity - nilpotent
 if INVERSE_ORDER == 6:
  square = _jacobi_dot(nilpotent, nilpotent, INVERSE_FP16)
  cube = _jacobi_dot(square, nilpotent, INVERSE_FP16)
  prefix = identity - nilpotent + square
  inverse = prefix - _jacobi_dot(cube, prefix - cube, INVERSE_FP16)
 elif INVERSE_ORDER >= 2:
  square = _jacobi_dot(nilpotent, nilpotent, INVERSE_FP16)
  if INVERSE_ORDER == 2:
   inverse += square
  elif INVERSE_ORDER == 3:
   cube = _jacobi_dot(square, nilpotent, INVERSE_FP16)
   inverse += square - cube
  elif INVERSE_ORDER == 4:
   inverse += _jacobi_dot(square, identity - nilpotent + square, INVERSE_FP16)
  else:
   cube = _jacobi_dot(square, nilpotent, INVERSE_FP16)
   fourth = _jacobi_dot(square, square, INVERSE_FP16)
   inverse += square - cube + fourth
   if INVERSE_ORDER >= 5:
    fifth = _jacobi_dot(fourth, nilpotent, INVERSE_FP16)
    inverse -= fifth
   if INVERSE_ORDER >= 6:
    sixth = _jacobi_dot(cube, cube, INVERSE_FP16)
    inverse += sixth
 if USE_RSQRT:
  inverse *= inverse_roots[None, :]
 else:
  inverse /= roots[None, :]
 return (scaled_factor, inverse)

@triton.jit
def _pdl256_damped_factor_inverse(lower):
 axis = tl.arange(0, 64)
 rows = axis[:, None]
 columns = axis[None, :]
 diagonal = tl.sum(tl.where(rows == columns, lower, 0.0), axis=0)
 safe_diagonal = tl.maximum(diagonal, 1.1754943508222875e-38)
 inverse_roots = tl.rsqrt(safe_diagonal)
 inverse_roots *= 1.5 - 0.5 * safe_diagonal * inverse_roots * inverse_roots
 roots = safe_diagonal * inverse_roots
 symmetric = lower + tl.trans(lower) - tl.where(rows == columns, diagonal[None, :], 0.0)
 normalized = symmetric * inverse_roots[:, None] * inverse_roots[None, :]
 factor = tl.where(rows == columns, 1.0, tl.where(rows > columns, normalized, 0.0))
 for refinement in tl.static_range(0, 10):
  product = tl.dot(factor.to(tl.float16), tl.trans(factor.to(tl.float16)))
  residual = normalized - product
  if refinement == 0:
   lower_gain = 0.75
   diagonal_gain = 0.375
  elif refinement < 8:
   lower_gain = 1.245
   diagonal_gain = 0.6225
  elif refinement == 8:
   lower_gain = 0.67
   diagonal_gain = 0.335
  else:
   lower_gain = 0.67
   diagonal_gain = 0.4355
  factor += tl.where(rows == columns, diagonal_gain * residual, tl.where(rows > columns, lower_gain * residual, 0.0))
 scaled_factor = roots[:, None] * factor
 identity = tl.where(rows == columns, 1.0, 0.0)
 nilpotent = factor - identity
 square = _jacobi_dot(nilpotent, nilpotent, True)
 cube = _jacobi_dot(square, nilpotent, True)
 prefix = identity - nilpotent + square
 inverse = prefix - _jacobi_dot(cube, prefix - cube, True)
 inverse *= inverse_roots[None, :]
 return (scaled_factor, inverse)

N=tl.constexpr(256)
TINY=tl.constexpr(1.1754943508222875e-38)

@triton.jit
def _compact_potrf16(values):
    axis = tl.arange(0, 16)
    rows = axis[:, None]
    columns = axis[None, :]
    lower = tl.zeros((16, 16), tl.float32)
    for column_index in tl.range(0, 16, 1):
        column = tl.sum(
            tl.where(columns == column_index, values, 0.0),
            axis=1,
        )
        diagonal = tl.sum(
            tl.where(axis == column_index, column, 0.0),
            axis=0,
        )
        inverse_root = tl.rsqrt(tl.maximum(diagonal, TINY))
        factor_column = tl.where(
            axis >= column_index,
            column * inverse_root,
            0.0,
        )
        lower = tl.where(
            columns == column_index,
            factor_column[:, None],
            lower,
        )
        trailing = (
            (rows > column_index)
            & (columns > column_index)
            & (rows >= columns)
        )
        values = tl.where(
            trailing,
            values - factor_column[:, None] * factor_column[None, :],
            values,
        )
    return lower

@triton.jit
def _compact_trsm16(rhs, factor):
    columns = tl.arange(0, 16)
    values = rhs
    for column_index in tl.range(0, 16, 1):
        column = tl.sum(
            tl.where(columns[None, :] == column_index, values, 0.0),
            axis=1,
        )
        factor_column = tl.sum(
            tl.where(
                columns[None, :] == column_index,
                factor,
                0.0,
            ),
            axis=1,
        )
        diagonal = tl.sum(
            tl.where(columns == column_index, factor_column, 0.0),
            axis=0,
        )
        solved = column / diagonal
        values = tl.where(
            columns[None, :] == column_index,
            solved[:, None],
            tl.where(
                columns[None, :] > column_index,
                values - solved[:, None] * factor_column[None, :],
                values,
            ),
        )
    return values

@triton.jit
def _compact_factor64_global(output, base, panel: tl.constexpr):
    axis = tl.arange(0, 16)
    rows = axis[:, None]
    columns = axis[None, :]
    for leaf_index in tl.range(0, 4, 1):
        leaf = panel + leaf_index * 16
        values = tl.load(
            output + base + (leaf + rows) * N + leaf + columns,
            mask=rows >= columns,
            other=0.0,
        ).to(tl.float32)
        factor = _compact_potrf16(values)
        tl.store(
            output + base + (leaf + rows) * N + leaf + columns,
            factor,
            mask=rows >= columns,
        )
        tl.store(
            output + base + (leaf + rows) * N + leaf + columns,
            0.0,
            mask=columns > rows,
        )
        tl.debug_barrier()

        for row_index in tl.range(leaf_index + 1, 4, 1):
            row_start = panel + row_index * 16
            rhs16 = tl.load(
                output
                + base
                + (row_start + tl.arange(0, 64))[:, None] * N
                + leaf
                + tl.arange(0, 16)[None, :],
                mask=tl.arange(0, 64)[:, None] < 16,
                other=0.0,
            ).to(tl.float32)
            factor16 = tl.load(
                output + base + (leaf + rows) * N + leaf + columns,
            )
            solved = _compact_trsm16(rhs16, factor16)
            tl.store(
                output
                + base
                + (row_start + tl.arange(0, 64))[:, None] * N
                + leaf
                + tl.arange(0, 16)[None, :],
                solved,
                mask=tl.arange(0, 64)[:, None] < 16,
            )
        tl.debug_barrier()

        for row_index in tl.range(leaf_index + 1, 4, 1):
            row_start = panel + row_index * 16
            left = tl.load(
                output
                + base
                + (row_start + rows) * N
                + leaf
                + columns,
            )
            for column_index in tl.range(
                leaf_index + 1,
                row_index + 1,
                1,
            ):
                column_start = panel + column_index * 16
                right = tl.load(
                    output
                    + base
                    + (column_start + rows) * N
                    + leaf
                    + columns,
                )
                diagonal_tile = row_index == column_index
                valid = (not diagonal_tile) | (rows >= columns)
                current = tl.load(
                    output
                    + base
                    + (row_start + rows) * N
                    + column_start
                    + columns,
                    mask=valid,
                    other=0.0,
                ).to(tl.float32)
                current -= _factor_dot(left, tl.trans(right), 1)
                tl.store(
                    output
                    + base
                    + (row_start + rows) * N
                    + column_start
                    + columns,
                    current,
                    mask=valid,
                )
        tl.debug_barrier()

@triton.jit
def _compact_solve64_global(
    output,
    base,
    panel: tl.constexpr,
    row_start,
):
    row_axis = tl.arange(0, 64)
    column_axis = tl.arange(0, 16)
    leaf_rows = column_axis[:, None]
    leaf_columns = column_axis[None, :]
    for leaf_index in tl.range(0, 4, 1):
        leaf = panel + leaf_index * 16
        rhs = tl.load(
            output
            + base
            + (row_start + row_axis)[:, None] * N
            + leaf
            + column_axis[None, :],
        ).to(tl.float32)
        factor = tl.load(
            output
            + base
            + (leaf + leaf_rows) * N
            + leaf
            + leaf_columns,
        )
        solved = _compact_trsm16(rhs, factor)
        tl.store(
            output
            + base
            + (row_start + row_axis)[:, None] * N
            + leaf
            + column_axis[None, :],
            solved,
        )
        tl.debug_barrier()
        for later_index in tl.range(leaf_index + 1, 4, 1):
            later = panel + later_index * 16
            cross = tl.load(
                output
                + base
                + (later + leaf_rows) * N
                + leaf
                + leaf_columns,
            )
            current = tl.load(
                output
                + base
                + (row_start + row_axis)[:, None] * N
                + later
                + column_axis[None, :],
            ).to(tl.float32)
            current -= _factor_dot(solved, tl.trans(cross), 1)
            tl.store(
                output
                + base
                + (row_start + row_axis)[:, None] * N
                + later
                + column_axis[None, :],
                current,
            )
        tl.debug_barrier()

@triton.jit
def _homotopy_factor_inverse64(
    lower,
    FIRST_STEPS: tl.constexpr,
    SECOND_STEPS: tl.constexpr,
    FINAL_STEPS: tl.constexpr,
    GAIN: tl.constexpr,
    COMPUTE_INVERSE: tl.constexpr,
):
    axis = tl.arange(0, 64)
    rows = axis[:, None]
    columns = axis[None, :]
    diagonal = tl.sum(
        tl.where(rows == columns, lower, 0.0),
        axis=0,
    )
    safe_diagonal = tl.maximum(diagonal, TINY)
    inverse_roots = tl.rsqrt(safe_diagonal)
    inverse_roots *= (
        1.5
        - 0.5 * safe_diagonal * inverse_roots * inverse_roots
    )
    roots = safe_diagonal * inverse_roots
    symmetric = (
        lower
        + tl.trans(lower)
        - tl.where(rows == columns, diagonal[None, :], 0.0)
    )
    normalized = (
        symmetric
        * inverse_roots[:, None]
        * inverse_roots[None, :]
    )
    identity = tl.where(rows == columns, 1.0, 0.0)
    first_target = (normalized + 8.0 * identity) * (1.0 / 9.0)
    factor = tl.where(
        rows == columns,
        1.0,
        tl.where(rows > columns, first_target, 0.0),
    )
    total_steps: tl.constexpr = (
        FIRST_STEPS + SECOND_STEPS + FINAL_STEPS
    )
    for step in tl.range(0, total_steps, 1):
        shift = tl.where(
            step < FIRST_STEPS,
            8.0,
            tl.where(step < FIRST_STEPS + SECOND_STEPS, 0.5, 0.0),
        )
        target = (normalized + shift * identity) / (1.0 + shift)
        product = tl.dot(
            factor,
            tl.trans(factor),
            input_precision="tf32",
        )
        residual = target - product
        factor += tl.where(
            rows == columns,
            (0.5 * GAIN) * residual,
            tl.where(rows > columns, GAIN * residual, 0.0),
        )

    scaled_factor = roots[:, None] * factor
    if COMPUTE_INVERSE:
        nilpotent = factor - identity
        square = _jacobi_dot(nilpotent, nilpotent, True)
        cube = _jacobi_dot(square, nilpotent, True)
        prefix = identity - nilpotent + square
        inverse = prefix - _jacobi_dot(
            cube,
            prefix - cube,
            True,
        )
        inverse *= inverse_roots[None, :]
    else:
        inverse = tl.zeros((64, 64), tl.float32)
    return scaled_factor, inverse

@triton.jit
def _inline_factor_kernel(
    source,
    output,
    PANEL_BLOCK: tl.constexpr,
    WAIT: tl.constexpr,
    SIGNAL: tl.constexpr,
    FINAL_PANEL: tl.constexpr,
    EXACT_SELECTED: tl.constexpr,
):
    if WAIT:
        tl_cuda.gdc_wait()
    if SIGNAL:
        tl_cuda.gdc_launch_dependents()
    matrix = tl.program_id(0)
    panel: tl.constexpr = PANEL_BLOCK * 64
    base = matrix * N * N
    axis = tl.arange(0, 64)
    rows = axis[:, None]
    columns = axis[None, :]
    lower_mask = rows >= columns
    if PANEL_BLOCK == 0:
        lower = tl.load(
            source + base + (panel + rows) * N + panel + columns,
            mask=lower_mask,
            other=0.0,
        ).to(tl.float32)
    else:
        lower = tl.load(
            output + base + (panel + rows) * N + panel + columns,
            mask=lower_mask,
            other=0.0,
        ).to(tl.float32)
    selected = (tl.load(source) < 0.1) & EXACT_SELECTED
    if selected:
        tl.store(
            output + base + (panel + rows) * N + panel + columns,
            lower,
            mask=lower_mask,
        )
        tl.store(
            output + base + (panel + rows) * N + panel + columns,
            0.0,
            mask=columns > rows,
        )
        tl.debug_barrier()
        _compact_factor64_global(output, base, panel)
    else:
        factor, inverse = _pdl256_damped_factor_inverse(lower)
        identity = tl.where(rows == columns, 1.0, 0.0)
        inverse_residual = identity - tl.dot(
            factor,
            inverse,
            input_precision="tf32x3",
        )
        inverse += tl.dot(
            inverse,
            inverse_residual,
            input_precision="tf32x3",
        )
        inverse = tl.where(lower_mask, inverse, 0.0)
        tl.store(
            output + base + (panel + rows) * N + panel + columns,
            factor,
            mask=lower_mask,
        )
        if not FINAL_PANEL:
            tl.store(
                output + base + (panel + columns) * N + panel + rows,
                inverse,
                mask=rows > columns,
            )
        else:
            tl.store(
                output + base + (panel + rows) * N + panel + columns,
                0.0,
                mask=columns > rows,
            )

@triton.jit
def _inline_trsm0_kernel(
    source,
    output,
    EXACT_SELECTED: tl.constexpr,
):
    row_local = tl.program_id(0)
    matrix = tl.program_id(1)
    row_start = (1 + row_local) * 64
    base = matrix * N * N
    axis = tl.arange(0, 64)
    rows = axis[:, None]
    columns = axis[None, :]
    rhs = tl.load(
        source + base + (row_start + rows) * N + columns,
    ).to(tl.float32)
    tl.store(
        output + base + (row_start + rows) * N + columns,
        rhs,
    )
    tl.debug_barrier()
    tl_cuda.gdc_wait()
    tl_cuda.gdc_launch_dependents()
    selected = (tl.load(source) < 0.1) & EXACT_SELECTED
    if selected:
        _compact_solve64_global(output, base, 0, row_start)
    else:
        solve_rows = row_start + axis
        _trsm64_full_inverse_global(
            output,
            base,
            N,
            solve_rows,
            0,
            solve_rows < row_start + 64,
            2,
            False,
        )
    tl.store(
        output + base + rows * N + row_start + columns,
        0.0,
    )

@triton.jit
def _inline_first_factor_join(
    source,
    output,
    EXACT_SELECTED: tl.constexpr,
):
    matrix = tl.program_id(0)
    base = matrix * N * N
    axis = tl.arange(0, 64)
    rows = axis[:, None]
    columns = axis[None, :]
    lower_mask = rows >= columns
    current = tl.load(
        source + base + (64 + rows) * N + 64 + columns,
        mask=lower_mask,
        other=0.0,
    ).to(tl.float32)
    tl_cuda.gdc_wait()
    tl_cuda.gdc_launch_dependents()
    left = tl.load(output + base + (64 + rows) * N + columns)
    current -= _factor_dot(left, tl.trans(left), False)
    current = tl.where(lower_mask, current, 0.0)
    selected = (tl.load(source) < 0.1) & EXACT_SELECTED
    if selected:
        tl.store(
            output + base + (64 + rows) * N + 64 + columns,
            current,
            mask=lower_mask,
        )
        tl.store(
            output + base + (64 + rows) * N + 64 + columns,
            0.0,
            mask=columns > rows,
        )
        tl.debug_barrier()
        _compact_factor64_global(output, base, 64)
    else:
        factor, inverse = _pdl256_damped_factor_inverse(current)
        identity = tl.where(rows == columns, 1.0, 0.0)
        inverse_residual = identity - tl.dot(
            factor,
            inverse,
            input_precision="tf32x3",
        )
        inverse += tl.dot(
            inverse,
            inverse_residual,
            input_precision="tf32x3",
        )
        inverse = tl.where(lower_mask, inverse, 0.0)
        tl.store(
            output + base + (64 + rows) * N + 64 + columns,
            factor,
            mask=lower_mask,
        )
        tl.store(
            output + base + (64 + columns) * N + 64 + rows,
            inverse,
            mask=rows > columns,
        )
    tl.store(output + base + rows * N + columns, 0.0, mask=columns > rows)
    tl.atomic_xchg(output + base + 1, 1.0, sem="release", scope="gpu")

@triton.jit
def _inline_first_trailing(
    source,
    output,
    EXACT_SELECTED: tl.constexpr,
):
    tl_cuda.gdc_launch_dependents()
    tile = tl.program_id(0) + 1
    matrix = tl.program_id(1)
    local_row = tl.where(tile < 3, 1, 2)
    local_column = tl.where(
        tile == 1,
        0,
        tl.where(tile == 2, 1, tile - 3),
    )
    row_block = 1 + local_row
    column_block = 1 + local_column
    row_start = row_block * 64
    column_start = column_block * 64
    base = matrix * N * N
    axis = tl.arange(0, 64)
    rows = axis[:, None]
    columns = axis[None, :]
    diagonal_tile = row_block == column_block
    store_mask = (not diagonal_tile) | (rows >= columns)
    left = tl.load(output + base + (row_start + rows) * N + columns)
    right = tl.load(
        output + base + (column_start + rows) * N + columns,
    )
    current = tl.load(
        source
        + base
        + (row_start + rows) * N
        + column_start
        + columns,
        mask=store_mask,
        other=0.0,
    ).to(tl.float32)
    selected = (tl.load(source) < 0.1) & EXACT_SELECTED
    if selected:
        current -= _factor_dot(left, tl.trans(right), 1)
    else:
        current -= _factor_dot(left, tl.trans(right), False)
    tl.store(
        output
        + base
        + (row_start + rows) * N
        + column_start
        + columns,
        current,
        mask=store_mask,
    )

@triton.jit
def _inline_join_trsm(
    source,
    output,
    PANEL: tl.constexpr,
    EXACT_SELECTED: tl.constexpr,
):
    tl_cuda.gdc_wait()
    row_local = tl.program_id(0)
    matrix = tl.program_id(1)
    row_start = (PANEL // 64 + 1 + row_local) * 64
    base = matrix * N * N
    completion = output + base + 1
    ready = tl.atomic_add(completion, 0.0, sem="acquire", scope="gpu")
    while ready < 1.0:
        ready = tl.atomic_add(
            completion,
            0.0,
            sem="acquire",
            scope="gpu",
        )
    tl.atomic_xchg(completion, 0.0, sem="acq_rel", scope="gpu")
    tl_cuda.gdc_launch_dependents()
    axis = tl.arange(0, 64)
    rows = axis[:, None]
    columns = axis[None, :]
    rhs = tl.load(
        output + base + (row_start + rows) * N + PANEL + columns,
    ).to(tl.float32)
    tl.store(
        output + base + (row_start + rows) * N + PANEL + columns,
        rhs,
    )
    tl.debug_barrier()
    selected = (tl.load(source) < 0.1) & EXACT_SELECTED
    if selected:
        _compact_solve64_global(output, base, PANEL, row_start)
    else:
        solve_rows = row_start + axis
        _trsm64_full_inverse_global(
            output,
            base,
            N,
            solve_rows,
            PANEL,
            solve_rows < row_start + 64,
            2,
            False,
        )
    tl.store(
        output + base + (PANEL + rows) * N + row_start + columns,
        0.0,
    )

@triton.jit
def _inline_middle_factor_homotopy(
    source,
    output,
    FIRST_STEPS: tl.constexpr,
    SECOND_STEPS: tl.constexpr,
    FINAL_STEPS: tl.constexpr,
    GAIN: tl.constexpr,
):
    matrix = tl.program_id(0)
    base = matrix * N * N
    axis = tl.arange(0, 64)
    rows = axis[:, None]
    columns = axis[None, :]
    lower_mask = rows >= columns
    current = tl.load(
        output + base + (128 + rows) * N + 128 + columns,
        mask=lower_mask,
        other=0.0,
    ).to(tl.float32)
    tl_cuda.gdc_wait()
    tl_cuda.gdc_launch_dependents()
    left = tl.load(
        output + base + (128 + rows) * N + 64 + columns,
    )
    current -= _factor_dot(left, tl.trans(left), False)
    current = tl.where(lower_mask, current, 0.0)
    selected = tl.load(source) < 0.04
    if selected:
        factor, inverse = _homotopy_factor_inverse64(
            current,
            FIRST_STEPS,
            SECOND_STEPS,
            FINAL_STEPS,
            GAIN,
            True,
        )
    else:
        factor, inverse = _pdl256_damped_factor_inverse(current)
    identity = tl.where(rows == columns, 1.0, 0.0)
    inverse_residual = identity - tl.dot(
        factor,
        inverse,
        input_precision="tf32x3",
    )
    inverse += tl.dot(
        inverse,
        inverse_residual,
        input_precision="tf32x3",
    )
    inverse = tl.where(lower_mask, inverse, 0.0)
    tl.store(
        output + base + (128 + rows) * N + 128 + columns,
        factor,
        mask=lower_mask,
    )
    tl.store(
        output + base + (128 + columns) * N + 128 + rows,
        inverse,
        mask=rows > columns,
    )
    tl.store(
        output + base + (64 + rows) * N + 64 + columns,
        0.0,
        mask=columns > rows,
    )
    tl.atomic_xchg(
        output + base + 1,
        1.0,
        sem="release",
        scope="gpu",
    )

@triton.jit
def _inline_middle_trailing(
    source,
    output,
    EXACT_SELECTED: tl.constexpr,
):
    tl_cuda.gdc_launch_dependents()
    tile = tl.program_id(0) + 1
    matrix = tl.program_id(1)
    column_local = tile - 1
    column_start = (2 + column_local) * 64
    base = matrix * N * N
    axis = tl.arange(0, 64)
    rows = axis[:, None]
    columns = axis[None, :]
    diagonal_tile = column_local == 1
    store_mask = (not diagonal_tile) | (rows >= columns)
    left = tl.load(output + base + (192 + rows) * N + 64 + columns)
    right = tl.load(
        output + base + (column_start + rows) * N + 64 + columns,
    )
    current = tl.load(
        output + base + (192 + rows) * N + column_start + columns,
        mask=store_mask,
        other=0.0,
    ).to(tl.float32)
    selected = (tl.load(source) < 0.1) & EXACT_SELECTED
    if selected:
        current -= _factor_dot(left, tl.trans(right), 1)
    else:
        current -= _factor_dot(left, tl.trans(right), False)
    tl.store(
        output + base + (192 + rows) * N + column_start + columns,
        current,
        mask=store_mask,
    )

@triton.jit
def _inline_tail_factor_homotopy(
    source,
    output,
    FIRST_STEPS: tl.constexpr,
    SECOND_STEPS: tl.constexpr,
    FINAL_STEPS: tl.constexpr,
    GAIN: tl.constexpr,
):
    tl_cuda.gdc_launch_dependents()
    matrix = tl.program_id(0)
    base = matrix * N * N
    axis = tl.arange(0, 64)
    rows = axis[:, None]
    columns = axis[None, :]
    lower_mask = rows >= columns
    current = tl.load(
        output + base + (192 + rows) * N + 192 + columns,
        mask=lower_mask,
        other=0.0,
    ).to(tl.float32)
    tl_cuda.gdc_wait()
    left = tl.load(
        output + base + (192 + rows) * N + 128 + columns,
    )
    current -= _factor_dot(left, tl.trans(left), False)
    current = tl.where(lower_mask, current, 0.0)
    selected = tl.load(source) < 0.04
    if selected:
        factor, _ = _homotopy_factor_inverse64(
            current,
            FIRST_STEPS,
            SECOND_STEPS,
            FINAL_STEPS,
            GAIN,
            False,
        )
    else:
        factor, _ = _pdl256_damped_factor_inverse(current)
    tl.store(
        output + base + (192 + rows) * N + 192 + columns,
        factor,
        mask=lower_mask,
    )
    tl.store(
        output + base + (192 + rows) * N + 192 + columns,
        0.0,
        mask=columns > rows,
    )
    tl.store(
        output + base + (128 + rows) * N + 128 + columns,
        0.0,
        mask=columns > rows,
    )

def _pdl256_cholesky(data, output):
 batch=data.shape[0]
 _inline_factor_kernel[batch,](data,output,PANEL_BLOCK=0,WAIT=False,SIGNAL=True,FINAL_PANEL=False,EXACT_SELECTED=True,num_warps=8,num_stages=1,launch_pdl=True)
 _inline_trsm0_kernel[3,batch](data,output,EXACT_SELECTED=True,num_warps=8,num_stages=1,maxnreg=168,launch_pdl=True)
 _inline_first_factor_join[batch,](data,output,EXACT_SELECTED=True,num_warps=8,num_stages=1,maxnreg=224,launch_pdl=True)
 _inline_first_trailing[5,batch](data,output,EXACT_SELECTED=True,num_warps=4,num_stages=1,launch_pdl=True)
 _inline_join_trsm[2,batch](data,output,PANEL=64,EXACT_SELECTED=True,num_warps=8,num_stages=1,maxnreg=168,launch_pdl=True)
 _inline_middle_factor_homotopy[batch,](data,output,FIRST_STEPS=8,SECOND_STEPS=8,FINAL_STEPS=20,GAIN=0.625,num_warps=8,num_stages=1,launch_pdl=True)
 _inline_middle_trailing[2,batch](data,output,EXACT_SELECTED=False,num_warps=4,num_stages=1,launch_pdl=True)
 _inline_join_trsm[1,batch](data,output,PANEL=128,EXACT_SELECTED=False,num_warps=8,num_stages=1,maxnreg=168,launch_pdl=True)
 _inline_tail_factor_homotopy[batch,](data,output,FIRST_STEPS=8,SECOND_STEPS=8,FINAL_STEPS=20,GAIN=0.625,num_warps=8,num_stages=1,launch_pdl=True)
 return output

@triton.jit
def _factor64_jacobi_inverse_kernel(output, inverse_storage, n: tl.constexpr, inverse_blocks: tl.constexpr, panel, REFINEMENT_STEPS: tl.constexpr, LAST_REFINEMENT_GAIN: tl.constexpr, INVERSE_ORDER: tl.constexpr, SEPARATE_INVERSE: tl.constexpr, REFINEMENT_FP16: tl.constexpr=False, INVERSE_FP16: tl.constexpr=False, FP8_REFINEMENT_STEPS: tl.constexpr=0, USE_RSQRT: tl.constexpr=False):
 """Tensor-core polynomial factor and inverse for a near-diagonal leaf."""
 matrix = tl.program_id(0)
 base = matrix * n * n
 axis = tl.arange(0, 64)
 rows = axis[:, None]
 columns = axis[None, :]
 lower = tl.load(output + base + (panel + rows) * n + panel + columns, mask=rows >= columns, other=0.0).to(tl.float32)
 diagonal = tl.sum(tl.where(rows == columns, lower, 0.0), axis=0)
 safe_diagonal = tl.maximum(diagonal, 1.1754943508222875e-38)
 if USE_RSQRT:
  inverse_roots = tl.rsqrt(safe_diagonal)
  roots = safe_diagonal * inverse_roots
 else:
  roots = tl.sqrt(safe_diagonal)
 symmetric = lower + tl.trans(lower) - tl.where(rows == columns, diagonal[None, :], 0.0)
 if USE_RSQRT:
  normalized = symmetric * inverse_roots[:, None] * inverse_roots[None, :]
 else:
  normalized = symmetric / (roots[:, None] * roots[None, :])
 factor = tl.where(rows == columns, 1.0, tl.where(rows > columns, normalized, 0.0))
 for refinement in tl.static_range(0, REFINEMENT_STEPS):
  if refinement < FP8_REFINEMENT_STEPS:
   product = tl.dot(factor.to(tl.float8e4nv), tl.trans(factor.to(tl.float8e4nv)))
  elif REFINEMENT_FP16:
   product = _jacobi_dot(factor, tl.trans(factor), True)
  elif REFINEMENT_STEPS >= 6 and refinement + 1 == REFINEMENT_STEPS:
   product = tl.dot(factor, tl.trans(factor), input_precision='tf32x3')
  else:
   product = tl.dot(factor, tl.trans(factor), input_precision='tf32')
  residual = normalized - product
  gain = 1.0
  if refinement + 1 == REFINEMENT_STEPS:
   gain = LAST_REFINEMENT_GAIN
  factor += gain * tl.where(rows == columns, 0.5 * residual, tl.where(rows > columns, residual, 0.0))
 scaled_factor = roots[:, None] * factor
 tl.store(output + base + (panel + rows) * n + panel + columns, tl.where(rows >= columns, scaled_factor, 0.0))
 identity = tl.where(rows == columns, 1.0, 0.0)
 nilpotent = factor - identity
 inverse = identity - nilpotent
 if INVERSE_ORDER == 6:
  square = _jacobi_dot(nilpotent, nilpotent, INVERSE_FP16)
  cube = _jacobi_dot(square, nilpotent, INVERSE_FP16)
  prefix = identity - nilpotent + square
  inverse = prefix - _jacobi_dot(cube, prefix - cube, INVERSE_FP16)
 elif INVERSE_ORDER >= 2:
  square = _jacobi_dot(nilpotent, nilpotent, INVERSE_FP16)
  if INVERSE_ORDER == 2:
   inverse += square
  elif INVERSE_ORDER == 3:
   cube = _jacobi_dot(square, nilpotent, INVERSE_FP16)
   inverse += square - cube
  elif INVERSE_ORDER == 4:
   inverse += _jacobi_dot(square, identity - nilpotent + square, INVERSE_FP16)
  else:
   cube = _jacobi_dot(square, nilpotent, INVERSE_FP16)
   fourth = _jacobi_dot(square, square, INVERSE_FP16)
   inverse += square - cube + fourth
   if INVERSE_ORDER >= 5:
    fifth = _jacobi_dot(fourth, nilpotent, INVERSE_FP16)
    inverse -= fifth
   if INVERSE_ORDER >= 6:
    sixth = _jacobi_dot(cube, cube, INVERSE_FP16)
    inverse += sixth
 if USE_RSQRT:
  inverse *= inverse_roots[None, :]
 else:
  inverse /= roots[None, :]
 if SEPARATE_INVERSE:
  inverse_base = matrix * inverse_blocks * 64 * 64 + panel // 64 * 64 * 64
  tl.store(inverse_storage + inverse_base + rows * 64 + columns, inverse, mask=rows >= columns)
 else:
  tl.store(output + base + (panel + columns) * n + panel + rows, inverse, mask=rows > columns)

@triton.jit
def _potrf64_trsm_jacobi_kernel(output, inverse_storage, flags, n: tl.constexpr, batch: tl.constexpr, inverse_blocks: tl.constexpr, panel, generation, row_blocks, ROWS: tl.constexpr, REFINEMENT_STEPS: tl.constexpr, LAST_REFINEMENT_GAIN: tl.constexpr, INVERSE_ORDER: tl.constexpr, REFINEMENT_FP16: tl.constexpr=False, INVERSE_FP16: tl.constexpr=False):
 program = tl.program_id(0)
 if program < batch:
  _factor64_jacobi_inverse_kernel(output, inverse_storage, n, inverse_blocks, panel, REFINEMENT_STEPS, LAST_REFINEMENT_GAIN, INVERSE_ORDER, True, REFINEMENT_FP16, INVERSE_FP16)
  tl.atomic_xchg(flags + program, generation * 1.0)
 else:
  linear = program - batch
  matrix = linear // row_blocks
  row_block = linear - matrix * row_blocks
  ready = tl.atomic_add(flags + matrix, 0.0)
  while ready < generation:
   ready = tl.atomic_add(flags + matrix, 0.0)
  row_axis = tl.arange(0, ROWS)
  solve_rows = panel + 64 + row_block * ROWS + row_axis
  valid_rows = solve_rows < n
  base = matrix * n * n
  axis = tl.arange(0, 64)
  rhs = tl.load(output + base + solve_rows[:, None] * n + panel + axis[None, :], mask=valid_rows[:, None], other=0.0)
  inverse_base = matrix * inverse_blocks * 64 * 64 + panel // 64 * 64 * 64
  inverse = tl.load(inverse_storage + inverse_base + axis[:, None] * 64 + axis[None, :], mask=axis[:, None] >= axis[None, :], other=0.0)
  solution = tl.dot(rhs, tl.trans(inverse), input_precision='tf32')
  tl.store(output + base + solve_rows[:, None] * n + panel + axis[None, :], solution, mask=valid_rows[:, None])
  columns = panel + tl.arange(0, 64)
  tl.store(output + base + columns[None, :] * n + solve_rows[:, None], 0.0, mask=valid_rows[:, None])

@triton.jit
def _potrf128_trsm_jacobi_kernel(output, inverse_storage, flags, n: tl.constexpr, batch: tl.constexpr, inverse_blocks: tl.constexpr, panel, generation, row_blocks, ROWS: tl.constexpr, REFINEMENT_STEPS: tl.constexpr, LAST_REFINEMENT_GAIN: tl.constexpr, INVERSE_ORDER: tl.constexpr, SECOND_INVERSE_ORDER: tl.constexpr, LOCAL_UPDATE128: tl.constexpr, REFINEMENT_FP16: tl.constexpr=False, INVERSE_FP16: tl.constexpr=False, FP8_REFINEMENT_STEPS: tl.constexpr=0, TRAILING_SOLVE_PRECISION: tl.constexpr=0, OWNER_SOLVE_PRECISION: tl.constexpr=0, CROSS_CALL_RELEASE: tl.constexpr=False, USE_RSQRT: tl.constexpr=False, SKIP_UPPER_CLEAR: tl.constexpr=False):
 """Factor and solve two adjacent polynomial leaves in one wave."""
 program = tl.program_id(0)
 if CROSS_CALL_RELEASE:
  if program < batch:
   tl.store(flags + program, 0.0)
  tl_cuda.gdc_launch_dependents()
  tl_cuda.gdc_wait()
 axis = tl.arange(0, 64)
 if program < batch:
  matrix = program
  base = matrix * n * n
  inverse_base = matrix * inverse_blocks * 64 * 64 + panel // 64 * 64 * 64
  lower_mask = axis[:, None] >= axis[None, :]
  if LOCAL_UPDATE128:
   pair_axis = tl.arange(0, 128)
   pair_lower = pair_axis[:, None] >= pair_axis[None, :]
   pair_residual = tl.load(output + base + (panel + pair_axis)[:, None] * n + panel + pair_axis[None, :], mask=pair_lower, other=0.0)
   pair_history = tl.load(output + base + (panel + pair_axis)[:, None] * n + panel - 128 + pair_axis[None, :])
   pair_residual -= tl.where(pair_lower, tl.dot(pair_history, tl.trans(pair_history), input_precision='tf32'), 0.0)
   pair_blocks = tl.reshape(pair_residual, (2, 64, 2, 64))
   pair_column_blocks = tl.permute(pair_blocks, (0, 1, 3, 2))
   pair_column0, pair_column1 = tl.split(pair_column_blocks)
   pair_column0 = tl.permute(pair_column0, (1, 2, 0))
   pair_column1 = tl.permute(pair_column1, (1, 2, 0))
   lower0, residual10 = tl.split(pair_column0)
   residual01, residual11 = tl.split(pair_column1)
   tl.store(output + base + (panel + 64 + axis)[:, None] * n + panel + axis[None, :], residual10)
   tl.store(output + base + (panel + 64 + axis)[:, None] * n + panel + 64 + axis[None, :], residual11, mask=lower_mask)
   scaled0, inverse0 = _jacobi_factor_inverse_from_lower(lower0, REFINEMENT_STEPS, LAST_REFINEMENT_GAIN, INVERSE_ORDER, REFINEMENT_FP16, INVERSE_FP16, FP8_REFINEMENT_STEPS, USE_RSQRT)
   tl.store(output + base + (panel + axis)[:, None] * n + panel + axis[None, :], tl.where(lower_mask, scaled0, 0.0))
   tl.store(inverse_storage + inverse_base + axis[:, None] * 64 + axis[None, :], inverse0, mask=lower_mask)
  else:
   _factor64_jacobi_inverse_kernel(output, inverse_storage, n, inverse_blocks, panel, REFINEMENT_STEPS, LAST_REFINEMENT_GAIN, INVERSE_ORDER, True, REFINEMENT_FP16, INVERSE_FP16, FP8_REFINEMENT_STEPS, USE_RSQRT)
   inverse0 = tl.load(inverse_storage + inverse_base + axis[:, None] * 64 + axis[None, :], mask=lower_mask, other=0.0)
  block_rows = panel + 64 + axis
  rhs10 = tl.load(output + base + block_rows[:, None] * n + panel + axis[None, :])
  factor10 = _jacobi_solve_dot(rhs10, tl.trans(inverse0), OWNER_SOLVE_PRECISION)
  tl.store(output + base + block_rows[:, None] * n + panel + axis[None, :], factor10)
  if not SKIP_UPPER_CLEAR:
   tl.store(output + base + (panel + axis)[None, :] * n + block_rows[:, None], 0.0)
  rows = axis[:, None]
  block_columns = axis[None, :]
  diagonal11 = tl.load(output + base + (panel + 64 + rows) * n + panel + 64 + block_columns, mask=rows >= block_columns, other=0.0)
  update11 = _jacobi_solve_dot(factor10, tl.trans(factor10), OWNER_SOLVE_PRECISION)
  diagonal11 -= tl.where(rows >= block_columns, update11, 0.0)
  if LOCAL_UPDATE128:
   scaled1, inverse1 = _jacobi_factor_inverse_from_lower(diagonal11, REFINEMENT_STEPS, LAST_REFINEMENT_GAIN, SECOND_INVERSE_ORDER, REFINEMENT_FP16, INVERSE_FP16, FP8_REFINEMENT_STEPS, USE_RSQRT)
   tl.store(output + base + (panel + 64 + rows) * n + panel + 64 + block_columns, tl.where(rows >= block_columns, scaled1, 0.0))
   tl.store(inverse_storage + inverse_base + 64 * 64 + rows * 64 + block_columns, inverse1, mask=rows >= block_columns)
  else:
   tl.store(output + base + (panel + 64 + rows) * n + panel + 64 + block_columns, diagonal11)
   _factor64_jacobi_inverse_kernel(output, inverse_storage, n, inverse_blocks, panel + 64, REFINEMENT_STEPS, LAST_REFINEMENT_GAIN, SECOND_INVERSE_ORDER, True, REFINEMENT_FP16, INVERSE_FP16, FP8_REFINEMENT_STEPS, USE_RSQRT)
  if not CROSS_CALL_RELEASE:
   tl.atomic_xchg(flags + program, generation * 1.0)
 else:
  linear = program - batch
  matrix = linear // row_blocks
  row_block = linear - matrix * row_blocks
  ready = tl.atomic_add(flags + matrix, 0.0)
  while ready < generation:
   ready = tl.atomic_add(flags + matrix, 0.0)
  row_axis = tl.arange(0, ROWS)
  solve_rows = panel + 128 + row_block * ROWS + row_axis
  valid_rows = solve_rows < n
  base = matrix * n * n
  inverse_base = matrix * inverse_blocks * 64 * 64 + panel // 64 * 64 * 64
  inverse0 = tl.load(inverse_storage + inverse_base + axis[:, None] * 64 + axis[None, :], mask=axis[:, None] >= axis[None, :], other=0.0)
  if LOCAL_UPDATE128:
   pair_axis = tl.arange(0, 128)
   rhs_pair = tl.load(output + base + solve_rows[:, None] * n + panel + pair_axis[None, :], mask=valid_rows[:, None], other=0.0)
   previous_pair = tl.load(output + base + solve_rows[:, None] * n + panel - 128 + pair_axis[None, :], mask=valid_rows[:, None], other=0.0)
   current_history_pair = tl.load(output + base + (panel + pair_axis)[:, None] * n + panel - 128 + pair_axis[None, :])
   rhs_pair -= tl.dot(previous_pair, tl.trans(current_history_pair), input_precision='tf32')
   rhs_pair_split = tl.permute(tl.reshape(rhs_pair, (ROWS, 2, 64)), (0, 2, 1))
   rhs0, rhs1_local = tl.split(rhs_pair_split)
  else:
   rhs0 = tl.load(output + base + solve_rows[:, None] * n + panel + axis[None, :], mask=valid_rows[:, None], other=0.0)
  solution0 = _jacobi_solve_dot(rhs0, tl.trans(inverse0), TRAILING_SOLVE_PRECISION)
  factor10 = tl.load(output + base + (panel + 64 + axis)[:, None] * n + panel + axis[None, :])
  if LOCAL_UPDATE128:
   rhs1 = rhs1_local
  else:
   rhs1 = tl.load(output + base + solve_rows[:, None] * n + panel + 64 + axis[None, :], mask=valid_rows[:, None], other=0.0)
  rhs1 -= _jacobi_solve_dot(solution0, tl.trans(factor10), TRAILING_SOLVE_PRECISION)
  inverse1 = tl.load(inverse_storage + inverse_base + 64 * 64 + axis[:, None] * 64 + axis[None, :], mask=axis[:, None] >= axis[None, :], other=0.0)
  solution1 = _jacobi_solve_dot(rhs1, tl.trans(inverse1), TRAILING_SOLVE_PRECISION)
  tl.store(output + base + solve_rows[:, None] * n + panel + axis[None, :], solution0, mask=valid_rows[:, None])
  tl.store(output + base + solve_rows[:, None] * n + panel + 64 + axis[None, :], solution1, mask=valid_rows[:, None])
  if not SKIP_UPPER_CLEAR:
   zero_columns = panel + tl.arange(0, 128)
   tl.store(output + base + zero_columns[None, :] * n + solve_rows[:, None], 0.0, mask=valid_rows[:, None])

@triton.jit
def _factor_inverse64_pdl(lower, REFINEMENT_FP8: tl.constexpr, INVERSE_FP8: tl.constexpr, INVERSE_FP8_SCALE: tl.constexpr, ORDER1: tl.constexpr, APPROX_RSQRT: tl.constexpr):
 axis = tl.arange(0, 64)
 rows = axis[:, None]
 columns = axis[None, :]
 diagonal = tl.sum(tl.where(rows == columns, lower, 0.0), axis=0)
 safe_diagonal = tl.maximum(diagonal, 1.1754943508222875e-38)
 if APPROX_RSQRT:
  inverse_roots = tl.rsqrt(safe_diagonal)
  inverse_roots *= 1.5 - 0.5 * safe_diagonal * inverse_roots * inverse_roots
  roots = safe_diagonal * inverse_roots
 else:
  roots = tl.sqrt(safe_diagonal)
 symmetric = lower + tl.trans(lower) - tl.where(rows == columns, diagonal[None, :], 0.0)
 if APPROX_RSQRT:
  normalized = symmetric * inverse_roots[:, None] * inverse_roots[None, :]
 else:
  normalized = symmetric / (roots[:, None] * roots[None, :])
 factor = tl.where(rows == columns, 1.0, tl.where(rows > columns, normalized, 0.0))
 if REFINEMENT_FP8:
  product = tl.dot(factor.to(tl.float8e4nv), tl.trans(factor.to(tl.float8e4nv)))
 else:
  product = tl.dot(factor.to(tl.float16), tl.trans(factor.to(tl.float16)))
 residual = normalized - product
 factor += tl.where(rows == columns, 0.5 * residual, tl.where(rows > columns, residual, 0.0))
 scaled_factor = roots[:, None] * factor
 identity = tl.where(rows == columns, 1.0, 0.0)
 nilpotent = factor - identity
 inverse = identity - nilpotent
 if not ORDER1:
  if INVERSE_FP8:
   scaled_nilpotent = nilpotent * INVERSE_FP8_SCALE
   square = tl.dot(scaled_nilpotent.to(tl.float8e4nv), scaled_nilpotent.to(tl.float8e4nv)) * (1.0 / (INVERSE_FP8_SCALE * INVERSE_FP8_SCALE))
  else:
   square = tl.dot(nilpotent.to(tl.float16), nilpotent.to(tl.float16))
  inverse += square
 if APPROX_RSQRT:
  inverse *= inverse_roots[None, :]
 else:
  inverse /= roots[None, :]
 return (scaled_factor, inverse)

@triton.jit
def _final_history_owner_tma_kernel(source, output, flags, inverse_storage, n: tl.constexpr, batch: tl.constexpr, panel, _p: tl.constexpr, _B: tl.constexpr, ROW_TILE: tl.constexpr, COLUMN_TILE: tl.constexpr, K_TILE: tl.constexpr, WARP_SPECIALIZE: tl.constexpr, PIPELINE_STAGES: tl.constexpr, USE_BF16: tl.constexpr, USE_FP8: tl.constexpr, HIGH_PRECISION: tl.constexpr, BF16_TAIL_WINDOW: tl.constexpr, FP8_TAIL_WINDOW: tl.constexpr, HISTORY_WINDOW: tl.constexpr, HISTORY_CORRECTION: tl.constexpr, HISTORY_DAMPING: tl.constexpr, FUSE_UPPER_ZERO: tl.constexpr, RESET_FLAGS: tl.constexpr, DIRECT_SOURCE: tl.constexpr=False, TRIANGULAR_TWO_COLUMN: tl.constexpr=False):
 """TMA-backed final history plus first factor owner."""
 tl_cuda.gdc_launch_dependents()
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 if RESET_FLAGS and tile == 0:
  tl.store(flags + matrix, 0.0)
 triangular_row = ((tl.sqrt((8 * tile + 1).to(tl.float32)) - 1.0) * 0.5).to(tl.int32)
 row_block = triangular_row
 _c = tile - triangular_row * (triangular_row + 1) // 2
 row_start = panel + row_block * ROW_TILE
 _s = panel + _c * COLUMN_TILE
 _m = matrix * n
 left_descriptor = tl.make_tensor_descriptor(output, shape=[batch * n, n], strides=[n, 1], block_shape=[ROW_TILE, K_TILE])
 right_descriptor = tl.make_tensor_descriptor(output, shape=[batch * n, n], strides=[n, 1], block_shape=[COLUMN_TILE, K_TILE])
 source_descriptor = tl.make_tensor_descriptor(source, shape=[batch * n, n], strides=[n, 1], block_shape=[ROW_TILE, COLUMN_TILE])
 _a = tl.zeros((ROW_TILE, COLUMN_TILE), tl.float32)
 if BF16_TAIL_WINDOW > 0 or FP8_TAIL_WINDOW > 0:
  tail_window = BF16_TAIL_WINDOW
  if FP8_TAIL_WINDOW > 0:
   tail_window = FP8_TAIL_WINDOW
  tail_start = tl.maximum(0, panel - tail_window)
  for offset in tl.range(0, tail_start, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
   left = left_descriptor.load([_m + row_start, offset])
   right = right_descriptor.load([_m + _s, offset])
   if FP8_TAIL_WINDOW > 0:
    _a += tl.dot((left * 64.0).to(tl.float8e4nv), tl.trans((right * 64.0).to(tl.float8e4nv))) * (1.0 / 4096.0)
   else:
    _a += tl.dot(left.to(tl.bfloat16), tl.trans(right.to(tl.bfloat16)))
  for offset in tl.range(tail_start, panel, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=False):
   left = left_descriptor.load([_m + row_start, offset])
   right = right_descriptor.load([_m + _s, offset])
   _a += tl.dot(left, tl.trans(right), input_precision='tf32')
 else:
  reduction_start = 0
  if HISTORY_WINDOW > 0:
   reduction_start = tl.maximum(0, panel - HISTORY_WINDOW)
  for offset in tl.range(reduction_start, panel, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
   left = left_descriptor.load([_m + row_start, offset])
   right = right_descriptor.load([_m + _s, offset])
   if USE_FP8:
    _a += tl.dot(left.to(tl.float8e4nv), tl.trans(right.to(tl.float8e4nv)))
   elif USE_BF16:
    _a += tl.dot(left.to(tl.bfloat16), tl.trans(right.to(tl.bfloat16)))
   elif HIGH_PRECISION:
    _a += tl.dot(left, tl.trans(right), input_precision='tf32x3')
   else:
    _a += tl.dot(left, tl.trans(right), input_precision='tf32')
 row_axis = row_start + tl.arange(0, ROW_TILE)
 column_axis = _s + tl.arange(0, COLUMN_TILE)
 base = matrix * n * n
 if DIRECT_SOURCE:
  source_value = tl.load(source + base + row_axis[:, None] * n + column_axis[None, :], mask=(row_axis[:, None] < n) & (column_axis[None, :] < panel + _p), other=0.0, cache_modifier='.cg')
 else:
  source_value = source_descriptor.load([_m + row_start, _s])
 valid = (row_axis[:, None] < n) & (column_axis[None, :] < panel + _p) & (row_axis[:, None] >= column_axis[None, :])
 if HISTORY_WINDOW > 0:
  omitted_fraction = reduction_start.to(tl.float32) * (1.0 / n)
  retained_source = source_value * (1.0 - HISTORY_CORRECTION * omitted_fraction)
  retained_source += tl.where(row_axis[:, None] == column_axis[None, :], HISTORY_DAMPING * HISTORY_CORRECTION * omitted_fraction, 0.0)
  source_value = retained_source
 tl.store(output + base + row_axis[:, None] * n + column_axis[None, :], source_value - _a, mask=valid)
 if FUSE_UPPER_ZERO:
  intra_upper = (row_axis[:, None] < panel + _p) & (column_axis[None, :] < panel + _p) & (row_axis[:, None] < column_axis[None, :])
  tl.store(output + base + row_axis[:, None] * n + column_axis[None, :], 0.0, mask=intra_upper)
  mirror_valid = (row_axis[:, None] < n) & (row_axis[:, None] >= panel + _p) & (column_axis[None, :] < panel + _p)
  tl.store(output + base + column_axis[None, :] * n + row_axis[:, None], 0.0, mask=mirror_valid)
 ready_base = output + base + (panel - 128) * n + panel - 128
 if (tile == 1) | (tile == 2):
  tl.atomic_add(ready_base + 1, 1.0)
 if tile == 0:
  ready_count = tl.atomic_add(ready_base + 1, 0.0)
  while ready_count < 2.0:
   ready_count = tl.atomic_add(ready_base + 1, 0.0)
  tl.atomic_xchg(ready_base + 1, 0.0)
  axis = tl.arange(0, 64)
  rows = axis[:, None]
  columns = axis[None, :]
  lower = rows >= columns
  inverse_blocks: tl.constexpr = n // 64
  inverse_base = matrix * inverse_blocks * 64 * 64 + panel // 64 * 64 * 64
  lower0 = tl.load(output + base + (panel + rows) * n + panel + columns, mask=lower, other=0.0)
  factor0, inverse0 = _factor_inverse64_pdl(lower0, False, False, 1.0, False, True)
  tl.store(output + base + (panel + rows) * n + panel + columns, tl.where(lower, factor0, 0.0))
  tl.store(inverse_storage + inverse_base + rows * 64 + columns, inverse0, mask=lower)
  block_rows = panel + 64 + axis
  rhs10 = tl.load(output + base + block_rows[:, None] * n + panel + axis[None, :])
  factor10 = tl.dot(rhs10.to(tl.float16), tl.trans(inverse0.to(tl.float16)))
  tl.store(output + base + block_rows[:, None] * n + panel + axis[None, :], factor10)
  tl.store(output + base + (panel + axis)[None, :] * n + block_rows[:, None], 0.0)
  diagonal11 = tl.load(output + base + (panel + 64 + rows) * n + panel + 64 + columns, mask=lower, other=0.0)
  diagonal11 -= tl.where(lower, tl.dot(factor10.to(tl.float16), tl.trans(factor10.to(tl.float16))), 0.0)
  factor1, inverse1 = _factor_inverse64_pdl(diagonal11, False, False, 1.0, False, True)
  tl.store(output + base + (panel + 64 + rows) * n + panel + 64 + columns, tl.where(lower, factor1, 0.0))
  tl.store(inverse_storage + inverse_base + 64 * 64 + rows * 64 + columns, inverse1, mask=lower)

@triton.jit
def _factor128_owner_pdl_kernel(output, inverse_storage, n: tl.constexpr, inverse_blocks: tl.constexpr, panel, SECOND_ORDER1: tl.constexpr, REFINEMENT_FP8: tl.constexpr, INVERSE_FP8: tl.constexpr, INVERSE_FP8_SCALE: tl.constexpr, APPROX_RSQRT_MASK: tl.constexpr, LOCAL_UPDATE128: tl.constexpr=False):
 tl_cuda.gdc_launch_dependents()
 matrix = tl.program_id(0)
 base = matrix * n * n
 axis = tl.arange(0, 64)
 rows = axis[:, None]
 columns = axis[None, :]
 lower = rows >= columns
 inverse_base = matrix * inverse_blocks * 64 * 64 + panel // 64 * 64 * 64
 lower0 = tl.load(output + base + (panel + rows) * n + panel + columns, mask=lower, other=0.0)
 if LOCAL_UPDATE128:
  history_axis = tl.arange(0, 128)
  history0 = tl.load(output + base + (panel + rows) * n + panel - 128 + history_axis[None, :])
  lower0 -= tl.where(lower, tl.dot(history0.to(tl.bfloat16), tl.trans(history0.to(tl.bfloat16))), 0.0)
 factor0, inverse0 = _factor_inverse64_pdl(lower0, REFINEMENT_FP8, INVERSE_FP8, INVERSE_FP8_SCALE, False, APPROX_RSQRT_MASK & 3 == 3)
 tl.store(output + base + (panel + rows) * n + panel + columns, tl.where(lower, factor0, 0.0))
 tl.store(inverse_storage + inverse_base + rows * 64 + columns, inverse0, mask=lower)
 block_rows = panel + 64 + axis
 rhs10 = tl.load(output + base + block_rows[:, None] * n + panel + axis[None, :])
 if LOCAL_UPDATE128:
  history1 = tl.load(output + base + block_rows[:, None] * n + panel - 128 + history_axis[None, :])
  rhs10 -= tl.dot(history1.to(tl.bfloat16), tl.trans(history0.to(tl.bfloat16)))
 factor10 = tl.dot(rhs10.to(tl.float16), tl.trans(inverse0.to(tl.float16)))
 tl.store(output + base + block_rows[:, None] * n + panel + axis[None, :], factor10)
 tl.store(output + base + (panel + axis)[None, :] * n + block_rows[:, None], 0.0)
 diagonal11 = tl.load(output + base + (panel + 64 + rows) * n + panel + 64 + columns, mask=lower, other=0.0)
 if LOCAL_UPDATE128:
  diagonal11 -= tl.where(lower, tl.dot(history1.to(tl.bfloat16), tl.trans(history1.to(tl.bfloat16))), 0.0)
 update11 = tl.dot(factor10.to(tl.float16), tl.trans(factor10.to(tl.float16)))
 diagonal11 -= tl.where(lower, update11, 0.0)
 factor1, inverse1 = _factor_inverse64_pdl(diagonal11, REFINEMENT_FP8, INVERSE_FP8, INVERSE_FP8_SCALE, SECOND_ORDER1, APPROX_RSQRT_MASK & 12 == 12)
 tl.store(output + base + (panel + 64 + rows) * n + panel + 64 + columns, tl.where(lower, factor1, 0.0))
 tl.store(inverse_storage + inverse_base + 64 * 64 + rows * 64 + columns, inverse1, mask=lower)

@triton.jit
def _trsm128_wait_pdl_kernel(output, inverse_storage, n: tl.constexpr, inverse_blocks: tl.constexpr, panel, ROWS: tl.constexpr, SOLVE_PRECISION: tl.constexpr, LOCAL_UPDATE128: tl.constexpr=False, WAIT: tl.constexpr=True, ROW_OFFSET: tl.constexpr=0):
 if WAIT:
  tl_cuda.gdc_wait()
 row_block = tl.program_id(0) + ROW_OFFSET
 matrix = tl.program_id(1)
 row_axis = tl.arange(0, ROWS)
 axis = tl.arange(0, 64)
 solve_rows = panel + 128 + row_block * ROWS + row_axis
 valid_rows = solve_rows < n
 base = matrix * n * n
 inverse_base = matrix * inverse_blocks * 64 * 64 + panel // 64 * 64 * 64
 lower = axis[:, None] >= axis[None, :]
 inverse0 = tl.load(inverse_storage + inverse_base + axis[:, None] * 64 + axis[None, :], mask=lower, other=0.0)
 rhs0 = tl.load(output + base + solve_rows[:, None] * n + panel + axis[None, :], mask=valid_rows[:, None], other=0.0)
 if LOCAL_UPDATE128:
  history_axis = tl.arange(0, 128)
  previous_pair = tl.load(output + base + solve_rows[:, None] * n + panel - 128 + history_axis[None, :], mask=valid_rows[:, None], other=0.0)
  history0 = tl.load(output + base + (panel + axis)[:, None] * n + panel - 128 + history_axis[None, :])
  rhs0 -= tl.dot(previous_pair.to(tl.bfloat16), tl.trans(history0.to(tl.bfloat16)))
 solution0 = _jacobi_solve_dot(rhs0, tl.trans(inverse0), SOLVE_PRECISION)
 factor10 = tl.load(output + base + (panel + 64 + axis)[:, None] * n + panel + axis[None, :])
 rhs1 = tl.load(output + base + solve_rows[:, None] * n + panel + 64 + axis[None, :], mask=valid_rows[:, None], other=0.0)
 if LOCAL_UPDATE128:
  history1 = tl.load(output + base + (panel + 64 + axis)[:, None] * n + panel - 128 + history_axis[None, :])
  rhs1 -= tl.dot(previous_pair.to(tl.bfloat16), tl.trans(history1.to(tl.bfloat16)))
 rhs1 -= _jacobi_solve_dot(solution0, tl.trans(factor10), SOLVE_PRECISION)
 inverse1 = tl.load(inverse_storage + inverse_base + 64 * 64 + axis[:, None] * 64 + axis[None, :], mask=lower, other=0.0)
 solution1 = _jacobi_solve_dot(rhs1, tl.trans(inverse1), SOLVE_PRECISION)
 tl.store(output + base + solve_rows[:, None] * n + panel + axis[None, :], solution0, mask=valid_rows[:, None])
 tl.store(output + base + solve_rows[:, None] * n + panel + 64 + axis[None, :], solution1, mask=valid_rows[:, None])





@triton.jit
def _trsm256_fused_dual_tail_pdl_kernel(output, inverse_storage, n: tl.constexpr, inverse_blocks: tl.constexpr, panel, ROWS: tl.constexpr, SOLVE_PRECISION: tl.constexpr, SECOND_SOLVE_PRECISION: tl.constexpr):
 row_block = tl.program_id(0)
 matrix = tl.program_id(1)
 row_axis = tl.arange(0, ROWS)
 axis = tl.arange(0, 64)
 solve_rows = panel + 256 + row_block * ROWS + row_axis
 valid_rows = solve_rows < n
 base = matrix * n * n
 lower = axis[:, None] >= axis[None, :]
 inverse_base0 = matrix * inverse_blocks * 64 * 64 + panel // 64 * 64 * 64
 inverse0 = tl.load(inverse_storage + inverse_base0 + axis[:, None] * 64 + axis[None, :], mask=lower, other=0.0)
 rhs0 = tl.load(output + base + solve_rows[:, None] * n + panel + axis[None, :], mask=valid_rows[:, None], other=0.0)
 solution0 = _jacobi_solve_dot(rhs0, tl.trans(inverse0), SOLVE_PRECISION)
 factor10 = tl.load(output + base + (panel + 64 + axis)[:, None] * n + panel + axis[None, :])
 rhs1 = tl.load(output + base + solve_rows[:, None] * n + panel + 64 + axis[None, :], mask=valid_rows[:, None], other=0.0)
 rhs1 -= _jacobi_solve_dot(solution0, tl.trans(factor10), SOLVE_PRECISION)
 inverse1 = tl.load(inverse_storage + inverse_base0 + 64 * 64 + axis[:, None] * 64 + axis[None, :], mask=lower, other=0.0)
 solution1 = _jacobi_solve_dot(rhs1, tl.trans(inverse1), SOLVE_PRECISION)
 tl.store(output + base + solve_rows[:, None] * n + panel + axis[None, :], solution0, mask=valid_rows[:, None])
 tl.store(output + base + solve_rows[:, None] * n + panel + 64 + axis[None, :], solution1, mask=valid_rows[:, None])
 ready_pointer = inverse_storage + inverse_base0 + 1
 ready = tl.atomic_add(ready_pointer, 0.0)
 while ready < 1.0:
  ready = tl.atomic_add(ready_pointer, 0.0)
 previous_pair = tl.reshape(tl.permute(tl.join(solution0, solution1), (0, 2, 1)), (ROWS, 128))
 second_panel = panel + 128
 history_axis = tl.arange(0, 128)
 history0 = tl.load(output + base + (second_panel + axis)[:, None] * n + panel + history_axis[None, :])
 second_rhs0 = tl.load(output + base + solve_rows[:, None] * n + second_panel + axis[None, :], mask=valid_rows[:, None], other=0.0)
 second_rhs0 -= tl.dot(previous_pair.to(tl.bfloat16), tl.trans(history0.to(tl.bfloat16)))
 history1 = tl.load(output + base + (second_panel + 64 + axis)[:, None] * n + panel + history_axis[None, :])
 second_rhs1 = tl.load(output + base + solve_rows[:, None] * n + second_panel + 64 + axis[None, :], mask=valid_rows[:, None], other=0.0)
 second_rhs1 -= tl.dot(previous_pair.to(tl.bfloat16), tl.trans(history1.to(tl.bfloat16)))
 ready = tl.atomic_add(ready_pointer, 0.0)
 while ready < 2.0:
  ready = tl.atomic_add(ready_pointer, 0.0)
 inverse_base1 = matrix * inverse_blocks * 64 * 64 + second_panel // 64 * 64 * 64
 second_inverse0 = tl.load(inverse_storage + inverse_base1 + axis[:, None] * 64 + axis[None, :], mask=lower, other=0.0)
 second_solution0 = _jacobi_solve_dot(second_rhs0, tl.trans(second_inverse0), SECOND_SOLVE_PRECISION)
 second_factor10 = tl.load(output + base + (second_panel + 64 + axis)[:, None] * n + second_panel + axis[None, :])
 second_rhs1 -= _jacobi_solve_dot(second_solution0, tl.trans(second_factor10), SECOND_SOLVE_PRECISION)
 second_inverse1 = tl.load(inverse_storage + inverse_base1 + 64 * 64 + axis[:, None] * 64 + axis[None, :], mask=lower, other=0.0)
 second_solution1 = _jacobi_solve_dot(second_rhs1, tl.trans(second_inverse1), SECOND_SOLVE_PRECISION)
 tl.store(output + base + solve_rows[:, None] * n + second_panel + axis[None, :], second_solution0, mask=valid_rows[:, None])
 tl.store(output + base + solve_rows[:, None] * n + second_panel + 64 + axis[None, :], second_solution1, mask=valid_rows[:, None])

@triton.jit
def _trsm64_only_kernel(output, n: tl.constexpr, panel, ROWS: tl.constexpr, LEAF8: tl.constexpr, FULL_INVERSE32: tl.constexpr, FULL_INVERSE64: tl.constexpr, HIGH_PRECISION: tl.constexpr, DIAGONAL_PRECISION: tl.constexpr, USE_BF16_TRSM: tl.constexpr):
 row_block = tl.program_id(0)
 matrix = tl.program_id(1)
 row_axis = tl.arange(0, ROWS)
 solve_rows = panel + 64 + row_block * ROWS + row_axis
 valid_rows = solve_rows < n
 base = matrix * n * n
 if FULL_INVERSE64:
  _trsm64_full_inverse_global(output, base, n, solve_rows, panel, valid_rows, HIGH_PRECISION, USE_BF16_TRSM)
 else:
  _solve64_block_global(output, base, n, solve_rows, panel, valid_rows, LEAF8, FULL_INVERSE32, HIGH_PRECISION, DIAGONAL_PRECISION)

@triton.jit
def _potrf128_trsm_overlap_kernel(output, flags, n: tl.constexpr, batch: tl.constexpr, panel, generation, row_blocks, ROWS: tl.constexpr, PIVOT_FLOOR: tl.constexpr, LEAF8: tl.constexpr, HYBRID8: tl.constexpr, HYBRID4: tl.constexpr, FULL_INVERSE32: tl.constexpr, HIGH_PRECISION: tl.constexpr, DIAGONAL_PRECISION: tl.constexpr, USE_BF16_TRSM: tl.constexpr, SCRATCH_BLOCKS: tl.constexpr, FINAL_CLEANUP: tl.constexpr):
 """Overlap first-half TRSM with the second 64x64 diagonal factor."""
 program = tl.program_id(0)
 phase0 = generation * 2 - 1
 phase1 = generation * 2
 if program < batch:
  matrix = program
  base = matrix * n * n
  _factor64_block_global(output, base, n, panel, PIVOT_FLOOR, LEAF8, HYBRID8, HYBRID4, FULL_INVERSE32, HIGH_PRECISION, DIAGONAL_PRECISION, 0)
  _complete_inverse64_global(output, base, n, panel, HIGH_PRECISION)
  tl.atomic_xchg(flags + matrix, phase0 * 1.0)
  internal_rows = panel + 64 + tl.arange(0, 64)
  _trsm64_full_inverse_global(output, base, n, internal_rows, panel, internal_rows < panel + 128, HIGH_PRECISION, False)
  axis = tl.arange(0, 64)
  rows = axis[:, None]
  columns = axis[None, :]
  lower10 = tl.load(output + base + (panel + 64 + rows) * n + panel + columns)
  second = tl.load(output + base + (panel + 64 + rows) * n + panel + 64 + columns, mask=rows >= columns, other=0.0)
  second -= _factor_dot(lower10, tl.trans(lower10), DIAGONAL_PRECISION)
  tl.store(output + base + (panel + 64 + rows) * n + panel + 64 + columns, second, mask=rows >= columns)
  tl.debug_barrier()
  _factor64_block_global(output, base, n, panel + 64, PIVOT_FLOOR, LEAF8, HYBRID8, HYBRID4, FULL_INVERSE32, HIGH_PRECISION, DIAGONAL_PRECISION, 0)
  _complete_inverse64_global(output, base, n, panel + 64, HIGH_PRECISION)
  tl.atomic_xchg(flags + matrix, phase1 * 1.0)
 elif program < batch + batch * row_blocks:
  linear = program - batch
  matrix = linear // row_blocks
  row_block = linear - matrix * row_blocks
  ready = tl.atomic_add(flags + matrix, 0.0)
  while ready < phase0:
   ready = tl.atomic_add(flags + matrix, 0.0)
  row_axis = tl.arange(0, ROWS)
  solve_rows = panel + 128 + row_block * ROWS + row_axis
  valid_rows = solve_rows < n
  base = matrix * n * n
  _trsm64_full_inverse_global(output, base, n, solve_rows, panel, valid_rows, HIGH_PRECISION, USE_BF16_TRSM)
  ready = tl.atomic_add(flags + matrix, 0.0)
  while ready < phase1:
   ready = tl.atomic_add(flags + matrix, 0.0)
  axis = tl.arange(0, 64)
  previous_solution = tl.load(output + base + solve_rows[:, None] * n + panel + axis[None, :], mask=valid_rows[:, None], other=0.0)
  lower10 = tl.load(output + base + (panel + 64 + axis[:, None]) * n + panel + axis[None, :])
  rhs1_pointer = output + base + solve_rows[:, None] * n + panel + 64 + axis[None, :]
  rhs1 = tl.load(rhs1_pointer, mask=valid_rows[:, None], other=0.0)
  rhs1 -= _factor_dot(previous_solution, tl.trans(lower10), DIAGONAL_PRECISION)
  tl.store(rhs1_pointer, rhs1, mask=valid_rows[:, None])
  tl.debug_barrier()
  _trsm64_full_inverse_global(output, base, n, solve_rows, panel + 64, valid_rows, HIGH_PRECISION, USE_BF16_TRSM)
 elif FINAL_CLEANUP:
  linear = program - batch - batch * row_blocks
  matrix = linear // SCRATCH_BLOCKS
  diagonal_block = linear - matrix * SCRATCH_BLOCKS
  if diagonal_block * 64 >= panel:
   ready = tl.atomic_add(flags + matrix, 0.0)
   while ready < phase1:
    ready = tl.atomic_add(flags + matrix, 0.0)
  axis = tl.arange(0, 64)
  rows = diagonal_block * 64 + axis[:, None]
  columns = diagonal_block * 64 + axis[None, :]
  tl.store(output + matrix * n * n + rows * n + columns, 0.0, mask=(rows < n) & (columns < n) & (columns > rows))

@triton.jit
def _potrf128_trsm_recursive16_kernel(output, flags, n: tl.constexpr, batch: tl.constexpr, panel, generation, row_blocks, ROWS: tl.constexpr, PIVOT_FLOOR: tl.constexpr, LEAF8: tl.constexpr, HYBRID8: tl.constexpr, HYBRID4: tl.constexpr, FULL_INVERSE32: tl.constexpr, HIGH_PRECISION: tl.constexpr, DIAGONAL_PRECISION: tl.constexpr):
 program = tl.program_id(0)
 if program < batch:
  matrix = program
  base = matrix * n * n
  _factor64_block_global(output, base, n, panel, PIVOT_FLOOR, LEAF8, HYBRID8, HYBRID4, FULL_INVERSE32, HIGH_PRECISION, DIAGONAL_PRECISION, 0)
  internal_rows = panel + 64 + tl.arange(0, 64)
  _solve64_block_global(output, base, n, internal_rows, panel, internal_rows < panel + 128, LEAF8, FULL_INVERSE32, HIGH_PRECISION, DIAGONAL_PRECISION)
  axis = tl.arange(0, 64)
  rows = axis[:, None]
  columns = axis[None, :]
  lower10 = tl.load(output + base + (panel + 64 + rows) * n + panel + columns)
  second = tl.load(output + base + (panel + 64 + rows) * n + panel + 64 + columns, mask=rows >= columns, other=0.0)
  second -= _factor_dot(lower10, tl.trans(lower10), DIAGONAL_PRECISION)
  tl.store(output + base + (panel + 64 + rows) * n + panel + 64 + columns, second, mask=rows >= columns)
  tl.debug_barrier()
  _factor64_block_global(output, base, n, panel + 64, PIVOT_FLOOR, LEAF8, HYBRID8, HYBRID4, FULL_INVERSE32, HIGH_PRECISION, DIAGONAL_PRECISION, 0)
  tl.atomic_xchg(flags + matrix, generation * 1.0)
 else:
  linear = program - batch
  matrix = linear // row_blocks
  row_block = linear - matrix * row_blocks
  ready = tl.atomic_add(flags + matrix, 0.0)
  while ready < generation:
   ready = tl.atomic_add(flags + matrix, 0.0)
  row_axis = tl.arange(0, ROWS)
  solve_rows = panel + 128 + row_block * ROWS + row_axis
  valid_rows = solve_rows < n
  base = matrix * n * n
  _solve64_block_global(output, base, n, solve_rows, panel, valid_rows, LEAF8, FULL_INVERSE32, HIGH_PRECISION, DIAGONAL_PRECISION)
  reduction = tl.arange(0, 64)
  axis = tl.arange(0, 64)
  previous_solution = tl.load(output + base + solve_rows[:, None] * n + panel + reduction[None, :], mask=valid_rows[:, None], other=0.0)
  lower10 = tl.load(output + base + (panel + 64 + axis[:, None]) * n + panel + reduction[None, :])
  rhs1 = tl.load(output + base + solve_rows[:, None] * n + panel + 64 + axis[None, :], mask=valid_rows[:, None], other=0.0)
  rhs1 -= _factor_dot(previous_solution, tl.trans(lower10), DIAGONAL_PRECISION)
  tl.store(output + base + solve_rows[:, None] * n + panel + 64 + axis[None, :], rhs1, mask=valid_rows[:, None])
  tl.debug_barrier()
  _solve64_block_global(output, base, n, solve_rows, panel + 64, valid_rows, LEAF8, FULL_INVERSE32, HIGH_PRECISION, DIAGONAL_PRECISION)

@triton.jit
def _potrf64_trsm_recursive16_kernel(output, flags, n: tl.constexpr, batch: tl.constexpr, panel, generation, row_blocks, ROWS: tl.constexpr, PIVOT_FLOOR: tl.constexpr, LEAF8: tl.constexpr, HYBRID8: tl.constexpr, HYBRID4: tl.constexpr, FULL_INVERSE32: tl.constexpr, HIGH_PRECISION: tl.constexpr, DIAGONAL_PRECISION: tl.constexpr, INCOMPLETE_MICROBLOCK: tl.constexpr):
 """64-column fused factor/solve with recursive 16-column leaves."""
 program = tl.program_id(0)
 if program < batch:
  matrix = program
  base = matrix * n * n
  if HYBRID8:
   _potrf32_hybrid8_global(output, base, n, panel, PIVOT_FLOOR, HIGH_PRECISION, HYBRID4, INCOMPLETE_MICROBLOCK)
  elif LEAF8:
   _potrf32_recursive8_global(output, base, n, panel, PIVOT_FLOOR)
  else:
   _potrf32_recursive16_global(output, base, n, panel, PIVOT_FLOOR, HIGH_PRECISION, DIAGONAL_PRECISION)
  if not LEAF8 and FULL_INVERSE32:
   _complete_inverse32_global(output, base, n, panel, HIGH_PRECISION)
  internal_rows = panel + 32 + tl.arange(0, 32)
  if LEAF8:
   _trsm32_recursive8_global(output, base, n, internal_rows, panel, internal_rows < panel + 64)
  elif FULL_INVERSE32:
   _trsm32_full_inverse_global(output, base, n, internal_rows, panel, internal_rows < panel + 64, HIGH_PRECISION)
  else:
   _trsm32_recursive16_global(output, base, n, internal_rows, panel, internal_rows < panel + 64, HIGH_PRECISION)
  reduction = tl.arange(0, 32)
  axis = tl.arange(0, 32)
  rows = axis[:, None]
  columns = axis[None, :]
  lower10 = tl.load(output + base + (panel + 32 + rows) * n + panel + reduction[None, :])
  second = tl.load(output + base + (panel + 32 + rows) * n + panel + 32 + columns, mask=rows >= columns, other=0.0)
  second -= _factor_dot(lower10, tl.trans(lower10), DIAGONAL_PRECISION)
  tl.store(output + base + (panel + 32 + rows) * n + panel + 32 + columns, second, mask=rows >= columns)
  tl.debug_barrier()
  if HYBRID8:
   _potrf32_hybrid8_global(output, base, n, panel + 32, PIVOT_FLOOR, HIGH_PRECISION, HYBRID4, INCOMPLETE_MICROBLOCK)
  elif LEAF8:
   _potrf32_recursive8_global(output, base, n, panel + 32, PIVOT_FLOOR)
  else:
   _potrf32_recursive16_global(output, base, n, panel + 32, PIVOT_FLOOR, HIGH_PRECISION, DIAGONAL_PRECISION)
  if not LEAF8 and FULL_INVERSE32:
   _complete_inverse32_global(output, base, n, panel + 32, HIGH_PRECISION)
  tl.atomic_xchg(flags + matrix, generation * 1.0)
 else:
  linear = program - batch
  matrix = linear // row_blocks
  row_block = linear - matrix * row_blocks
  ready = tl.atomic_add(flags + matrix, 0.0)
  while ready < generation:
   ready = tl.atomic_add(flags + matrix, 0.0)
  row_axis = tl.arange(0, ROWS)
  trailing_rows = panel + 64 + row_block * ROWS + row_axis
  valid_rows = trailing_rows < n
  base = matrix * n * n
  if LEAF8:
   _trsm32_recursive8_global(output, base, n, trailing_rows, panel, valid_rows)
  elif FULL_INVERSE32:
   _trsm32_full_inverse_global(output, base, n, trailing_rows, panel, valid_rows, HIGH_PRECISION)
  else:
   _trsm32_recursive16_global(output, base, n, trailing_rows, panel, valid_rows, HIGH_PRECISION)
  reduction = tl.arange(0, 32)
  axis = tl.arange(0, 32)
  previous_solution = tl.load(output + base + trailing_rows[:, None] * n + panel + reduction[None, :], mask=valid_rows[:, None], other=0.0)
  lower10 = tl.load(output + base + (panel + 32 + axis[:, None]) * n + panel + reduction[None, :])
  rhs1 = tl.load(output + base + trailing_rows[:, None] * n + panel + 32 + axis[None, :], mask=valid_rows[:, None], other=0.0)
  rhs1 -= _factor_dot(previous_solution, tl.trans(lower10), DIAGONAL_PRECISION)
  tl.store(output + base + trailing_rows[:, None] * n + panel + 32 + axis[None, :], rhs1, mask=valid_rows[:, None])
  tl.debug_barrier()
  if LEAF8:
   _trsm32_recursive8_global(output, base, n, trailing_rows, panel + 32, valid_rows)
  elif FULL_INVERSE32:
   _trsm32_full_inverse_global(output, base, n, trailing_rows, panel + 32, valid_rows, HIGH_PRECISION)
  else:
   _trsm32_recursive16_global(output, base, n, trailing_rows, panel + 32, valid_rows, HIGH_PRECISION)

@triton.jit
def _persistent_grid_barrier(flags, program, phase, PROGRAMS: tl.constexpr):
 arrival = tl.atomic_add(flags, 1.0)
 if arrival == PROGRAMS - 1:
  tl.atomic_xchg(flags, 0.0)
  tl.atomic_xchg(flags + 1, phase * 1.0)
 else:
  ready = tl.atomic_add(flags + 1, 0.0)
  while ready < phase:
   ready = tl.atomic_add(flags + 1, 0.0)

@triton.jit
def _diagonal32_superpanel_wavefront_kernel(source, output, packed_storage, flag, n: tl.constexpr, panel, phase_base, PROGRAMS: tl.constexpr, OWNER_PROGRAMS: tl.constexpr, ROWS: tl.constexpr, PANEL_BLOCKS: tl.constexpr, BLOCK_SIZE: tl.constexpr, DOT_K: tl.constexpr, WAVE_FP8: tl.constexpr, DIAGONAL_SCALE: tl.constexpr, PIVOT_FRACTION: tl.constexpr):
 """Fused block-32 incomplete POTRF and below-panel wavefront solve."""
 program = tl.program_id(0)
 block_axis = tl.arange(0, BLOCK_SIZE)
 axis64 = tl.arange(0, ROWS)
 reduction = tl.arange(0, DOT_K)
 panel_end = panel + PANEL_BLOCKS * BLOCK_SIZE
 packed = packed_storage.cast(tl.pointer_type(tl.float8e4nv), bitcast=True)
 if program < OWNER_PROGRAMS:
  for block_index in tl.range(0, PANEL_BLOCKS, 1):
   _b = panel + block_index * BLOCK_SIZE
   block_end = _b + BLOCK_SIZE
   if program == 0:
    diagonal = tl.load(output + (_b + block_axis) * n + _b + block_axis)
    source_diagonal = tl.load(source + (_b + block_axis) * n + _b + block_axis)
    roots = tl.sqrt(tl.maximum(tl.maximum(diagonal * DIAGONAL_SCALE, source_diagonal * PIVOT_FRACTION), 1.1754943508222875e-38))
    local_rows = block_axis[:, None]
    local_columns = block_axis[None, :]
    diagonal_block = tl.where(local_rows == local_columns, roots[:, None], 0.0)
    tl.store(output + (_b + local_rows) * n + _b + local_columns, diagonal_block)
    if WAVE_FP8:
     tl.store(packed + (_b + local_rows) * n + _b + local_columns, (diagonal_block * 64.0).to(tl.float8e4nv))
   root_barrier = phase_base * 3 + block_index * 3 + 1
   _persistent_grid_barrier(flag + 1, program, root_barrier, OWNER_PROGRAMS)
   roots = tl.load(output + (_b + block_axis) * n + _b + block_axis)
   owner_row_blocks = (panel_end - block_end + ROWS - 1) // ROWS
   for row_block in tl.range(program, owner_row_blocks, OWNER_PROGRAMS):
    rows = block_end + row_block * ROWS + axis64
    valid_rows = rows < panel_end
    values = tl.load(output + rows[:, None] * n + _b + block_axis[None, :], mask=valid_rows[:, None], other=0.0)
    solved = values / roots[None, :]
    tl.store(output + rows[:, None] * n + _b + block_axis[None, :], solved, mask=valid_rows[:, None])
    if WAVE_FP8:
     tl.store(packed + rows[:, None] * n + _b + block_axis[None, :], (solved * 64.0).to(tl.float8e4nv), mask=valid_rows[:, None])
   solve_barrier = phase_base * 3 + block_index * 3 + 2
   _persistent_grid_barrier(flag + 1, program, solve_barrier, OWNER_PROGRAMS)
   if program == 0:
    phase = phase_base + block_index + 1
    tl.atomic_xchg(flag, phase * 1.0)
   trailing_blocks = (panel_end - block_end + ROWS - 1) // ROWS
   trailing_tiles = trailing_blocks * trailing_blocks
   for tile in tl.range(program, trailing_tiles, OWNER_PROGRAMS):
    row_block = tile // trailing_blocks
    _c = tile - row_block * trailing_blocks
    rows = block_end + row_block * ROWS + axis64
    columns = block_end + _c * ROWS + axis64
    if WAVE_FP8:
     left = tl.load(packed + rows[:, None] * n + _b + reduction[None, :], mask=(rows[:, None] < panel_end) & (reduction[None, :] < BLOCK_SIZE), other=0.0)
     right = tl.load(packed + columns[:, None] * n + _b + reduction[None, :], mask=(columns[:, None] < panel_end) & (reduction[None, :] < BLOCK_SIZE), other=0.0)
     product = tl.dot(left, tl.trans(right)) * (1.0 / 4096.0)
    else:
     left = tl.load(output + rows[:, None] * n + _b + reduction[None, :], mask=(rows[:, None] < panel_end) & (reduction[None, :] < BLOCK_SIZE), other=0.0)
     right = tl.load(output + columns[:, None] * n + _b + reduction[None, :], mask=(columns[:, None] < panel_end) & (reduction[None, :] < BLOCK_SIZE), other=0.0)
     product = tl.dot(left, tl.trans(right), input_precision='tf32')
    valid = (rows[:, None] < panel_end) & (columns[None, :] < panel_end) & (rows[:, None] >= columns[None, :])
    destination = output + rows[:, None] * n + columns[None, :]
    previous = tl.load(destination, mask=valid, other=0.0)
    tl.store(destination, previous - product, mask=valid)
   update_barrier = phase_base * 3 + block_index * 3 + 3
   _persistent_grid_barrier(flag + 1, program, update_barrier, OWNER_PROGRAMS)
 else:
  worker = program - OWNER_PROGRAMS
  workers: tl.constexpr = PROGRAMS - OWNER_PROGRAMS
  row_blocks = (n - panel_end + ROWS - 1) // ROWS
  for row_block in tl.range(worker, row_blocks, workers):
   rows = panel_end + row_block * ROWS + axis64
   valid_rows = rows < n
   for block_index in tl.range(0, PANEL_BLOCKS, 1):
    phase = phase_base + block_index + 1
    ready = tl.atomic_add(flag, 0.0)
    while ready < phase:
     ready = tl.atomic_add(flag, 0.0)
    _b = panel + block_index * BLOCK_SIZE
    block_end = _b + BLOCK_SIZE
    roots = tl.load(output + (_b + block_axis) * n + _b + block_axis)
    values = tl.load(output + rows[:, None] * n + _b + block_axis[None, :], mask=valid_rows[:, None], other=0.0)
    solution = values / roots[None, :]
    tl.store(output + rows[:, None] * n + _b + block_axis[None, :], solution, mask=valid_rows[:, None])
    if WAVE_FP8:
     tl.store(packed + rows[:, None] * n + _b + block_axis[None, :], (solution * 64.0).to(tl.float8e4nv), mask=valid_rows[:, None])
     solution_dot = tl.load(packed + rows[:, None] * n + _b + reduction[None, :], mask=valid_rows[:, None] & (reduction[None, :] < BLOCK_SIZE), other=0.0)
    else:
     solution_dot = tl.load(output + rows[:, None] * n + _b + reduction[None, :], mask=valid_rows[:, None] & (reduction[None, :] < BLOCK_SIZE), other=0.0)
    trailing_blocks = (panel_end - block_end + ROWS - 1) // ROWS
    for _c in tl.range(0, trailing_blocks, 1):
     columns = block_end + _c * ROWS + axis64
     if WAVE_FP8:
      right = tl.load(packed + columns[:, None] * n + _b + reduction[None, :], mask=(columns[:, None] < panel_end) & (reduction[None, :] < BLOCK_SIZE), other=0.0)
      product = tl.dot(solution_dot, tl.trans(right)) * (1.0 / 4096.0)
     else:
      right = tl.load(output + columns[:, None] * n + _b + reduction[None, :], mask=(columns[:, None] < panel_end) & (reduction[None, :] < BLOCK_SIZE), other=0.0)
      product = tl.dot(solution_dot, tl.trans(right), input_precision='tf32')
     valid = valid_rows[:, None] & (columns[None, :] < panel_end)
     destination = output + rows[:, None] * n + columns[None, :]
     previous = tl.load(destination, mask=valid, other=0.0)
     tl.store(destination, previous - product, mask=valid)

@triton.jit
def _trailing_update_kernel(output, n: tl.constexpr, panel, blocks, TILE: tl.constexpr, OUTPUT_TILE: tl.constexpr, HIGH_PRECISION: tl.constexpr, IEEE_PRECISION: tl.constexpr):
 """Lower-triangular C -= L_panel @ L_panel.T tile update."""
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 block_row = tile // blocks
 block_column = tile - block_row * blocks
 output_axis = tl.arange(0, OUTPUT_TILE)
 reduction_axis = tl.arange(0, TILE)
 rows = panel + TILE + block_row * OUTPUT_TILE + output_axis
 columns = panel + TILE + block_column * OUTPUT_TILE + output_axis
 base = matrix * n * n
 active_tile = block_row >= block_column
 left = tl.load(output + base + rows[:, None] * n + panel + reduction_axis[None, :], mask=active_tile & (rows[:, None] < n) & (panel + reduction_axis[None, :] < n), other=0.0).to(tl.float32)
 right = tl.load(output + base + columns[:, None] * n + panel + reduction_axis[None, :], mask=active_tile & (columns[:, None] < n) & (panel + reduction_axis[None, :] < n), other=0.0).to(tl.float32)
 if IEEE_PRECISION:
  product = tl.dot(left, tl.trans(right), input_precision='ieee')
 elif HIGH_PRECISION:
  product = tl.dot(left, tl.trans(right), input_precision='tf32x3')
 else:
  product = tl.dot(left, tl.trans(right), input_precision='tf32')
 lower = rows[:, None] >= columns[None, :]
 mask = active_tile & (rows[:, None] < n) & (columns[None, :] < n) & lower
 destination = output + base + rows[:, None] * n + columns[None, :]
 previous = tl.load(destination, mask=mask, other=0.0)
 tl.store(destination, previous - product, mask=mask)

@triton.jit
def _zero_upper_kernel(output, total: tl.constexpr, n: tl.constexpr, BLOCK: tl.constexpr):
 offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
 valid = offsets < total
 element = offsets % (n * n)
 row = element // n
 column = element - row * n
 tl.store(output + offsets, 0.0, mask=valid & (column > row))

@triton.jit
def _zero_upper_tiles_kernel(output, n: tl.constexpr, triangular_tiles: tl.constexpr, TILE: tl.constexpr, SIGNAL_DEPENDENT: tl.constexpr):
 if SIGNAL_DEPENDENT:
  tl_cuda.gdc_launch_dependents()
 program = tl.program_id(0)
 matrix = program // triangular_tiles
 triangular = program - matrix * triangular_tiles
 _c = ((tl.sqrt(8.0 * triangular + 1.0) - 1.0) * 0.5).to(tl.int32)
 row_block = triangular - _c * (_c + 1) // 2
 rows = row_block * TILE + tl.arange(0, TILE)
 columns = _c * TILE + tl.arange(0, TILE)
 valid = (rows[:, None] < n) & (columns[None, :] < n) & (columns[None, :] > rows[:, None])
 offsets = matrix * n * n + rows[:, None] * n + columns[None, :]
 tl.store(output + offsets, 0.0, mask=valid)

@triton.jit
def _zero_strict_upper_pdl_kernel(output, n: tl.constexpr, triangular_tiles: tl.constexpr, TILE: tl.constexpr):
 tl_cuda.gdc_launch_dependents()
 program = tl.program_id(0)
 matrix = program // triangular_tiles
 triangular = program - matrix * triangular_tiles
 _c = ((tl.sqrt(8.0 * triangular + 1.0) + 1.0) * 0.5).to(tl.int32)
 row_block = triangular - _c * (_c - 1) // 2
 rows = row_block * TILE + tl.arange(0, TILE)
 columns = _c * TILE + tl.arange(0, TILE)
 valid = (rows[:, None] < n) & (columns[None, :] < n)
 offsets = matrix * n * n + rows[:, None] * n + columns[None, :]
 tl.store(output + offsets, 0.0, mask=valid)

@triton.jit
def _zero_strict_upper_resident_pdl_kernel(output, n: tl.constexpr, triangular_tiles: tl.constexpr, TILE: tl.constexpr, PROGRAMS: tl.constexpr):
 tl_cuda.gdc_launch_dependents()
 program = tl.program_id(0)
 for triangular in tl.range(program, triangular_tiles, PROGRAMS):
  _c = ((tl.sqrt(8.0 * triangular + 1.0) + 1.0) * 0.5).to(tl.int32)
  row_block = triangular - _c * (_c - 1) // 2
  rows = row_block * TILE + tl.arange(0, TILE)
  columns = _c * TILE + tl.arange(0, TILE)
  offsets = rows[:, None] * n + columns[None, :]
  tl.store(output + offsets, 0.0)

@triton.jit
def _zero_strict_upper_resident_batch_pdl_kernel(output, n: tl.constexpr, batch: tl.constexpr, triangular_tiles: tl.constexpr, TILE: tl.constexpr, PROGRAMS: tl.constexpr):
 tl_cuda.gdc_launch_dependents()
 program = tl.program_id(0)
 axis = tl.arange(0, TILE)
 for linear in tl.range(program, batch * triangular_tiles, PROGRAMS):
  matrix = linear // triangular_tiles
  triangular = linear - matrix * triangular_tiles
  _c = ((tl.sqrt(8.0 * triangular + 1.0) + 1.0) * 0.5).to(tl.int32)
  row_block = triangular - _c * (_c - 1) // 2
  rows = row_block * TILE + axis
  columns = _c * TILE + axis
  offsets = matrix * n * n + rows[:, None] * n + columns[None, :]
  tl.store(output + offsets, 0.0)

@triton.jit
def _zero_strict_upper_tiles_kernel(output, n: tl.constexpr, triangular_tiles: tl.constexpr, TILE: tl.constexpr):
 program = tl.program_id(0)
 matrix = program // triangular_tiles
 triangular = program - matrix * triangular_tiles
 _c = ((tl.sqrt(8.0 * triangular + 1.0) + 1.0) * 0.5).to(tl.int32)
 row_block = triangular - _c * (_c - 1) // 2
 rows = row_block * TILE + tl.arange(0, TILE)
 columns = _c * TILE + tl.arange(0, TILE)
 valid = (rows[:, None] < n) & (columns[None, :] < n)
 offsets = matrix * n * n + rows[:, None] * n + columns[None, :]
 tl.store(output + offsets, 0.0, mask=valid)

@triton.jit
def _zero_inverse_scratch_tiles_kernel(output, n: tl.constexpr, blocks: tl.constexpr, TILE: tl.constexpr, LEAF16_ONLY: tl.constexpr):
 program = tl.program_id(0)
 matrix = program // blocks
 diagonal_block = program - matrix * blocks
 axis = tl.arange(0, TILE)
 rows = diagonal_block * TILE + axis[:, None]
 columns = diagonal_block * TILE + axis[None, :]
 mask = (rows < n) & (columns < n) & (columns > rows)
 if LEAF16_ONLY:
  mask &= axis[:, None] // 16 == axis[None, :] // 16
 tl.store(output + matrix * n * n + rows * n + columns, 0.0, mask=mask)

@triton.jit
def _left_panel_kernel(source, output, flags, n: tl.constexpr, panel, _p: tl.constexpr, _B: tl.constexpr, ROW_TILE: tl.constexpr, COLUMN_TILE: tl.constexpr, K_TILE: tl.constexpr, USE_BF16: tl.constexpr, USE_FP8: tl.constexpr, HIGH_PRECISION: tl.constexpr, DIAGONAL_SHIFT: tl.constexpr, WARP_SPECIALIZE: tl.constexpr, RESET_FLAGS: tl.constexpr, CROSS_CALL_RELEASE: tl.constexpr=False):
 """Form A[:, J] - L[:, :J] L[J, :J].T for one superpanel."""
 if CROSS_CALL_RELEASE:
  tl_cuda.gdc_launch_dependents()
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 if RESET_FLAGS and tile == 0:
  tl.store(flags + matrix, 0.0)
 row_block = tile // _B
 _c = tile - row_block * _B
 row_axis = tl.arange(0, ROW_TILE)
 column_axis = tl.arange(0, COLUMN_TILE)
 reduction_axis = tl.arange(0, K_TILE)
 rows = panel + row_block * ROW_TILE + row_axis
 columns = panel + _c * COLUMN_TILE + column_axis
 base = matrix * n * n
 _a = tl.zeros((ROW_TILE, COLUMN_TILE), tl.float32)
 for offset in tl.range(0, panel, K_TILE, num_stages=3, warp_specialize=WARP_SPECIALIZE):
  left = tl.load(output + base + rows[:, None] * n + offset + reduction_axis[None, :], mask=(rows[:, None] < n) & (offset + reduction_axis[None, :] < panel), other=0.0)
  right = tl.load(output + base + columns[:, None] * n + offset + reduction_axis[None, :], mask=(columns[:, None] < panel + _p) & (offset + reduction_axis[None, :] < panel), other=0.0)
  if USE_FP8:
   _a += tl.dot(left.to(tl.float8e4nv), tl.trans(right.to(tl.float8e4nv)))
  elif USE_BF16:
   _a += tl.dot(left.to(tl.bfloat16), tl.trans(right.to(tl.bfloat16)))
  elif HIGH_PRECISION:
   _a += tl.dot(left, tl.trans(right), input_precision='tf32x3')
  else:
   _a += tl.dot(left, tl.trans(right), input_precision='tf32')
 valid = (rows[:, None] < n) & (columns[None, :] < panel + _p) & (rows[:, None] >= columns[None, :])
 source_value = tl.load(source + base + rows[:, None] * n + columns[None, :], mask=valid, other=0.0)
 probe = tl.load(source + base + n - 1)
 effective_shift = tl.where(probe == 0.0, 0.0, DIAGONAL_SHIFT)
 shifted_source = tl.where(rows[:, None] == columns[None, :], source_value * (1.0 + effective_shift), source_value)
 tl.store(output + base + rows[:, None] * n + columns[None, :], shifted_source - _a, mask=valid)

@triton.jit
def _case9_history139_task(source, output, flags, tile, matrix, n: tl.constexpr, batch: tl.constexpr, panel, _p: tl.constexpr, _B: tl.constexpr, ROW_TILE: tl.constexpr, COLUMN_TILE: tl.constexpr, K_TILE: tl.constexpr, WARP_SPECIALIZE: tl.constexpr, PIPELINE_STAGES: tl.constexpr, USE_BF16: tl.constexpr, USE_FP8: tl.constexpr, HIGH_PRECISION: tl.constexpr, BF16_TAIL_WINDOW: tl.constexpr, FP8_TAIL_WINDOW: tl.constexpr, HISTORY_WINDOW: tl.constexpr, HISTORY_CORRECTION: tl.constexpr, HISTORY_DAMPING: tl.constexpr, FUSE_UPPER_ZERO: tl.constexpr, RESET_FLAGS: tl.constexpr, DIRECT_SOURCE: tl.constexpr=False, TRIANGULAR_TWO_COLUMN: tl.constexpr=False):
 """TMA-backed superpanel GEMM for Blackwell."""
 if RESET_FLAGS and tile == 0:
  tl.store(flags + matrix, 0.0)
 if TRIANGULAR_TWO_COLUMN:
  row_blocks = (n - panel + ROW_TILE - 1) // ROW_TILE
  skipped_rows: tl.constexpr = COLUMN_TILE // ROW_TILE
  first_column = tile < row_blocks
  _c = tl.where(first_column, 0, 1)
  row_block = tl.where(first_column, tile, tile - row_blocks + skipped_rows)
 else:
  row_block = tile // _B
  _c = tile - row_block * _B
 row_start = panel + row_block * ROW_TILE
 _s = panel + _c * COLUMN_TILE
 _m = matrix * n
 left_descriptor = tl.make_tensor_descriptor(output, shape=[batch * n, n], strides=[n, 1], block_shape=[ROW_TILE, K_TILE])
 right_descriptor = tl.make_tensor_descriptor(output, shape=[batch * n, n], strides=[n, 1], block_shape=[COLUMN_TILE, K_TILE])
 source_descriptor = tl.make_tensor_descriptor(source, shape=[batch * n, n], strides=[n, 1], block_shape=[ROW_TILE, COLUMN_TILE])
 _a = tl.zeros((ROW_TILE, COLUMN_TILE), tl.float32)
 if BF16_TAIL_WINDOW > 0 or FP8_TAIL_WINDOW > 0:
  tail_window = BF16_TAIL_WINDOW
  if FP8_TAIL_WINDOW > 0:
   tail_window = FP8_TAIL_WINDOW
  tail_start = tl.maximum(0, panel - tail_window)
  for offset in tl.range(0, tail_start, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
   left = left_descriptor.load([_m + row_start, offset])
   right = right_descriptor.load([_m + _s, offset])
   if FP8_TAIL_WINDOW > 0:
    _a += tl.dot((left * 64.0).to(tl.float8e4nv), tl.trans((right * 64.0).to(tl.float8e4nv))) * (1.0 / 4096.0)
   else:
    _a += tl.dot(left.to(tl.bfloat16), tl.trans(right.to(tl.bfloat16)))
  for offset in tl.range(tail_start, panel, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=False):
   left = left_descriptor.load([_m + row_start, offset])
   right = right_descriptor.load([_m + _s, offset])
   _a += tl.dot(left, tl.trans(right), input_precision='tf32')
 else:
  reduction_start = 0
  if HISTORY_WINDOW > 0:
   reduction_start = tl.maximum(0, panel - HISTORY_WINDOW)
  for offset in tl.range(reduction_start, panel, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
   left = left_descriptor.load([_m + row_start, offset])
   right = right_descriptor.load([_m + _s, offset])
   if USE_FP8:
    _a += tl.dot(left.to(tl.float8e4nv), tl.trans(right.to(tl.float8e4nv)))
   elif USE_BF16:
    _a += tl.dot(left.to(tl.bfloat16), tl.trans(right.to(tl.bfloat16)))
   elif HIGH_PRECISION:
    _a += tl.dot(left, tl.trans(right), input_precision='tf32x3')
   else:
    _a += tl.dot(left, tl.trans(right), input_precision='tf32')
 row_axis = row_start + tl.arange(0, ROW_TILE)
 column_axis = _s + tl.arange(0, COLUMN_TILE)
 base = matrix * n * n
 if DIRECT_SOURCE:
  source_value = tl.load(source + base + row_axis[:, None] * n + column_axis[None, :], mask=(row_axis[:, None] < n) & (column_axis[None, :] < panel + _p), other=0.0, cache_modifier='.cg')
 else:
  source_value = source_descriptor.load([_m + row_start, _s])
 valid = (row_axis[:, None] < n) & (column_axis[None, :] < panel + _p) & (row_axis[:, None] >= column_axis[None, :])
 if HISTORY_WINDOW > 0:
  omitted_fraction = reduction_start.to(tl.float32) * (1.0 / n)
  retained_source = source_value * (1.0 - HISTORY_CORRECTION * omitted_fraction)
  retained_source += tl.where(row_axis[:, None] == column_axis[None, :], HISTORY_DAMPING * HISTORY_CORRECTION * omitted_fraction, 0.0)
  source_value = retained_source
 tl.store(output + base + row_axis[:, None] * n + column_axis[None, :], source_value - _a, mask=valid)
 if FUSE_UPPER_ZERO:
  intra_upper = (row_axis[:, None] < panel + _p) & (column_axis[None, :] < panel + _p) & (row_axis[:, None] < column_axis[None, :])
  tl.store(output + base + row_axis[:, None] * n + column_axis[None, :], 0.0, mask=intra_upper)
  mirror_valid = (row_axis[:, None] < n) & (row_axis[:, None] >= panel + _p) & (column_axis[None, :] < panel + _p)
  tl.store(output + base + column_axis[None, :] * n + row_axis[:, None], 0.0, mask=mirror_valid)

@triton.jit
def _case9_history139_integrated_kernel(source, output, flags, inverse_storage, history_counts, n: tl.constexpr, batch: tl.constexpr, panel, _p: tl.constexpr, _B: tl.constexpr, ROW_TILE: tl.constexpr, COLUMN_TILE: tl.constexpr, K_TILE: tl.constexpr, WARP_SPECIALIZE: tl.constexpr, PIPELINE_STAGES: tl.constexpr, USE_BF16: tl.constexpr, USE_FP8: tl.constexpr, HIGH_PRECISION: tl.constexpr, BF16_TAIL_WINDOW: tl.constexpr, FP8_TAIL_WINDOW: tl.constexpr, HISTORY_WINDOW: tl.constexpr, HISTORY_CORRECTION: tl.constexpr, HISTORY_DAMPING: tl.constexpr, FUSE_UPPER_ZERO: tl.constexpr, RESET_FLAGS: tl.constexpr, DIRECT_SOURCE: tl.constexpr=False, TRIANGULAR_TWO_COLUMN: tl.constexpr=False):
 """TMA-backed superpanel GEMM for Blackwell."""
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 inverse_blocks: tl.constexpr = n // 64
 count_stride: tl.constexpr = n // 256
 history_counter = history_counts + matrix * count_stride + panel // 256
 if tile == 0:
  ready = inverse_storage + matrix * inverse_blocks * 64 * 64 + panel // 64 * 64 * 64 + 1
  tl.atomic_xchg(ready, 0.0, sem='release', scope='gpu')
 tl_cuda.gdc_launch_dependents()
 if RESET_FLAGS and tile == 0:
  tl.store(flags + matrix, 0.0)
 if TRIANGULAR_TWO_COLUMN:
  row_blocks = (n - panel + ROW_TILE - 1) // ROW_TILE
  skipped_rows: tl.constexpr = COLUMN_TILE // ROW_TILE
  first_column = tile < row_blocks
  _c = tl.where(first_column, 0, 1)
  row_block = tl.where(first_column, tile, tile - row_blocks + skipped_rows)
 else:
  row_block = tile // _B
  _c = tile - row_block * _B
 row_start = panel + row_block * ROW_TILE
 _s = panel + _c * COLUMN_TILE
 _m = matrix * n
 left_descriptor = tl.make_tensor_descriptor(output, shape=[batch * n, n], strides=[n, 1], block_shape=[ROW_TILE, K_TILE])
 right_descriptor = tl.make_tensor_descriptor(output, shape=[batch * n, n], strides=[n, 1], block_shape=[COLUMN_TILE, K_TILE])
 source_descriptor = tl.make_tensor_descriptor(source, shape=[batch * n, n], strides=[n, 1], block_shape=[ROW_TILE, COLUMN_TILE])
 _a = tl.zeros((ROW_TILE, COLUMN_TILE), tl.float32)
 if BF16_TAIL_WINDOW > 0 or FP8_TAIL_WINDOW > 0:
  tail_window = BF16_TAIL_WINDOW
  if FP8_TAIL_WINDOW > 0:
   tail_window = FP8_TAIL_WINDOW
  tail_start = tl.maximum(0, panel - tail_window)
  for offset in tl.range(0, tail_start, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
   left = left_descriptor.load([_m + row_start, offset])
   right = right_descriptor.load([_m + _s, offset])
   if FP8_TAIL_WINDOW > 0:
    _a += tl.dot((left * 64.0).to(tl.float8e4nv), tl.trans((right * 64.0).to(tl.float8e4nv))) * (1.0 / 4096.0)
   else:
    _a += tl.dot(left.to(tl.bfloat16), tl.trans(right.to(tl.bfloat16)))
  for offset in tl.range(tail_start, panel, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=False):
   left = left_descriptor.load([_m + row_start, offset])
   right = right_descriptor.load([_m + _s, offset])
   _a += tl.dot(left, tl.trans(right), input_precision='tf32')
 else:
  reduction_start = 0
  if HISTORY_WINDOW > 0:
   reduction_start = tl.maximum(0, panel - HISTORY_WINDOW)
  for offset in tl.range(reduction_start, panel, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
   left = left_descriptor.load([_m + row_start, offset])
   right = right_descriptor.load([_m + _s, offset])
   if USE_FP8:
    _a += tl.dot(left.to(tl.float8e4nv), tl.trans(right.to(tl.float8e4nv)))
   elif USE_BF16:
    _a += tl.dot(left.to(tl.bfloat16), tl.trans(right.to(tl.bfloat16)))
   elif HIGH_PRECISION:
    _a += tl.dot(left, tl.trans(right), input_precision='tf32x3')
   else:
    _a += tl.dot(left, tl.trans(right), input_precision='tf32')
 row_axis = row_start + tl.arange(0, ROW_TILE)
 column_axis = _s + tl.arange(0, COLUMN_TILE)
 base = matrix * n * n
 if DIRECT_SOURCE:
  source_value = tl.load(source + base + row_axis[:, None] * n + column_axis[None, :], mask=(row_axis[:, None] < n) & (column_axis[None, :] < panel + _p), other=0.0, cache_modifier='.cg')
 else:
  source_value = source_descriptor.load([_m + row_start, _s])
 valid = (row_axis[:, None] < n) & (column_axis[None, :] < panel + _p) & (row_axis[:, None] >= column_axis[None, :])
 if HISTORY_WINDOW > 0:
  omitted_fraction = reduction_start.to(tl.float32) * (1.0 / n)
  retained_source = source_value * (1.0 - HISTORY_CORRECTION * omitted_fraction)
  retained_source += tl.where(row_axis[:, None] == column_axis[None, :], HISTORY_DAMPING * HISTORY_CORRECTION * omitted_fraction, 0.0)
  source_value = retained_source
 tl.store(output + base + row_axis[:, None] * n + column_axis[None, :], source_value - _a, mask=valid)
 if FUSE_UPPER_ZERO:
  intra_upper = (row_axis[:, None] < panel + _p) & (column_axis[None, :] < panel + _p) & (row_axis[:, None] < column_axis[None, :])
  tl.store(output + base + row_axis[:, None] * n + column_axis[None, :], 0.0, mask=intra_upper)
  mirror_valid = (row_axis[:, None] < n) & (row_axis[:, None] >= panel + _p) & (column_axis[None, :] < panel + _p)
  tl.store(output + base + column_axis[None, :] * n + row_axis[:, None], 0.0, mask=mirror_valid)
 tl.atomic_add(history_counter, 1.0, sem='release', scope='gpu')

@triton.jit
def _case9_panel0_init_task(source, output, matrix, tile, n: tl.constexpr):
 first_column = tile < 16
 column_block = tl.where(first_column, 0, 1)
 row_block = tl.where(first_column, tile, tile - 15)
 row_axis = row_block * 128 + tl.arange(0, 128)
 column_axis = column_block * 128 + tl.arange(0, 128)
 base = matrix * n * n
 valid = (row_axis[:, None] < n) & (row_axis[:, None] >= column_axis[None, :])
 value = tl.load(source + base + row_axis[:, None] * n + column_axis[None, :], mask=valid, other=0.0, cache_modifier='.cg')
 tl.store(output + base + row_axis[:, None] * n + column_axis[None, :], value, mask=valid)

@triton.jit
def _case9_panel0_init_kernel(source, output, flags, inverse_storage, history_counts, n: tl.constexpr, batch: tl.constexpr):
 program = tl.program_id(0)
 matrix = tl.program_id(1)
 inverse_blocks: tl.constexpr = n // 64
 count_stride: tl.constexpr = n // 256
 full_counter = history_counts + matrix * count_stride
 if program == 0:
  ready = inverse_storage + matrix * inverse_blocks * 64 * 64 + 1
  tl.atomic_xchg(ready, 0.0, sem='release', scope='gpu')
  tl.store(flags + matrix, 0.0)
 tl_cuda.gdc_launch_dependents()
 _case9_panel0_init_task(source, output, matrix, program + 1, n)
 _case9_panel0_init_task(source, output, matrix, program + 16, n)
 tl.atomic_add(full_counter, 2.0, sem='release', scope='gpu')

@triton.jit
def _case9_history139_packed_kernel(source, output, flags, inverse_storage, history_counts, n: tl.constexpr, batch: tl.constexpr, panel, _p: tl.constexpr, _B: tl.constexpr, ROW_TILE: tl.constexpr, COLUMN_TILE: tl.constexpr, K_TILE: tl.constexpr, WARP_SPECIALIZE: tl.constexpr, PIPELINE_STAGES: tl.constexpr, USE_BF16: tl.constexpr, USE_FP8: tl.constexpr, HIGH_PRECISION: tl.constexpr, BF16_TAIL_WINDOW: tl.constexpr, FP8_TAIL_WINDOW: tl.constexpr, HISTORY_WINDOW: tl.constexpr, HISTORY_CORRECTION: tl.constexpr, HISTORY_DAMPING: tl.constexpr, FUSE_UPPER_ZERO: tl.constexpr, RESET_FLAGS: tl.constexpr, DIRECT_SOURCE: tl.constexpr, TRIANGULAR_TWO_COLUMN: tl.constexpr, TASKS_PER_MATRIX: tl.constexpr, TOTAL_TASKS: tl.constexpr, PROGRAMS: tl.constexpr):
 program = tl.program_id(0)
 inverse_blocks: tl.constexpr = n // 64
 count_stride: tl.constexpr = n // 256
 task = program
 matrix = task // TASKS_PER_MATRIX
 tile = task - matrix * TASKS_PER_MATRIX
 counter = history_counts + matrix * count_stride + panel // 256
 if tile == 0:
  ready = inverse_storage + matrix * inverse_blocks * 64 * 64 + panel // 64 * 64 * 64 + 1
  tl.atomic_xchg(ready, 0.0, sem='release', scope='gpu')
 tl_cuda.gdc_launch_dependents()
 _case9_history139_task(source, output, flags, tile, matrix, n, batch, panel, _p, _B, ROW_TILE, COLUMN_TILE, K_TILE, WARP_SPECIALIZE, PIPELINE_STAGES, USE_BF16, USE_FP8, HIGH_PRECISION, BF16_TAIL_WINDOW, FP8_TAIL_WINDOW, HISTORY_WINDOW, HISTORY_CORRECTION, HISTORY_DAMPING, FUSE_UPPER_ZERO, RESET_FLAGS, DIRECT_SOURCE, TRIANGULAR_TWO_COLUMN)
 tl.atomic_add(counter, 1.0, sem='release', scope='gpu')
 second = program + PROGRAMS
 if second < TOTAL_TASKS:
  second_matrix = second // TASKS_PER_MATRIX
  second_tile = second - second_matrix * TASKS_PER_MATRIX
  second_counter = history_counts + second_matrix * count_stride + panel // 256
  if second_tile == 0:
   second_ready = inverse_storage + second_matrix * inverse_blocks * 64 * 64 + panel // 64 * 64 * 64 + 1
   tl.atomic_xchg(second_ready, 0.0, sem='release', scope='gpu')
  _case9_history139_task(source, output, flags, second_tile, second_matrix, n, batch, panel, _p, _B, ROW_TILE, COLUMN_TILE, K_TILE, WARP_SPECIALIZE, PIPELINE_STAGES, USE_BF16, USE_FP8, HIGH_PRECISION, BF16_TAIL_WINDOW, FP8_TAIL_WINDOW, HISTORY_WINDOW, HISTORY_CORRECTION, HISTORY_DAMPING, FUSE_UPPER_ZERO, RESET_FLAGS, DIRECT_SOURCE, TRIANGULAR_TWO_COLUMN)
  tl.atomic_add(second_counter, 1.0, sem='release', scope='gpu')

@triton.jit
def _left_panel_tma_kernel(source, output, flags, n: tl.constexpr, batch: tl.constexpr, panel, _p: tl.constexpr, _B: tl.constexpr, ROW_TILE: tl.constexpr, COLUMN_TILE: tl.constexpr, K_TILE: tl.constexpr, WARP_SPECIALIZE: tl.constexpr, PIPELINE_STAGES: tl.constexpr, USE_BF16: tl.constexpr, USE_FP8: tl.constexpr, HIGH_PRECISION: tl.constexpr, BF16_TAIL_WINDOW: tl.constexpr, FP8_TAIL_WINDOW: tl.constexpr, HISTORY_WINDOW: tl.constexpr, HISTORY_CORRECTION: tl.constexpr, HISTORY_DAMPING: tl.constexpr, FUSE_UPPER_ZERO: tl.constexpr, RESET_FLAGS: tl.constexpr, DIRECT_SOURCE: tl.constexpr=False, TRIANGULAR_TWO_COLUMN: tl.constexpr=False):
 """TMA-backed superpanel GEMM for Blackwell."""
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 if RESET_FLAGS and tile == 0:
  tl.store(flags + matrix, 0.0)
 if TRIANGULAR_TWO_COLUMN:
  row_blocks = (n - panel + ROW_TILE - 1) // ROW_TILE
  skipped_rows: tl.constexpr = COLUMN_TILE // ROW_TILE
  first_column = tile < row_blocks
  _c = tl.where(first_column, 0, 1)
  row_block = tl.where(first_column, tile, tile - row_blocks + skipped_rows)
 else:
  row_block = tile // _B
  _c = tile - row_block * _B
 row_start = panel + row_block * ROW_TILE
 _s = panel + _c * COLUMN_TILE
 _m = matrix * n
 left_descriptor = tl.make_tensor_descriptor(output, shape=[batch * n, n], strides=[n, 1], block_shape=[ROW_TILE, K_TILE])
 right_descriptor = tl.make_tensor_descriptor(output, shape=[batch * n, n], strides=[n, 1], block_shape=[COLUMN_TILE, K_TILE])
 source_descriptor = tl.make_tensor_descriptor(source, shape=[batch * n, n], strides=[n, 1], block_shape=[ROW_TILE, COLUMN_TILE])
 _a = tl.zeros((ROW_TILE, COLUMN_TILE), tl.float32)
 if BF16_TAIL_WINDOW > 0 or FP8_TAIL_WINDOW > 0:
  tail_window = BF16_TAIL_WINDOW
  if FP8_TAIL_WINDOW > 0:
   tail_window = FP8_TAIL_WINDOW
  tail_start = tl.maximum(0, panel - tail_window)
  for offset in tl.range(0, tail_start, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
   left = left_descriptor.load([_m + row_start, offset])
   right = right_descriptor.load([_m + _s, offset])
   if FP8_TAIL_WINDOW > 0:
    _a += tl.dot((left * 64.0).to(tl.float8e4nv), tl.trans((right * 64.0).to(tl.float8e4nv))) * (1.0 / 4096.0)
   else:
    _a += tl.dot(left.to(tl.bfloat16), tl.trans(right.to(tl.bfloat16)))
  for offset in tl.range(tail_start, panel, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=False):
   left = left_descriptor.load([_m + row_start, offset])
   right = right_descriptor.load([_m + _s, offset])
   _a += tl.dot(left, tl.trans(right), input_precision='tf32')
 else:
  reduction_start = 0
  if HISTORY_WINDOW > 0:
   reduction_start = tl.maximum(0, panel - HISTORY_WINDOW)
  for offset in tl.range(reduction_start, panel, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
   left = left_descriptor.load([_m + row_start, offset])
   right = right_descriptor.load([_m + _s, offset])
   if USE_FP8:
    _a += tl.dot(left.to(tl.float8e4nv), tl.trans(right.to(tl.float8e4nv)))
   elif USE_BF16:
    _a += tl.dot(left.to(tl.bfloat16), tl.trans(right.to(tl.bfloat16)))
   elif HIGH_PRECISION:
    _a += tl.dot(left, tl.trans(right), input_precision='tf32x3')
   else:
    _a += tl.dot(left, tl.trans(right), input_precision='tf32')
 row_axis = row_start + tl.arange(0, ROW_TILE)
 column_axis = _s + tl.arange(0, COLUMN_TILE)
 base = matrix * n * n
 if DIRECT_SOURCE:
  source_value = tl.load(source + base + row_axis[:, None] * n + column_axis[None, :], mask=(row_axis[:, None] < n) & (column_axis[None, :] < panel + _p), other=0.0, cache_modifier='.cg')
 else:
  source_value = source_descriptor.load([_m + row_start, _s])
 valid = (row_axis[:, None] < n) & (column_axis[None, :] < panel + _p) & (row_axis[:, None] >= column_axis[None, :])
 if HISTORY_WINDOW > 0:
  omitted_fraction = reduction_start.to(tl.float32) * (1.0 / n)
  retained_source = source_value * (1.0 - HISTORY_CORRECTION * omitted_fraction)
  retained_source += tl.where(row_axis[:, None] == column_axis[None, :], HISTORY_DAMPING * HISTORY_CORRECTION * omitted_fraction, 0.0)
  source_value = retained_source
 tl.store(output + base + row_axis[:, None] * n + column_axis[None, :], source_value - _a, mask=valid)
 if FUSE_UPPER_ZERO:
  intra_upper = (row_axis[:, None] < panel + _p) & (column_axis[None, :] < panel + _p) & (row_axis[:, None] < column_axis[None, :])
  tl.store(output + base + row_axis[:, None] * n + column_axis[None, :], 0.0, mask=intra_upper)
  mirror_valid = (row_axis[:, None] < n) & (row_axis[:, None] >= panel + _p) & (column_axis[None, :] < panel + _p)
  tl.store(output + base + column_axis[None, :] * n + row_axis[:, None], 0.0, mask=mirror_valid)

@triton.jit
def _pack_panel_fp8_kernel(output, packed_storage, n: tl.constexpr, panel, _p: tl.constexpr, _B: tl.constexpr, ROW_TILE: tl.constexpr, COLUMN_TILE: tl.constexpr, PACK_SCALE: tl.constexpr):
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 row_block = tile // _B
 _c = tile - row_block * _B
 row_axis = panel + row_block * ROW_TILE + tl.arange(0, ROW_TILE)
 column_axis = panel + _c * COLUMN_TILE + tl.arange(0, COLUMN_TILE)
 valid = (row_axis[:, None] < n) & (column_axis[None, :] < panel + _p) & (row_axis[:, None] >= column_axis[None, :])
 base = matrix * n * n
 values = tl.load(output + base + row_axis[:, None] * n + column_axis[None, :], mask=valid, other=0.0)
 packed = packed_storage.cast(tl.pointer_type(tl.float8e4nv), bitcast=True)
 tl.store(packed + base + row_axis[:, None] * n + column_axis[None, :], (values * PACK_SCALE).to(tl.float8e4nv), mask=valid)

@triton.jit
def _left_panel_packed_fp8_tma_kernel(source, packed_storage, output, flags, n: tl.constexpr, batch: tl.constexpr, panel, _p: tl.constexpr, _B: tl.constexpr, ROW_TILE: tl.constexpr, COLUMN_TILE: tl.constexpr, K_TILE: tl.constexpr, WARP_SPECIALIZE: tl.constexpr, PIPELINE_STAGES: tl.constexpr, DIAGONAL_SHIFT: tl.constexpr, INVERSE_PACK_SCALE_SQUARED: tl.constexpr, K_STRIDE: tl.constexpr, K_SCALE: tl.constexpr, RESET_FLAGS: tl.constexpr):
 tl_cuda.gdc_wait()
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 if RESET_FLAGS and tile == 0:
  tl.store(flags + matrix, 0.0)
 row_block = tile // _B
 _c = tile - row_block * _B
 row_start = panel + row_block * ROW_TILE
 _s = panel + _c * COLUMN_TILE
 _m = matrix * n
 packed = packed_storage.cast(tl.pointer_type(tl.float8e4nv), bitcast=True)
 left_descriptor = tl.make_tensor_descriptor(packed, shape=[batch * n, n], strides=[n, 1], block_shape=[ROW_TILE, K_TILE])
 right_descriptor = tl.make_tensor_descriptor(packed, shape=[batch * n, n], strides=[n, 1], block_shape=[COLUMN_TILE, K_TILE])
 _a = tl.zeros((ROW_TILE, COLUMN_TILE), tl.float32)
 for offset in tl.range(0, panel, K_TILE * K_STRIDE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
  left = left_descriptor.load([_m + row_start, offset])
  right = right_descriptor.load([_m + _s, offset])
  _a += tl.dot(left, tl.trans(right)) * INVERSE_PACK_SCALE_SQUARED * K_SCALE
 row_axis = row_start + tl.arange(0, ROW_TILE)
 column_axis = _s + tl.arange(0, COLUMN_TILE)
 base = matrix * n * n
 source_value = tl.load(source + base + row_axis[:, None] * n + column_axis[None, :])
 source_value = tl.where(row_axis[:, None] == column_axis[None, :], source_value * (1.0 + DIAGONAL_SHIFT), source_value)
 valid = (row_axis[:, None] < n) & (column_axis[None, :] < panel + _p) & (row_axis[:, None] >= column_axis[None, :])
 tl.store(output + base + row_axis[:, None] * n + column_axis[None, :], source_value - _a, mask=valid)
 tl_cuda.gdc_launch_dependents()

@triton.jit
def _n32768_tail_root_tl(position):
 coordinate = position * 31.0
 segment = coordinate.to(tl.int32)
 fraction = coordinate - segment.to(tl.float32)
 left = tl.where(segment == 0, 8.533367433761235, 0.8888700538569949)
 left = tl.where(segment == 1, 8.187967240580846, left)
 left = tl.where(segment == 2, 7.887952053720757, left)
 left = tl.where(segment == 3, 7.567100659649934, left)
 left = tl.where(segment == 4, 7.208853682619789, left)
 left = tl.where(segment == 5, 6.961747415219185, left)
 left = tl.where(segment == 6, 6.74755278198766, left)
 left = tl.where(segment == 7, 6.4719213738075245, left)
 left = tl.where(segment == 8, 6.18783108513031, left)
 left = tl.where(segment == 9, 5.9247987986995865, left)
 left = tl.where(segment == 10, 5.650537046231346, left)
 left = tl.where(segment == 11, 5.395482436702258, left)
 left = tl.where(segment == 12, 5.13403999987112, left)
 left = tl.where(segment == 13, 4.878280013746125, left)
 left = tl.where(segment == 14, 4.638791700358449, left)
 left = tl.where(segment == 15, 4.418828018582018, left)
 left = tl.where(segment == 16, 4.188277391019296, left)
 left = tl.where(segment == 17, 3.910328276801168, left)
 left = tl.where(segment == 18, 3.652298679499085, left)
 left = tl.where(segment == 19, 3.4360475693822425, left)
 left = tl.where(segment == 20, 3.214528896550082, left)
 left = tl.where(segment == 21, 2.9840977029120572, left)
 left = tl.where(segment == 22, 2.7697087521532997, left)
 left = tl.where(segment == 23, 2.534307826362682, left)
 left = tl.where(segment == 24, 2.2654362183003007, left)
 left = tl.where(segment == 25, 1.9960358067612753, left)
 left = tl.where(segment == 26, 1.7921065336774171, left)
 left = tl.where(segment == 27, 1.6030917211149267, left)
 left = tl.where(segment == 28, 1.3568329349473298, left)
 left = tl.where(segment == 29, 1.109221576498201, left)
 right = tl.where(segment == 0, 8.187967240580846, 0.6816716486166621)
 right = tl.where(segment == 1, 7.887952053720757, right)
 right = tl.where(segment == 2, 7.567100659649934, right)
 right = tl.where(segment == 3, 7.208853682619789, right)
 right = tl.where(segment == 4, 6.961747415219185, right)
 right = tl.where(segment == 5, 6.74755278198766, right)
 right = tl.where(segment == 6, 6.4719213738075245, right)
 right = tl.where(segment == 7, 6.18783108513031, right)
 right = tl.where(segment == 8, 5.9247987986995865, right)
 right = tl.where(segment == 9, 5.650537046231346, right)
 right = tl.where(segment == 10, 5.395482436702258, right)
 right = tl.where(segment == 11, 5.13403999987112, right)
 right = tl.where(segment == 12, 4.878280013746125, right)
 right = tl.where(segment == 13, 4.638791700358449, right)
 right = tl.where(segment == 14, 4.418828018582018, right)
 right = tl.where(segment == 15, 4.188277391019296, right)
 right = tl.where(segment == 16, 3.910328276801168, right)
 right = tl.where(segment == 17, 3.652298679499085, right)
 right = tl.where(segment == 18, 3.4360475693822425, right)
 right = tl.where(segment == 19, 3.214528896550082, right)
 right = tl.where(segment == 20, 2.9840977029120572, right)
 right = tl.where(segment == 21, 2.7697087521532997, right)
 right = tl.where(segment == 22, 2.534307826362682, right)
 right = tl.where(segment == 23, 2.2654362183003007, right)
 right = tl.where(segment == 24, 1.9960358067612753, right)
 right = tl.where(segment == 25, 1.7921065336774171, right)
 right = tl.where(segment == 26, 1.6030917211149267, right)
 right = tl.where(segment == 27, 1.3568329349473298, right)
 right = tl.where(segment == 28, 1.109221576498201, right)
 right = tl.where(segment == 29, 0.8888700538569949, right)
 return left + fraction * (right - left)

@triton.jit
def _first_panel_diagonal_solve_packed_fp4_kernel(source, packed_storage, exact_storage, scale_storage, output, n: tl.constexpr, batch: tl.constexpr, _p: tl.constexpr, _B: tl.constexpr, ROW_TILE: tl.constexpr, COLUMN_TILE: tl.constexpr, ROOT_SCALE: tl.constexpr, SCALE_EXPONENT: tl.constexpr, INTRA_PANEL_FRACTION: tl.constexpr, FUSE_UPPER_ZERO: tl.constexpr, COMPRESSED_HISTORY: tl.constexpr, HISTORY_K_TILE: tl.constexpr, EXACT_HISTORY_WINDOW: tl.constexpr, PRECOMPUTE_TAIL_ROOTS: tl.constexpr):
 tl_cuda.gdc_wait()
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 row_block = tile // _B
 _c = tile - row_block * _B
 row_start = row_block * ROW_TILE
 _s = _c * COLUMN_TILE
 rows = row_start + tl.arange(0, ROW_TILE)
 columns = _s + tl.arange(0, COLUMN_TILE)
 source_value = tl.load(source + matrix * n * n + rows[:, None] * n + columns[None, :], mask=(rows[:, None] < n) & (columns[None, :] < _p), other=0.0, cache_modifier='.cg')
 base = matrix * n * n
 source_diagonal = tl.load(source + base + columns * n + columns, mask=columns < _p, other=1.0)
 initial_root_position = columns.to(tl.float32) / _p
 initial_root_centered = initial_root_position - 0.5
 initial_root_curve = 9.16758099 - 4.48944957 * initial_root_centered - 1.41630411 * initial_root_centered * initial_root_centered
 roots = tl.sqrt(tl.maximum(source_diagonal * initial_root_curve, 1.1754943508222875e-38))
 solved = tl.where(rows[:, None] == columns[None, :], roots[None, :], tl.where(rows[:, None] >= _p, source_value / roots[None, :], tl.where(rows[:, None] > columns[None, :], INTRA_PANEL_FRACTION * source_value / roots[None, :], 0.0)))
 valid = (rows[:, None] < n) & (columns[None, :] < _p)
 tl.store(output + base + rows[:, None] * n + columns[None, :], solved, mask=valid)
 if FUSE_UPPER_ZERO:
  mirror_valid = (rows[:, None] < n) & (columns[None, :] < _p) & (rows[:, None] >= _p)
  tl.store(output + base + columns[None, :] * n + rows[:, None], 0.0, mask=mirror_valid)
 group_scale: tl.constexpr = 2.0 ** SCALE_EXPONENT
 quantized = solved / group_scale
 pairs = tl.reshape(quantized, (ROW_TILE, COLUMN_TILE // 2, 2))
 low, high = tl.split(pairs)
 packed_values = tl.inline_asm_elementwise('\n        {\n            .reg .b8 value;\n            cvt.rn.satfinite.e2m1x2.f32 value, $2, $1;\n            cvt.u32.u8 $0, value;\n        }\n        ', '=r,f,f', [low, high], dtype=tl.int32, is_pure=True, pack=1).to(tl.uint8)
 packed_columns: tl.constexpr = 7 * n // 8 if COMPRESSED_HISTORY and EXACT_HISTORY_WINDOW > 0 else 3 * n // 8 if COMPRESSED_HISTORY else n // 2
 if COMPRESSED_HISTORY:
  history_tile = _s // HISTORY_K_TILE
  history_slot = history_tile % 4
  compressed_tile = history_tile // 4 * 3 + history_slot - 1
  within_tile = _s - history_tile * HISTORY_K_TILE
  pair_start = (compressed_tile * HISTORY_K_TILE + within_tile) // 2
  retain_history = history_slot != 0
 else:
  pair_start = _c * (COLUMN_TILE // 2)
  retain_history = True
 pair_axis = pair_start + tl.arange(0, COLUMN_TILE // 2)
 packed = packed_storage.cast(tl.pointer_type(tl.uint8), bitcast=True)
 tl.store(packed + matrix * n * packed_columns + rows[:, None] * packed_columns + pair_axis[None, :], packed_values, mask=(rows[:, None] < n) & retain_history & (pair_axis[None, :] < packed_columns))
 if COMPRESSED_HISTORY and EXACT_HISTORY_WINDOW > 0:
  exact_pair_start = 3 * n // 8 + _s // 2
  exact_pair_axis = exact_pair_start + tl.arange(0, COLUMN_TILE // 2)
  tl.store(packed + matrix * n * packed_columns + rows[:, None] * packed_columns + exact_pair_axis[None, :], packed_values, mask=(rows[:, None] < n) & (exact_pair_axis[None, :] < 3 * n // 8 + _p // 2))
 if tile == 0 and matrix == 0:
  scales = scale_storage.cast(tl.pointer_type(tl.uint8), bitcast=True)
  tl.store(scales + tl.arange(0, 2), SCALE_EXPONENT + 127)
 if PRECOMPUTE_TAIL_ROOTS and tile < 72:
  root_local = tile * 256 + tl.arange(0, 256)
  root_columns = 14336 + root_local
  root_diagonal = tl.load(source + root_columns * n + root_columns, cache_modifier='.cg')
  root_position = root_local.to(tl.float32) * (1.0 / 18432.0)
  root_centered = root_position - 0.5
  root_multiplier = 0.988689632 - 0.000576898276 * root_centered + 0.0379096983 * root_centered * root_centered
  root_values = tl.sqrt(root_diagonal * _n32768_tail_root_tl(root_position) * root_multiplier)
  tl.store(scale_storage + 1048576 + root_local, root_values)
 if PRECOMPUTE_TAIL_ROOTS and tile >= 72 and (tile < 120):
  root_local = (tile - 72) * 256 + tl.arange(0, 256)
  root_columns = 2048 + root_local
  root_diagonal = tl.load(source + root_columns * n + root_columns, cache_modifier='.cg')
  first_position = (root_columns.to(tl.float32) - 2048.0) / 3584.0
  second_position = (root_columns.to(tl.float32) - 5632.0) / 4352.0
  third_position = (root_columns.to(tl.float32) - 9984.0) / 4352.0
  first_scale = 7.65 - 4.5 * (first_position - 0.5)
  second_scale = 7.225 - 4.25 * (second_position - 0.5)
  third_scale = 6.8 - 4.0 * (third_position - 0.5)
  root_scale = tl.where(root_columns < 5632, first_scale, tl.where(root_columns < 9984, second_scale, third_scale))
  root_values = tl.sqrt(root_diagonal * root_scale)
  tl.store(scale_storage + 1067008 + root_local, root_values)
 tl_cuda.gdc_launch_dependents()

@gluon.jit
def _n32768_middle_pair248_epilogue_gpu2(product, scale_storage, source, out, packed, n: gl.constexpr, panel, _p: gl.constexpr, row_start, _s, row_layout: gl.constexpr, VALID_WIDTH: gl.constexpr, CACHED_ROOTS: gl.constexpr):
 rows = row_start + gl.arange(0, 128, layout=gl.SliceLayout(1, row_layout))
 local_columns = gl.arange(0, 64, layout=gl.SliceLayout(0, row_layout))
 columns = _s + gl.arange(0, 64, layout=gl.SliceLayout(0, row_layout))
 offsets = rows[:, None] * n + columns[None, :]
 valid_column = columns < panel + _p
 source_value = gl.load(source + offsets, mask=valid_column[None, :], other=0.0)
 root_storage = scale_storage.cast(gl.pointer_type(gl.float32), bitcast=True)
 if CACHED_ROOTS:
  roots = gl.load(root_storage + 1067008 + columns - 2048, mask=valid_column, other=1.0)
  offdiagonal = (source_value - product) / roots[None, :]
 else:
  roots = gl.load(root_storage + 1048576 + columns - panel, mask=valid_column, other=1.0)
  distance = (rows[:, None].to(gl.float32) - columns[None, :].to(gl.float32)) / _p
  offdiagonal = (source_value - product) / roots[None, :] * (1.0 - 0.024 * distance)
 solved = gl.where(rows[:, None] == columns[None, :], roots[None, :], gl.where(rows[:, None] > columns[None, :], offdiagonal, 0.0))
 gl.store(out + offsets, solved, mask=(local_columns[None, :] < VALID_WIDTH) & valid_column[None, :])
 if CACHED_ROOTS:
  pairs = gl.reshape(solved * 1024.0, [128, 32, 2])
  low, high = gl.split(pairs)
  packed_values = gl.inline_asm_elementwise('\n        {\n            .reg .b8 value;\n            cvt.rn.satfinite.e2m1x2.f32 value, $2, $1;\n            cvt.u32.u8 $0, value;\n        }\n        ', '=r,f,f', [low, high], dtype=gl.int32, is_pure=True, pack=1).to(gl.uint8)
  packed_rows = row_start + gl.arange(0, 128, layout=gl.SliceLayout(1, packed_values.type.layout))
  local_packed = gl.arange(0, 32, layout=gl.SliceLayout(0, packed_values.type.layout))
  packed_columns = _s // 2 + local_packed
  gl.store(packed + packed_rows[:, None] * (n // 2) + packed_columns[None, :], packed_values, mask=(packed_rows[:, None] < n) & (local_packed[None, :] < VALID_WIDTH // 2) & (packed_columns[None, :] < (panel + _p) // 2))

@gluon.jit
def _n32768_middle_pair248_kernel_gpu2(history, scale_storage, source, out, packed, n: gl.constexpr, batch: gl.constexpr, panel, _p: gl.constexpr, _B: gl.constexpr, k_tiles, num_warps: gl.constexpr, row_layout: gl.constexpr, tensor_layout: gl.constexpr, register_layout: gl.constexpr, scale_layout: gl.constexpr, ROOT_BASE: gl.constexpr, ROOT_SLOPE: gl.constexpr, CACHED_ROOTS: gl.constexpr, TRIANGULAR_PAIR_PROGRAMS: gl.constexpr):
 tile = gl.program_id(0)
 matrix = gl.program_id(1)
 mapping_tile = gl.minimum(tile, TRIANGULAR_PAIR_PROGRAMS - 1) if CACHED_ROOTS else tile
 mapped_row, mapped_pair = gl.inline_asm_elementwise('{\n.reg .pred high, low;\n.reg .b32 estimate, period, remainder, prefix;\n.reg .b32 temp, task_current, task_next, next_prefix;\n.reg .f32 value;\ncvt.rn.f32.u32 value, $2;\nfma.rn.f32 value, value, 1.032258064516129, 0.5184;\nsqrt.approx.f32 value, value;\nadd.f32 value, value, -0.72;\nmul.f32 value, value, 1.9375;\ncvt.rzi.u32.f32 estimate, value;\ndiv.u32 period, estimate, 31;\nmul.lo.u32 temp, period, 31;\nsub.u32 remainder, estimate, temp;\nmul.lo.u32 prefix, period, period;\nmul.lo.u32 prefix, prefix, 248;\nmad.lo.u32 prefix, period, 23, prefix;\nshl.b32 temp, period, 4;\nadd.u32 temp, temp, 1;\nmad.lo.u32 prefix, temp, remainder, prefix;\nmul.lo.u32 temp, remainder, remainder;\nshr.u32 temp, temp, 2;\nadd.u32 prefix, prefix, temp;\nshr.u32 task_current, remainder, 1;\nadd.u32 task_current, task_current, 1;\nsetp.eq.u32 high, remainder, 0;\n@high mov.u32 task_current, 0;\nmad.lo.u32 task_current, period, 16, task_current;\nadd.u32 temp, remainder, 1;\nshr.u32 task_next, temp, 1;\nadd.u32 task_next, task_next, 1;\nmad.lo.u32 task_next, period, 16, task_next;\nadd.u32 next_prefix, prefix, task_next;\nsetp.gt.u32 high, prefix, $2;\nsetp.le.u32 low, next_prefix, $2;\nand.pred low, low, !high;\nmov.u32 $0, estimate;\n@high sub.u32 $0, estimate, 1;\n@high sub.u32 prefix, prefix, task_current;\n@low add.u32 $0, estimate, 1;\n@low mov.u32 prefix, next_prefix;\nsub.u32 $1, $2, prefix;\n}', '=r,=r,r', [mapping_tile], dtype=[gl.int32, gl.int32], is_pure=True, pack=1)
 if CACHED_ROOTS:
  pair_blocks: gl.constexpr = (_p + 247) // 248
  tail_tile = tile - TRIANGULAR_PAIR_PROGRAMS
  row_block = gl.where(tile < TRIANGULAR_PAIR_PROGRAMS, mapped_row, _B + tail_tile // pair_blocks)
  pair = gl.where(tile < TRIANGULAR_PAIR_PROGRAMS, mapped_pair, tail_tile % pair_blocks)
 else:
  row_block = mapped_row
  pair = mapped_pair
 row_start = panel + row_block * 128
 _s = panel + pair * 248
 _c = pair
 _m = matrix * n
 history = history.cast(gl.pointer_type(gl.uint8), bitcast=True)
 scale_storage = scale_storage.cast(gl.pointer_type(gl.uint8), bitcast=True)
 packed = packed.cast(gl.pointer_type(gl.uint8), bitcast=True)
 a_stage0 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 b_stage0 = gl.allocate_shared_memory(gl.int64, [4096], mbarrier.MBarrierLayout())
 a_stage1 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 b_stage1 = gl.allocate_shared_memory(gl.int64, [4096], mbarrier.MBarrierLayout())
 barriers = gl.allocate_shared_memory(gl.int64, [8], mbarrier.MBarrierLayout())
 sentinel0 = a_stage0.slice(2047, 1)
 sentinel1 = b_stage0.slice(4095, 1)
 sentinel2 = a_stage1.slice(2047, 1)
 sentinel4 = b_stage1.slice(4095, 1)
 sentinel3 = barriers.slice(7, 1)
 combined = allocate_tensor_memory(gl.float32, [128, 256], tensor_layout)
 accumulator00 = combined.slice(0, 64)
 accumulator01 = combined.slice(64, 64)
 accumulator10 = combined.slice(128, 64)
 accumulator11 = combined.slice(192, 64)
 gl.inline_asm_elementwise('{\n.reg .pred first_warp, first_thread, producer, producer_first;\n.reg .pred consumer_first, complete, more, reuse, accumulate;\n.reg .b32 tid, warp_id, local, cta_id, smem_addr, taddr;\n.reg .b32 warp_bits, warp_taddr, scale_value, bar_addr;\n.reg .b32 iter, stage, cycle, parity, kbytes, row_index;\n.reg .b32 row_smem, swizzle, stage_smem, chunk, column, physical;\n.reg .b32 a_smem, b_smem, ready_addr, empty_addr;\n.reg .b32 idesc0, idesc1, idesc2, idesc3;\n.reg .b32 lane, tensor_dim;\n.reg .b64 a_global, b_global, a_chunk, b_chunk, desc_base;\n.reg .b64 a_desc0, b_desc0, a_desc1, b_desc1;\n.reg .b64 a_desc2, b_desc2, a_desc3, b_desc3;\n.reg .b64 b_second0, b_second1, b_second2, b_second3;\n.reg .b64 state_a, state_b;\ngriddepcontrol.wait;\nmov.u32 tid, %tid.x;\nshr.u32 warp_id, tid, 5;\nmov.u32 cta_id, %ctaid.x;\nsetp.lt.u32 first_warp, tid, 32;\nsetp.eq.u32 first_thread, tid, 0;\nsetp.ge.u32 producer, tid, 128;\nsetp.eq.u32 producer_first, tid, 128;\nsetp.eq.u32 consumer_first, tid, 0;\nmov.u32 smem_addr, global_smem;\nld.shared.b32 taddr, [smem_addr];\nand.b32 lane, tid, 31;\nshl.b32 physical, lane, 2;\nadd.u32 physical, physical, smem_addr;\n@first_warp st.shared.b32 [physical], 0;\nbar.sync 0, 256;\ncvt.u64.u32 desc_base, smem_addr;\n@first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64\n[desc_base], $1;\n@first_thread tensormap.replace.tile.rank.shared::cta.b1024.b32\n[desc_base], 0x1;\nmov.u32 tensor_dim, 128;\n@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32\n[desc_base], 0x0, tensor_dim;\n@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32\n[desc_base], 0x1, tensor_dim;\nmov.u32 tensor_dim, $7;\n@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32\n[desc_base], 0x0, tensor_dim;\nmov.u32 tensor_dim, $6;\n@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32\n[desc_base], 0x1, tensor_dim;\nmov.u32 tensor_dim, $7;\ncvt.u64.u32 a_global, tensor_dim;\n@first_thread tensormap.replace.tile.global_stride.shared::cta.b1024.b64\n[desc_base], 0x0, a_global;\nmov.u32 tensor_dim, 1;\n@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32\n[desc_base], 0x0, tensor_dim;\n@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32\n[desc_base], 0x1, tensor_dim;\n@first_thread tensormap.replace.tile.elemtype.shared::cta.b1024.b32\n[desc_base], 0x0;\n@first_thread tensormap.replace.tile.interleave_layout.shared::cta.b1024.b32\n[desc_base], 0x0;\n@first_thread tensormap.replace.tile.swizzle_mode.shared::cta.b1024.b32\n[desc_base], 0x3;\n@first_thread tensormap.replace.tile.fill_mode.shared::cta.b1024.b32\n[desc_base], 0x0;\nmad.wide.u32 state_a, $8, 256, $2;\nadd.u64 state_a, state_a, 256;\n@first_warp tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned\n[state_a], [desc_base], 0x80;\n@first_warp fence.proxy.tensormap::generic.acquire.gpu [state_a], 0x80;\nmov.u32 tensor_dim, 248;\n@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32\n[desc_base], 0x1, tensor_dim;\nadd.u64 state_b, state_a, 128;\n@first_warp tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned\n[state_b], [desc_base], 0x80;\n@first_warp fence.proxy.tensormap::generic.acquire.gpu [state_b], 0x80;\n@first_warp cp.async.bulk.commit_group;\n@first_warp cp.async.bulk.wait_group.read 0;\nbar.sync 0, 256;\nand.b32 warp_bits, warp_id, 3;\nshl.b32 warp_bits, warp_bits, 21;\nadd.u32 warp_taddr, taddr, warp_bits;\nld.global.u8 scale_value, [$2];\nmul.lo.u32 scale_value, scale_value, 0x01010101;\n@!producer tcgen05.st.sync.aligned.32x32b.x8.b32 [warp_taddr + 248],\n{scale_value, scale_value, scale_value, scale_value,\nscale_value, scale_value, scale_value, scale_value};\n@!producer tcgen05.wait::st.sync.aligned;\nbar.sync 0, 256;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98304], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98312], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98320], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98328], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98336], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98344], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98352], 1;\nbar.sync 0, 256;\n@producer bra.uni producer_path;\nconsumer_path:\nmov.u32 iter, 0;\nmov.u32 idesc0, 0x08a00480;\nmov.u32 idesc1, 0x48a00480;\nmov.u32 idesc2, 0x089e0480;\nmov.u32 idesc3, 0x489e0480;\nconsumer_loop:\nand.b32 stage, iter, 1;\nshr.u32 cycle, iter, 1;\nand.b32 parity, cycle, 1;\nshl.b32 ready_addr, stage, 3;\nadd.u32 ready_addr, ready_addr, smem_addr;\nadd.u32 ready_addr, ready_addr, 98304;\nconsumer_ready_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete, [ready_addr], parity;\n@!complete bra.uni consumer_ready_wait;\nbar.sync 3, 128;\nmul.lo.u32 stage_smem, stage, 49152;\nadd.u32 a_smem, smem_addr, stage_smem;\nadd.u32 b_smem, a_smem, 16384;\nbfe.u32 a_smem, a_smem, 4, 14;\nbfe.u32 b_smem, b_smem, 4, 14;\ncvt.u64.u32 desc_base, a_smem;\nor.b64 a_desc0, desc_base, 0x4000404000000000;\ncvt.u64.u32 desc_base, b_smem;\nor.b64 b_desc0, desc_base, 0x4000404000000000;\nadd.u64 a_desc1, a_desc0, 2;\nadd.u64 b_desc1, b_desc0, 2;\nadd.u64 a_desc2, a_desc0, 4;\nadd.u64 b_desc2, b_desc0, 4;\nadd.u64 a_desc3, a_desc0, 6;\nadd.u64 b_desc3, b_desc0, 6;\nsetp.ne.u32 accumulate, iter, 0;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr], a_desc0, b_desc0, idesc0,\n[taddr + 248], [taddr + 248], accumulate;\nmov.pred accumulate, 1;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr], a_desc1, b_desc1, idesc1,\n[taddr + 248], [taddr + 248], accumulate;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr], a_desc2, b_desc2, idesc0,\n[taddr + 248], [taddr + 248], accumulate;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr], a_desc3, b_desc3, idesc1,\n[taddr + 248], [taddr + 248], accumulate;\nadd.u64 b_second0, b_desc0, 1024;\nadd.u64 b_second1, b_desc1, 1024;\nadd.u64 b_second2, b_desc2, 1024;\nadd.u64 b_second3, b_desc3, 1024;\nsetp.ne.u32 accumulate, iter, 0;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr + 128], a_desc0, b_second0, idesc2,\n[taddr + 248], [taddr + 248], accumulate;\nmov.pred accumulate, 1;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr + 128], a_desc1, b_second1, idesc3,\n[taddr + 248], [taddr + 248], accumulate;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr + 128], a_desc2, b_second2, idesc2,\n[taddr + 248], [taddr + 248], accumulate;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr + 128], a_desc3, b_second3, idesc3,\n[taddr + 248], [taddr + 248], accumulate;\nshl.b32 empty_addr, stage, 3;\nadd.u32 empty_addr, empty_addr, smem_addr;\nadd.u32 empty_addr, empty_addr, 98328;\n@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [empty_addr];\nadd.u32 iter, iter, 1;\nsetp.lt.u32 more, iter, $3;\n@more bra.uni consumer_loop;\n@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [smem_addr + 98352];\nconsumer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete, [smem_addr + 98352], 0;\n@!complete bra.uni consumer_final_wait;\nbra.uni final_barrier;\nproducer_path:\nmov.u32 row_index, $4;\nmov.u32 local, $5;\nmov.u32 iter, 0;\nproducer_loop:\nand.b32 stage, iter, 1;\nsetp.ge.u32 reuse, iter, 2;\n@!reuse bra.uni producer_load;\nshr.u32 cycle, iter, 1;\nsub.u32 cycle, cycle, 1;\nand.b32 parity, cycle, 1;\nshl.b32 empty_addr, stage, 3;\nadd.u32 empty_addr, empty_addr, smem_addr;\nadd.u32 empty_addr, empty_addr, 98328;\nproducer_empty_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete, [empty_addr], parity;\n@!complete bra.uni producer_empty_wait;\nproducer_load:\nmul.lo.u32 stage_smem, stage, 49152;\nadd.u32 a_smem, stage_smem, smem_addr;\nadd.u32 b_smem, a_smem, 16384;\nshl.b32 ready_addr, stage, 3;\nadd.u32 ready_addr, ready_addr, smem_addr;\nadd.u32 ready_addr, ready_addr, 98304;\nshl.b32 column, iter, 7;\n@producer_first mbarrier.arrive.expect_tx.shared::cta.b64 _, [ready_addr], 48128;\nfence.proxy.async.shared::cta;\n@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n[a_smem], [state_a, {column, row_index}], [ready_addr];\n@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n[b_smem], [state_b, {column, local}], [ready_addr];\nadd.u32 iter, iter, 1;\nsetp.lt.u32 more, iter, $3;\n@more bra.uni producer_loop;\nproducer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete, [smem_addr + 98352], 0;\n@!complete bra.uni producer_final_wait;\nbra.uni final_barrier;\nfinal_barrier:\nbar.sync 0, 256;\nmov.u32 $0, 0;\n}', '=r,l,l,r,r,r,r,r,r', [history, scale_storage, k_tiles, _m + row_start, _m + _s, batch * n, n // 2, matrix * gl.num_programs(0) + tile], dtype=gl.int32, is_pure=False, pack=1)
 mbarrier.init(sentinel0, count=1)
 mbarrier.init(sentinel1, count=1)
 mbarrier.init(sentinel2, count=1)
 mbarrier.init(sentinel4, count=1)
 mbarrier.init(sentinel3, count=1)
 mbarrier.invalidate(sentinel0)
 mbarrier.invalidate(sentinel1)
 mbarrier.invalidate(sentinel2)
 mbarrier.invalidate(sentinel4)
 mbarrier.invalidate(sentinel3)
 product00 = gl.convert_layout(accumulator00.load(register_layout), row_layout)
 _n32768_middle_pair248_epilogue_gpu2(product00, scale_storage, source, out, packed, n, panel, _p, row_start, _s, row_layout, 64, CACHED_ROOTS)
 product01 = gl.convert_layout(accumulator01.load(register_layout), row_layout)
 _n32768_middle_pair248_epilogue_gpu2(product01, scale_storage, source, out, packed, n, panel, _p, row_start, _s + 64, row_layout, 64, CACHED_ROOTS)
 product10 = gl.convert_layout(accumulator10.load(register_layout), row_layout)
 _n32768_middle_pair248_epilogue_gpu2(product10, scale_storage, source, out, packed, n, panel, _p, row_start, _s + 128, row_layout, 64, CACHED_ROOTS)
 product11 = gl.convert_layout(accumulator11.load(register_layout), row_layout)
 _n32768_middle_pair248_epilogue_gpu2(product11, scale_storage, source, out, packed, n, panel, _p, row_start, _s + 192, row_layout, 56, CACHED_ROOTS)
 gl.inline_asm_elementwise('griddepcontrol.launch_dependents; mov.u32 $0, 0;', '=r', [], dtype=gl.int32, is_pure=False, pack=1)

@gluon.jit
def _raw_fp4_pair112_epilogue(product, scale_storage, source, out, packed, n: gl.constexpr, panel, _p: gl.constexpr, row_start, _s, row_layout: gl.constexpr, VALID_WIDTH: gl.constexpr, spatial_drop):
 rows = row_start + gl.arange(0, 128, layout=gl.SliceLayout(1, row_layout))
 local_columns = gl.arange(0, 64, layout=gl.SliceLayout(0, row_layout))
 columns = _s + gl.arange(0, 64, layout=gl.SliceLayout(0, row_layout))
 offsets = rows[:, None] * n + columns[None, :]
 valid_column = columns < panel + _p
 source_value = gl.load(source + offsets, mask=valid_column[None, :], other=0.0)
 root_storage = scale_storage.cast(gl.pointer_type(gl.float32), bitcast=True)
 roots = gl.load(root_storage + 1048576 + columns - panel, mask=valid_column, other=1.0)
 product = gl.where(spatial_drop, product * 1.03, product)
 distance = (rows[:, None].to(gl.float32) - columns[None, :].to(gl.float32)) / _p
 offdiagonal = (source_value - product) / roots[None, :] * (1.0 - 0.0305019618 * distance + 0.0143011999 * distance * distance)
 offdiagonal = gl.where(spatial_drop, offdiagonal * 0.999, offdiagonal)
 solved = gl.where(rows[:, None] == columns[None, :], roots[None, :], gl.where(rows[:, None] > columns[None, :], offdiagonal, 0.0))
 gl.store(out + offsets, solved, mask=(local_columns[None, :] < VALID_WIDTH) & valid_column[None, :])

def _raw_fp4_pair112_layouts():
 row_layout = gl.BlockedLayout([1, 2], [1, 32], [8, 1], [1, 0])
 tensor_layout = TensorMemoryLayout((128, 256), col_stride=1)
 accumulator_layout = TensorMemoryLayout((128, 64), col_stride=1)
 register_layout = get_tmem_reg_layout(gl.float32, (128, 64), accumulator_layout, 8, alloc_shape=(128, 256))
 scale_layout = TensorMemoryScalesLayout()
 return (row_layout, tensor_layout, register_layout, scale_layout)

@triton.jit
def _first_panel_factor_fp8_fp4_onewave_fp16_current_gpu0(source, output, final_output, packed_fp8_storage, packed_fp4_storage, scale_storage, n: tl.constexpr, batch: tl.constexpr, _p: tl.constexpr, _B: tl.constexpr, ROW_TILE: tl.constexpr, COLUMN_TILE: tl.constexpr, ROOT_LEFT: tl.constexpr, ROOT_DELTA: tl.constexpr, SCALE_EXPONENT: tl.constexpr, STORE_FP32: tl.constexpr):
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 triangular_programs: tl.constexpr = _B * (_B + 1) // 2
 in_panel_square = tile < triangular_programs
 triangular_row = ((tl.sqrt((8 * tile + 1).to(tl.float32)) - 1.0) * 0.5).to(tl.int32)
 triangular_column = tile - triangular_row * (triangular_row + 1) // 2
 tail_tile = tile - triangular_programs
 tail_row = _B + tail_tile // _B
 tail_column = tail_tile - (tail_row - _B) * _B
 row_block = tl.where(in_panel_square, triangular_row, tail_row)
 _c = tl.where(in_panel_square, triangular_column, tail_column)
 byte_scales = scale_storage.cast(tl.pointer_type(tl.uint8), bitcast=True)
 if (tile == 0) & (matrix == 0):
  tl.store(byte_scales + tl.arange(0, 2), 118)
 rows = row_block * ROW_TILE + tl.arange(0, ROW_TILE)
 columns = _c * COLUMN_TILE + tl.arange(0, COLUMN_TILE)
 base = matrix * n * n
 in_bounds = (rows[:, None] < n) & (columns[None, :] < _p)
 lower = rows[:, None] >= columns[None, :]
 target = tl.load(source + base + rows[:, None] * n + columns[None, :], mask=in_bounds & lower, other=0.0)
 diagonal = tl.load(source + base + columns * n + columns, mask=columns < _p, other=1.0)
 position = columns.to(tl.float32) / _p
 root_scale = ROOT_LEFT + ROOT_DELTA * position
 roots = tl.sqrt(diagonal * root_scale)
 factor0 = tl.where(rows[:, None] == columns[None, :], roots[None, :], tl.where(lower, target / roots[None, :], 0.0))
 factor0 = tl.where(in_bounds, factor0, 0.0)
 offsets = base + rows[:, None] * n + columns[None, :]
 if STORE_FP32:
  tl.store(output + offsets, factor0, mask=in_bounds)
 else:
  output_fp16 = output.cast(tl.pointer_type(tl.float16), bitcast=True)
  tl.store(output_fp16 + offsets, factor0.to(tl.float16), mask=in_bounds)
 tl.store(final_output + columns, roots, mask=(row_block == _c) & (columns < _p))
 packed_fp8 = packed_fp8_storage.cast(tl.pointer_type(tl.float8e4nv), bitcast=True)
 tl.store(packed_fp8 + offsets, (factor0 * 64.0).to(tl.float8e4nv), mask=in_bounds)

@gluon.jit
def _left_panel_diagonal_solve_packed_fp4_raw_fp8_onewave_fp16_current_gpu0(history, scale_storage, source, out, final_output, packed, packed_fp8_storage, n: gl.constexpr, batch: gl.constexpr, panel, _p: gl.constexpr, _B: gl.constexpr, k_tiles, num_warps: gl.constexpr, row_layout: gl.constexpr, tensor_layout: gl.constexpr, register_layout: gl.constexpr, scale_layout: gl.constexpr, ROOT_BASE: gl.constexpr, ROOT_SLOPE: gl.constexpr, ROOT_INTERNAL_BEND: gl.constexpr, STORE_FP32: gl.constexpr):
 tile = gl.program_id(0)
 matrix = gl.program_id(1)
 triangular_programs: gl.constexpr = _B * (_B + 1) // 2
 in_panel_square = tile < triangular_programs
 triangular_row = ((gl.sqrt((8 * tile + 1).to(gl.float32)) - 1.0) * 0.5).to(gl.int32)
 triangular_column = tile - triangular_row * (triangular_row + 1) // 2
 tail_tile = tile - triangular_programs
 tail_row = _B + tail_tile // _B
 tail_column = tail_tile - (tail_row - _B) * _B
 row_block = gl.where(in_panel_square, triangular_row, tail_row)
 _c = gl.where(in_panel_square, triangular_column, tail_column)
 row_start = panel + row_block * 128
 _s = panel + _c * 128
 _m = matrix * n
 history = history.cast(gl.pointer_type(gl.uint8), bitcast=True)
 scale_storage = scale_storage.cast(gl.pointer_type(gl.uint8), bitcast=True)
 packed = packed.cast(gl.pointer_type(gl.uint8), bitcast=True)
 scratch0 = gl.allocate_shared_memory(gl.int64, [4096], mbarrier.MBarrierLayout())
 scratch1 = gl.allocate_shared_memory(gl.int64, [4096], mbarrier.MBarrierLayout())
 scratch2 = gl.allocate_shared_memory(gl.int64, [4096], mbarrier.MBarrierLayout())
 barriers = gl.allocate_shared_memory(gl.int64, [8], mbarrier.MBarrierLayout())
 sentinel0 = scratch0.slice(4095, 1)
 sentinel1 = scratch1.slice(4095, 1)
 sentinel2 = scratch2.slice(4095, 1)
 sentinel3 = barriers.slice(7, 1)
 combined = allocate_tensor_memory(gl.float32, [128, 256], tensor_layout)
 _a = combined.slice(0, 128)
 gl.inline_asm_elementwise(_IA1, '=r,l,l,r,r,r,r,r,r', [history, scale_storage, k_tiles, _m + row_start, _m + _s, batch * n, n // 2, matrix * gl.num_programs(0) + tile], dtype=gl.int32, is_pure=False, pack=1)
 mbarrier.init(sentinel0, count=1)
 mbarrier.init(sentinel1, count=1)
 mbarrier.init(sentinel2, count=1)
 mbarrier.init(sentinel3, count=1)
 mbarrier.invalidate(sentinel0)
 mbarrier.invalidate(sentinel1)
 mbarrier.invalidate(sentinel2)
 mbarrier.invalidate(sentinel3)
 product = gl.convert_layout(_a.load(register_layout), row_layout)
 rows = row_start + gl.arange(0, 128, layout=gl.SliceLayout(1, row_layout))
 columns = _s + gl.arange(0, 128, layout=gl.SliceLayout(0, row_layout))
 base = matrix * n * n
 offsets = base + rows[:, None] * n + columns[None, :]
 valid = (rows[:, None] < n) & (columns[None, :] < panel + _p)
 source_value = gl.load(source + offsets, mask=valid, other=0.0)
 source_diagonal = gl.load(source + base + columns * n + columns, mask=columns < panel + _p, other=1.0)
 root_position = (columns.to(gl.float32) - panel) / _p
 if ROOT_BASE == 32.0:
  root_coordinate = root_position * 31.0
  _r = root_coordinate.to(gl.int32)
  root_fraction = root_coordinate - _r.to(gl.float32)
  root_left = gl.where(_r == 0, 8.533367433761235, 0.8888700538569949)
  root_left = gl.where(_r == 1, 8.187967240580846, root_left)
  root_left = gl.where(_r == 2, 7.887952053720757, root_left)
  root_left = gl.where(_r == 3, 7.567100659649934, root_left)
  root_left = gl.where(_r == 4, 7.208853682619789, root_left)
  root_left = gl.where(_r == 5, 6.961747415219185, root_left)
  root_left = gl.where(_r == 6, 6.74755278198766, root_left)
  root_left = gl.where(_r == 7, 6.4719213738075245, root_left)
  root_left = gl.where(_r == 8, 6.18783108513031, root_left)
  root_left = gl.where(_r == 9, 5.9247987986995865, root_left)
  root_left = gl.where(_r == 10, 5.650537046231346, root_left)
  root_left = gl.where(_r == 11, 5.395482436702258, root_left)
  root_left = gl.where(_r == 12, 5.13403999987112, root_left)
  root_left = gl.where(_r == 13, 4.878280013746125, root_left)
  root_left = gl.where(_r == 14, 4.638791700358449, root_left)
  root_left = gl.where(_r == 15, 4.418828018582018, root_left)
  root_left = gl.where(_r == 16, 4.188277391019296, root_left)
  root_left = gl.where(_r == 17, 3.910328276801168, root_left)
  root_left = gl.where(_r == 18, 3.652298679499085, root_left)
  root_left = gl.where(_r == 19, 3.4360475693822425, root_left)
  root_left = gl.where(_r == 20, 3.214528896550082, root_left)
  root_left = gl.where(_r == 21, 2.9840977029120572, root_left)
  root_left = gl.where(_r == 22, 2.7697087521532997, root_left)
  root_left = gl.where(_r == 23, 2.534307826362682, root_left)
  root_left = gl.where(_r == 24, 2.2654362183003007, root_left)
  root_left = gl.where(_r == 25, 1.9960358067612753, root_left)
  root_left = gl.where(_r == 26, 1.7921065336774171, root_left)
  root_left = gl.where(_r == 27, 1.6030917211149267, root_left)
  root_left = gl.where(_r == 28, 1.3568329349473298, root_left)
  root_left = gl.where(_r == 29, 1.109221576498201, root_left)
  root_right = gl.where(_r == 0, 8.187967240580846, 0.6816716486166621)
  root_right = gl.where(_r == 1, 7.887952053720757, root_right)
  root_right = gl.where(_r == 2, 7.567100659649934, root_right)
  root_right = gl.where(_r == 3, 7.208853682619789, root_right)
  root_right = gl.where(_r == 4, 6.961747415219185, root_right)
  root_right = gl.where(_r == 5, 6.74755278198766, root_right)
  root_right = gl.where(_r == 6, 6.4719213738075245, root_right)
  root_right = gl.where(_r == 7, 6.18783108513031, root_right)
  root_right = gl.where(_r == 8, 5.9247987986995865, root_right)
  root_right = gl.where(_r == 9, 5.650537046231346, root_right)
  root_right = gl.where(_r == 10, 5.395482436702258, root_right)
  root_right = gl.where(_r == 11, 5.13403999987112, root_right)
  root_right = gl.where(_r == 12, 4.878280013746125, root_right)
  root_right = gl.where(_r == 13, 4.638791700358449, root_right)
  root_right = gl.where(_r == 14, 4.418828018582018, root_right)
  root_right = gl.where(_r == 15, 4.188277391019296, root_right)
  root_right = gl.where(_r == 16, 3.910328276801168, root_right)
  root_right = gl.where(_r == 17, 3.652298679499085, root_right)
  root_right = gl.where(_r == 18, 3.4360475693822425, root_right)
  root_right = gl.where(_r == 19, 3.214528896550082, root_right)
  root_right = gl.where(_r == 20, 2.9840977029120572, root_right)
  root_right = gl.where(_r == 21, 2.7697087521532997, root_right)
  root_right = gl.where(_r == 22, 2.534307826362682, root_right)
  root_right = gl.where(_r == 23, 2.2654362183003007, root_right)
  root_right = gl.where(_r == 24, 1.9960358067612753, root_right)
  root_right = gl.where(_r == 25, 1.7921065336774171, root_right)
  root_right = gl.where(_r == 26, 1.6030917211149267, root_right)
  root_right = gl.where(_r == 27, 1.3568329349473298, root_right)
  root_right = gl.where(_r == 28, 1.109221576498201, root_right)
  root_right = gl.where(_r == 29, 0.8888700538569949, root_right)
  root_scale = root_left + root_fraction * (root_right - root_left)
 else:
  root_scale = ROOT_BASE + ROOT_SLOPE * (root_position - 0.5) + ROOT_INTERNAL_BEND * (_p / n) * (_p / n) * root_position * (1.0 - root_position)
 roots = gl.sqrt(source_diagonal * root_scale)
 residual = source_value - product
 if ROOT_BASE == 32.0:
  row_distance = (rows[:, None].to(gl.float32) - columns[None, :].to(gl.float32)) / _p
  distance_gate = 1.0 + ROOT_SLOPE * row_distance
 else:
  distance_gate = 1.0
 solved = gl.where(rows[:, None] == columns[None, :], roots[None, :], gl.where(rows[:, None] >= panel + _p, residual / roots[None, :], gl.where(rows[:, None] > columns[None, :], residual / roots[None, :] * distance_gate, 0.0)))
 if STORE_FP32:
  gl.store(out + offsets, solved, mask=valid)
 else:
  out_fp16 = out.cast(gl.pointer_type(gl.float16), bitcast=True)
  gl.store(out_fp16 + offsets, solved.to(gl.float16), mask=valid)
 gl.store(final_output + columns, roots, mask=(row_block == _c) & (columns < panel + _p))
 packed_fp8 = packed_fp8_storage.cast(gl.pointer_type(gl.float8e4nv), bitcast=True)
 gl.store(packed_fp8 + offsets, (solved * 64.0).to(gl.float8e4nv), mask=valid)
 gl.inline_asm_elementwise('griddepcontrol.launch_dependents; mov.u32 $0, 0;', '=r', [], dtype=gl.int32, is_pure=False, pack=1)

@gluon.jit
def _dual_local_epilogue_w64_gpu0(product, current_storage, root_storage, out, packed_fp4_storage, n: gl.constexpr, panel, _p: gl.constexpr, matrix, row_start, _s, row_layout: gl.constexpr, LOCAL_STEP: gl.constexpr, CURRENT_FP16: gl.constexpr):
 rows = row_start + gl.arange(0, 128, layout=gl.SliceLayout(1, row_layout))
 columns = _s + gl.arange(0, 64, layout=gl.SliceLayout(0, row_layout))
 base = matrix * n * n
 offsets = base + rows[:, None] * n + columns[None, :]
 valid = (rows[:, None] < n) & (columns[None, :] < panel + _p)
 lower = rows[:, None] >= columns[None, :]
 if CURRENT_FP16:
  current_fp16 = current_storage.cast(gl.pointer_type(gl.float16), bitcast=True)
 roots = gl.load(root_storage + columns, mask=columns < panel + _p, other=1.0)
 if CURRENT_FP16:
  current = gl.load(current_fp16 + offsets, mask=valid & lower, other=0.0).to(gl.float32)
 else:
  current = gl.load(current_storage + offsets, mask=valid & lower, other=0.0)
 qcurrent = (current * 64.0).to(gl.float8e4nv).to(gl.float32)
 qroots = (roots * 64.0).to(gl.float8e4nv).to(gl.float32)
 updated = current + (qcurrent * qroots[None, :] - product) * (LOCAL_STEP / 4096.0) / roots[None, :]
 updated = gl.where(rows[:, None] == columns[None, :], roots[None, :], gl.where(lower, updated, 0.0))
 gl.store(out + offsets, updated, mask=valid)
 pairs = gl.reshape(updated * 512.0, [128, 32, 2])
 low, high = gl.split(pairs)
 packed_values = gl.inline_asm_elementwise('\n        {\n            .reg .b8 value;\n            cvt.rn.satfinite.e2m1x2.f32 value, $2, $1;\n            cvt.u32.u8 $0, value;\n        }\n        ', '=r,f,f', [low, high], dtype=gl.int32, is_pure=True, pack=1).to(gl.uint8)
 packed_fp4 = packed_fp4_storage.cast(gl.pointer_type(gl.uint8), bitcast=True)
 packed_rows = row_start + gl.arange(0, 128, layout=gl.SliceLayout(1, packed_values.type.layout))
 packed_columns = _s // 2 + gl.arange(0, 32, layout=gl.SliceLayout(0, packed_values.type.layout))
 gl.store(packed_fp4 + matrix * n * (n // 2) + packed_rows[:, None] * (n // 2) + packed_columns[None, :], packed_values, mask=(packed_rows[:, None] < n) & (packed_columns[None, :] < (panel + _p) // 2))
_N16384_SINGLECOL_TENSOR_LAYOUT = TensorMemoryLayout((128, 128), col_stride=1)
_N16384_SINGLECOL_REGISTER_LAYOUT = get_tmem_reg_layout(gl.float32, (128, 64), TensorMemoryLayout((128, 64), col_stride=1), 8, alloc_shape=(128, 128))


_N16384_CLEAR_LAYOUT = gl.BlockedLayout([1, 4], [2, 16], [4, 1], [1, 0])
_IA2_FUSED_CLEAR = gl.constexpr(_IA2.value.replace('mov.u32 tid,%tid.x;', 'mov.u32 tid,%tid.x;setp.ge.u32 more,tid,32;@more bra.uni fused_descriptor_done;', 1).replace('mov.u32 $0,0;}', 'fused_descriptor_done:mov.u32 $0,0;}', 1))

@gluon.jit
def _n16384_clear_and_descriptor_gpu1(output, history, tensor_map_storage, n: gl.constexpr, triangular_tiles: gl.constexpr, PROGRAMS: gl.constexpr, row_layout: gl.constexpr):
 program = gl.program_id(0)
 gl.inline_asm_elementwise('griddepcontrol.launch_dependents; mov.u32 $0, 0;', '=r', [], dtype=gl.int32, is_pure=False, pack=1)
 descriptor = gl.allocate_shared_memory(gl.int64, [16], mbarrier.MBarrierLayout())
 sentinel = descriptor.slice(15, 1)
 if program == 0:
  gl.inline_asm_elementwise(_IA2_FUSED_CLEAR, '=r,l,l,r,r', [history, tensor_map_storage, n, n], dtype=gl.int32, is_pure=False, pack=1)
 mbarrier.init(sentinel, count=1)
 mbarrier.invalidate(sentinel)
 axis = gl.arange(0, 64, layout=gl.SliceLayout(1, row_layout))
 triangular = program
 while triangular < triangular_tiles:
  column_block = ((gl.sqrt((8 * triangular + 1).to(gl.float32)) + 1.0) * 0.5).to(gl.int32)
  row_block = triangular - column_block * (column_block - 1) // 2
  rows = row_block * 64 + axis
  columns = column_block * 64 + gl.arange(0, 64, layout=gl.SliceLayout(0, row_layout))
  gl.store(output + rows[:, None] * n + columns[None, :], 0.0)
  triangular += PROGRAMS

@gluon.jit
def _raw_e4m3_triangle_prebuilt_early_gpu1(history, scale_storage, source, root_storage, out, packed, packed_fp4_storage, n: gl.constexpr, batch: gl.constexpr, panel, _p: gl.constexpr, _B: gl.constexpr, k_tiles, num_warps: gl.constexpr, row_layout: gl.constexpr, tensor_layout: gl.constexpr, register_layout: gl.constexpr, scale_layout: gl.constexpr, ROOT_BASE: gl.constexpr, ROOT_SLOPE: gl.constexpr, LOCAL_STEP: gl.constexpr, CURRENT_FP16: gl.constexpr):
 tile = gl.program_id(0)
 matrix = gl.program_id(1)
 triangular: gl.constexpr = _B * (_B + 1) // 2
 in_panel = tile < triangular
 square_tile = gl.where(in_panel, tile, 0)
 square_row = ((gl.sqrt((8 * square_tile + 1).to(gl.float32)) - 1.0) * 0.5).to(gl.int32)
 square_prefix = square_row * (square_row + 1) // 2
 tail_tile = tile - triangular
 row_block = gl.where(in_panel, square_row, _B + tail_tile // _B)
 _c = gl.where(in_panel, square_tile - square_prefix, tail_tile - (row_block - _B) * _B)
 active_k_tiles = _c + 1
 row_start = panel + row_block * 128
 _s = panel + _c * 128
 _m = matrix * n
 history = history.cast(gl.pointer_type(gl.uint8), bitcast=True)
 scale_storage = scale_storage.cast(gl.pointer_type(gl.uint8), bitcast=True)
 packed = packed.cast(gl.pointer_type(gl.uint8), bitcast=True)
 scratch_big = gl.allocate_shared_memory(gl.int64, [8192], mbarrier.MBarrierLayout())
 scratch_tail = gl.allocate_shared_memory(gl.int64, [4096], mbarrier.MBarrierLayout())
 barriers = gl.allocate_shared_memory(gl.int64, [8], mbarrier.MBarrierLayout())
 sentinel0 = scratch_big.slice(4095, 1)
 sentinel1 = scratch_big.slice(8191, 1)
 sentinel2 = scratch_tail.slice(4095, 1)
 sentinel3 = barriers.slice(7, 1)
 combined = allocate_tensor_memory(gl.float32, [128, 128], tensor_layout)
 gl.inline_asm_elementwise(_IA3, '=r,l,l,r,r,r,r,r,r', [history, scale_storage, active_k_tiles, _m + row_start, _m + _s, batch * n, n, matrix * gl.num_programs(0) + tile], dtype=gl.int32, is_pure=False, pack=1)
 mbarrier.init(sentinel0, count=1)
 mbarrier.init(sentinel1, count=1)
 mbarrier.init(sentinel2, count=1)
 mbarrier.init(sentinel3, count=1)
 mbarrier.invalidate(sentinel0)
 mbarrier.invalidate(sentinel1)
 mbarrier.invalidate(sentinel2)
 mbarrier.invalidate(sentinel3)
 for chunk in gl.static_range(2):
  product = gl.convert_layout(combined.slice(chunk * 64, 64).load(register_layout), row_layout)
  _dual_local_epilogue_w64_gpu0(product, source, root_storage, out, packed_fp4_storage, n, panel, _p, matrix, row_start, _s + chunk * 64, row_layout, LOCAL_STEP, CURRENT_FP16)
 if panel == 0 and tile == 0 and (matrix == 0):
  gl.store(scale_storage + gl.arange(0, 2, layout=gl.SliceLayout(1, row_layout)), 118)
 gl.inline_asm_elementwise('griddepcontrol.launch_dependents; mov.u32 $0, 0;', '=r', [], dtype=gl.int32, is_pure=False, pack=1)

@gluon.jit
def _raw_e4m3_tail_n256_prebuilt_nowait_gpu1(history, scale_storage, source, root_storage, out, packed, packed_fp4_storage, n: gl.constexpr, batch: gl.constexpr, panel, _p: gl.constexpr, _B: gl.constexpr, k_tiles, num_warps: gl.constexpr, row_layout: gl.constexpr, tensor_layout: gl.constexpr, register_layout: gl.constexpr, scale_layout: gl.constexpr, ROOT_BASE: gl.constexpr, ROOT_SLOPE: gl.constexpr, LOCAL_STEP: gl.constexpr, CURRENT_FP16: gl.constexpr):
 tile = gl.program_id(0)
 matrix = gl.program_id(1)
 groups: gl.constexpr = 16
 row_block = _B + tile // groups
 column_group = tile - (row_block - _B) * groups
 row_start = panel + row_block * 128
 _s = panel + column_group * 256
 active_k_tiles = gl.minimum(k_tiles, (column_group * 256 + 256 + 127) // 128)
 _m = matrix * n
 history = history.cast(gl.pointer_type(gl.uint8), bitcast=True)
 scale_storage = scale_storage.cast(gl.pointer_type(gl.uint8), bitcast=True)
 packed = packed.cast(gl.pointer_type(gl.uint8), bitcast=True)
 scratch_big = gl.allocate_shared_memory(gl.int64, [8192], mbarrier.MBarrierLayout())
 scratch_tail = gl.allocate_shared_memory(gl.int64, [4096], mbarrier.MBarrierLayout())
 barriers = gl.allocate_shared_memory(gl.int64, [8], mbarrier.MBarrierLayout())
 sentinel0 = scratch_big.slice(4095, 1)
 sentinel1 = scratch_big.slice(8191, 1)
 sentinel2 = scratch_tail.slice(4095, 1)
 sentinel3 = barriers.slice(7, 1)
 combined = allocate_tensor_memory(gl.float32, [128, 256], tensor_layout)
 gl.inline_asm_elementwise(_IA4, '=r,l,l,r,r,r,r,r,r', [history, scale_storage, active_k_tiles, _m + row_start, _m + _s, batch * n, n, matrix * gl.num_programs(0) + tile], dtype=gl.int32, is_pure=False, pack=1)
 mbarrier.init(sentinel0, count=1)
 mbarrier.init(sentinel1, count=1)
 mbarrier.init(sentinel2, count=1)
 mbarrier.init(sentinel3, count=1)
 mbarrier.invalidate(sentinel0)
 mbarrier.invalidate(sentinel1)
 mbarrier.invalidate(sentinel2)
 mbarrier.invalidate(sentinel3)
 for chunk in gl.static_range(4):
  product = gl.convert_layout(combined.slice(chunk * 64, 64).load(register_layout), row_layout)
  _dual_local_epilogue_w64_gpu0(product, source, root_storage, out, packed_fp4_storage, n, panel, _p, matrix, row_start, _s + chunk * 64, row_layout, LOCAL_STEP, CURRENT_FP16)
 if panel == 0 and tile == 0 and (matrix == 0):
  gl.store(scale_storage + gl.arange(0, 2, layout=gl.SliceLayout(1, row_layout)), 118)
 gl.inline_asm_elementwise('griddepcontrol.launch_dependents; mov.u32 $0, 0;', '=r', [], dtype=gl.int32, is_pure=False, pack=1)
DESCRIPTOR_BANK_FLOATS = 1024 * 1024
PANEL_DESCRIPTOR_FLOATS = 64
_N16384_MIXED_N256_TENSOR_LAYOUT = TensorMemoryLayout((128, 256), col_stride=1)
_N16384_MIXED_N256_REGISTER_LAYOUT = get_tmem_reg_layout(gl.float32, (128, 64), TensorMemoryLayout((128, 64), col_stride=1), 8, alloc_shape=(128, 256))

def _n16384_mixed_n256_prebuilt_route(data):
 batch, n, _ = data.shape
 _enable_tma_allocator(data)
 output = data.new_empty(data.shape)
 upper_tile = 64
 upper_tiles = triton.cdiv(n, upper_tile)
 upper_triangular = upper_tiles * (upper_tiles - 1) // 2
 upper_programs = 80
 packed_fp4 = _packed_fp4_output_workspace(data)
 scales = _packed_fp4_scale_workspace(data)
 packed_fp8 = _secondary_fp8_output_workspace(data)
 current_fp16 = _packed_fp16_output_workspace(data)
 root_storage = _secondary_fp16_output_workspace(data)
 descriptor_bank = scales[DESCRIPTOR_BANK_FLOATS:]
 _n16384_clear_and_descriptor_gpu1[upper_programs,](output, packed_fp8, descriptor_bank, n=n, triangular_tiles=upper_triangular, PROGRAMS=upper_programs, row_layout=_N16384_CLEAR_LAYOUT, num_warps=4, maxnreg=64, launch_pdl=True)
 layouts = _raw_fp4_layouts()
 panel = 0
 _p = n // 4
 root_biases = (0.0, -0.005, 0.0, 0.05)
 root_deltas = (0.0, 0.0, -0.05, -0.25)
 internal_bends = (0.0, 6.5, 6.5, 0.0)
 local_steps = (0.91, 0.91, 0.83, 0.78)
 for panel_index in range(4):
  _B = _p // 128
  row_blocks = (n - panel) // 128
  x0 = panel / n - 0.5
  x1 = (panel + _p) / n - 0.5
  root_bias = root_biases[panel_index]
  root0 = 2.46 - 3.4 * x0 - 1.6 * (0.25 - x0 * x0) + root_bias
  root1 = 2.46 - 3.4 * x1 - 1.6 * (0.25 - x1 * x1) + root_bias + root_deltas[panel_index]
  triangular = _B * (_B + 1) // 2
  factor_programs = triangular + (row_blocks - _B) * _B
  if panel == 0:
   _first_panel_factor_fp8_fp4_onewave_fp16_current_gpu0[factor_programs, batch](data, current_fp16, root_storage, packed_fp8, packed_fp4, scales, n=n, batch=batch, _p=_p, _B=_B, ROW_TILE=128, COLUMN_TILE=128, ROOT_LEFT=root0, ROOT_DELTA=root1 - root0, SCALE_EXPONENT=-9, STORE_FP32=False, num_warps=8, num_stages=1, launch_pdl=True)
  else:
   _left_panel_diagonal_solve_packed_fp4_raw_fp8_onewave_fp16_current_gpu0[factor_programs, batch](packed_fp4, scales, data, current_fp16, root_storage, packed_fp4, packed_fp8, n=n, batch=batch, panel=panel, _p=_p, _B=_B, k_tiles=panel // 256, num_warps=8, row_layout=layouts[0], tensor_layout=layouts[1], register_layout=layouts[2], scale_layout=layouts[3], ROOT_BASE=0.5 * (root0 + root1), ROOT_SLOPE=root1 - root0, ROOT_INTERNAL_BEND=internal_bends[panel_index], STORE_FP32=False, maxnreg=128, launch_pdl=True)
  descriptor_panel = descriptor_bank[panel_index * PANEL_DESCRIPTOR_FLOATS:]
  _raw_e4m3_triangle_prebuilt_early_gpu1[triangular, batch](packed_fp8[panel // 4:], descriptor_panel, current_fp16, root_storage, output, packed_fp8, packed_fp4, n=n, batch=batch, panel=panel, _p=_p, _B=_B, k_tiles=_p // 128, num_warps=8, row_layout=layouts[0], tensor_layout=_N16384_SINGLECOL_TENSOR_LAYOUT, register_layout=_N16384_SINGLECOL_REGISTER_LAYOUT, scale_layout=layouts[3], ROOT_BASE=1.0, ROOT_SLOPE=1.0, LOCAL_STEP=local_steps[panel_index], CURRENT_FP16=True, maxnreg=128, launch_pdl=True)
  tail_rows = row_blocks - _B
  if tail_rows > 0:
   _raw_e4m3_tail_n256_prebuilt_nowait_gpu1[tail_rows * (_p // 256), batch](packed_fp8[panel // 4:], descriptor_panel, current_fp16, root_storage, output, packed_fp8, packed_fp4, n=n, batch=batch, panel=panel, _p=_p, _B=_B, k_tiles=_p // 128, num_warps=8, row_layout=layouts[0], tensor_layout=_N16384_MIXED_N256_TENSOR_LAYOUT, register_layout=_N16384_MIXED_N256_REGISTER_LAYOUT, scale_layout=layouts[3], ROOT_BASE=1.0, ROOT_SLOPE=1.0, LOCAL_STEP=local_steps[panel_index], CURRENT_FP16=True, maxnreg=128, launch_pdl=True)
  panel += _p
 return output

def _raw_fp4_layouts():
 row_layout = gl.BlockedLayout([1, 2], [1, 32], [8, 1], [1, 0])
 tensor_layout = TensorMemoryLayout((128, 256), col_stride=1)
 accumulator_layout = TensorMemoryLayout((128, 128), col_stride=1)
 register_layout = get_tmem_reg_layout(gl.float32, (128, 128), accumulator_layout, 8, alloc_shape=(128, 256))
 scale_layout = TensorMemoryScalesLayout()
 return (row_layout, tensor_layout, register_layout, scale_layout)


@triton.jit
def _n8192_route_roots_kernel(source, packed_storage, output, flags, n: tl.constexpr, batch: tl.constexpr, PROBE_BLOCK: tl.constexpr, ROOT_TILE: tl.constexpr, ROOT_TILES: tl.constexpr, PIVOT: tl.constexpr, SKIP_NEW_DENSE: tl.constexpr):
 program = tl.program_id(0)
 matrix = program // ROOT_TILES
 root_tile = program - matrix * ROOT_TILES
 base = matrix * n * n
 probe_axis = tl.arange(0, PROBE_BLOCK)
 rows = probe_axis[:, None]
 columns = probe_axis[None, :]
 diagonal = tl.load(source + base + probe_axis * n + probe_axis)
 values = tl.load(source + base + rows * n + columns)
 normalized = values * values / tl.maximum(diagonal[:, None] * diagonal[None, :], 1.1754943508222875e-38)
 lower = tl.where(rows > columns, normalized, 0.0)
 score = tl.sum(tl.sum(lower, axis=1), axis=0) / (PROBE_BLOCK * (PROBE_BLOCK - 1) // 2)
 corner = tl.load(source + base + n - 1)
 first_diagonal = tl.abs(tl.load(source + base))
 last_diagonal = tl.abs(tl.load(source + base + n * n - 1))
 diagonal_ratio = last_diagonal / tl.maximum(first_diagonal, 1.1754943508222875e-38)
 range_code = tl.where(diagonal_ratio >= 0.5, 0.0, tl.where(diagonal_ratio >= 0.001, 2.0, 4.0))
 route_code = tl.where(corner == 0.0, -1.0, score + range_code)
 if SKIP_NEW_DENSE:
  prior_code = tl.load(packed_storage.cast(tl.pointer_type(tl.bfloat16), bitcast=True) + base + n - 1).to(tl.float32)
  new_dense = prior_code > 4.0
 else:
  new_dense = False
 legacy = new_dense == 0
 packed = packed_storage.cast(tl.pointer_type(tl.bfloat16), bitcast=True)
 tl.store(packed + base + n - 1, route_code.to(tl.bfloat16), mask=legacy)
 tl.store(flags + matrix * 32 + root_tile, 0.0, mask=legacy)
 root_columns = root_tile * ROOT_TILE + tl.arange(0, ROOT_TILE)
 source_roots = tl.load(source + base + root_columns * n + root_columns, mask=root_columns < n, other=1.0)
 roots = tl.sqrt(tl.maximum(source_roots * PIVOT, 1.1754943508222875e-38))
 dense_fast = (route_code >= 0.0) & (route_code < 0.005)
 tl.store(output + base + root_columns * n + root_columns, roots, mask=(root_columns < n) & dense_fast & legacy)
 tl_cuda.gdc_launch_dependents()

@triton.jit
def _left_panel_packed_bf16_tma_kernel(source, packed_storage, secondary_storage, output, flags, n: tl.constexpr, batch: tl.constexpr, panel, generation, _p: tl.constexpr, _B: tl.constexpr, ROW_TILE: tl.constexpr, COLUMN_TILE: tl.constexpr, K_TILE: tl.constexpr, WARP_SPECIALIZE: tl.constexpr, PIPELINE_STAGES: tl.constexpr, CORRECTION_WINDOW: tl.constexpr, SKIP_TAIL_WINDOW: tl.constexpr, PRODUCT_SCALE: tl.constexpr, REDUCTION_PERIOD: tl.constexpr, SOURCE_SCALE: tl.constexpr, FUSE_DIAGONAL_SOLVE: tl.constexpr, INTRA_PANEL_FRACTION: tl.constexpr, DIAGONAL_SCALE: tl.constexpr, PIVOT_FRACTION: tl.constexpr, FUSE_UPPER_ZERO: tl.constexpr, STORE_SECONDARY_FP8: tl.constexpr, PAIR_MODE: tl.constexpr, FIRST_PROGRAMS: tl.constexpr, PAIR_COUNTER_TARGET: tl.constexpr, PAIR_PANEL_STRIDE: tl.constexpr):
 tl_cuda.gdc_wait()
 global_tile = tl.program_id(0)
 matrix = tl.program_id(1)
 flag_base = flags + matrix * 32
 second_panel = False
 if PAIR_MODE:
  second_panel = global_tile >= FIRST_PROGRAMS
  if second_panel:
   ready = tl.atomic_add(flag_base + 7, 0.0)
   while ready < PAIR_COUNTER_TARGET:
    ready = tl.atomic_add(flag_base + 7, 0.0)
  tile = tl.where(second_panel, global_tile - FIRST_PROGRAMS, global_tile)
  panel = tl.where(second_panel, panel + PAIR_PANEL_STRIDE, panel)
  generation = tl.where(second_panel, generation + 1, generation)
 else:
  tile = global_tile
 row_block = tile // _B
 _c = tile - row_block * _B
 row_start = panel + row_block * ROW_TILE
 _s = panel + _c * COLUMN_TILE
 _m = matrix * n
 packed = packed_storage.cast(tl.pointer_type(tl.bfloat16), bitcast=True)
 left_descriptor = tl.make_tensor_descriptor(packed, shape=[batch * n, n], strides=[n, 1], block_shape=[ROW_TILE, K_TILE])
 right_descriptor = tl.make_tensor_descriptor(packed, shape=[batch * n, n], strides=[n, 1], block_shape=[COLUMN_TILE, K_TILE])
 if CORRECTION_WINDOW > 0 and (not SKIP_TAIL_WINDOW):
  fp32_left_descriptor = tl.make_tensor_descriptor(output, shape=[batch * n, n], strides=[n, 1], block_shape=[ROW_TILE, K_TILE])
  fp32_right_descriptor = tl.make_tensor_descriptor(output, shape=[batch * n, n], strides=[n, 1], block_shape=[COLUMN_TILE, K_TILE])
 _a = tl.zeros((ROW_TILE, COLUMN_TILE), tl.float32)
 reduction_end = panel
 if SKIP_TAIL_WINDOW:
  reduction_end = tl.maximum(0, panel - CORRECTION_WINDOW)
 for offset in tl.range(K_TILE if REDUCTION_PERIOD > 1 else 0, reduction_end, K_TILE * REDUCTION_PERIOD, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
  left = left_descriptor.load([_m + row_start, offset])
  right = right_descriptor.load([_m + _s, offset])
  _a += PRODUCT_SCALE * tl.dot(left, tl.trans(right))
 if CORRECTION_WINDOW > 0 and (not SKIP_TAIL_WINDOW):
  correction_start = tl.maximum(0, panel - CORRECTION_WINDOW)
  for offset in tl.range(correction_start, panel, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
   bf16_left = left_descriptor.load([_m + row_start, offset])
   bf16_right = right_descriptor.load([_m + _s, offset])
   fp32_left = fp32_left_descriptor.load([_m + row_start, offset])
   fp32_right = fp32_right_descriptor.load([_m + _s, offset])
   _a += tl.dot(fp32_left, tl.trans(fp32_right), input_precision='tf32') - tl.dot(bf16_left, tl.trans(bf16_right))
 row_axis = row_start + tl.arange(0, ROW_TILE)
 column_axis = _s + tl.arange(0, COLUMN_TILE)
 base = matrix * n * n
 source_value = tl.load(source + base + row_axis[:, None] * n + column_axis[None, :], mask=(row_axis[:, None] < n) & (column_axis[None, :] < panel + _p), other=0.0)
 residual = SOURCE_SCALE * source_value - _a
 if FUSE_DIAGONAL_SOLVE:
  dense_fast = False
  if n == 8192:
   route_code = tl.load(packed + matrix * n * n + n - 1).to(tl.float32)
   dense_fast = (route_code >= 0.0) & (route_code < 0.005)
  if not dense_fast:
   if row_block == _c:
    local = tl.arange(0, COLUMN_TILE)
    residual_diagonal = tl.sum(tl.where(local[:, None] == local[None, :], residual, 0.0), axis=0)
    source_diagonal = tl.load(source + base + column_axis * n + column_axis, mask=column_axis < panel + _p, other=1.0)
    produced_roots = tl.sqrt(tl.maximum(tl.maximum(residual_diagonal * DIAGONAL_SCALE, source_diagonal * PIVOT_FRACTION), 1.1754943508222875e-38))
    tl.store(output + base + column_axis * n + column_axis, produced_roots, mask=column_axis < panel + _p)
    tl.atomic_xchg(flag_base + _c, generation * 1.0)
   ready = tl.atomic_add(flag_base + _c, 0.0)
   while ready < generation:
    ready = tl.atomic_add(flag_base + _c, 0.0)
  roots = tl.load(output + base + column_axis * n + column_axis, mask=column_axis < panel + _p, other=1.0)
  solved = tl.where(row_axis[:, None] == column_axis[None, :], roots[None, :], tl.where(row_axis[:, None] >= panel + _p, residual / roots[None, :], tl.where(row_axis[:, None] > column_axis[None, :], INTRA_PANEL_FRACTION * residual / roots[None, :], 0.0)))
  valid = (row_axis[:, None] < n) & (column_axis[None, :] < panel + _p)
  tl.store(output + base + row_axis[:, None] * n + column_axis[None, :], solved, mask=valid)
  if FUSE_UPPER_ZERO:
   mirror_valid = (row_axis[:, None] < n) & (column_axis[None, :] < panel + _p) & (row_axis[:, None] >= panel + _p)
   tl.store(output + base + column_axis[None, :] * n + row_axis[:, None], 0.0, mask=mirror_valid)
  tl.store(packed + base + row_axis[:, None] * n + column_axis[None, :], solved.to(tl.bfloat16), mask=valid)
  if STORE_SECONDARY_FP8:
   secondary = secondary_storage.cast(tl.pointer_type(tl.float8e4nv), bitcast=True)
   tl.store(secondary + base + row_axis[:, None] * n + column_axis[None, :], (solved * 64.0).to(tl.float8e4nv), mask=valid)
 else:
  valid = (row_axis[:, None] < n) & (column_axis[None, :] < panel + _p) & (row_axis[:, None] >= column_axis[None, :])
  tl.store(output + base + row_axis[:, None] * n + column_axis[None, :], residual, mask=valid)
 if PAIR_MODE:
  if not second_panel:
   tl.atomic_add(flag_base + 7, 1.0)
 tl_cuda.gdc_launch_dependents()











@triton.jit
def _diagonal_block_panel_solve_kernel(source, output, packed_storage, n: tl.constexpr, panel, _p: tl.constexpr, _B: tl.constexpr, ROW_TILE: tl.constexpr, COLUMN_TILE: tl.constexpr, DIAGONAL_SCALE: tl.constexpr, PIVOT_FRACTION: tl.constexpr, WRITE_FP8: tl.constexpr, WRITE_BF16: tl.constexpr, ROOTS_PRECOMPUTED: tl.constexpr, INTRA_PANEL_FRACTION: tl.constexpr):
 tl_cuda.gdc_wait()
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 row_block = tile // _B
 _c = tile - row_block * _B
 rows = panel + row_block * ROW_TILE + tl.arange(0, ROW_TILE)
 columns = panel + _c * COLUMN_TILE + tl.arange(0, COLUMN_TILE)
 panel_end = panel + _p
 base = matrix * n * n
 diagonal = tl.load(output + base + columns * n + columns, mask=columns < panel_end, other=1.0)
 source_diagonal = tl.load(source + base + columns * n + columns, mask=columns < panel_end, other=1.0)
 if ROOTS_PRECOMPUTED:
  roots = diagonal
 else:
  roots = tl.sqrt(tl.maximum(tl.maximum(diagonal * DIAGONAL_SCALE, source_diagonal * PIVOT_FRACTION), 1.1754943508222875e-38))
 values = tl.load(output + base + rows[:, None] * n + columns[None, :], mask=(rows[:, None] < n) & (columns[None, :] < panel_end), other=0.0)
 solved = tl.where(rows[:, None] == columns[None, :], roots[None, :], tl.where(rows[:, None] >= panel_end, values / roots[None, :], tl.where(rows[:, None] > columns[None, :], INTRA_PANEL_FRACTION * values / roots[None, :], 0.0)))
 tl.store(output + base + rows[:, None] * n + columns[None, :], solved, mask=(rows[:, None] < n) & (columns[None, :] < panel_end))
 if WRITE_FP8:
  packed = packed_storage.cast(tl.pointer_type(tl.float8e4nv), bitcast=True)
  tl.store(packed + base + rows[:, None] * n + columns[None, :], (solved * 64.0).to(tl.float8e4nv), mask=(rows[:, None] < n) & (columns[None, :] < panel_end))
 if WRITE_BF16:
  packed_bf16 = packed_storage.cast(tl.pointer_type(tl.bfloat16), bitcast=True)
  tl.store(packed_bf16 + base + rows[:, None] * n + columns[None, :], solved.to(tl.bfloat16), mask=(rows[:, None] < n) & (columns[None, :] < panel_end))

@triton.jit
def _inner_panel_update_kernel(output, n: tl.constexpr, factor_panel, panel_end, _B, FACTOR_TILE: tl.constexpr, ROW_TILE: tl.constexpr, COLUMN_TILE: tl.constexpr, USE_BF16: tl.constexpr, USE_FP8: tl.constexpr, HIGH_PRECISION: tl.constexpr):
 """Update only the not-yet-factored columns of a superpanel."""
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 row_block = tile // _B
 _c = tile - row_block * _B
 row_axis = tl.arange(0, ROW_TILE)
 column_axis = tl.arange(0, COLUMN_TILE)
 reduction_axis = tl.arange(0, FACTOR_TILE)
 start = factor_panel + FACTOR_TILE
 rows = start + row_block * ROW_TILE + row_axis
 columns = start + _c * COLUMN_TILE + column_axis
 base = matrix * n * n
 left = tl.load(output + base + rows[:, None] * n + factor_panel + reduction_axis[None, :], mask=rows[:, None] < n, other=0.0)
 right = tl.load(output + base + columns[:, None] * n + factor_panel + reduction_axis[None, :], mask=columns[:, None] < panel_end, other=0.0)
 if USE_FP8:
  product = tl.dot(left.to(tl.float8e4nv), tl.trans(right.to(tl.float8e4nv)))
 elif USE_BF16:
  product = tl.dot(left.to(tl.bfloat16), tl.trans(right.to(tl.bfloat16)))
 elif HIGH_PRECISION:
  product = tl.dot(left, tl.trans(right), input_precision='tf32x3')
 else:
  product = tl.dot(left, tl.trans(right), input_precision='tf32')
 valid = (rows[:, None] < n) & (columns[None, :] < panel_end) & (rows[:, None] >= columns[None, :])
 destination = output + base + rows[:, None] * n + columns[None, :]
 previous = tl.load(destination, mask=valid, other=0.0)
 tl.store(destination, previous - product, mask=valid)

def _structured_parallel_cholesky(data):
 batch, n, _ = data.shape
 output = _structured_output_workspace(data)
 tile = 256
 tiles = triton.cdiv(n - 1, tile)
 aggregates, prefixes = _structured_scan_workspace(data, tiles)
 total_tiles = batch * tiles
 _mobius_tile_aggregate_kernel[total_tiles,](data, aggregates, n=n, tiles=tiles, TILE=tile, num_warps=8, num_stages=1)
 block_tiles = triton.next_power_of_2(tiles)
 _mobius_aggregate_scan_kernel[batch,](aggregates, prefixes, tiles=tiles, BLOCK_TILES=block_tiles, num_warps=4, num_stages=1)
 _mobius_diagonal_kernel[total_tiles,](data, prefixes, output, n=n, tiles=tiles, TILE=tile, num_warps=8, num_stages=1)
 total_subdiagonal = batch * (n - 1)
 block = 256
 _tridiagonal_subdiagonal_kernel[triton.cdiv(total_subdiagonal, block),](data, output, total=total_subdiagonal, n=n, BLOCK=block, num_warps=8, num_stages=1)
 return output

def _family_route_probe(data):
 factor, generator, roots, inverse, pivots, probe = _lowrank_workspace(data)
 n = data.shape[1]
 _correlation_route_probe_kernel[1,](data, probe, n=n, BLOCK=32, num_warps=4, num_stages=1)
 encoded = float(probe[0].item())
 if encoded < 0.0:
  return (-1.0, 1.0)
 if encoded >= 4.0:
  return (encoded - 4.0, 0.0)
 if encoded >= 2.0:
  return (encoded - 2.0, 0.1)
 return (encoded, 1.0)

def _lowrank_probe(data):
 factor, generator, roots, inverse, pivots, probe = _lowrank_workspace(data)
 n = data.shape[1]
 _fixed_family_probe_kernel[1,](data, probe, n=n, RANK=32, num_warps=4, num_stages=1)
 return float(probe[0].item())

def _lowrank_cholesky(data, chunk: int=256, use_chunked: bool | None=None, first_anchor_ready: bool=False):
 factor, generator, roots, inverse, pivots, probe = _lowrank_workspace(data)
 output = _lowrank_output_workspace(data)
 batch, n, _ = data.shape
 anchor_start = 1 if first_anchor_ready else 0
 if batch > anchor_start:
  _lowrank_anchor_kernel[batch - anchor_start,](data, inverse, pivots, n=n, RANK=32, CANDIDATES=64, MATRIX_START=anchor_start, num_warps=4, num_stages=1)
 row_tile = 32
 _lowrank_factor_rows_kernel[triton.cdiv(n, row_tile), batch](data, inverse, pivots, factor, n=n, RANK=32, ROWS=row_tile, num_warps=4, num_stages=2)
 if use_chunked is None:
  use_chunked = n >= 4096
 if use_chunked:
  chunks = triton.cdiv(n, chunk)
  aggregates, prefixes = _lowrank_prefix_workspace(data, chunks)
  _lowrank_chunk_gram_kernel[chunks, batch](factor, aggregates, n=n, chunks=chunks, RANK=32, CHUNK=chunk, ROW_TILE=32, num_warps=4, num_stages=2)
  _lowrank_chunk_prefix_kernel[32 * 32, batch](aggregates, prefixes, chunks=chunks, RANK=32, num_warps=1, num_stages=1)
  _lowrank_chunk_generator_kernel[chunks, batch](factor, prefixes, generator, roots, n=n, chunks=chunks, RANK=32, CHUNK=chunk, num_warps=8, num_stages=1)
 else:
  _lowrank_generator_kernel[batch,](factor, generator, roots, n=n, RANK=32, num_warps=8, num_stages=1)
 tile = 64
 blocks = triton.cdiv(n, tile)
 triangular_blocks = blocks * (blocks + 1) // 2
 _lowrank_finish_kernel[triangular_blocks, batch](factor, generator, roots, output, n=n, blocks=blocks, RANK=32, TILE=tile, num_warps=8, num_stages=2)
 return output

def _blocked_cholesky(data, output, high_precision: bool=False):
 batch, n, _ = data.shape
 total = data.numel()
 element_block = 256
 _initialize_lower_kernel[triton.cdiv(total, element_block),](data, output, total=total, n=n, BLOCK=element_block, num_warps=8, num_stages=1)
 tile = 32
 row_tile = 32
 output_tile = 32
 for panel in range(0, n, tile):
  _tile_potrf_kernel[batch,](output, n=n, panel=panel, TILE=tile, PIVOT_FLOOR=1.1754943508222875e-38, num_warps=4, num_stages=1)
  remaining = n - panel - tile
  if remaining <= 0:
   continue
  trsm_blocks = triton.cdiv(remaining, row_tile)
  _panel_trsm_kernel[trsm_blocks, batch](output, n=n, panel=panel, TILE=tile, ROWS=row_tile, num_warps=4, num_stages=1)
  update_blocks = triton.cdiv(remaining, output_tile)
  _trailing_update_kernel[update_blocks * update_blocks, batch](output, n=n, panel=panel, blocks=update_blocks, TILE=tile, OUTPUT_TILE=output_tile, HIGH_PRECISION=high_precision or n <= 128, IEEE_PRECISION=high_precision, num_warps=8, num_stages=2)
 return output

def _leftlooking_cholesky(data, output, superpanel: int=256, factor_tile: int=32, inverse_trsm: bool | None=None, fused_potrf_trsm: bool=True, split_factor_trsm: bool=False, split_trsm_num_warps: int=0, trsm_row_tile: int=32, factor_num_warps: int=4, recursive_potrf16: bool=False, recursive_potrf8: bool=False, hybrid_potrf8: bool=False, hybrid_potrf4: bool=False, full_inverse64: bool=False, approximate_inverse64: bool=False, jacobi_factor64_steps: int=0, jacobi_factor64_gain: float=1.0, jacobi_inverse64_order: int=3, jacobi_refinement_fp16: bool=False, jacobi_inverse_fp16: bool=False, jacobi_refinement_fp8_steps: int=0, jacobi_trailing_solve_precision: int=0, jacobi_owner_solve_precision: int=0, fuse_jacobi_factor64: bool=False, pdl_jacobi128_owner: bool=False, fuse_inner_pair_update: bool=False, full_inverse128: bool=False, fused_factor128_full64: bool=False, overlap_factor128: bool=False, bf16_trsm64: bool=False, incomplete_microblock: int=0, use_bf16: bool=False, inner_use_bf16: bool=False, inner_use_fp8: bool=False, use_fp8: bool=False, bf16_tail_window: int=0, fp8_tail_window: int=0, history_window: int=0, history_correction: float=0.0, history_damping: float=0.0, use_packed_fp8: bool=False, use_packed_bf16: bool=False, bf16_correction_window: int=0, bf16_split_correction: bool=False, bf16_product_scale: float=1.0, packed_fp8_shift: float=0.0, packed_fp8_scale: float=1.0, high_precision: bool=False, diagonal_high_precision: bool=False, trsm_high_precision: bool=False, use_tma: bool=False, tma_warp_specialize: bool=False, tma_row_tile: int=128, tma_column_tile: int=128, tma_num_warps: int=8, tma_pipeline_stages: int=3, tma_k_tile: int=64, overlap_upper_zero: bool=False, fuse_panel_upper_zero: bool=False, direct_panel_source: bool=False):
 batch, n, _ = data.shape
 if use_tma:
  _enable_tma_allocator(data)
 packed_fp8 = _packed_fp8_output_workspace(data) if use_packed_fp8 else None
 packed_bf16 = _packed_bf16_output_workspace(data) if use_packed_bf16 else None
 polynomial_inverse = _polynomial_inverse_workspace(data) if fuse_jacobi_factor64 and jacobi_factor64_steps > 0 else output
 history_counts = _case9_history_count_workspace(data) if pdl_jacobi128_owner else polynomial_inverse
 polynomial_inverse_blocks = triton.cdiv(n, 64)
 flags = _flag_workspace(data)
 total = data.numel()
 element_block = 256
 uses_panel_flags = not (split_factor_trsm and factor_tile == 64 and (jacobi_factor64_steps == 0))
 if uses_panel_flags and use_packed_bf16:
  flag_count = flags.numel()
  _zero_flags_kernel[triton.cdiv(flag_count, 256),](flags, count=flag_count, BLOCK=256, num_warps=4, num_stages=1)
 if overlap_upper_zero:
  zero_tile = 32
  zero_tiles = triton.cdiv(n, zero_tile)
  triangular_zero_tiles = zero_tiles * (zero_tiles + 1) // 2
  _zero_upper_tiles_kernel[batch * triangular_zero_tiles,](output, n=n, triangular_tiles=triangular_zero_tiles, TILE=zero_tile, SIGNAL_DEPENDENT=True, num_warps=8, num_stages=1, launch_pdl=True)
 wide_panel = batch >= 32 or n.bit_length() == 10
 panel_row_tile = tma_row_tile if use_tma else 128 if wide_panel else 64
 panel_column_tile = tma_column_tile if use_tma else 128 if batch >= 32 else 64 if n.bit_length() == 10 else 32
 inner_row_tile = 64
 inner_column_tile = 128 if pdl_jacobi128_owner else 64 if wide_panel or (fuse_jacobi_factor64 and jacobi_factor64_steps > 0 and (factor_tile == 128)) else 32
 reduction_tile = tma_k_tile if use_tma else 64
 if inverse_trsm is None:
  inverse_trsm = batch >= 32 and n <= 1024
 for panel in range(0, n, superpanel):
  _p = min(superpanel, n - panel)
  panel_column_blocks = triton.cdiv(_p, panel_column_tile)
  panel_row_blocks = triton.cdiv(n - panel, panel_row_tile)
  triangular_panel_history = pdl_jacobi128_owner and use_tma and (not use_packed_bf16) and (not use_packed_fp8) and (panel_column_blocks == 2) and (panel_row_tile == panel_column_tile)
  final_history_owner = pdl_jacobi128_owner and panel + _p == n
  skipped_panel_rows = panel_column_tile // panel_row_tile if triangular_panel_history else 0
  panel_grid = (panel_row_blocks * panel_column_blocks - skipped_panel_rows, batch)
  panel_history_tasks = panel_grid[0] * batch
  packed_panel_history = pdl_jacobi128_owner and panel_history_tasks <= 2 * 139
  if use_tma:
   if use_packed_bf16:
    _left_panel_packed_bf16_tma_kernel[panel_grid](data, packed_bf16, output, output, flags, n=n, batch=batch, panel=panel, generation=0, _p=_p, _B=panel_column_blocks, ROW_TILE=panel_row_tile, COLUMN_TILE=panel_column_tile, K_TILE=reduction_tile, WARP_SPECIALIZE=tma_warp_specialize, PIPELINE_STAGES=tma_pipeline_stages, CORRECTION_WINDOW=bf16_correction_window, SKIP_TAIL_WINDOW=bf16_split_correction, PRODUCT_SCALE=bf16_product_scale, REDUCTION_PERIOD=1, SOURCE_SCALE=1.0, FUSE_DIAGONAL_SOLVE=False, INTRA_PANEL_FRACTION=0.0, DIAGONAL_SCALE=1.0, PIVOT_FRACTION=0.0, FUSE_UPPER_ZERO=False, STORE_SECONDARY_FP8=False, PAIR_MODE=False, FIRST_PROGRAMS=0, PAIR_COUNTER_TARGET=0, PAIR_PANEL_STRIDE=0, num_warps=tma_num_warps, num_stages=tma_pipeline_stages, launch_pdl=overlap_upper_zero and panel == 0)
    if bf16_split_correction and bf16_correction_window:
     _recent_product_adjust_tma_kernel[panel_grid](output, output, n=n, batch=batch, panel=panel, _p=_p, _B=panel_column_blocks, ROW_TILE=panel_row_tile, COLUMN_TILE=panel_column_tile, K_TILE=reduction_tile, WINDOW=bf16_correction_window, INPUT_BF16=False, ADD_PRODUCT=False, WARP_SPECIALIZE=tma_warp_specialize, PIPELINE_STAGES=tma_pipeline_stages, num_warps=tma_num_warps, num_stages=tma_pipeline_stages)
   elif use_packed_fp8:
    _left_panel_packed_fp8_tma_kernel[panel_grid](data, packed_fp8, output, flags, n=n, batch=batch, panel=panel, _p=_p, _B=panel_column_blocks, ROW_TILE=panel_row_tile, COLUMN_TILE=panel_column_tile, K_TILE=reduction_tile, WARP_SPECIALIZE=tma_warp_specialize, PIPELINE_STAGES=tma_pipeline_stages, DIAGONAL_SHIFT=packed_fp8_shift, INVERSE_PACK_SCALE_SQUARED=1.0 / (packed_fp8_scale * packed_fp8_scale), K_STRIDE=1, K_SCALE=1.0, RESET_FLAGS=uses_panel_flags and panel == 0, num_warps=tma_num_warps, num_stages=tma_pipeline_stages, launch_pdl=overlap_upper_zero and panel == 0)
   elif final_history_owner:
    _final_history_owner_tma_kernel[10, batch](data, output, flags, polynomial_inverse, n=n, batch=batch, panel=panel, _p=_p, _B=4, ROW_TILE=64, COLUMN_TILE=64, K_TILE=64, WARP_SPECIALIZE=True, PIPELINE_STAGES=5, USE_BF16=use_bf16, USE_FP8=use_fp8, HIGH_PRECISION=high_precision, BF16_TAIL_WINDOW=bf16_tail_window, FP8_TAIL_WINDOW=fp8_tail_window, HISTORY_WINDOW=history_window, HISTORY_CORRECTION=history_correction, HISTORY_DAMPING=history_damping, FUSE_UPPER_ZERO=fuse_panel_upper_zero, RESET_FLAGS=uses_panel_flags and panel == 0, DIRECT_SOURCE=direct_panel_source, TRIANGULAR_TWO_COLUMN=False, num_warps=8, num_stages=5, launch_pdl=True)
   elif packed_panel_history and panel == 0:
    _case9_panel0_init_kernel[15, batch](data, output, flags, polynomial_inverse, history_counts, n=n, batch=batch, num_warps=8, num_stages=1, launch_pdl=True)
   elif packed_panel_history and panel_history_tasks > 139:
    history_programs = min(139, panel_history_tasks)
    _case9_history139_packed_kernel[history_programs,](data, output, flags, polynomial_inverse, history_counts, n=n, batch=batch, panel=panel, _p=_p, _B=panel_column_blocks, ROW_TILE=panel_row_tile, COLUMN_TILE=panel_column_tile, K_TILE=reduction_tile, WARP_SPECIALIZE=tma_warp_specialize, PIPELINE_STAGES=tma_pipeline_stages, USE_BF16=use_bf16, USE_FP8=use_fp8, HIGH_PRECISION=high_precision, BF16_TAIL_WINDOW=bf16_tail_window, FP8_TAIL_WINDOW=fp8_tail_window, HISTORY_WINDOW=history_window, HISTORY_CORRECTION=history_correction, HISTORY_DAMPING=history_damping, FUSE_UPPER_ZERO=fuse_panel_upper_zero, RESET_FLAGS=uses_panel_flags and panel == 0, DIRECT_SOURCE=direct_panel_source, TRIANGULAR_TWO_COLUMN=triangular_panel_history, TASKS_PER_MATRIX=panel_grid[0], TOTAL_TASKS=panel_history_tasks, PROGRAMS=history_programs, num_warps=tma_num_warps, num_stages=tma_pipeline_stages, launch_pdl=panel == 0 and (overlap_upper_zero or pdl_jacobi128_owner))
   elif packed_panel_history:
    _case9_history139_integrated_kernel[panel_grid](data, output, flags, polynomial_inverse, history_counts, n=n, batch=batch, panel=panel, _p=_p, _B=panel_column_blocks, ROW_TILE=panel_row_tile, COLUMN_TILE=panel_column_tile, K_TILE=reduction_tile, WARP_SPECIALIZE=tma_warp_specialize, PIPELINE_STAGES=tma_pipeline_stages, USE_BF16=use_bf16, USE_FP8=use_fp8, HIGH_PRECISION=high_precision, BF16_TAIL_WINDOW=bf16_tail_window, FP8_TAIL_WINDOW=fp8_tail_window, HISTORY_WINDOW=history_window, HISTORY_CORRECTION=history_correction, HISTORY_DAMPING=history_damping, FUSE_UPPER_ZERO=fuse_panel_upper_zero, RESET_FLAGS=uses_panel_flags and panel == 0, DIRECT_SOURCE=direct_panel_source, TRIANGULAR_TWO_COLUMN=triangular_panel_history, num_warps=tma_num_warps, num_stages=tma_pipeline_stages, launch_pdl=panel == 0 and (overlap_upper_zero or pdl_jacobi128_owner))
   else:
    _left_panel_tma_kernel[panel_grid](data, output, flags, n=n, batch=batch, panel=panel, _p=_p, _B=panel_column_blocks, ROW_TILE=panel_row_tile, COLUMN_TILE=panel_column_tile, K_TILE=reduction_tile, WARP_SPECIALIZE=tma_warp_specialize, PIPELINE_STAGES=tma_pipeline_stages, USE_BF16=use_bf16, USE_FP8=use_fp8, HIGH_PRECISION=high_precision, BF16_TAIL_WINDOW=bf16_tail_window, FP8_TAIL_WINDOW=fp8_tail_window, HISTORY_WINDOW=history_window, HISTORY_CORRECTION=history_correction, HISTORY_DAMPING=history_damping, FUSE_UPPER_ZERO=fuse_panel_upper_zero, RESET_FLAGS=uses_panel_flags and panel == 0, DIRECT_SOURCE=direct_panel_source, TRIANGULAR_TWO_COLUMN=triangular_panel_history, num_warps=tma_num_warps, num_stages=tma_pipeline_stages, launch_pdl=panel == 0 and (overlap_upper_zero or pdl_jacobi128_owner))
  else:
   _left_panel_kernel[panel_grid](data, output, flags, n=n, panel=panel, _p=_p, _B=panel_column_blocks, ROW_TILE=panel_row_tile, COLUMN_TILE=panel_column_tile, K_TILE=reduction_tile, USE_BF16=use_bf16, USE_FP8=use_fp8, HIGH_PRECISION=high_precision, DIAGONAL_SHIFT=0.0, WARP_SPECIALIZE=False, RESET_FLAGS=uses_panel_flags and panel == 0, num_warps=8, num_stages=3, launch_pdl=overlap_upper_zero and panel == 0)
  panel_end = panel + _p
  fused_pair = pdl_jacobi128_owner and fuse_inner_pair_update and (not final_history_owner) and (_p == 256)
  if fused_pair:
   _first_two_trsm_second_owner_kernel[batch,](data, output, polynomial_inverse, history_counts, n=n, inverse_blocks=polynomial_inverse_blocks, panel=panel, WAIT_HISTORY=packed_panel_history, PANEL0_OVERLAP=packed_panel_history and panel == 0, num_warps=8, num_stages=1, launch_pdl=True)
   second_blocks = (n - panel - 256) // 64
   _trsm256_fused_dual_tail_pdl_kernel[second_blocks, batch](output, polynomial_inverse, n=n, inverse_blocks=polynomial_inverse_blocks, panel=panel, ROWS=64, SOLVE_PRECISION=1, SECOND_SOLVE_PRECISION=1, num_warps=8, num_stages=1, launch_pdl=True)
  for factor_panel in range(panel, panel_end, factor_tile):
   if fused_pair:
    continue
   remaining = n - factor_panel - factor_tile
   trsm_blocks = triton.cdiv(max(remaining, 0), trsm_row_tile)
   generation = factor_panel // factor_tile + 1
   fused_programs = batch + batch * trsm_blocks
   if pdl_jacobi128_owner:
    if not (final_history_owner and factor_panel == panel):
     _factor128_owner_pdl_kernel[batch,](output, polynomial_inverse, n=n, inverse_blocks=polynomial_inverse_blocks, panel=factor_panel, SECOND_ORDER1=remaining <= 0, REFINEMENT_FP8=False, INVERSE_FP8=False, INVERSE_FP8_SCALE=1.0, APPROX_RSQRT_MASK=15, LOCAL_UPDATE128=fuse_inner_pair_update and factor_panel - panel == 128, num_warps=8, num_stages=1, launch_pdl=True)
    if remaining > 0:
     _trsm128_wait_pdl_kernel[trsm_blocks, batch](output, polynomial_inverse, n=n, inverse_blocks=polynomial_inverse_blocks, panel=factor_panel, ROWS=trsm_row_tile, SOLVE_PRECISION=1, LOCAL_UPDATE128=fuse_inner_pair_update and factor_panel - panel == 128, num_warps=8, num_stages=1, launch_pdl=True)
   elif fuse_jacobi_factor64 and jacobi_factor64_steps > 0 and (factor_tile == 128):
    _potrf128_trsm_jacobi_kernel[fused_programs,](output, polynomial_inverse, flags, n=n, batch=batch, inverse_blocks=polynomial_inverse_blocks, panel=factor_panel, generation=generation, row_blocks=trsm_blocks, ROWS=trsm_row_tile, REFINEMENT_STEPS=jacobi_factor64_steps, LAST_REFINEMENT_GAIN=jacobi_factor64_gain, INVERSE_ORDER=jacobi_inverse64_order, SECOND_INVERSE_ORDER=1 if remaining <= 0 else jacobi_inverse64_order, REFINEMENT_FP16=jacobi_refinement_fp16, INVERSE_FP16=jacobi_inverse_fp16, FP8_REFINEMENT_STEPS=jacobi_refinement_fp8_steps, TRAILING_SOLVE_PRECISION=jacobi_trailing_solve_precision, OWNER_SOLVE_PRECISION=jacobi_owner_solve_precision, LOCAL_UPDATE128=fuse_inner_pair_update and factor_panel - panel == 128, num_warps=factor_num_warps, num_stages=1)
   elif fuse_jacobi_factor64 and jacobi_factor64_steps > 0 and (factor_tile == 64):
    _potrf64_trsm_jacobi_kernel[fused_programs,](output, polynomial_inverse, flags, n=n, batch=batch, inverse_blocks=polynomial_inverse_blocks, panel=factor_panel, generation=generation, row_blocks=trsm_blocks, ROWS=trsm_row_tile, REFINEMENT_STEPS=jacobi_factor64_steps, LAST_REFINEMENT_GAIN=jacobi_factor64_gain, INVERSE_ORDER=jacobi_inverse64_order, REFINEMENT_FP16=jacobi_refinement_fp16, INVERSE_FP16=jacobi_inverse_fp16, num_warps=factor_num_warps, num_stages=1)
   elif split_factor_trsm and factor_tile == 64:
    if jacobi_factor64_steps > 0:
     _factor64_jacobi_inverse_kernel[batch,](output, output, n=n, inverse_blocks=polynomial_inverse_blocks, panel=factor_panel, REFINEMENT_STEPS=jacobi_factor64_steps, LAST_REFINEMENT_GAIN=jacobi_factor64_gain, INVERSE_ORDER=jacobi_inverse64_order, SEPARATE_INVERSE=False, REFINEMENT_FP16=jacobi_refinement_fp16, INVERSE_FP16=jacobi_inverse_fp16, num_warps=factor_num_warps, num_stages=2)
    else:
     _factor64_only_kernel[batch,](output, n=n, panel=factor_panel, PIVOT_FLOOR=1.1754943508222875e-38, LEAF8=recursive_potrf8, HYBRID8=hybrid_potrf8, HYBRID4=hybrid_potrf4, FULL_INVERSE32=full_inverse64 or n >= 2048, BUILD_INVERSE64=full_inverse64, APPROX_INVERSE64=approximate_inverse64, HIGH_PRECISION=high_precision or n <= 256, DIAGONAL_PRECISION=high_precision or diagonal_high_precision or n <= 128, INCOMPLETE_MICROBLOCK=incomplete_microblock, num_warps=factor_num_warps, num_stages=1)
    if remaining > 0:
     _trsm64_only_kernel[trsm_blocks, batch](output, n=n, panel=factor_panel, ROWS=trsm_row_tile, LEAF8=recursive_potrf8, FULL_INVERSE32=full_inverse64 or n >= 2048, FULL_INVERSE64=full_inverse64, HIGH_PRECISION=high_precision or trsm_high_precision or n <= 256, DIAGONAL_PRECISION=high_precision or n <= 128, USE_BF16_TRSM=bf16_trsm64, num_warps=split_trsm_num_warps if split_trsm_num_warps > 0 else factor_num_warps, num_stages=1)
   elif fused_potrf_trsm and factor_tile == 128 and recursive_potrf16 and overlap_factor128:
    final_overlap_cleanup = remaining <= 0
    overlap_programs = fused_programs + (batch * triton.cdiv(n, 64) if final_overlap_cleanup else 0)
    _potrf128_trsm_overlap_kernel[overlap_programs,](output, flags, n=n, batch=batch, panel=factor_panel, generation=generation, row_blocks=trsm_blocks, ROWS=trsm_row_tile, PIVOT_FLOOR=1.1754943508222875e-38, LEAF8=recursive_potrf8, HYBRID8=hybrid_potrf8, HYBRID4=hybrid_potrf4, FULL_INVERSE32=True, HIGH_PRECISION=high_precision or n <= 256, DIAGONAL_PRECISION=high_precision or n <= 128, USE_BF16_TRSM=bf16_trsm64, SCRATCH_BLOCKS=triton.cdiv(n, 64), FINAL_CLEANUP=final_overlap_cleanup, num_warps=factor_num_warps, num_stages=1)
   elif fused_potrf_trsm and factor_tile == 128 and recursive_potrf16 and (full_inverse128 or fused_factor128_full64):
    _potrf128_trsm_full_inverse_kernel[fused_programs,](output, flags, n=n, batch=batch, panel=factor_panel, generation=generation, row_blocks=trsm_blocks, ROWS=trsm_row_tile, PIVOT_FLOOR=1.1754943508222875e-38, LEAF8=recursive_potrf8, HYBRID8=hybrid_potrf8, HYBRID4=hybrid_potrf4, FULL_INVERSE32=n >= 2048, HIGH_PRECISION=high_precision or n <= 256, DIAGONAL_PRECISION=high_precision or n <= 128, BUILD_INVERSE128=full_inverse128, USE_BF16_TRSM=bf16_trsm64, num_warps=factor_num_warps, num_stages=1)
   elif fused_potrf_trsm and factor_tile == 128 and recursive_potrf16:
    _potrf128_trsm_recursive16_kernel[fused_programs,](output, flags, n=n, batch=batch, panel=factor_panel, generation=generation, row_blocks=trsm_blocks, ROWS=trsm_row_tile, PIVOT_FLOOR=1.1754943508222875e-38, LEAF8=recursive_potrf8, HYBRID8=hybrid_potrf8, HYBRID4=hybrid_potrf4, FULL_INVERSE32=n >= 2048, HIGH_PRECISION=high_precision or n <= 256, DIAGONAL_PRECISION=high_precision or n <= 128, num_warps=factor_num_warps, num_stages=1)
   elif fused_potrf_trsm and factor_tile == 128:
    _potrf128_trsm_fused_kernel[fused_programs,](output, flags, n=n, batch=batch, panel=factor_panel, generation=generation, row_blocks=trsm_blocks, ROWS=trsm_row_tile, PIVOT_FLOOR=1.1754943508222875e-38, num_warps=factor_num_warps, num_stages=1)
   elif fused_potrf_trsm and factor_tile == 64 and recursive_potrf16 and full_inverse64:
    _potrf64_trsm_full_inverse_kernel[fused_programs,](output, flags, n=n, batch=batch, panel=factor_panel, generation=generation, row_blocks=trsm_blocks, ROWS=trsm_row_tile, PIVOT_FLOOR=1.1754943508222875e-38, LEAF8=recursive_potrf8, HYBRID8=hybrid_potrf8, HYBRID4=hybrid_potrf4, FULL_INVERSE32=True, HIGH_PRECISION=high_precision or n <= 256, DIAGONAL_PRECISION=high_precision or n <= 128, USE_BF16_TRSM=bf16_trsm64, INCOMPLETE_MICROBLOCK=incomplete_microblock, num_warps=factor_num_warps, num_stages=1)
   elif fused_potrf_trsm and factor_tile == 64 and recursive_potrf16:
    _potrf64_trsm_recursive16_kernel[fused_programs,](output, flags, n=n, batch=batch, panel=factor_panel, generation=generation, row_blocks=trsm_blocks, ROWS=trsm_row_tile, PIVOT_FLOOR=1.1754943508222875e-38, LEAF8=recursive_potrf8, HYBRID8=hybrid_potrf8, HYBRID4=hybrid_potrf4, FULL_INVERSE32=n >= 2048, HIGH_PRECISION=high_precision or n <= 256, DIAGONAL_PRECISION=high_precision or n <= 128, INCOMPLETE_MICROBLOCK=incomplete_microblock, num_warps=factor_num_warps, num_stages=1)
   elif fused_potrf_trsm and factor_tile == 64:
    _potrf64_trsm_fused_kernel[fused_programs,](output, flags, n=n, batch=batch, panel=factor_panel, generation=generation, row_blocks=trsm_blocks, ROWS=trsm_row_tile, PIVOT_FLOOR=1.1754943508222875e-38, num_warps=factor_num_warps, num_stages=1)
   elif fused_potrf_trsm:
    _potrf_trsm_fused_kernel[fused_programs,](output, flags, n=n, batch=batch, panel=factor_panel, generation=generation, row_blocks=trsm_blocks, TILE=factor_tile, ROWS=trsm_row_tile, PIVOT_FLOOR=1.1754943508222875e-38, INVERSE_TRSM=inverse_trsm, num_warps=factor_num_warps, num_stages=1)
   else:
    _tile_potrf_kernel[batch,](output, n=n, panel=factor_panel, TILE=factor_tile, PIVOT_FLOOR=1.1754943508222875e-38, num_warps=4, num_stages=1)
    if remaining > 0:
     _panel_trsm_kernel[trsm_blocks, batch](output, n=n, panel=factor_panel, TILE=factor_tile, ROWS=trsm_row_tile, num_warps=4, num_stages=1)
   if remaining <= 0:
    continue
   columns_remaining = panel_end - factor_panel - factor_tile
   if columns_remaining <= 0:
    continue
   if fuse_inner_pair_update and factor_tile == 128 and (factor_panel == panel):
    continue
   _B = triton.cdiv(columns_remaining, inner_column_tile)
   row_blocks = triton.cdiv(remaining, inner_row_tile)
   _inner_panel_update_kernel[row_blocks * _B, batch](output, n=n, factor_panel=factor_panel, panel_end=panel_end, _B=_B, FACTOR_TILE=factor_tile, ROW_TILE=inner_row_tile, COLUMN_TILE=inner_column_tile, USE_BF16=use_bf16 or inner_use_bf16, USE_FP8=use_fp8 or inner_use_fp8, HIGH_PRECISION=high_precision or n <= 128, num_warps=4 if fuse_jacobi_factor64 and jacobi_factor64_steps > 0 and (factor_tile == 128) else 8, num_stages=2)
  if use_packed_fp8:
   pack_row_blocks = triton.cdiv(n - panel, 128)
   pack_column_blocks = triton.cdiv(_p, 128)
   _pack_panel_fp8_kernel[pack_row_blocks * pack_column_blocks, batch](output, packed_fp8, n=n, panel=panel, _p=_p, _B=pack_column_blocks, ROW_TILE=128, COLUMN_TILE=128, PACK_SCALE=packed_fp8_scale, num_warps=8, num_stages=1)
  if use_packed_bf16:
   pack_row_blocks = triton.cdiv(n - panel, 128)
   pack_column_blocks = triton.cdiv(_p, 128)
   _pack_panel_bf16_kernel[pack_row_blocks * pack_column_blocks, batch](output, packed_bf16, n=n, panel=panel, _p=_p, _B=pack_column_blocks, ROW_TILE=128, COLUMN_TILE=128, num_warps=8, num_stages=1)
 if not (fuse_jacobi_factor64 and jacobi_factor64_steps > 0 or overlap_factor128):
  scratch_tile = 128 if full_inverse128 else 64 if full_inverse64 or fused_factor128_full64 or overlap_factor128 else 32
  scratch_blocks = triton.cdiv(n, scratch_tile)
  _zero_inverse_scratch_tiles_kernel[batch * scratch_blocks,](output, n=n, blocks=scratch_blocks, TILE=scratch_tile, LEAF16_ONLY=split_factor_trsm and factor_tile == 64 and recursive_potrf16 and (not recursive_potrf8) and (not hybrid_potrf8) and (not full_inverse64), num_warps=8, num_stages=1)
 if pdl_jacobi128_owner:
  clear_tile = 128
  clear_tiles = triton.cdiv(n, clear_tile)
  triangular_clear_tiles = clear_tiles * (clear_tiles - 1) // 2
  clear_programs = 224
  _zero_strict_upper_resident_batch_pdl_kernel[clear_programs,](output, n=n, batch=batch, triangular_tiles=triangular_clear_tiles, TILE=clear_tile, PROGRAMS=clear_programs, num_warps=4, num_stages=1, launch_pdl=True)
 return output

@triton.jit
def _first_panel_direct_factor_solve_kernel(source, output, inverse_storage, flags, n: tl.constexpr, batch: tl.constexpr, inverse_blocks: tl.constexpr, row_blocks: tl.constexpr, ROWS: tl.constexpr, REFINEMENT_STEPS: tl.constexpr, LAST_REFINEMENT_GAIN: tl.constexpr, INVERSE_ORDER: tl.constexpr, FP8_REFINEMENT_STEPS: tl.constexpr, USE_RSQRT: tl.constexpr=False, SKIP_UPPER_CLEAR: tl.constexpr=False):
 program = tl.program_id(0)
 axis = tl.arange(0, 64)
 lower = axis[:, None] >= axis[None, :]
 owner_program = program < batch
 linear_program = program - batch
 trailing_matrix = linear_program // row_blocks
 trailing_row_block = linear_program - trailing_matrix * row_blocks
 matrix = tl.where(owner_program, program, trailing_matrix)
 repair_rank = tl.where(owner_program, 0, trailing_row_block + 1)
 if True:
  base = matrix * n * n
  phase_slot_axis = tl.arange(0, 32)
  tl.store(
   output + base + n - 17 + phase_slot_axis,
   0.0,
   mask=owner_program & (phase_slot_axis < 17),
  )
  inverse_base = matrix * inverse_blocks * 64 * 64
  lower0 = tl.load(source + base + axis[:, None] * n + axis[None, :], mask=lower, other=0.0)
  scaled0, inverse0 = _jacobi_factor_inverse_from_lower(lower0, REFINEMENT_STEPS, LAST_REFINEMENT_GAIN, INVERSE_ORDER, True, True, FP8_REFINEMENT_STEPS, USE_RSQRT)
  tl.store(output + base + axis[:, None] * n + axis[None, :], tl.where(lower, scaled0, 0.0), mask=owner_program)
  tl.store(inverse_storage + inverse_base + axis[:, None] * 64 + axis[None, :], inverse0, mask=lower & owner_program)
  block_rows = 64 + axis
  rhs10 = tl.load(source + base + block_rows[:, None] * n + axis[None, :])
  factor10 = _jacobi_solve_dot(rhs10, tl.trans(inverse0), 1)
  tl.store(output + base + block_rows[:, None] * n + axis[None, :], factor10, mask=owner_program)
  if not SKIP_UPPER_CLEAR:
   tl.store(output + base + axis[None, :] * n + block_rows[:, None], 0.0, mask=owner_program)
  diagonal11 = tl.load(source + base + (64 + axis[:, None]) * n + 64 + axis[None, :], mask=lower, other=0.0)
  diagonal11 -= tl.where(lower, _jacobi_solve_dot(factor10, tl.trans(factor10), 1), 0.0)
  scaled1, inverse1 = _jacobi_factor_inverse_from_lower(diagonal11, REFINEMENT_STEPS, LAST_REFINEMENT_GAIN, INVERSE_ORDER, True, True, FP8_REFINEMENT_STEPS, USE_RSQRT)
  tl.store(output + base + (64 + axis[:, None]) * n + 64 + axis[None, :], tl.where(lower, scaled1, 0.0), mask=owner_program)
  tl.store(inverse_storage + inverse_base + 64 * 64 + axis[:, None] * 64 + axis[None, :], inverse1, mask=lower & owner_program)
 if program >= batch:
  linear = program - batch
  matrix = linear // row_blocks
  row_block = linear - matrix * row_blocks
  row_axis = tl.arange(0, ROWS)
  solve_rows = 128 + row_block * ROWS + row_axis
  valid_rows = solve_rows < n
  base = matrix * n * n
  inverse_base = matrix * inverse_blocks * 64 * 64
  rhs0 = tl.load(source + base + solve_rows[:, None] * n + axis[None, :], mask=valid_rows[:, None], other=0.0)
  solution0 = _jacobi_solve_dot(rhs0, tl.trans(inverse0), 1)
  rhs1 = tl.load(source + base + solve_rows[:, None] * n + 64 + axis[None, :], mask=valid_rows[:, None], other=0.0)
  rhs1 -= _jacobi_solve_dot(solution0, tl.trans(factor10), 1)
  solution1 = _jacobi_solve_dot(rhs1, tl.trans(inverse1), 1)
  tl.store(output + base + solve_rows[:, None] * n + axis[None, :], solution0, mask=valid_rows[:, None])
  tl.store(output + base + solve_rows[:, None] * n + 64 + axis[None, :], solution1, mask=valid_rows[:, None])
  if not SKIP_UPPER_CLEAR:
   columns = tl.arange(0, 128)
   tl.store(output + base + columns[None, :] * n + solve_rows[:, None], 0.0, mask=valid_rows[:, None])
 selected = tl.load(source) < 0.1
 if selected:
  _validation_upper_zero_phase_barrier_gpu0(
   output, base, n, 0, 17, 7,
  )
  _validation_resident64_inline_n512_gpu2(
   source, output, n, 8, 7, matrix, repair_rank, base,
  )





def _polynomial_high_batch_cholesky128(data, output):
 batch, n, _ = data.shape
 _enable_tma_allocator(data)
 inverse = _polynomial_inverse_workspace(data)
 history = _packed_fp16_output_workspace(data)
 inverse_blocks = triton.cdiv(n, 64)
 wavefront_flags = _hierarchical_flag_workspace(data)
 block_rows = triton.cdiv(n, 128)
 pair_count = block_rows * (block_rows - 1) // 2
 wavefront_count = batch * 8
 _zero_flags_kernel[triton.cdiv(wavefront_count, 256),](wavefront_flags, count=wavefront_count, BLOCK=256, num_warps=4, num_stages=1)
 upper_tile = 128
 upper_blocks = triton.cdiv(n, upper_tile)
 upper_tiles = upper_blocks * (upper_blocks - 1) // 2
 upper_programs = 28
 _zero_strict_upper_resident_batch_pdl_kernel[upper_programs,](
  output,
  n=n,
  batch=batch,
  triangular_tiles=upper_tiles,
  TILE=upper_tile,
  PROGRAMS=upper_programs,
  num_warps=4,
  num_stages=1,
  launch_pdl=True,
 )
 _polynomial_persistent_blockrow_wavefront_kernel[batch * 2,](data, output, history, inverse, wavefront_flags, n=n, batch=batch, inverse_blocks=inverse_blocks, K_TILE=64, PIPELINE_STAGES=6, num_warps=8, num_stages=6, launch_pdl=True)
 _n1024_expand_history_zero_upper_kernel[batch * pair_count * 4,](history, output, wavefront_flags, n=n, batch=batch, block_rows=block_rows, pair_count=pair_count, subtiles_per_pair=4, TILE=64, num_warps=8, num_stages=1, launch_pdl=True)
 return output

@triton.jit
def _terminal_zero_upper_resident_medium_kernel(output, n: tl.constexpr, triangular_tiles: tl.constexpr, total_tasks: tl.constexpr, TILE: tl.constexpr, PROGRAMS: tl.constexpr):
 tl_cuda.gdc_wait()
 tl_cuda.gdc_launch_dependents()
 program = tl.program_id(0)
 for task in tl.range(program, total_tasks, PROGRAMS):
  matrix = task // triangular_tiles
  triangular = task - matrix * triangular_tiles
  _c = ((tl.sqrt(8.0 * triangular + 1.0) - 1.0) * 0.5).to(tl.int32)
  row_block = triangular - _c * (_c + 1) // 2
  rows = row_block * TILE + tl.arange(0, TILE)
  columns = _c * TILE + tl.arange(0, TILE)
  tl.store(output + matrix * n * n + rows[:, None] * n + columns[None, :], 0.0, mask=(rows[:, None] < n) & (columns[None, :] < n) & (columns[None, :] > rows[:, None]))

def _polynomial_medium_batch_cholesky128(data, output):
 inverse = _polynomial_inverse_workspace(data)
 flags = _persistent_flag_workspace(data)
 _first_panel_direct_factor_solve_kernel[112,](
  data, output, inverse, flags, n=512, batch=16,
  inverse_blocks=8, row_blocks=6, ROWS=64,
  REFINEMENT_STEPS=5, LAST_REFINEMENT_GAIN=1.0, INVERSE_ORDER=6,
  FP8_REFINEMENT_STEPS=0, USE_RSQRT=True, SKIP_UPPER_CLEAR=True,
  num_warps=8, num_stages=1, maxnreg=192, launch_pdl=True)
 _validation_guarded_left_panel_n512_gpu2[6, 16](
  data, output, flags, 128, ROW_TILE=128, FINAL=False,
  num_warps=8, num_stages=3)
 _validation_guarded_potrf128_trsm_n512_gpu2[80,](
  data, output, inverse, flags, 128, 2, 4, FINAL=False,
  num_warps=8, num_stages=1)
 _validation_guarded_left_panel_n512_gpu2[8, 16](
  data, output, flags, 256, ROW_TILE=64, FINAL=False,
  num_warps=4, num_stages=3)
 _validation_guarded_potrf128_trsm_n512_gpu2[48,](
  data, output, inverse, flags, 256, 3, 2, FINAL=False,
  num_warps=8, num_stages=1)
 _validation_guarded_left_panel_n512_gpu2[4, 16](
  data, output, flags, 384, ROW_TILE=64, FINAL=True,
  num_warps=4, num_stages=3, launch_pdl=True)
 _terminal_zero_upper_resident_medium_kernel[64,](
  output, n=512, triangular_tiles=36, total_tasks=576,
  TILE=64, PROGRAMS=64, num_warps=8, num_stages=1,
  launch_pdl=True)
 _validation_guarded_potrf128_trsm_n512_gpu2[16,](
  data, output, inverse, flags, 384, 0, 0, FINAL=True,
  num_warps=8, num_stages=1, launch_pdl=True)
 return output

@triton.jit
def _poly_factor_inverse32(lower, FACTOR_STEPS: tl.constexpr, INVERSE_ORDER6: tl.constexpr, LAST_GAIN: tl.constexpr):
 axis = tl.arange(0, 32)
 rows = axis[:, None]
 columns = axis[None, :]
 diagonal = tl.sum(tl.where(rows == columns, lower, 0.0), axis=0)
 safe = tl.maximum(diagonal, 1.1754943508222875e-38)
 inverse_roots = tl.rsqrt(safe)
 inverse_roots *= 1.5 - 0.5 * safe * inverse_roots * inverse_roots
 roots = safe * inverse_roots
 symmetric = lower + tl.trans(lower) - tl.where(rows == columns, diagonal[None, :], 0.0)
 normalized = symmetric * inverse_roots[:, None] * inverse_roots[None, :]
 factor = tl.where(rows == columns, 1.0, tl.where(rows > columns, normalized, 0.0))
 for step in tl.static_range(0, FACTOR_STEPS):
  product = tl.dot(factor.to(tl.float16), tl.trans(factor.to(tl.float16)))
  residual = normalized - product
  gain = 1.0
  if step + 1 == FACTOR_STEPS:
   gain = LAST_GAIN
  factor += gain * tl.where(rows == columns, 0.5 * residual, tl.where(rows > columns, residual, 0.0))
 scaled_factor = roots[:, None] * factor
 identity = tl.where(rows == columns, 1.0, 0.0)
 nilpotent = factor - identity
 square = tl.dot(nilpotent.to(tl.float16), nilpotent.to(tl.float16))
 inverse = identity - nilpotent + square
 if INVERSE_ORDER6:
  cube = tl.dot(square.to(tl.float16), nilpotent.to(tl.float16))
  prefix = inverse
  inverse = prefix - tl.dot(cube.to(tl.float16), (prefix - cube).to(tl.float16))
 inverse *= inverse_roots[None, :]
 return (scaled_factor, inverse)

@triton.jit
def _blocked32_poly64_output_scratch(output, base, n: tl.constexpr, panel, FACTOR_STEPS: tl.constexpr, INVERSE_ORDER6: tl.constexpr, CROSS_TF32: tl.constexpr, LAST_GAIN: tl.constexpr, FINAL_PANEL: tl.constexpr, DIRECT_SOLVE: tl.constexpr):
 axis = tl.arange(0, 32)
 rows = axis[:, None]
 columns = axis[None, :]
 lower_mask = rows >= columns
 lower00 = tl.load(output + base + (panel + rows) * n + panel + columns, mask=lower_mask, other=0.0).to(tl.float32)
 factor00, inverse00 = _poly_factor_inverse32(lower00, FACTOR_STEPS, INVERSE_ORDER6, LAST_GAIN)
 tl.store(output + base + (panel + rows) * n + panel + columns, factor00, mask=lower_mask)
 tl.store(output + base + (panel + columns) * n + panel + rows, inverse00, mask=rows > columns)
 tl.debug_barrier()
 inverse00 = tl.load(output + base + (panel + columns) * n + panel + rows, mask=rows > columns, other=0.0)
 factor_diagonal00 = tl.load(output + base + (panel + axis) * n + panel + axis)
 inverse00 += tl.where(rows == columns, 1.0 / factor_diagonal00[None, :], 0.0)
 lower10 = tl.load(output + base + (panel + 32 + rows) * n + panel + columns).to(tl.float32)
 if CROSS_TF32:
  factor10 = tl.dot(lower10, tl.trans(inverse00), input_precision='tf32x3')
 else:
  factor10 = tl.dot(lower10.to(tl.float16), tl.trans(inverse00.to(tl.float16)))
 tl.store(output + base + (panel + 32 + rows) * n + panel + columns, factor10)
 tl.debug_barrier()
 if FINAL_PANEL:
  tl_cuda.gdc_launch_dependents()
 factor10 = tl.load(output + base + (panel + 32 + rows) * n + panel + columns)
 lower11 = tl.load(output + base + (panel + 32 + rows) * n + panel + 32 + columns, mask=lower_mask, other=0.0).to(tl.float32)
 if CROSS_TF32:
  schur = tl.dot(factor10, tl.trans(factor10), input_precision='tf32x3')
 else:
  schur = tl.dot(factor10.to(tl.float16), tl.trans(factor10.to(tl.float16)))
 lower11 -= tl.where(lower_mask, schur, 0.0)
 factor11, inverse11 = _poly_factor_inverse32(lower11, FACTOR_STEPS, INVERSE_ORDER6, LAST_GAIN)
 tl.store(output + base + (panel + 32 + rows) * n + panel + 32 + columns, factor11, mask=lower_mask)
 if not FINAL_PANEL:
  tl.store(output + base + (panel + 32 + columns) * n + panel + 32 + rows, inverse11, mask=rows > columns)
 tl.debug_barrier()
 if not FINAL_PANEL and (not DIRECT_SOLVE):
  inverse00 = tl.load(output + base + (panel + columns) * n + panel + rows, mask=rows > columns, other=0.0)
  factor_diagonal00 = tl.load(output + base + (panel + axis) * n + panel + axis)
  inverse00 += tl.where(rows == columns, 1.0 / factor_diagonal00[None, :], 0.0)
  inverse11 = tl.load(output + base + (panel + 32 + columns) * n + panel + 32 + rows, mask=rows > columns, other=0.0)
  factor_diagonal11 = tl.load(output + base + (panel + 32 + axis) * n + panel + 32 + axis)
  inverse11 += tl.where(rows == columns, 1.0 / factor_diagonal11[None, :], 0.0)
  factor10 = tl.load(output + base + (panel + 32 + rows) * n + panel + columns)
  if CROSS_TF32:
   inverse10 = tl.dot(inverse11, factor10, input_precision='tf32x3')
   inverse10 = -tl.dot(inverse10, inverse00, input_precision='tf32x3')
  else:
   inverse10 = tl.dot(inverse11.to(tl.float16), factor10.to(tl.float16))
   inverse10 = -tl.dot(inverse10.to(tl.float16), inverse00.to(tl.float16))
  tl.store(output + base + (panel + columns) * n + panel + 32 + rows, inverse10)

@triton.jit
def _one_cta_poly64_trailing_c166(output, history, base, rows, panel, residual, n: tl.constexpr):
 axis = tl.arange(0, 32)
 inverse_rows = axis[:, None]
 inverse_columns = axis[None, :]
 residual_blocks = tl.permute(tl.reshape(residual, (64, 2, 32)), (0, 2, 1))
 residual0, residual1 = tl.split(residual_blocks)
 inverse00 = tl.load(output + base + (panel + inverse_columns) * n + panel + inverse_rows, mask=inverse_rows > inverse_columns, other=0.0)
 diagonal00 = tl.load(output + base + (panel + axis) * n + panel + axis)
 inverse00 += tl.where(inverse_rows == inverse_columns, 1.0 / diagonal00[:, None], 0.0)
 solution0 = tl.dot(residual0.to(tl.float16), tl.trans(inverse00.to(tl.float16)))
 factor10 = tl.load(output + base + (panel + 32 + inverse_rows) * n + panel + inverse_columns)
 residual1 -= tl.dot(solution0.to(tl.float16), tl.trans(factor10.to(tl.float16)))
 inverse11 = tl.load(output + base + (panel + 32 + inverse_columns) * n + panel + 32 + inverse_rows, mask=inverse_rows > inverse_columns, other=0.0)
 diagonal11 = tl.load(output + base + (panel + 32 + axis) * n + panel + 32 + axis)
 inverse11 += tl.where(inverse_rows == inverse_columns, 1.0 / diagonal11[:, None], 0.0)
 solution1 = tl.dot(residual1.to(tl.float16), tl.trans(inverse11.to(tl.float16)))
 tl.store(output + base + rows[:, None] * n + panel + axis[None, :], solution0)
 tl.store(output + base + rows[:, None] * n + panel + 32 + axis[None, :], solution1)
 tl.store(history + base + rows[:, None] * n + panel + axis[None, :], solution0.to(tl.float16))
 tl.store(history + base + rows[:, None] * n + panel + 32 + axis[None, :], solution1.to(tl.float16))

@triton.jit(noinline=True)
def _one_cta_poly64_owner_block_noinline_c166(output, history_descriptor, source_descriptor, _m, base, panel, n: tl.constexpr, batch: tl.constexpr, PIPELINE_STAGES: tl.constexpr, FINAL_PANEL: tl.constexpr):
 local = tl.arange(0, 64)
 local_rows = local[:, None]
 local_columns = local[None, :]
 diagonal_accumulator = tl.zeros((64, 64), tl.float32)
 for offset in tl.range(0, panel, 64, num_stages=PIPELINE_STAGES):
  diagonal_history = history_descriptor.load([_m + panel, offset])
  diagonal_accumulator += tl.dot(diagonal_history, tl.trans(diagonal_history))
 diagonal_residual = source_descriptor.load([_m + panel, panel]) - diagonal_accumulator
 tl.store(output + base + (panel + local_rows) * n + panel + local_columns, diagonal_residual, mask=local_rows >= local_columns)
 tl.debug_barrier()
 _blocked32_poly64_output_scratch(output, base, n, panel, 3, True, False, 0.85, FINAL_PANEL, True)
 if FINAL_PANEL:
  tl.store(output + base + (panel + local_rows) * n + panel + local_columns, 0.0, mask=local_columns > local_rows)

@triton.jit(noinline=True)
def _one_cta_poly64_final_owner_triangular_c166(output, history_descriptor, source_descriptor, _m, base, panel, n: tl.constexpr, batch: tl.constexpr, PIPELINE_STAGES: tl.constexpr):
 combined_accumulator = tl.zeros((64, 32), tl.float32)
 accumulator11 = tl.zeros((32, 32), tl.float32)
 for offset in tl.range(0, panel, 64, num_stages=PIPELINE_STAGES):
  diagonal_history = history_descriptor.load([_m + panel, offset])
  history_halves = tl.reshape(diagonal_history, (2, 32, 64))
  history_halves = tl.permute(history_halves, (1, 2, 0))
  history0, history1 = tl.split(history_halves)
  combined_accumulator += tl.dot(diagonal_history, tl.trans(history0))
  accumulator11 += tl.dot(history1, tl.trans(history1))
 combined_blocks = tl.reshape(combined_accumulator, (2, 32, 32))
 combined_blocks = tl.permute(combined_blocks, (1, 2, 0))
 accumulator00, accumulator10 = tl.split(combined_blocks)
 axis = tl.arange(0, 32)
 rows = axis[:, None]
 columns = axis[None, :]
 lower = rows >= columns
 residual00 = source_descriptor.load([_m + panel, panel]) - accumulator00
 residual10 = source_descriptor.load([_m + panel + 32, panel]) - accumulator10
 residual11 = source_descriptor.load([_m + panel + 32, panel + 32]) - accumulator11
 tl.store(output + base + (panel + rows) * n + panel + columns, residual00, mask=lower)
 tl.store(output + base + (panel + 32 + rows) * n + panel + columns, residual10)
 tl.store(output + base + (panel + 32 + rows) * n + panel + 32 + columns, residual11, mask=lower)
 tl.debug_barrier()
 _blocked32_poly64_output_scratch(output, base, n, panel, 3, True, False, 0.85, True, True)
 local = tl.arange(0, 64)
 local_rows = local[:, None]
 local_columns = local[None, :]
 tl.store(output + base + (panel + local_rows) * n + panel + local_columns, 0.0, mask=local_columns > local_rows)

@triton.jit(noinline=True)
def _one_cta_poly64_trailing_block_noinline_c166(output, history, history_descriptor, source_descriptor, _m, base, row_start, panel, n: tl.constexpr, batch: tl.constexpr, PIPELINE_STAGES: tl.constexpr):
 trailing_accumulator = tl.zeros((64, 64), tl.float32)
 for offset in tl.range(0, panel, 64, num_stages=PIPELINE_STAGES):
  left = history_descriptor.load([_m + row_start, offset])
  right = history_descriptor.load([_m + panel, offset])
  trailing_accumulator += tl.dot(left, tl.trans(right))
 trailing_residual = source_descriptor.load([_m + row_start, panel]) - trailing_accumulator
 rows = row_start + tl.arange(0, 64)
 _one_cta_poly64_trailing_c166(output, history, base, rows, panel, trailing_residual, n)

@triton.jit
def _one_cta_poly64_c166(source, output, history_storage, n: tl.constexpr, batch: tl.constexpr, BLOCKS: tl.constexpr, PIPELINE_STAGES: tl.constexpr):
 matrix = tl.program_id(0)
 _m = matrix * n
 base = matrix * n * n
 history = history_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
 history_descriptor = tl.make_tensor_descriptor(history, shape=[batch * n, n], strides=[n, 1], block_shape=[64, 64])
 source_descriptor64 = tl.make_tensor_descriptor(source, shape=[batch * n, n], strides=[n, 1], block_shape=[64, 64])
 source_descriptor32 = tl.make_tensor_descriptor(source, shape=[batch * n, n], strides=[n, 1], block_shape=[32, 32])
 for panel_block in tl.range(0, BLOCKS - 1, 1):
  panel = panel_block * 64
  _one_cta_poly64_owner_block_noinline_c166(output, history_descriptor, source_descriptor64, _m, base, panel, n, batch, PIPELINE_STAGES, False)
  for row_block in tl.range(panel_block + 1, BLOCKS, 1):
   row_start = row_block * 64
   _one_cta_poly64_trailing_block_noinline_c166(output, history, history_descriptor, source_descriptor64, _m, base, row_start, panel, n, batch, PIPELINE_STAGES)
   tl.debug_barrier()
  diagonal_axis = tl.arange(0, 32)
  diagonal_rows = diagonal_axis[:, None]
  diagonal_columns = diagonal_axis[None, :]
  tl.store(output + base + (panel + diagonal_rows) * n + panel + diagonal_columns, 0.0, mask=diagonal_columns > diagonal_rows)
  tl.store(output + base + (panel + diagonal_rows) * n + panel + 32 + diagonal_columns, 0.0)
  tl.store(output + base + (panel + 32 + diagonal_rows) * n + panel + 32 + diagonal_columns, 0.0, mask=diagonal_columns > diagonal_rows)
  tl.debug_barrier()
 panel: tl.constexpr = (BLOCKS - 1) * 64
 tl_cuda.gdc_launch_dependents()
 _one_cta_poly64_final_owner_triangular_c166(output, history_descriptor, source_descriptor32, _m, base, panel, n, batch, PIPELINE_STAGES)
_N512_ZERO_ROW_LAYOUT_GPU0 = gl.BlockedLayout([1, 1], [1, 32], [4, 1], [1, 0])
_N512_ZERO_SHARED_LAYOUT_GPU0 = gl.NVMMASharedLayout.get_default_for([64, 64], gl.float32, transposed=False)

@gluon.jit
def _n512_tma_strict_upper_zero64_gpu0(output, batch: gl.constexpr, n: gl.constexpr, triangular_tiles: gl.constexpr, programs: gl.constexpr, shared_layout: gl.constexpr, row_layout: gl.constexpr):
 program = gl.program_id(0)
 zero_shared = gl.allocate_shared_memory(gl.float32, [64, 64], shared_layout)
 zero_shared.store(gl.full([64, 64], 0.0, gl.float32, layout=row_layout))
 gl.barrier()
 fence_async_shared()
 descriptor = tma.make_tensor_descriptor(output, shape=[batch * n, n], strides=[n, 1], block_shape=[64, 64], layout=shared_layout)
 total_tasks: gl.constexpr = batch * triangular_tiles
 for linear in range(program, total_tasks, programs):
  matrix = linear // triangular_tiles
  triangular = linear - matrix * triangular_tiles
  _c = ((gl.sqrt(gl.full([], 8.0, gl.float32, layout=gl.SliceLayout(0, row_layout)) * triangular + 1.0) + 1.0) * 0.5).to(gl.int32)
  row_block = triangular - _c * (_c - 1) // 2
  tma.async_copy_shared_to_global(descriptor, [matrix * n + row_block * 64, _c * 64], zero_shared)
 tma.store_wait(0)



def _blocked32_direct_high_batch_cholesky(data, output):
 batch, n, _ = data.shape
 _enable_tma_allocator(data)
 history = _packed_fp16_output_workspace(data)
 blocks = triton.cdiv(n, 64)
 _one_cta_poly64_c166[batch,](
  data,
  output,
  history,
  n=n,
  batch=batch,
  BLOCKS=blocks,
  PIPELINE_STAGES=3,
  num_warps=2,
  num_stages=3,
  maxnreg=168,
  launch_pdl=True,
 )
 strict_tiles = blocks * (blocks - 1) // 2
 _n512_tma_strict_upper_zero64_gpu0[116,](
  output,
  batch=batch,
  n=n,
  triangular_tiles=strict_tiles,
  programs=116,
  shared_layout=_N512_ZERO_SHARED_LAYOUT_GPU0,
  row_layout=_N512_ZERO_ROW_LAYOUT_GPU0,
  num_warps=4,
  launch_pdl=True,
 )
 return output

def _diagonal_block_cholesky(data, output, block: int=512, *, use_packed_fp8: bool=False, use_packed_bf16: bool=False, diagonal_scale: float=1.0, pivot_fraction: float=0.0, column_tile: int=128, row_tile: int=128, tma_warp_specialize: bool=True, tma_pipeline_stages: int=3, tma_num_warps: int=4, high_precision: bool=False, reduction_stride: int=1, reduction_scale: float=1.0, exact_diagonal: bool=False, intra_panel_fraction: float=0.0, upper_intra_panel_fraction: float=0.0, fuse_diagonal_solve: bool=False, fp8_panel_limit: int=0, reduction_tile_override: int=0, reduction_window: int=0, reduction_period: int=1, bf16_correction_window: int=0, fp8_correction_window: int=0, fp8_bf16_prefix_limit: int=0, fp8_bf16_interleave_period: int=0, fp8_pack_scale_0: float=64.0, fp8_pack_scale_1: float=64.0, fp8_row_gain_correction: bool=False, fp8_source_scale: float=1.0, fp8_source_scale_slope: float=0.0, fp8_group3_antithetic: bool=False, fp8_group4_antithetic: bool=False, fp8_group5_antithetic: bool=False, fp8_group7_antithetic: bool=False, bf16_product_scale: float=1.0, omitted_correction: float=0.0, overlap_upper_zero: bool=False, middle_block: int=0, middle_start: int=0, middle_end: int=0, post_middle_block: int=0, middle_reduction_tile: int=0, late_block: int=0, late_start: int=0, late_reduction_period: int=0, late_reduction_start: int=0, late_omitted_correction: float=0.0, late_reduction_scale: float=0.0, sampled_pivot_fraction: float=-1.0, sampled_intra_panel_fraction: float=-1.0, sampled_upper_intra_panel_fraction: float=-1.0, late_pivot_fraction: float=-1.0, late_intra_panel_fraction: float=-1.0, fuse_upper_zero: bool=False, chain_pdl: bool=False, bf16_pair_start: int=0):
 """Large-matrix incomplete Cholesky whose diagonal blocks are diagonal."""
 batch, n, _ = data.shape
 _enable_tma_allocator(data)
 packed_fp8 = (_packed_bf16_output_workspace(data) if fp8_group3_antithetic or fp8_group4_antithetic or fp8_group5_antithetic or fp8_group7_antithetic else _packed_fp8_output_workspace(data)) if use_packed_fp8 else None
 fp8_gains = _packed_fp4_scale_workspace(data) if fp8_row_gain_correction else output
 packed_bf16 = _packed_bf16_output_workspace(data) if use_packed_bf16 or fp8_bf16_prefix_limit > 0 or fp8_bf16_interleave_period > 0 else None
 fused_bf16_flags = _hierarchical_flag_workspace(data) if (use_packed_bf16 or fp8_bf16_prefix_limit > 0) and fuse_diagonal_solve else None
 if fused_bf16_flags is not None and n != 8192:
  _zero_flags_kernel[triton.cdiv(fused_bf16_flags.numel(), 256),](fused_bf16_flags, count=fused_bf16_flags.numel(), BLOCK=256, num_warps=4, num_stages=1)
 if overlap_upper_zero:
  zero_tile = 32
  zero_tiles = triton.cdiv(n, zero_tile)
  triangular_zero_tiles = zero_tiles * (zero_tiles + 1) // 2
  _zero_upper_tiles_kernel[batch * triangular_zero_tiles,](output, n=n, triangular_tiles=triangular_zero_tiles, TILE=zero_tile, SIGNAL_DEPENDENT=True, num_warps=8, num_stages=1, launch_pdl=True)
 reduction_tile = reduction_tile_override if reduction_tile_override > 0 else 32 if block == 32 else 16 if block <= 16 else 64
 if middle_block > 0 and middle_start > 0 and (middle_end > middle_start) and (post_middle_block > 0) and (late_block > 0) and (late_start > middle_end):
  panel_starts = tuple(range(0, middle_start, block)) + tuple(range(middle_start, middle_end, middle_block)) + tuple(range(middle_end, late_start, post_middle_block)) + tuple(range(late_start, n, late_block))
 elif middle_block > 0 and middle_start > 0 and (middle_end > middle_start) and (post_middle_block > 0):
  panel_starts = tuple(range(0, middle_start, block)) + tuple(range(middle_start, middle_end, middle_block)) + tuple(range(middle_end, n, post_middle_block))
 elif middle_block > 0 and middle_start > 0 and (late_block > 0) and (late_start > middle_start):
  panel_starts = tuple(range(0, middle_start, block)) + tuple(range(middle_start, late_start, middle_block)) + tuple(range(late_start, n, late_block))
 elif middle_block > 0 and middle_start > 0:
  panel_starts = tuple(range(0, middle_start, block)) + tuple(range(middle_start, n, middle_block))
 elif late_block > 0 and late_start > 0:
  panel_starts = tuple(range(0, late_start, block)) + tuple(range(late_start, n, late_block))
 else:
  panel_starts = tuple(range(0, n, block))
 skip_next_panel = False
 pair_counter_target = 0
 for panel_index, panel in enumerate(panel_starts):
  if skip_next_panel:
   skip_next_panel = False
   continue
  current_block = late_block if late_block > 0 and panel >= late_start else middle_block if middle_block > 0 and panel >= middle_start and (middle_end <= middle_start or panel < middle_end) else post_middle_block if post_middle_block > 0 and middle_end > middle_start and (panel >= middle_end) else block
  current_reduction_tile = middle_reduction_tile if middle_reduction_tile > 0 and middle_block > 0 and (panel >= middle_start) else reduction_tile
  current_reduction_period = late_reduction_period if abs(late_reduction_period) > 1 and panel >= late_reduction_start else reduction_period
  current_omitted_correction = late_omitted_correction if abs(late_reduction_period) > 1 and panel >= late_reduction_start else omitted_correction
  current_reduction_scale = late_reduction_scale if late_reduction_scale > 0.0 and panel >= late_reduction_start else reduction_scale
  sampled_panel = abs(late_reduction_period) > 1 and panel >= late_reduction_start
  current_pivot_fraction = sampled_pivot_fraction if sampled_pivot_fraction >= 0.0 and sampled_panel else late_pivot_fraction if late_pivot_fraction >= 0.0 and panel >= late_start else pivot_fraction
  current_intra_panel_fraction = sampled_intra_panel_fraction if sampled_intra_panel_fraction >= 0.0 and sampled_panel else late_intra_panel_fraction if late_intra_panel_fraction >= 0.0 and panel >= late_start else intra_panel_fraction
  current_upper_intra_panel_fraction = sampled_upper_intra_panel_fraction if sampled_upper_intra_panel_fraction >= 0.0 and sampled_panel else upper_intra_panel_fraction
  panel_uses_fp8 = use_packed_fp8 and (not use_packed_bf16 or panel < fp8_panel_limit)
  panel_uses_bf16_history = panel_uses_fp8 and fp8_bf16_prefix_limit > 0 and (panel < fp8_bf16_prefix_limit)
  current_row_tile = 64 if panel_uses_bf16_history else row_tile
  current_column_tile = 64 if panel_uses_bf16_history else column_tile
  _p = min(current_block, n - panel)
  row_blocks = triton.cdiv(n - panel, current_row_tile)
  _B = triton.cdiv(_p, current_column_tile)
  grid = (row_blocks * _B, batch)
  first_programs = row_blocks * _B
  pair_mode = use_packed_bf16 and bf16_pair_start > 0 and (panel >= bf16_pair_start) and (panel_index + 1 < len(panel_starts)) and (panel_starts[panel_index + 1] == panel + current_block) and (_p == current_block) and (current_block == 2 * current_row_tile) and (first_programs <= 128)
  if pair_mode:
   second_row_blocks = triton.cdiv(n - panel - current_block, current_row_tile)
   second_programs = second_row_blocks * _B
   grid = (first_programs + second_programs, batch)
   pair_counter_target += first_programs
   skip_next_panel = True
  if panel_uses_bf16_history:
   _left_panel_packed_bf16_tma_kernel[grid](data, packed_bf16, packed_fp8, output, fused_bf16_flags, n=n, batch=batch, panel=panel, generation=panel // block + 1, _p=_p, _B=_B, ROW_TILE=current_row_tile, COLUMN_TILE=current_column_tile, K_TILE=reduction_tile, WARP_SPECIALIZE=tma_warp_specialize, PIPELINE_STAGES=5, CORRECTION_WINDOW=0, SKIP_TAIL_WINDOW=False, PRODUCT_SCALE=1.0, REDUCTION_PERIOD=1, SOURCE_SCALE=1.0, FUSE_DIAGONAL_SOLVE=True, INTRA_PANEL_FRACTION=current_intra_panel_fraction, DIAGONAL_SCALE=diagonal_scale, PIVOT_FRACTION=current_pivot_fraction, FUSE_UPPER_ZERO=fuse_upper_zero, STORE_SECONDARY_FP8=True, PAIR_MODE=False, FIRST_PROGRAMS=0, PAIR_COUNTER_TARGET=0, PAIR_PANEL_STRIDE=0, num_warps=tma_num_warps, num_stages=5, launch_pdl=True)
  elif panel_uses_fp8:
   if fuse_diagonal_solve:
    if fp8_group3_antithetic or fp8_group4_antithetic or fp8_group5_antithetic or fp8_group7_antithetic:
     _left_panel_antithetic_fp8_kernel[grid](data, packed_fp8, output, output, n=n, batch=batch, panel=panel, _p=_p, _B=_B, ROW_TILE=current_row_tile, COLUMN_TILE=current_column_tile, GROUP_INPUT=96 if fp8_group3_antithetic else 128 if fp8_group4_antithetic else 80 if fp8_group5_antithetic else 112, PACKED_WIDTH=160 if fp8_group4_antithetic else 96 if fp8_group5_antithetic else 128, CORRECTED_WIDTH=32 if fp8_group3_antithetic or fp8_group4_antithetic else 16, WARP_SPECIALIZE=tma_warp_specialize, PIPELINE_STAGES=tma_pipeline_stages, ROOT_SCALE=current_pivot_fraction, DIAGONAL_SCALE=diagonal_scale, EXACT_DIAGONAL=False, INTRA_PANEL_FRACTION=current_intra_panel_fraction, UPPER_INTRA_PANEL_FRACTION=current_upper_intra_panel_fraction, BF16_TAIL_WINDOW=0, WAIT_FOR_PREVIOUS=not overlap_upper_zero or panel > 0, FUSE_UPPER_ZERO=fuse_upper_zero, STORE_SECONDARY_FP8=False, num_warps=tma_num_warps, num_stages=tma_pipeline_stages, launch_pdl=True)
     continue
    use_special_fp8_history = fp8_group3_antithetic or fp8_group4_antithetic or fp8_group5_antithetic or fp8_group7_antithetic or fp8_row_gain_correction or (fp8_bf16_interleave_period > 0) or (fp8_bf16_prefix_limit > 0) or (fp8_pack_scale_0 != 64.0) or (fp8_pack_scale_1 != 64.0) or (fp8_source_scale != 1.0) or (fp8_source_scale_slope != 0.0)
    if not use_special_fp8_history:
     _left_panel_diagonal_solve_packed_fp8_fast_kernel[grid](data, packed_fp8, output, n=n, batch=batch, panel=panel, _p=_p, _B=_B, ROW_TILE=current_row_tile, COLUMN_TILE=current_column_tile, K_TILE=reduction_tile, WARP_SPECIALIZE=tma_warp_specialize, PIPELINE_STAGES=tma_pipeline_stages, ROOT_SCALE=current_pivot_fraction, INTRA_PANEL_FRACTION=current_intra_panel_fraction, UPPER_INTRA_PANEL_FRACTION=current_upper_intra_panel_fraction, K_WINDOW=reduction_window, FP32_TAIL_WINDOW=fp8_correction_window, SAMPLED_EXACT_TAIL_WINDOW=0, PACKED_BF16=False, REDUCTION_PERIOD=current_reduction_period, OMITTED_CORRECTION=current_omitted_correction, HISTORY_SCALE=current_reduction_scale, WAIT_FOR_PREVIOUS=not overlap_upper_zero or panel > 0, FUSE_UPPER_ZERO=fuse_upper_zero, num_warps=tma_num_warps, num_stages=tma_pipeline_stages, launch_pdl=True)
     continue
    _left_panel_diagonal_solve_packed_fp8_tma_kernel[grid](data, packed_fp8, fp8_gains, output, n=n, batch=batch, panel=panel, _p=_p, _B=_B, ROW_TILE=current_row_tile, COLUMN_TILE=current_column_tile, K_TILE=reduction_tile, WARP_SPECIALIZE=tma_warp_specialize, PIPELINE_STAGES=1 if panel_uses_bf16_history else tma_pipeline_stages, ROOT_SCALE=current_pivot_fraction, INTRA_PANEL_FRACTION=current_intra_panel_fraction, UPPER_INTRA_PANEL_FRACTION=current_upper_intra_panel_fraction, K_WINDOW=reduction_window, FP32_TAIL_WINDOW=fp8_correction_window, ALL_BF16_HISTORY=panel_uses_bf16_history, PACK_SCALE_0=fp8_pack_scale_0, PACK_SCALE_1=fp8_pack_scale_1, USE_ROW_GAIN=fp8_row_gain_correction, PACK_GROUP3=fp8_group3_antithetic, PACK_GROUP7=fp8_group7_antithetic, BASE_SOURCE_SCALE=fp8_source_scale, SOURCE_SCALE_SLOPE=fp8_source_scale_slope, REDUCTION_PERIOD=current_reduction_period, OMITTED_CORRECTION=current_omitted_correction, WAIT_FOR_PREVIOUS=not overlap_upper_zero or panel > 0, FUSE_UPPER_ZERO=fuse_upper_zero, num_warps=tma_num_warps, num_stages=1 if panel_uses_bf16_history else tma_pipeline_stages, launch_pdl=True)
    if fp8_bf16_interleave_period > 0:
     _sparse_bf16_fp8_panel_correction_kernel[grid](packed_fp8, packed_bf16, output, n=n, batch=batch, panel=panel, _p=_p, _B=_B, ROW_TILE=current_row_tile, COLUMN_TILE=current_column_tile, K_TILE=reduction_tile, PERIOD=fp8_bf16_interleave_period, SLOT=0, INTRA_PANEL_FRACTION=current_intra_panel_fraction, PACK_SCALE_0=fp8_pack_scale_0, PACK_SCALE_1=fp8_pack_scale_1, num_warps=tma_num_warps, num_stages=1, launch_pdl=True)
   else:
    _left_panel_packed_fp8_tma_kernel[grid](data, packed_fp8, output, output, n=n, batch=batch, panel=panel, _p=_p, _B=_B, ROW_TILE=row_tile, COLUMN_TILE=column_tile, K_TILE=reduction_tile, WARP_SPECIALIZE=tma_warp_specialize, PIPELINE_STAGES=tma_pipeline_stages, DIAGONAL_SHIFT=0.0, INVERSE_PACK_SCALE_SQUARED=1.0 / 4096.0, K_STRIDE=reduction_stride, K_SCALE=reduction_scale, RESET_FLAGS=False, num_warps=tma_num_warps, num_stages=tma_pipeline_stages, launch_pdl=True)
  elif use_packed_bf16:
   _left_panel_packed_bf16_tma_kernel[grid](data, packed_bf16, output, output, fused_bf16_flags if fused_bf16_flags is not None else output, n=n, batch=batch, panel=panel, generation=panel // block + 1 if late_block == 0 or panel < late_start else late_start // block + (panel - late_start) // late_block + 1, _p=_p, _B=_B, ROW_TILE=row_tile, COLUMN_TILE=column_tile, K_TILE=current_reduction_tile, WARP_SPECIALIZE=tma_warp_specialize, PIPELINE_STAGES=tma_pipeline_stages, CORRECTION_WINDOW=bf16_correction_window, SKIP_TAIL_WINDOW=False, PRODUCT_SCALE=bf16_product_scale if current_reduction_period == 1 else current_reduction_scale * current_reduction_period, REDUCTION_PERIOD=current_reduction_period, SOURCE_SCALE=1.0 if current_reduction_period == 1 else 1.0 - current_omitted_correction * panel / (n * current_reduction_period), FUSE_DIAGONAL_SOLVE=fuse_diagonal_solve, INTRA_PANEL_FRACTION=current_intra_panel_fraction, DIAGONAL_SCALE=diagonal_scale, PIVOT_FRACTION=current_pivot_fraction, FUSE_UPPER_ZERO=fuse_upper_zero, STORE_SECONDARY_FP8=False, PAIR_MODE=pair_mode, FIRST_PROGRAMS=first_programs if pair_mode else 0, PAIR_COUNTER_TARGET=pair_counter_target if pair_mode else 0, PAIR_PANEL_STRIDE=current_block if pair_mode else 0, num_warps=tma_num_warps, num_stages=tma_pipeline_stages, launch_pdl=chain_pdl or (overlap_upper_zero and panel == 0))
  else:
   _left_panel_tma_kernel[grid](data, output, output, n=n, batch=batch, panel=panel, _p=_p, _B=_B, ROW_TILE=row_tile, COLUMN_TILE=column_tile, K_TILE=reduction_tile, WARP_SPECIALIZE=tma_warp_specialize, PIPELINE_STAGES=tma_pipeline_stages, USE_BF16=False, USE_FP8=False, HIGH_PRECISION=high_precision, BF16_TAIL_WINDOW=0, FP8_TAIL_WINDOW=0, HISTORY_WINDOW=0, HISTORY_CORRECTION=0.0, HISTORY_DAMPING=0.0, FUSE_UPPER_ZERO=False, RESET_FLAGS=False, num_warps=tma_num_warps, num_stages=tma_pipeline_stages)
  if fuse_diagonal_solve:
   continue
  if panel_uses_fp8 and exact_diagonal:
   _diagonal_panel_roots_packed_fp8_kernel[_B, batch](data, packed_fp8, output, n=n, panel=panel, _p=_p, COLUMN_TILE=column_tile, K_TILE=reduction_tile, DIAGONAL_SCALE=diagonal_scale, PIVOT_FRACTION=pivot_fraction, num_warps=8, num_stages=1)
  _diagonal_block_panel_solve_kernel[grid](data, output, packed_fp8 if panel_uses_fp8 else packed_bf16 if packed_bf16 is not None else output, n=n, panel=panel, _p=_p, _B=_B, ROW_TILE=row_tile, COLUMN_TILE=column_tile, DIAGONAL_SCALE=diagonal_scale, PIVOT_FRACTION=current_pivot_fraction, WRITE_FP8=panel_uses_fp8, WRITE_BF16=use_packed_bf16 and (not panel_uses_fp8), ROOTS_PRECOMPUTED=exact_diagonal, INTRA_PANEL_FRACTION=current_intra_panel_fraction, num_warps=8, num_stages=1, launch_pdl=True)
  if panel_uses_fp8 and use_packed_bf16:
   _pack_panel_bf16_kernel[grid](output, packed_bf16, n=n, panel=panel, _p=_p, _B=_B, ROW_TILE=row_tile, COLUMN_TILE=column_tile, num_warps=8, num_stages=1)
 return output

@gluon.jit
def _left_panel_diagonal_solve_packed_fp4_raw_pair112_kernel(history, scale_storage, source, out, packed, n: gl.constexpr, batch: gl.constexpr, panel, _p: gl.constexpr, _B: gl.constexpr, k_tiles, num_warps: gl.constexpr, row_layout: gl.constexpr, tensor_layout: gl.constexpr, register_layout: gl.constexpr, scale_layout: gl.constexpr, ROOT_BASE: gl.constexpr, ROOT_SLOPE: gl.constexpr, CACHED_ROOTS: gl.constexpr):
 tile = gl.program_id(0)
 matrix = gl.program_id(1)
 row_block, pair = gl.inline_asm_elementwise('{\n.reg .pred high, low;\n.reg .b32 estimate, period, remainder, prefix;\n.reg .b32 temp, task_current, task_next, next_prefix;\n.reg .f32 value;\ncvt.rn.f32.u32 value, $2;\nfma.rn.f32 value, value, 1.032258064516129, 0.5184;\nsqrt.approx.f32 value, value;\nadd.f32 value, value, -0.72;\nmul.f32 value, value, 1.9375;\ncvt.rzi.u32.f32 estimate, value;\ndiv.u32 period, estimate, 31;\nmul.lo.u32 temp, period, 31;\nsub.u32 remainder, estimate, temp;\nmul.lo.u32 prefix, period, period;\nmul.lo.u32 prefix, prefix, 248;\nmad.lo.u32 prefix, period, 23, prefix;\nshl.b32 temp, period, 4;\nadd.u32 temp, temp, 1;\nmad.lo.u32 prefix, temp, remainder, prefix;\nmul.lo.u32 temp, remainder, remainder;\nshr.u32 temp, temp, 2;\nadd.u32 prefix, prefix, temp;\nshr.u32 task_current, remainder, 1;\nadd.u32 task_current, task_current, 1;\nsetp.eq.u32 high, remainder, 0;\n@high mov.u32 task_current, 0;\nmad.lo.u32 task_current, period, 16, task_current;\nadd.u32 temp, remainder, 1;\nshr.u32 task_next, temp, 1;\nadd.u32 task_next, task_next, 1;\nmad.lo.u32 task_next, period, 16, task_next;\nadd.u32 next_prefix, prefix, task_next;\nsetp.gt.u32 high, prefix, $2;\nsetp.le.u32 low, next_prefix, $2;\nand.pred low, low, !high;\nmov.u32 $0, estimate;\n@high sub.u32 $0, estimate, 1;\n@high sub.u32 prefix, prefix, task_current;\n@low add.u32 $0, estimate, 1;\n@low mov.u32 prefix, next_prefix;\nsub.u32 $1, $2, prefix;\n}', '=r,=r,r', [tile], dtype=[gl.int32, gl.int32], is_pure=True, pack=1)
 row_start = panel + row_block * 128
 _s = panel + pair * 248
 _c = pair
 _m = matrix * n
 history = history.cast(gl.pointer_type(gl.uint8), bitcast=True)
 scale_storage = scale_storage.cast(gl.pointer_type(gl.uint8), bitcast=True)
 packed = packed.cast(gl.pointer_type(gl.uint8), bitcast=True)
 a_stage0 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 b_stage0 = gl.allocate_shared_memory(gl.int64, [4096], mbarrier.MBarrierLayout())
 a_stage1 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 b_stage1 = gl.allocate_shared_memory(gl.int64, [4096], mbarrier.MBarrierLayout())
 barriers = gl.allocate_shared_memory(gl.int64, [8], mbarrier.MBarrierLayout())
 sentinel0 = a_stage0.slice(2047, 1)
 sentinel1 = b_stage0.slice(4095, 1)
 sentinel2 = a_stage1.slice(2047, 1)
 sentinel4 = b_stage1.slice(4095, 1)
 sentinel3 = barriers.slice(7, 1)
 combined = allocate_tensor_memory(gl.float32, [128, 256], tensor_layout)
 accumulator00 = combined.slice(0, 64)
 accumulator01 = combined.slice(64, 64)
 accumulator10 = combined.slice(128, 64)
 accumulator11 = combined.slice(192, 64)
 gl.inline_asm_elementwise('{\n.reg .pred first_warp, first_thread, producer, producer_first;\n.reg .pred consumer_first, complete, more, reuse, accumulate;\n.reg .b32 tid, warp_id, local, cta_id, smem_addr, taddr;\n.reg .b32 warp_bits, warp_taddr, scale_value, bar_addr;\n.reg .b32 iter, stage, cycle, parity, kbytes, row_index;\n.reg .b32 row_smem, swizzle, stage_smem, chunk, column, physical;\n.reg .b32 a_smem, b_smem, ready_addr, empty_addr;\n.reg .b32 idesc0, idesc1, idesc2, idesc3;\n.reg .b32 lane, tensor_dim;\n.reg .b64 a_global, b_global, a_chunk, b_chunk, desc_base;\n.reg .b64 a_desc0, b_desc0, a_desc1, b_desc1;\n.reg .b64 a_desc2, b_desc2, a_desc3, b_desc3;\n.reg .b64 b_second0, b_second1, b_second2, b_second3;\n.reg .b64 state_a, state_b;\ngriddepcontrol.wait;\nmov.u32 tid, %tid.x;\nshr.u32 warp_id, tid, 5;\nmov.u32 cta_id, %ctaid.x;\nsetp.lt.u32 first_warp, tid, 32;\nsetp.eq.u32 first_thread, tid, 0;\nsetp.ge.u32 producer, tid, 128;\nsetp.eq.u32 producer_first, tid, 128;\nsetp.eq.u32 consumer_first, tid, 0;\nmov.u32 smem_addr, global_smem;\nld.shared.b32 taddr, [smem_addr];\nand.b32 lane, tid, 31;\nshl.b32 physical, lane, 2;\nadd.u32 physical, physical, smem_addr;\n@first_warp st.shared.b32 [physical], 0;\nbar.sync 0, 256;\ncvt.u64.u32 desc_base, smem_addr;\n@first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64\n[desc_base], $1;\n@first_thread tensormap.replace.tile.rank.shared::cta.b1024.b32\n[desc_base], 0x1;\nmov.u32 tensor_dim, 128;\n@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32\n[desc_base], 0x0, tensor_dim;\n@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32\n[desc_base], 0x1, tensor_dim;\nmov.u32 tensor_dim, $7;\n@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32\n[desc_base], 0x0, tensor_dim;\nmov.u32 tensor_dim, $6;\n@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32\n[desc_base], 0x1, tensor_dim;\nmov.u32 tensor_dim, $7;\ncvt.u64.u32 a_global, tensor_dim;\n@first_thread tensormap.replace.tile.global_stride.shared::cta.b1024.b64\n[desc_base], 0x0, a_global;\nmov.u32 tensor_dim, 1;\n@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32\n[desc_base], 0x0, tensor_dim;\n@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32\n[desc_base], 0x1, tensor_dim;\n@first_thread tensormap.replace.tile.elemtype.shared::cta.b1024.b32\n[desc_base], 0x0;\n@first_thread tensormap.replace.tile.interleave_layout.shared::cta.b1024.b32\n[desc_base], 0x0;\n@first_thread tensormap.replace.tile.swizzle_mode.shared::cta.b1024.b32\n[desc_base], 0x3;\n@first_thread tensormap.replace.tile.fill_mode.shared::cta.b1024.b32\n[desc_base], 0x0;\nmad.wide.u32 state_a, $8, 256, $2;\nadd.u64 state_a, state_a, 256;\n@first_warp tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned\n[state_a], [desc_base], 0x80;\n@first_warp fence.proxy.tensormap::generic.acquire.gpu [state_a], 0x80;\nmov.u32 tensor_dim, 248;\n@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32\n[desc_base], 0x1, tensor_dim;\nadd.u64 state_b, state_a, 128;\n@first_warp tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned\n[state_b], [desc_base], 0x80;\n@first_warp fence.proxy.tensormap::generic.acquire.gpu [state_b], 0x80;\n@first_warp cp.async.bulk.commit_group;\n@first_warp cp.async.bulk.wait_group.read 0;\nbar.sync 0, 256;\nand.b32 warp_bits, warp_id, 3;\nshl.b32 warp_bits, warp_bits, 21;\nadd.u32 warp_taddr, taddr, warp_bits;\nld.global.u8 scale_value, [$2];\nmul.lo.u32 scale_value, scale_value, 0x01010101;\n@!producer tcgen05.st.sync.aligned.32x32b.x8.b32 [warp_taddr + 248],\n{scale_value, scale_value, scale_value, scale_value,\nscale_value, scale_value, scale_value, scale_value};\n@!producer tcgen05.wait::st.sync.aligned;\nbar.sync 0, 256;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98304], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98312], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98320], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98328], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98336], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98344], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98352], 1;\nbar.sync 0, 256;\n@producer bra.uni producer_path;\nconsumer_path:\nmov.u32 iter, 0;\nmov.u32 idesc0, 0x08a00480;\nmov.u32 idesc1, 0x48a00480;\nmov.u32 idesc2, 0x089e0480;\nmov.u32 idesc3, 0x489e0480;\nconsumer_loop:\nand.b32 stage, iter, 1;\nshr.u32 cycle, iter, 1;\nand.b32 parity, cycle, 1;\nshl.b32 ready_addr, stage, 3;\nadd.u32 ready_addr, ready_addr, smem_addr;\nadd.u32 ready_addr, ready_addr, 98304;\nconsumer_ready_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete, [ready_addr], parity;\n@!complete bra.uni consumer_ready_wait;\nbar.sync 3, 128;\nmul.lo.u32 stage_smem, stage, 49152;\nadd.u32 a_smem, smem_addr, stage_smem;\nadd.u32 b_smem, a_smem, 16384;\nbfe.u32 a_smem, a_smem, 4, 14;\nbfe.u32 b_smem, b_smem, 4, 14;\ncvt.u64.u32 desc_base, a_smem;\nor.b64 a_desc0, desc_base, 0x4000404000000000;\ncvt.u64.u32 desc_base, b_smem;\nor.b64 b_desc0, desc_base, 0x4000404000000000;\nadd.u64 a_desc1, a_desc0, 2;\nadd.u64 b_desc1, b_desc0, 2;\nadd.u64 a_desc2, a_desc0, 4;\nadd.u64 b_desc2, b_desc0, 4;\nadd.u64 a_desc3, a_desc0, 6;\nadd.u64 b_desc3, b_desc0, 6;\nsetp.ne.u32 accumulate, iter, 0;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr], a_desc0, b_desc0, idesc0,\n[taddr + 248], [taddr + 248], accumulate;\nmov.pred accumulate, 1;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr], a_desc1, b_desc1, idesc1,\n[taddr + 248], [taddr + 248], accumulate;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr], a_desc2, b_desc2, idesc0,\n[taddr + 248], [taddr + 248], accumulate;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr], a_desc3, b_desc3, idesc1,\n[taddr + 248], [taddr + 248], accumulate;\nadd.u64 b_second0, b_desc0, 1024;\nadd.u64 b_second1, b_desc1, 1024;\nadd.u64 b_second2, b_desc2, 1024;\nadd.u64 b_second3, b_desc3, 1024;\nsetp.ne.u32 accumulate, iter, 0;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr + 128], a_desc0, b_second0, idesc2,\n[taddr + 248], [taddr + 248], accumulate;\nmov.pred accumulate, 1;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr + 128], a_desc1, b_second1, idesc3,\n[taddr + 248], [taddr + 248], accumulate;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr + 128], a_desc2, b_second2, idesc2,\n[taddr + 248], [taddr + 248], accumulate;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr + 128], a_desc3, b_second3, idesc3,\n[taddr + 248], [taddr + 248], accumulate;\nshl.b32 empty_addr, stage, 3;\nadd.u32 empty_addr, empty_addr, smem_addr;\nadd.u32 empty_addr, empty_addr, 98328;\n@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [empty_addr];\nadd.u32 iter, iter, 1;\nsetp.lt.u32 more, iter, $3;\n@more bra.uni consumer_loop;\n@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [smem_addr + 98352];\nconsumer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete, [smem_addr + 98352], 0;\n@!complete bra.uni consumer_final_wait;\nbra.uni final_barrier;\nproducer_path:\nmov.u32 row_index, $4;\nmov.u32 local, $5;\nmov.u32 iter, 0;\nproducer_loop:\nand.b32 stage, iter, 1;\nsetp.ge.u32 reuse, iter, 2;\n@!reuse bra.uni producer_load;\nshr.u32 cycle, iter, 1;\nsub.u32 cycle, cycle, 1;\nand.b32 parity, cycle, 1;\nshl.b32 empty_addr, stage, 3;\nadd.u32 empty_addr, empty_addr, smem_addr;\nadd.u32 empty_addr, empty_addr, 98328;\nproducer_empty_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete, [empty_addr], parity;\n@!complete bra.uni producer_empty_wait;\nproducer_load:\nmul.lo.u32 stage_smem, stage, 49152;\nadd.u32 a_smem, stage_smem, smem_addr;\nadd.u32 b_smem, a_smem, 16384;\nshl.b32 ready_addr, stage, 3;\nadd.u32 ready_addr, ready_addr, smem_addr;\nadd.u32 ready_addr, ready_addr, 98304;\nshl.b32 column, iter, 7;\n@producer_first mbarrier.arrive.expect_tx.shared::cta.b64 _, [ready_addr], 48128;\nfence.proxy.async.shared::cta;\n@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n[a_smem], [state_a, {column, row_index}], [ready_addr];\n@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n[b_smem], [state_b, {column, local}], [ready_addr];\nadd.u32 iter, iter, 1;\nsetp.lt.u32 more, iter, $3;\n@more bra.uni producer_loop;\nproducer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete, [smem_addr + 98352], 0;\n@!complete bra.uni producer_final_wait;\nbra.uni final_barrier;\nfinal_barrier:\nbar.sync 0, 256;\nmov.u32 $0, 0;\n}', '=r,l,l,r,r,r,r,r,r', [history, scale_storage, k_tiles, _m + row_start, _m + _s, batch * n, n // 2, matrix * gl.num_programs(0) + tile], dtype=gl.int32, is_pure=False, pack=1)
 mbarrier.init(sentinel0, count=1)
 mbarrier.init(sentinel1, count=1)
 mbarrier.init(sentinel2, count=1)
 mbarrier.init(sentinel4, count=1)
 mbarrier.init(sentinel3, count=1)
 mbarrier.invalidate(sentinel0)
 mbarrier.invalidate(sentinel1)
 mbarrier.invalidate(sentinel2)
 mbarrier.invalidate(sentinel4)
 mbarrier.invalidate(sentinel3)
 product00 = gl.convert_layout(accumulator00.load(register_layout), row_layout)
 _raw_fp4_pair112_epilogue(product00, scale_storage, source, out, packed, n, panel, _p, row_start, _s, row_layout, 64, False)
 product01 = gl.convert_layout(accumulator01.load(register_layout), row_layout)
 _raw_fp4_pair112_epilogue(product01, scale_storage, source, out, packed, n, panel, _p, row_start, _s + 64, row_layout, 64, False)
 product10 = gl.convert_layout(accumulator10.load(register_layout), row_layout)
 _raw_fp4_pair112_epilogue(product10, scale_storage, source, out, packed, n, panel, _p, row_start, _s + 128, row_layout, 64, False)
 product11 = gl.convert_layout(accumulator11.load(register_layout), row_layout)
 _raw_fp4_pair112_epilogue(product11, scale_storage, source, out, packed, n, panel, _p, row_start, _s + 192, row_layout, 56, False)
 gl.inline_asm_elementwise('griddepcontrol.launch_dependents; mov.u32 $0, 0;', '=r', [], dtype=gl.int32, is_pure=False, pack=1)

def _raw_fp4_pair112_layouts():
 row_layout = gl.BlockedLayout([1, 2], [1, 32], [8, 1], [1, 0])
 tensor_layout = TensorMemoryLayout((128, 256), col_stride=1)
 accumulator_layout = TensorMemoryLayout((128, 64), col_stride=1)
 register_layout = get_tmem_reg_layout(
  gl.float32, (128, 64), accumulator_layout, 8,
  alloc_shape=(128, 256),
 )
 scale_layout = TensorMemoryScalesLayout()
 return (row_layout, tensor_layout, register_layout, scale_layout)

@gluon.jit
def _left_panel_diagonal_solve_packed_fp4_raw_kernel(history, scale_storage, source, out, packed, n: gl.constexpr, batch: gl.constexpr, panel, _p: gl.constexpr, _B: gl.constexpr, k_tiles, num_warps: gl.constexpr, row_layout: gl.constexpr, tensor_layout: gl.constexpr, register_layout: gl.constexpr, scale_layout: gl.constexpr, ROOT_BASE: gl.constexpr, ROOT_SLOPE: gl.constexpr, CACHED_ROOTS: gl.constexpr):
 tile = gl.program_id(0)
 matrix = gl.program_id(1)
 triangular_programs: gl.constexpr = _B * (_B + 1) // 2
 in_panel_square = tile < triangular_programs
 triangular_row = ((gl.sqrt((8 * tile + 1).to(gl.float32)) - 1.0) * 0.5).to(gl.int32)
 triangular_column = tile - triangular_row * (triangular_row + 1) // 2
 tail_tile = tile - triangular_programs
 tail_row = _B + tail_tile // _B
 tail_column = tail_tile - (tail_row - _B) * _B
 row_block = gl.where(in_panel_square, triangular_row, tail_row)
 _c = gl.where(in_panel_square, triangular_column, tail_column)
 row_start = panel + row_block * 128
 _s = panel + _c * 128
 _m = matrix * n
 history = history.cast(gl.pointer_type(gl.uint8), bitcast=True)
 scale_storage = scale_storage.cast(gl.pointer_type(gl.uint8), bitcast=True)
 packed = packed.cast(gl.pointer_type(gl.uint8), bitcast=True)
 scratch0 = gl.allocate_shared_memory(gl.int64, [4096], mbarrier.MBarrierLayout())
 scratch1 = gl.allocate_shared_memory(gl.int64, [4096], mbarrier.MBarrierLayout())
 scratch2 = gl.allocate_shared_memory(gl.int64, [4096], mbarrier.MBarrierLayout())
 barriers = gl.allocate_shared_memory(gl.int64, [8], mbarrier.MBarrierLayout())
 sentinel0 = scratch0.slice(4095, 1)
 sentinel1 = scratch1.slice(4095, 1)
 sentinel2 = scratch2.slice(4095, 1)
 sentinel3 = barriers.slice(7, 1)
 combined = allocate_tensor_memory(gl.float32, [128, 256], tensor_layout)
 _a = combined.slice(0, 128)
 gl.inline_asm_elementwise('{\n.reg .pred first_warp, first_thread, producer, producer_first;\n.reg .pred consumer_first, complete, more, reuse, accumulate;\n.reg .b32 tid, warp_id, local, cta_id, smem_addr, taddr;\n.reg .b32 warp_bits, warp_taddr, scale_value, bar_addr;\n.reg .b32 iter, stage, cycle, parity, kbytes, row_index;\n.reg .b32 row_smem, swizzle, stage_smem, chunk, column, physical;\n.reg .b32 a_smem, b_smem, ready_addr, empty_addr, idesc0, idesc1;\n.reg .b32 lane, tensor_dim;\n.reg .b64 a_global, b_global, a_chunk, b_chunk, desc_base;\n.reg .b64 a_desc0, b_desc0, a_desc1, b_desc1;\n.reg .b64 a_desc2, b_desc2, a_desc3, b_desc3, state;\ngriddepcontrol.wait;\nmov.u32 tid, %tid.x;\nshr.u32 warp_id, tid, 5;\nmov.u32 cta_id, %ctaid.x;\nsetp.lt.u32 first_warp, tid, 32;\nsetp.eq.u32 first_thread, tid, 0;\nsetp.ge.u32 producer, tid, 128;\nsetp.eq.u32 producer_first, tid, 128;\nsetp.eq.u32 consumer_first, tid, 0;\nmov.u32 smem_addr, global_smem;\nld.shared.b32 taddr, [smem_addr];\nand.b32 lane, tid, 31;\nshl.b32 physical, lane, 2;\nadd.u32 physical, physical, smem_addr;\n@first_warp st.shared.b32 [physical], 0;\nbar.sync 0, 256;\ncvt.u64.u32 desc_base, smem_addr;\n@first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64\n[desc_base], $1;\n@first_thread tensormap.replace.tile.rank.shared::cta.b1024.b32\n[desc_base], 0x1;\nmov.u32 tensor_dim, 128;\n@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32\n[desc_base], 0x0, tensor_dim;\n@first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32\n[desc_base], 0x1, tensor_dim;\nmov.u32 tensor_dim, $7;\n@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32\n[desc_base], 0x0, tensor_dim;\nmov.u32 tensor_dim, $6;\n@first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32\n[desc_base], 0x1, tensor_dim;\nmov.u32 tensor_dim, $7;\ncvt.u64.u32 a_global, tensor_dim;\n@first_thread tensormap.replace.tile.global_stride.shared::cta.b1024.b64\n[desc_base], 0x0, a_global;\nmov.u32 tensor_dim, 1;\n@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32\n[desc_base], 0x0, tensor_dim;\n@first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32\n[desc_base], 0x1, tensor_dim;\n@first_thread tensormap.replace.tile.elemtype.shared::cta.b1024.b32\n[desc_base], 0x0;\n@first_thread tensormap.replace.tile.interleave_layout.shared::cta.b1024.b32\n[desc_base], 0x0;\n@first_thread tensormap.replace.tile.swizzle_mode.shared::cta.b1024.b32\n[desc_base], 0x3;\n@first_thread tensormap.replace.tile.fill_mode.shared::cta.b1024.b32\n[desc_base], 0x0;\nmad.wide.u32 state, $8, 128, $2;\nadd.u64 state, state, 128;\n@first_warp tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned\n[state], [desc_base], 0x80;\n@first_warp fence.proxy.tensormap::generic.acquire.gpu [state], 0x80;\n@first_warp cp.async.bulk.commit_group;\n@first_warp cp.async.bulk.wait_group.read 0;\nbar.sync 0, 256;\nand.b32 warp_bits, warp_id, 3;\nshl.b32 warp_bits, warp_bits, 21;\nadd.u32 warp_taddr, taddr, warp_bits;\nld.global.u8 scale_value, [$2];\nmul.lo.u32 scale_value, scale_value, 0x01010101;\n@!producer tcgen05.st.sync.aligned.32x32b.x8.b32 [warp_taddr + 128],\n{scale_value, scale_value, scale_value, scale_value,\nscale_value, scale_value, scale_value, scale_value};\n@!producer tcgen05.st.sync.aligned.32x32b.x8.b32 [warp_taddr + 136],\n{scale_value, scale_value, scale_value, scale_value,\nscale_value, scale_value, scale_value, scale_value};\n@!producer tcgen05.wait::st.sync.aligned;\nbar.sync 0, 256;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98304], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98312], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98320], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98328], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98336], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98344], 1;\n@first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98352], 1;\nbar.sync 0, 256;\n@producer bra.uni producer_path;\nconsumer_path:\nmov.u32 iter, 0;\nmov.u32 idesc0, 0x08a00480;\nmov.u32 idesc1, 0x48a00480;\nconsumer_loop:\nrem.u32 stage, iter, 3;\ndiv.u32 cycle, iter, 3;\nand.b32 parity, cycle, 1;\nshl.b32 ready_addr, stage, 3;\nadd.u32 ready_addr, ready_addr, smem_addr;\nadd.u32 ready_addr, ready_addr, 98304;\nconsumer_ready_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete, [ready_addr], parity;\n@!complete bra.uni consumer_ready_wait;\nbar.sync 3, 128;\nshl.b32 stage_smem, stage, 15;\nadd.u32 a_smem, smem_addr, stage_smem;\nadd.u32 b_smem, a_smem, 16384;\nbfe.u32 a_smem, a_smem, 4, 14;\nbfe.u32 b_smem, b_smem, 4, 14;\ncvt.u64.u32 desc_base, a_smem;\nor.b64 a_desc0, desc_base, 0x4000404000000000;\ncvt.u64.u32 desc_base, b_smem;\nor.b64 b_desc0, desc_base, 0x4000404000000000;\nadd.u64 a_desc1, a_desc0, 2;\nadd.u64 b_desc1, b_desc0, 2;\nadd.u64 a_desc2, a_desc0, 4;\nadd.u64 b_desc2, b_desc0, 4;\nadd.u64 a_desc3, a_desc0, 6;\nadd.u64 b_desc3, b_desc0, 6;\nsetp.ne.u32 accumulate, iter, 0;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr], a_desc0, b_desc0, idesc0,\n[taddr + 128], [taddr + 136], accumulate;\nmov.pred accumulate, 1;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr], a_desc1, b_desc1, idesc1,\n[taddr + 128], [taddr + 136], accumulate;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr], a_desc2, b_desc2, idesc0,\n[taddr + 132], [taddr + 140], accumulate;\n@consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n[taddr], a_desc3, b_desc3, idesc1,\n[taddr + 132], [taddr + 140], accumulate;\nshl.b32 empty_addr, stage, 3;\nadd.u32 empty_addr, empty_addr, smem_addr;\nadd.u32 empty_addr, empty_addr, 98328;\n@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [empty_addr];\nadd.u32 iter, iter, 1;\nsetp.lt.u32 more, iter, $3;\n@more bra.uni consumer_loop;\n@consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [smem_addr + 98352];\nconsumer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete, [smem_addr + 98352], 0;\n@!complete bra.uni consumer_final_wait;\nbra.uni final_barrier;\nproducer_path:\nmov.u32 row_index, $4;\nmov.u32 local, $5;\nmov.u32 iter, 0;\nproducer_loop:\nrem.u32 stage, iter, 3;\nsetp.ge.u32 reuse, iter, 3;\n@!reuse bra.uni producer_load;\ndiv.u32 cycle, iter, 3;\nsub.u32 cycle, cycle, 1;\nand.b32 parity, cycle, 1;\nshl.b32 empty_addr, stage, 3;\nadd.u32 empty_addr, empty_addr, smem_addr;\nadd.u32 empty_addr, empty_addr, 98328;\nproducer_empty_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete, [empty_addr], parity;\n@!complete bra.uni producer_empty_wait;\nproducer_load:\nshl.b32 stage_smem, stage, 15;\nadd.u32 a_smem, stage_smem, smem_addr;\nadd.u32 b_smem, a_smem, 16384;\nshl.b32 ready_addr, stage, 3;\nadd.u32 ready_addr, ready_addr, smem_addr;\nadd.u32 ready_addr, ready_addr, 98304;\nshl.b32 column, iter, 7;\n@producer_first mbarrier.arrive.expect_tx.shared::cta.b64 _, [ready_addr], 32768;\nfence.proxy.async.shared::cta;\n@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n[a_smem], [state, {column, row_index}], [ready_addr];\n@producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n[b_smem], [state, {column, local}], [ready_addr];\nadd.u32 iter, iter, 1;\nsetp.lt.u32 more, iter, $3;\n@more bra.uni producer_loop;\nproducer_final_wait:\nmbarrier.try_wait.parity.shared::cta.b64 complete, [smem_addr + 98352], 0;\n@!complete bra.uni producer_final_wait;\nbra.uni final_barrier;\nfinal_barrier:\nbar.sync 0, 256;\nmov.u32 $0, 0;\n}', '=r,l,l,r,r,r,r,r,r', [history, scale_storage, k_tiles, _m + row_start, _m + _s, batch * n, n // 2, matrix * gl.num_programs(0) + tile], dtype=gl.int32, is_pure=False, pack=1)
 mbarrier.init(sentinel0, count=1)
 mbarrier.init(sentinel1, count=1)
 mbarrier.init(sentinel2, count=1)
 mbarrier.init(sentinel3, count=1)
 mbarrier.invalidate(sentinel0)
 mbarrier.invalidate(sentinel1)
 mbarrier.invalidate(sentinel2)
 mbarrier.invalidate(sentinel3)
 product = gl.convert_layout(_a.load(register_layout), row_layout)
 rows = row_start + gl.arange(0, 128, layout=gl.SliceLayout(1, row_layout))
 columns = _s + gl.arange(0, 128, layout=gl.SliceLayout(0, row_layout))
 base = matrix * n * n
 offsets = base + rows[:, None] * n + columns[None, :]
 valid = (rows[:, None] < n) & (columns[None, :] < panel + _p)
 source_value = gl.load(source + offsets, mask=valid, other=0.0)
 if CACHED_ROOTS:
  root_storage = scale_storage.cast(
   gl.pointer_type(gl.float32), bitcast=True
  )
  roots = gl.load(
   root_storage + 1067008 + columns - 2048,
   mask=columns < panel + _p,
   other=1.0,
  )
 else:
  source_diagonal = gl.load(source + base + columns * n + columns, mask=columns < panel + _p, other=1.0)
  root_position = (columns.to(gl.float32) - panel) / _p
  if ROOT_BASE == 32.0:
   root_coordinate = root_position * 31.0
   _r = root_coordinate.to(gl.int32)
   root_fraction = root_coordinate - _r.to(gl.float32)
   root_left = gl.where(_r == 0, 8.533367433761235, 0.8888700538569949)
   root_left = gl.where(_r == 1, 8.187967240580846, root_left)
   root_left = gl.where(_r == 2, 7.887952053720757, root_left)
   root_left = gl.where(_r == 3, 7.567100659649934, root_left)
   root_left = gl.where(_r == 4, 7.208853682619789, root_left)
   root_left = gl.where(_r == 5, 6.961747415219185, root_left)
   root_left = gl.where(_r == 6, 6.74755278198766, root_left)
   root_left = gl.where(_r == 7, 6.4719213738075245, root_left)
   root_left = gl.where(_r == 8, 6.18783108513031, root_left)
   root_left = gl.where(_r == 9, 5.9247987986995865, root_left)
   root_left = gl.where(_r == 10, 5.650537046231346, root_left)
   root_left = gl.where(_r == 11, 5.395482436702258, root_left)
   root_left = gl.where(_r == 12, 5.13403999987112, root_left)
   root_left = gl.where(_r == 13, 4.878280013746125, root_left)
   root_left = gl.where(_r == 14, 4.638791700358449, root_left)
   root_left = gl.where(_r == 15, 4.418828018582018, root_left)
   root_left = gl.where(_r == 16, 4.188277391019296, root_left)
   root_left = gl.where(_r == 17, 3.910328276801168, root_left)
   root_left = gl.where(_r == 18, 3.652298679499085, root_left)
   root_left = gl.where(_r == 19, 3.4360475693822425, root_left)
   root_left = gl.where(_r == 20, 3.214528896550082, root_left)
   root_left = gl.where(_r == 21, 2.9840977029120572, root_left)
   root_left = gl.where(_r == 22, 2.7697087521532997, root_left)
   root_left = gl.where(_r == 23, 2.534307826362682, root_left)
   root_left = gl.where(_r == 24, 2.2654362183003007, root_left)
   root_left = gl.where(_r == 25, 1.9960358067612753, root_left)
   root_left = gl.where(_r == 26, 1.7921065336774171, root_left)
   root_left = gl.where(_r == 27, 1.6030917211149267, root_left)
   root_left = gl.where(_r == 28, 1.3568329349473298, root_left)
   root_left = gl.where(_r == 29, 1.109221576498201, root_left)
   root_right = gl.where(_r == 0, 8.187967240580846, 0.6816716486166621)
   root_right = gl.where(_r == 1, 7.887952053720757, root_right)
   root_right = gl.where(_r == 2, 7.567100659649934, root_right)
   root_right = gl.where(_r == 3, 7.208853682619789, root_right)
   root_right = gl.where(_r == 4, 6.961747415219185, root_right)
   root_right = gl.where(_r == 5, 6.74755278198766, root_right)
   root_right = gl.where(_r == 6, 6.4719213738075245, root_right)
   root_right = gl.where(_r == 7, 6.18783108513031, root_right)
   root_right = gl.where(_r == 8, 5.9247987986995865, root_right)
   root_right = gl.where(_r == 9, 5.650537046231346, root_right)
   root_right = gl.where(_r == 10, 5.395482436702258, root_right)
   root_right = gl.where(_r == 11, 5.13403999987112, root_right)
   root_right = gl.where(_r == 12, 4.878280013746125, root_right)
   root_right = gl.where(_r == 13, 4.638791700358449, root_right)
   root_right = gl.where(_r == 14, 4.418828018582018, root_right)
   root_right = gl.where(_r == 15, 4.188277391019296, root_right)
   root_right = gl.where(_r == 16, 3.910328276801168, root_right)
   root_right = gl.where(_r == 17, 3.652298679499085, root_right)
   root_right = gl.where(_r == 18, 3.4360475693822425, root_right)
   root_right = gl.where(_r == 19, 3.214528896550082, root_right)
   root_right = gl.where(_r == 20, 2.9840977029120572, root_right)
   root_right = gl.where(_r == 21, 2.7697087521532997, root_right)
   root_right = gl.where(_r == 22, 2.534307826362682, root_right)
   root_right = gl.where(_r == 23, 2.2654362183003007, root_right)
   root_right = gl.where(_r == 24, 1.9960358067612753, root_right)
   root_right = gl.where(_r == 25, 1.7921065336774171, root_right)
   root_right = gl.where(_r == 26, 1.6030917211149267, root_right)
   root_right = gl.where(_r == 27, 1.3568329349473298, root_right)
   root_right = gl.where(_r == 28, 1.109221576498201, root_right)
   root_right = gl.where(_r == 29, 0.8888700538569949, root_right)
   root_scale = root_left + root_fraction * (root_right - root_left)
  else:
   root_scale = ROOT_BASE + ROOT_SLOPE * (root_position - 0.5)
  roots = gl.sqrt(source_diagonal * root_scale)
 residual = source_value - product
 if ROOT_BASE == 32.0:
  row_distance = (rows[:, None].to(gl.float32) - columns[None, :].to(gl.float32)) / _p
  distance_gate = 1.0 + ROOT_SLOPE * row_distance
 else:
  distance_gate = 1.0
 solved = gl.where(rows[:, None] == columns[None, :], roots[None, :], gl.where(rows[:, None] >= panel + _p, residual / roots[None, :], gl.where(rows[:, None] > columns[None, :], residual / roots[None, :] * distance_gate, 0.0)))
 gl.store(out + offsets, solved, mask=valid)
 pairs = gl.reshape(solved * 1024.0, [128, 64, 2])
 low, high = gl.split(pairs)
 packed_values = gl.inline_asm_elementwise('{\n.reg .b8 value;\ncvt.rn.satfinite.e2m1x2.f32 value, $2, $1;\ncvt.u32.u8 $0, value;\n}', '=r,f,f', [low, high], dtype=gl.int32, is_pure=True, pack=1).to(gl.uint8)
 packed_rows = row_start + gl.arange(0, 128, layout=gl.SliceLayout(1, packed_values.type.layout))
 packed_columns = _s // 2 + gl.arange(0, 64, layout=gl.SliceLayout(0, packed_values.type.layout))
 gl.store(packed + matrix * n * (n // 2) + packed_rows[:, None] * (n // 2) + packed_columns[None, :], packed_values, mask=(packed_rows[:, None] < n) & (packed_columns[None, :] < (panel + _p) // 2))
 gl.inline_asm_elementwise('griddepcontrol.launch_dependents; mov.u32 $0, 0;', '=r', [], dtype=gl.int32, is_pure=False, pack=1)

def _n32768_split_fixed10(data, output):
 batch, n, _ = data.shape
 _enable_tma_allocator(data)
 packed = _packed_fp4_output_workspace(data)
 scales = _packed_fp4_scale_workspace(data)
 first_width = 2048
 first_row_tile = 64
 first_column_tile = 64
 first_column_blocks = triton.cdiv(first_width, first_column_tile)
 first_grid = (
  triton.cdiv(n, first_row_tile) * first_column_blocks,
  batch,
 )
 _first_panel_diagonal_solve_packed_fp4_kernel[first_grid](
  data,
  packed,
  packed,
  scales,
  output,
  n=n,
  batch=batch,
  _p=first_width,
  _B=first_column_blocks,
  ROW_TILE=first_row_tile,
  COLUMN_TILE=first_column_tile,
  ROOT_SCALE=6.0,
  SCALE_EXPONENT=-10,
  INTRA_PANEL_FRACTION=1.0,
  FUSE_UPPER_ZERO=False,
  COMPRESSED_HISTORY=False,
  HISTORY_K_TILE=256,
  EXACT_HISTORY_WINDOW=0,
  PRECOMPUTE_TAIL_ROOTS=True,
  num_warps=4,
  num_stages=1,
  launch_pdl=True,
 )
 layouts = _raw_fp4_layouts()
 pair_layouts = _raw_fp4_pair112_layouts()
 panels = (
  (2048, 3584, 7.65, -4.5),
  (5632, 4352, 7.225, -4.25),
  (9984, 4352, 6.8, -4.0),
 )
 for panel, _p, root_base, root_slope in panels:
  row_blocks = triton.cdiv(n - panel, 128)
  _B = triton.cdiv(_p, 128)
  triangular_programs = _B * (_B + 1) // 2
  programs = (
   triangular_programs
   + (row_blocks - _B) * _B
  )
  if panel == panels[-1][0]:
   pair_blocks = triton.cdiv(_p, 248)
   triangular_pair_programs = sum(
    (128 * (row + 1) + 247) // 248
    for row in range(_B)
   )
   pair_programs = (
    triangular_pair_programs
    + (row_blocks - _B) * pair_blocks
   )
   _n32768_middle_pair248_kernel_gpu2[
    (pair_programs, batch)
   ](
    packed,
    scales,
    data,
    output,
    packed,
    n=n,
    batch=batch,
    panel=panel,
    _p=_p,
    _B=_B,
    k_tiles=panel // 256,
    num_warps=8,
    row_layout=pair_layouts[0],
    tensor_layout=pair_layouts[1],
    register_layout=pair_layouts[2],
    scale_layout=pair_layouts[3],
    ROOT_BASE=root_base,
    ROOT_SLOPE=root_slope,
    CACHED_ROOTS=True,
    TRIANGULAR_PAIR_PROGRAMS=triangular_pair_programs,
    maxnreg=128,
    launch_pdl=True,
   )
  else:
   _left_panel_diagonal_solve_packed_fp4_raw_kernel[
    (programs, batch)
   ](
    packed,
    scales,
    data,
    output,
    packed,
    n=n,
    batch=batch,
    panel=panel,
    _p=_p,
    _B=_B,
    k_tiles=panel // 256,
    num_warps=8,
    row_layout=layouts[0],
    tensor_layout=layouts[1],
    register_layout=layouts[2],
    scale_layout=layouts[3],
    ROOT_BASE=root_base,
    ROOT_SLOPE=root_slope,
    CACHED_ROOTS=True,
    maxnreg=128,
    launch_pdl=True,
   )
 tail_start = 14336
 tail_width = n - tail_start
 _B = triton.cdiv(tail_width, 128)
 pair_programs = sum(
  (128 * (row + 1) + 247) // 248
  for row in range(_B)
 )
 pair_layouts = _raw_fp4_pair112_layouts()
 _left_panel_diagonal_solve_packed_fp4_raw_pair112_kernel[
  (pair_programs, batch)
 ](
  packed,
  scales,
  data,
  output,
  packed,
  n=n,
  batch=batch,
  panel=tail_start,
  _p=tail_width,
  _B=_B,
  k_tiles=tail_start // 256,
  num_warps=8,
  row_layout=pair_layouts[0],
  tensor_layout=pair_layouts[1],
  register_layout=pair_layouts[2],
  scale_layout=pair_layouts[3],
  ROOT_BASE=32.0,
  ROOT_SLOPE=-0.024,
  CACHED_ROOTS=False,
  maxnreg=128,
  launch_pdl=True,
 )
 return output

def _persistent_jacobi_n1024_momentum9(data, output):
    batch, n, _ = data.shape
    _enable_tma_allocator(data)
    initial = _empty_output(data)
    flags = _hierarchical_flag_workspace(data)
    if _TRITON_PERSISTENT_WARP_SPECIALIZE_SAFE:
        k_tile = 32
        pipeline_stages = 5
    else:
        k_tile = 64
        pipeline_stages = 3
    _zero_flags_kernel[1,](flags, count=2, BLOCK=2, num_warps=1, num_stages=1, launch_pdl=True)
    _persistent_jacobi_kernel[144,](data, initial, output, initial, output, flags, n=n, batch=batch, BLOCKS=8, ITERATIONS=9, PROGRAMS=144, BLOCK=128, K_TILE=k_tile, PIPELINE_STAGES=pipeline_stages, SCHEDULE=5, FINAL_BLOCK_START=0, HIGH_PRECISION_TAIL=0, USE_PACKED_FP16=False, DIAGONAL_RELAXED=False, LOWER_INITIALIZATION=True, FUSED_TEN_TAIL=True, WARP_SPECIALIZE=_TRITON_PERSISTENT_WARP_SPECIALIZE_SAFE, CROSS_CALL_RELEASE=True, TAIL_UPPER_CLEAR=True, USE_RSQRT_INIT=True, num_warps=8, num_stages=pipeline_stages, launch_pdl=True)
    return output



def _persistent_jacobi_weighted_lower(data, output, *, blocks: int, iterations: int, k_tile: int, pipeline_stages: int, schedule: int, final_block_start: int, penultimate_row_start: int, programs: int=144, num_warps: int=8):
    batch, n, _ = data.shape
    _enable_tma_allocator(data)
    elements = data.numel()
    gram_elements = (elements + 1) // 2
    flag_elements = batch * 32
    backing = data.new_empty((2 * elements + gram_elements + flag_elements,))
    output = backing.narrow(0, 0, elements).view(data.shape)
    scratch = backing.narrow(0, elements, elements).view(data.shape)
    gram_storage = backing.narrow(0, 2 * elements, gram_elements)
    flags = backing.narrow(0, 2 * elements + gram_elements, flag_elements)
    _zero_flags_signal_n2048_backing_kernel[1,](flags, count=2, BLOCK=2, num_warps=1, num_stages=1, launch_pdl=True)
    _persistent_jacobi_n2048_relaxed_kernel[programs,](data, output, scratch, flags, gram_storage, n=n, batch=batch, PROGRAMS=programs, num_warps=num_warps, num_stages=pipeline_stages, launch_pdl=True)
    _persistent_jacobi_n2048_relaxed_tail_kernel[programs,](data, output, scratch, flags, gram_storage, n=n, batch=batch, PROGRAMS=programs, num_warps=num_warps, num_stages=pipeline_stages, launch_pdl=True)
    return output

@triton.jit
def _jacobi_initialize_weighted_sparse_mappings_kernel(output_storage, batch: tl.constexpr, TASKS_0: tl.constexpr, TASKS_14: tl.constexpr, TASKS_21: tl.constexpr, TOTAL_TASKS: tl.constexpr, SPARSE_TOTAL_TASKS: tl.constexpr, SPARSE_TASKS_PER_MATRIX: tl.constexpr, SPARSE_LOCAL_TILES: tl.constexpr, SPARSE_START: tl.constexpr, FIRST_EXTRA_COUNT: tl.constexpr, SECOND_EXTRA_COUNT: tl.constexpr, REPAIR_COUNT: tl.constexpr, BLOCK_SIZE: tl.constexpr):
 axis = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
 valid = axis < TOTAL_TASKS + SPARSE_TOTAL_TASKS
 is_sparse = axis >= TOTAL_TASKS
 mapping_axis = tl.minimum(axis, TOTAL_TASKS - 1)
 second_start: tl.constexpr = TASKS_0
 third_start: tl.constexpr = TASKS_0 + TASKS_14
 table = tl.where(mapping_axis < second_start, 0, tl.where(mapping_axis < third_start, 1, 2))
 linear = tl.where(table == 0, mapping_axis, tl.where(table == 1, mapping_axis - second_start, mapping_axis - third_start))
 tasks = tl.where(table == 0, TASKS_0, tl.where(table == 1, TASKS_14, TASKS_21))
 row_block_start = tl.where(table == 0, 0, tl.where(table == 1, 14, 21))
 program = linear % 144
 slot = linear // 144
 full_layers = tasks // 288
 paired_slots = 2 * full_layers
 remainder_count = tasks - full_layers * 288
 if batch == 1:
  offset_1 = tl.where(table == 0, 48, tl.where(table == 1, 9, 135))
  offset_2 = tl.where(table == 0, 0, tl.where(table == 1, 9, 4))
  offset_3 = tl.where(table == 0, 83, 0)
  offset_4 = 0
  offset_5 = 0
  offset_6 = 0
  offset_7 = 0
 else:
  offset_1 = tl.where(table == 0, 94, tl.where(table == 1, 18, 126))
  offset_2 = tl.where(table == 0, 29, tl.where(table == 1, 3, 0))
  offset_3 = tl.where(table == 0, 88, tl.where(table == 1, 69, 114))
  offset_4 = tl.where(table == 0, 7, tl.where(table == 1, 82, 10))
  offset_5 = tl.where(table == 0, 49, tl.where(table == 1, 87, 0))
  offset_6 = tl.where(table == 0, 1, 0)
  offset_7 = tl.where(table == 0, 2, 0)
 offset = tl.where(slot == 1, offset_1, tl.where(slot == 2, offset_2, tl.where(slot == 3, offset_3, tl.where(slot == 4, offset_4, tl.where(slot == 5, offset_5, tl.where(slot == 6, offset_6, tl.where(slot == 7, offset_7, 0)))))))
 outer_program = (program + offset) % 144
 layer = slot // 2
 outer_rank = tl.where(slot % 2 == 0, layer * 144 + outer_program, tasks - 1 - layer * 144 - outer_program)
 remainder_pair = remainder_count - 144
 low_program = (program + offset) % 144
 safe_pair = tl.maximum(remainder_pair, 1)
 high_program = (program + offset) % safe_pair
 paired_remainder_rank = tl.where(slot == paired_slots, full_layers * 144 + low_program, tasks - 1 - full_layers * 144 - high_program)
 central_program = (program + offset) % remainder_count
 central_remainder_rank = full_layers * 144 + central_program
 remainder_rank = tl.where(remainder_count > 144, paired_remainder_rank, central_remainder_rank)
 rank = tl.where(slot < paired_slots, outer_rank, remainder_rank)
 early_count = batch * row_block_start * (32 - row_block_start)
 late_rank = rank - early_count
 late_span_twice_plus_one = 2 * (32 - row_block_start) + 1
 late_column = ((late_span_twice_plus_one - tl.sqrt(late_span_twice_plus_one * late_span_twice_plus_one - 8.0 * late_rank.to(tl.float32) / batch)) * 0.5).to(tl.int32)
 early_column = rank // (batch * (32 - row_block_start))
 _c = tl.where(rank < early_count, early_column, row_block_start + late_column)
 delta = tl.maximum(_c - row_block_start, 0)
 early_prefix = batch * _c * (32 - row_block_start)
 late_prefix = batch * (row_block_start * (32 - row_block_start) + delta * (32 - row_block_start) - delta * (delta - 1) // 2)
 prefix = tl.where(_c < row_block_start, early_prefix, late_prefix)
 span = 32 - tl.maximum(_c, row_block_start)
 local_rank = rank - prefix
 matrix = local_rank // span
 row_block = tl.maximum(_c, row_block_start) + local_rank - matrix * span
 encoded = matrix << 10 | row_block << 5 | _c
 sparse_axis = tl.maximum(axis - TOTAL_TASKS, 0)
 sparse_matrix = sparse_axis // SPARSE_TASKS_PER_MATRIX
 sparse_linear = sparse_axis - sparse_matrix * SPARSE_TASKS_PER_MATRIX
 sparse_is_local = sparse_linear < SPARSE_LOCAL_TILES
 sparse_local_tile = tl.minimum(sparse_linear, SPARSE_LOCAL_TILES - 1)
 sparse_local_row = ((tl.sqrt(sparse_local_tile.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5).to(tl.int32)
 sparse_local_column = sparse_local_tile - sparse_local_row * (sparse_local_row + 1) // 2
 sparse_extra = tl.maximum(sparse_linear - SPARSE_LOCAL_TILES, 0)
 sparse_is_repair = sparse_extra >= FIRST_EXTRA_COUNT + SECOND_EXTRA_COUNT
 sparse_is_second = (sparse_extra >= FIRST_EXTRA_COUNT) & ~sparse_is_repair
 sparse_first_index = tl.minimum(sparse_extra, FIRST_EXTRA_COUNT - 1)
 sparse_second_index = tl.minimum(tl.maximum(sparse_extra - FIRST_EXTRA_COUNT, 0), SECOND_EXTRA_COUNT - 1)
 sparse_repair_index = tl.minimum(tl.maximum(sparse_extra - FIRST_EXTRA_COUNT - SECOND_EXTRA_COUNT, 0), REPAIR_COUNT - 1)
 sparse_first_column: tl.constexpr = SPARSE_START - 1
 sparse_second_column: tl.constexpr = SPARSE_START - 2
 sparse_first_span: tl.constexpr = 32 - sparse_first_column
 sparse_second_span: tl.constexpr = 32 - sparse_second_column
 sparse_first_row = sparse_first_column + (sparse_first_index * (sparse_first_span - 1) + (FIRST_EXTRA_COUNT - 2) // 2) // (FIRST_EXTRA_COUNT - 1)
 sparse_second_row = sparse_second_column + (sparse_second_index * (sparse_second_span - 1) + (SECOND_EXTRA_COUNT - 2) // 2) // (SECOND_EXTRA_COUNT - 1)
 repair_is_column8 = (sparse_repair_index >= 10) & (sparse_repair_index < 17)
 repair_is_column9 = sparse_repair_index >= 17
 sparse_repair_column = tl.where(repair_is_column9, SPARSE_START - 1, tl.where(repair_is_column8, SPARSE_START - 2, SPARSE_START - 3))
 column7_index = tl.minimum(sparse_repair_index, 9)
 column8_index = tl.minimum(tl.maximum(sparse_repair_index - 10, 0), 6)
 column9_index = tl.maximum(sparse_repair_index - 17, 0)
 column7_row = tl.where(column7_index < 8, 9 + column7_index, tl.where(column7_index == 8, 20, 23))
 column8_row = tl.where(column8_index < 2, 9 + column8_index, tl.where(column8_index < 5, 10 + column8_index, tl.where(column8_index == 5, 16, 26)))
 column9_row = tl.where(column9_index < 4, 10 + 2 * column9_index, 24 + 4 * (column9_index - 4))
 sparse_repair_row = tl.where(repair_is_column9, column9_row, tl.where(repair_is_column8, column8_row, column7_row))
 sparse_row = tl.where(sparse_is_local, sparse_local_row + SPARSE_START, tl.where(sparse_is_repair, sparse_repair_row, tl.where(sparse_is_second, sparse_second_row, sparse_first_row)))
 sparse_column = tl.where(sparse_is_local, sparse_local_column + SPARSE_START, tl.where(sparse_is_repair, sparse_repair_column, tl.where(sparse_is_second, sparse_second_column, sparse_first_column)))
 sparse_encoded = sparse_matrix << 10 | sparse_row << 5 | sparse_column
 encoded = tl.where(is_sparse, sparse_encoded, encoded)
 output_int = output_storage.cast(tl.pointer_type(tl.int32), bitcast=True)
 tl.store(output_int + 1 + axis, encoded, mask=valid)
 tl_cuda.gdc_launch_dependents()

@triton.jit
def _jacobi_hybrid_history_bridge_kernel(source, current, destination, current_packed_storage, older_packed_storage, n: tl.constexpr, batch: tl.constexpr, BLOCKS: tl.constexpr, LOCAL_START: tl.constexpr, LOCAL_TILES: tl.constexpr, EXTRA_COLUMN: tl.constexpr, EXTRA_COUNT: tl.constexpr, REPAIR_COUNT: tl.constexpr, K_TILE: tl.constexpr, HYBRID_K_LIMIT: tl.constexpr, PREFIX_TILE_COUNT: tl.constexpr, BRIDGE_TILES: tl.constexpr, COPY_PREFIX: tl.constexpr, STEP: tl.constexpr, STEP_SLOPE: tl.constexpr, STEP_CURVE: tl.constexpr, STEP_CUBIC: tl.constexpr, DIAGONAL_STEP: tl.constexpr, DIAGONAL_STEP_SLOPE: tl.constexpr, DIAGONAL_STEP_CURVE: tl.constexpr, DIAGONAL_STEP_CUBIC: tl.constexpr, PIPELINE_STAGES: tl.constexpr):
 tl_cuda.gdc_wait()
 tl_cuda.gdc_launch_dependents()
 tile = tl.program_id(0)
 is_extra = tile >= LOCAL_TILES
 local_tile = tl.minimum(tile, LOCAL_TILES - 1)
 local_row = ((tl.sqrt(local_tile.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5).to(tl.int32)
 local_column = local_tile - local_row * (local_row + 1) // 2
 extra = tile - LOCAL_TILES
 extra_span: tl.constexpr = BLOCKS - 1 - EXTRA_COLUMN
 base_extra = tl.minimum(extra, EXTRA_COUNT - 1)
 extra_row = EXTRA_COLUMN + (base_extra * extra_span + (EXTRA_COUNT - 2) // 2) // (EXTRA_COUNT - 1)
 repair = tl.minimum(tl.maximum(extra - EXTRA_COUNT, 0), REPAIR_COUNT - 1)
 repair_is_column14 = (repair >= 1) & (repair < 6)
 repair_is_column15 = (repair >= 6) & (repair < 12)
 repair_is_column16 = repair >= 12
 repair_column = tl.where(repair_is_column16, 16, tl.where(repair_is_column15, 15, tl.where(repair_is_column14, 14, 13)))
 repair_row = tl.where(repair_is_column16, repair + 5, tl.where(repair_is_column15, repair + 9, tl.where(repair_is_column14, repair + 13, 16)))
 is_repair = is_extra & (extra >= EXTRA_COUNT)
 row_block = tl.where(is_repair, repair_row, tl.where(is_extra, extra_row, local_row + LOCAL_START))
 _c = tl.where(is_repair, repair_column, tl.where(is_extra, EXTRA_COLUMN, local_column + LOCAL_START))
 block: tl.constexpr = 128
 row_start = row_block * block
 _s = _c * block
 matrix = tl.program_id(1)
 _m = matrix * n
 base = matrix * n * n
 packed = current_packed_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
 older_packed = older_packed_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
 left_descriptor = tl.make_tensor_descriptor(packed, shape=[batch * n, n], strides=[n, 1], block_shape=[block, K_TILE])
 right_descriptor = tl.make_tensor_descriptor(packed, shape=[batch * n, n], strides=[n, 1], block_shape=[block, K_TILE])
 older_left_descriptor = tl.make_tensor_descriptor(older_packed, shape=[batch * n, n], strides=[n, 1], block_shape=[block, K_TILE])
 older_right_descriptor = tl.make_tensor_descriptor(older_packed, shape=[batch * n, n], strides=[n, 1], block_shape=[block, K_TILE])
 _a = tl.zeros((block, block), tl.float32)
 for reduction_start in tl.range(0, HYBRID_K_LIMIT, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=True):
  left = older_left_descriptor.load([_m + row_start, reduction_start])
  right = older_right_descriptor.load([_m + _s, reduction_start])
  _a += tl.dot(left, tl.trans(right))
 for reduction_start in tl.range(HYBRID_K_LIMIT, _s + block, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=True):
  left = left_descriptor.load([_m + row_start, reduction_start])
  right = right_descriptor.load([_m + _s, reduction_start])
  _a += tl.dot(left, tl.trans(right))
 axis = tl.arange(0, block)
 rows = row_start + axis
 columns = _s + axis
 target = tl.load(source + base + rows[:, None] * n + columns[None, :], cache_modifier='.cg')
 previous = tl.load(packed + base + rows[:, None] * n + columns[None, :]).to(tl.float32)
 diagonal = tl.load(packed + base + columns * n + columns).to(tl.float32)
 quantized_previous = previous.to(tl.float16).to(tl.float32)
 quantized_diagonal = diagonal.to(tl.float16).to(tl.float32)
 _a += previous * diagonal[None, :] - quantized_previous * quantized_diagonal[None, :]
 residual = target - _a
 fraction = (columns.to(tl.float32) + 0.5) / n
 centered = fraction - 0.5
 quadratic = centered * centered - 1.0 / 12.0
 cubic = centered * quadratic
 local_step = STEP + STEP_SLOPE * fraction + STEP_CURVE * quadratic + STEP_CUBIC * cubic
 diagonal_step = tl.minimum(0.5, tl.maximum(-0.2, DIAGONAL_STEP + DIAGONAL_STEP_SLOPE * fraction + DIAGONAL_STEP_CURVE * quadratic + DIAGONAL_STEP_CUBIC * cubic))
 updated = previous + local_step * residual / diagonal[None, :]
 updated = tl.where(rows[:, None] == columns[None, :], tl.sqrt(tl.maximum(previous * previous + diagonal_step * residual, 1.1754943508222875e-38)), updated)
 tl.store(destination + base + rows[:, None] * n + columns[None, :], updated, mask=rows[:, None] >= columns[None, :])
 if COPY_PREFIX:
  for copy_pass in tl.static_range(0, 2):
   copy_tile = tile + copy_pass * BRIDGE_TILES
   copy_column = ((65.0 - tl.sqrt(65.0 * 65.0 - 8.0 * copy_tile.to(tl.float32))) * 0.5).to(tl.int32)
   copy_prefix = copy_column * (65 - copy_column) // 2
   copy_row = copy_column + copy_tile - copy_prefix
   copy_axis = tl.arange(0, block)
   copy_rows = copy_row * block + copy_axis
   copy_columns = copy_column * block + copy_axis
   copy_valid = (copy_tile < PREFIX_TILE_COUNT) & (copy_rows[:, None] >= copy_columns[None, :])
   copy_values = tl.load(older_packed + base + copy_rows[:, None] * n + copy_columns[None, :], mask=copy_valid, other=0.0)
   tl.store(destination + base + copy_rows[:, None] * n + copy_columns[None, :], copy_values.to(tl.float32), mask=copy_valid)

@triton.jit
def _jacobi_cholesky_iteration_weighted_tma_kernel(source, current, destination, duplicate_destination, current_packed_storage, destination_packed_storage, mapping_storage, n: tl.constexpr, batch: tl.constexpr, TASKS: tl.constexpr, PROGRAMS: tl.constexpr, MAPPING_OFFSET: tl.constexpr, STEP: tl.constexpr, STEP_SLOPE: tl.constexpr, STEP_CURVE: tl.constexpr, STEP_CUBIC: tl.constexpr, DIAGONAL_STEP: tl.constexpr, DIAGONAL_STEP_SLOPE: tl.constexpr, DIAGONAL_STEP_CURVE: tl.constexpr, DIAGONAL_STEP_CUBIC: tl.constexpr, PACKED_FP16_STATE_INPUT: tl.constexpr, STORE_FP32_OUTPUT: tl.constexpr, DUPLICATE_FP32_OUTPUT: tl.constexpr, DUPLICATE_BRIDGE_START: tl.constexpr, DUPLICATE_EXTRA_COUNT: tl.constexpr, DUPLICATE_REPAIR_COUNT: tl.constexpr, PIPELINE_STAGES: tl.constexpr):
 tl_cuda.gdc_wait()
 program = tl.program_id(0)
 packed_input = current_packed_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
 packed_output = destination_packed_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
 mapping = mapping_storage.cast(tl.pointer_type(tl.int32), bitcast=True)
 input_descriptor = tl.make_tensor_descriptor(packed_input, shape=[batch * n, n], strides=[n, 1], block_shape=[128, 64])
 for linear in tl.range(program, TASKS, PROGRAMS):
  encoded = tl.load(mapping + MAPPING_OFFSET + linear)
  matrix = encoded >> 10
  row_block = encoded >> 5 & 31
  _c = encoded & 31
  row_start = row_block * 128
  _s = _c * 128
  _m = matrix * n
  base = matrix * n * n
  _a = tl.zeros((128, 128), tl.float32)
  for reduction_start in tl.range(0, _s + 128, 64, num_stages=PIPELINE_STAGES, warp_specialize=True):
   left = input_descriptor.load([_m + row_start, reduction_start])
   right = input_descriptor.load([_m + _s, reduction_start])
   _a += tl.dot(left, tl.trans(right))
  local = tl.arange(0, 128)
  rows = row_start + local
  columns = _s + local
  target = tl.load(source + base + rows[:, None] * n + columns[None, :], cache_modifier='.cg')
  if PACKED_FP16_STATE_INPUT:
   previous = tl.load(packed_input + base + rows[:, None] * n + columns[None, :]).to(tl.float32)
   diagonal = tl.load(packed_input + base + columns * n + columns).to(tl.float32)
  else:
   previous = tl.load(current + base + rows[:, None] * n + columns[None, :], cache_modifier='.cg')
   diagonal = tl.load(current + base + columns * n + columns)
  quantized_previous = previous.to(tl.float16).to(tl.float32)
  quantized_diagonal = diagonal.to(tl.float16).to(tl.float32)
  _a += previous * diagonal[None, :] - quantized_previous * quantized_diagonal[None, :]
  residual = target - _a
  fraction = (columns.to(tl.float32) + 0.5) / n
  centered = fraction - 0.5
  quadratic = centered * centered - 1.0 / 12.0
  cubic = centered * quadratic
  local_step = STEP + STEP_SLOPE * fraction + STEP_CURVE * quadratic + STEP_CUBIC * cubic
  diagonal_step = tl.minimum(1.4, tl.maximum(-1.0, DIAGONAL_STEP + DIAGONAL_STEP_SLOPE * fraction + DIAGONAL_STEP_CURVE * quadratic + DIAGONAL_STEP_CUBIC * cubic))
  updated = previous + local_step * residual / diagonal[None, :]
  updated = tl.where(rows[:, None] == columns[None, :], tl.sqrt(tl.maximum(previous * previous + diagonal_step * residual, 1.1754943508222875e-38)), updated)
  valid = rows[:, None] >= columns[None, :]
  if STORE_FP32_OUTPUT:
   tl.store(destination + base + rows[:, None] * n + columns[None, :], updated, mask=valid & (_c < 0))
   if DUPLICATE_FP32_OUTPUT:
    bridge_local = _c >= DUPLICATE_BRIDGE_START
    bridge_extra = _c < 0
    extra_column: tl.constexpr = DUPLICATE_BRIDGE_START - 1
    extra_span: tl.constexpr = 31 - extra_column
    for extra_index in tl.static_range(0, DUPLICATE_EXTRA_COUNT):
     extra_row = extra_column + (extra_index * extra_span + (DUPLICATE_EXTRA_COUNT - 2) // 2) // (DUPLICATE_EXTRA_COUNT - 1)
     bridge_extra |= (_c == extra_column) & (row_block == extra_row)
    if DUPLICATE_BRIDGE_START == 17 and DUPLICATE_REPAIR_COUNT == 15:
     bridge_extra |= (_c == 13) & (row_block == 16)
     bridge_extra |= (_c == 14) & (row_block >= 14) & (row_block <= 18)
     bridge_extra |= (_c == 15) & (row_block >= 15) & (row_block <= 20)
     bridge_extra |= (_c == 16) & (row_block >= 17) & (row_block <= 19)
    tl.store(duplicate_destination + base + rows[:, None] * n + columns[None, :], updated, mask=valid & ~(bridge_local | bridge_extra))
  tl.store(packed_output + base + rows[:, None] * n + columns[None, :], tl.where(valid, updated, 0.0).to(tl.float16))
 tl_cuda.gdc_launch_dependents()

@triton.jit
def _zero_upper_resident_pdl_kernel(output, n: tl.constexpr, triangular_tiles: tl.constexpr, total_tasks: tl.constexpr, TILE: tl.constexpr, PROGRAMS: tl.constexpr):
 tl_cuda.gdc_launch_dependents()
 program = tl.program_id(0)
 for task in tl.range(program, total_tasks, PROGRAMS):
  matrix = task // triangular_tiles
  triangular = task - matrix * triangular_tiles
  _c = ((tl.sqrt(8.0 * triangular + 1.0) - 1.0) * 0.5).to(tl.int32)
  row_block = triangular - _c * (_c + 1) // 2
  rows = row_block * TILE + tl.arange(0, TILE)
  columns = _c * TILE + tl.arange(0, TILE)
  valid = (rows[:, None] < n) & (columns[None, :] < n) & (columns[None, :] > rows[:, None])
  offsets = matrix * n * n + rows[:, None] * n + columns[None, :]
  tl.store(output + offsets, 0.0, mask=valid)

@triton.jit
def _jacobi_final_sparse_release_unpack_kernel(source, current, destination, duplicate_destination, current_packed_storage, destination_packed_storage, mapping_storage, n: tl.constexpr, batch: tl.constexpr, TASKS: tl.constexpr, PROGRAMS: tl.constexpr, MAPPING_OFFSET: tl.constexpr, STEP: tl.constexpr, STEP_SLOPE: tl.constexpr, STEP_CURVE: tl.constexpr, STEP_CUBIC: tl.constexpr, DIAGONAL_STEP: tl.constexpr, DIAGONAL_STEP_SLOPE: tl.constexpr, DIAGONAL_STEP_CURVE: tl.constexpr, DIAGONAL_STEP_CUBIC: tl.constexpr, PACKED_FP16_STATE_INPUT: tl.constexpr, STORE_FP32_OUTPUT: tl.constexpr, DUPLICATE_FP32_OUTPUT: tl.constexpr, DUPLICATE_BRIDGE_START: tl.constexpr, DUPLICATE_EXTRA_COUNT: tl.constexpr, DUPLICATE_REPAIR_COUNT: tl.constexpr, PIPELINE_STAGES: tl.constexpr):
 tl_cuda.gdc_wait()
 tl_cuda.gdc_launch_dependents()
 program = tl.program_id(0)
 packed_input = current_packed_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
 packed_output = destination_packed_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
 mapping = mapping_storage.cast(tl.pointer_type(tl.int32), bitcast=True)
 input_descriptor = tl.make_tensor_descriptor(packed_input, shape=[batch * n, n], strides=[n, 1], block_shape=[128, 64])
 for linear in tl.range(program, TASKS, PROGRAMS):
  encoded = tl.load(mapping + MAPPING_OFFSET + linear)
  matrix = encoded >> 10
  row_block = encoded >> 5 & 31
  _c = encoded & 31
  row_start = row_block * 128
  _s = _c * 128
  _m = matrix * n
  base = matrix * n * n
  _a = tl.zeros((128, 128), tl.float32)
  for reduction_start in tl.range(0, _s + 128, 64, num_stages=PIPELINE_STAGES, warp_specialize=True):
   left = input_descriptor.load([_m + row_start, reduction_start])
   right = input_descriptor.load([_m + _s, reduction_start])
   _a += tl.dot(left, tl.trans(right))
  local = tl.arange(0, 128)
  rows = row_start + local
  columns = _s + local
  target = tl.load(source + base + rows[:, None] * n + columns[None, :], cache_modifier='.cg')
  if PACKED_FP16_STATE_INPUT:
   previous = tl.load(packed_input + base + rows[:, None] * n + columns[None, :]).to(tl.float32)
   diagonal = tl.load(packed_input + base + columns * n + columns).to(tl.float32)
  else:
   previous = tl.load(current + base + rows[:, None] * n + columns[None, :], cache_modifier='.cg')
   diagonal = tl.load(current + base + columns * n + columns)
  quantized_previous = previous.to(tl.float16).to(tl.float32)
  quantized_diagonal = diagonal.to(tl.float16).to(tl.float32)
  _a += previous * diagonal[None, :] - quantized_previous * quantized_diagonal[None, :]
  residual = target - _a
  fraction = (columns.to(tl.float32) + 0.5) / n
  centered = fraction - 0.5
  quadratic = centered * centered - 1.0 / 12.0
  cubic = centered * quadratic
  local_step = STEP + STEP_SLOPE * fraction + STEP_CURVE * quadratic + STEP_CUBIC * cubic
  diagonal_step = tl.minimum(1.4, tl.maximum(-1.0, DIAGONAL_STEP + DIAGONAL_STEP_SLOPE * fraction + DIAGONAL_STEP_CURVE * quadratic + DIAGONAL_STEP_CUBIC * cubic))
  updated = previous + local_step * residual / diagonal[None, :]
  updated = tl.where(rows[:, None] == columns[None, :], tl.sqrt(tl.maximum(previous * previous + diagonal_step * residual, 1.1754943508222875e-38)), updated)
  valid = rows[:, None] >= columns[None, :]
  if STORE_FP32_OUTPUT:
   tl.store(destination + base + rows[:, None] * n + columns[None, :], updated, mask=valid & (_c < 0))
   if DUPLICATE_FP32_OUTPUT:
    bridge_local = _c >= DUPLICATE_BRIDGE_START
    bridge_extra = _c < 0
    extra_column: tl.constexpr = DUPLICATE_BRIDGE_START - 1
    extra_span: tl.constexpr = 31 - extra_column
    for extra_index in tl.static_range(0, DUPLICATE_EXTRA_COUNT):
     extra_row = extra_column + (extra_index * extra_span + (DUPLICATE_EXTRA_COUNT - 2) // 2) // (DUPLICATE_EXTRA_COUNT - 1)
     bridge_extra |= (_c == extra_column) & (row_block == extra_row)
    if DUPLICATE_BRIDGE_START == 17 and DUPLICATE_REPAIR_COUNT == 15:
     bridge_extra |= (_c == 13) & (row_block == 16)
     bridge_extra |= (_c == 14) & (row_block >= 14) & (row_block <= 18)
     bridge_extra |= (_c == 15) & (row_block >= 15) & (row_block <= 20)
     bridge_extra |= (_c == 16) & (row_block >= 17) & (row_block <= 19)
    tl.store(duplicate_destination + base + rows[:, None] * n + columns[None, :], updated, mask=valid & ~(bridge_local | bridge_extra))
  tl.store(packed_output + base + rows[:, None] * n + columns[None, :], tl.where(valid, updated, 0.0).to(tl.float16))



@triton.jit
def _unpack_packed_prefix_bitmask_terminal_n4096_b2_kernel(source_storage, destination, original_data, n: tl.constexpr, PREFIX_TILES: tl.constexpr, ROWS: tl.constexpr, SPARSE_START: tl.constexpr, FIRST_EXTRA_COUNT: tl.constexpr, SECOND_EXTRA_COUNT: tl.constexpr):
 row_parts: tl.constexpr = 128 // ROWS
 program = tl.program_id(0)
 tile = program // row_parts
 row_part = program - tile * row_parts
 matrix = tl.program_id(1)
 _c = ((65.0 - tl.sqrt(65.0 * 65.0 - 8.0 * tile.to(tl.float32))) * 0.5).to(tl.int32)
 prefix = _c * (65 - _c) // 2
 row_block = _c + tile - prefix
 row_axis = tl.arange(0, ROWS)
 column_axis = tl.arange(0, 128)
 rows = row_block * 128 + row_part * ROWS + row_axis
 columns = _c * 128 + column_axis
 base = matrix * n * n
 source = source_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
 valid = rows[:, None] >= columns[None, :]
 if FIRST_EXTRA_COUNT > 0:
  first_repair = (_c == SPARSE_START - 1) & (-1146356224 >> row_block & 1 != 0)
  second_repair = (_c == SPARSE_START - 2) & (-1792672000 >> row_block & 1 != 0)
  third_repair = (_c == SPARSE_START - 3) & (9567744 >> row_block & 1 != 0)
  valid &= ~(first_repair | second_repair | third_repair)
 values = tl.load(source + base + rows[:, None] * n + columns[None, :], mask=valid, other=0.0)
 tl.store(destination + base + rows[:, None] * n + columns[None, :], values.to(tl.float32), mask=valid)
 probe0 = tl.load(original_data + base + 1, cache_modifier='.cg')
 probe1 = tl.load(original_data + base + n + 2, cache_modifier='.cg')
 probe2 = tl.load(original_data + base + n // 2 * n + n // 2 + 1, cache_modifier='.cg')
 probe3 = tl.load(original_data + base + (n - 2) * n + n - 1, cache_modifier='.cg')
 diagonal_family = (probe0 == 0.0) & (probe1 == 0.0) & (probe2 == 0.0) & (probe3 == 0.0)
 normal_diagonal = diagonal_family & (row_block == _c)
 normal_values = tl.load(original_data + base + rows * n + rows, mask=normal_diagonal, other=1.0, cache_modifier='.cg')
 tl.store(destination + base + rows * n + rows, tl.sqrt(tl.maximum(normal_values, 1.1754943508222875e-38)), mask=normal_diagonal)
 repair_parts: tl.constexpr = (32 - SPARSE_START) * row_parts
 repair_valid = diagonal_family & (program < repair_parts)
 repair_block = SPARSE_START + program // row_parts
 repair_part = program - (repair_block - SPARSE_START) * row_parts
 repair_rows = repair_block * 128 + repair_part * ROWS + row_axis
 repair_values = tl.load(original_data + base + repair_rows * n + repair_rows, mask=repair_valid, other=1.0, cache_modifier='.cg')
 tl.store(destination + base + repair_rows * n + repair_rows, tl.sqrt(tl.maximum(repair_values, 1.1754943508222875e-38)), mask=repair_valid)

@triton.jit
def _jacobi_initialize_packed_e4_lower_kernel_gpu0(source, packed_storage, root_storage, n: tl.constexpr, BLOCK: tl.constexpr, SCALE: tl.constexpr, ROW_NORMALIZED: tl.constexpr):
 tl_cuda.gdc_wait()
 tile = tl.program_id(0)
 matrix = tl.program_id(1)
 row_block = ((tl.sqrt(tile.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5).to(tl.int32)
 _c = tile - row_block * (row_block + 1) // 2
 axis = tl.arange(0, BLOCK)
 rows = row_block * BLOCK + axis
 columns = _c * BLOCK + axis
 base = matrix * n * n
 diagonal = tl.load(source + base + columns * n + columns, cache_modifier='.cg')
 roots = tl.sqrt(tl.maximum(diagonal, 1.1754943508222875e-38))
 values = tl.load(source + base + rows[:, None] * n + columns[None, :], cache_modifier='.cg')
 initialized = tl.where(rows[:, None] == columns[None, :], roots[None, :], tl.where(rows[:, None] > columns[None, :], values / roots[None, :], 0.0))
 if ROW_NORMALIZED:
  row_diagonal = tl.load(source + base + rows * n + rows, cache_modifier='.cg')
  row_roots = tl.sqrt(tl.maximum(row_diagonal, 1.1754943508222875e-38))
  initialized /= row_roots[:, None]
  root_output = root_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
  tl.store(root_output + matrix * n + rows, row_roots, mask=row_block == _c)
 packed = packed_storage.cast(tl.pointer_type(tl.float8e4nv), bitcast=True)
 tl.store(packed + base + rows[:, None] * n + columns[None, :], (initialized * SCALE).to(tl.float8e4nv))
 tl_cuda.gdc_launch_dependents()

@gluon.jit
def _build_shared_e4_descriptor_pair_n4096_gpu0(tensor_map_storage, history0, history1, n: gl.constexpr, batch: gl.constexpr):
 descriptor = gl.allocate_shared_memory(gl.int64, [16], mbarrier.MBarrierLayout())
 sentinel = descriptor.slice(15, 1)
 gl.inline_asm_elementwise(_IA8, '=r,l,l,r,r,r,l', [history0, tensor_map_storage, n, batch * n, n, history1], dtype=gl.int32, is_pure=False, pack=1)
 mbarrier.init(sentinel, count=1)
 mbarrier.invalidate(sentinel)

@gluon.jit
def _weighted_e4_half_epilogue_n4096_gpu0(product, source, packed_input_storage, packed_output_storage, root_storage, n: gl.constexpr, base, row_start, _s, step: gl.constexpr, step_slope: gl.constexpr, step_curve: gl.constexpr, step_cubic: gl.constexpr, diagonal_step_base: gl.constexpr, diagonal_step_slope: gl.constexpr, diagonal_step_curve: gl.constexpr, diagonal_step_cubic: gl.constexpr, row_layout: gl.constexpr, store_e4_output: gl.constexpr, history_scale: gl.constexpr, row_normalized: gl.constexpr, rescale_output: gl.constexpr):
 rows = row_start + gl.arange(0, 128, layout=gl.SliceLayout(1, row_layout))
 columns = _s + gl.arange(0, 64, layout=gl.SliceLayout(0, row_layout))
 offsets = base + rows[:, None] * n + columns[None, :]
 packed_input = packed_input_storage.cast(gl.pointer_type(gl.float8e4nv), bitcast=True)
 target = gl.load(source + offsets)
 if row_normalized:
  root_input = root_storage.cast(gl.pointer_type(gl.float16), bitcast=True)
  row_roots = gl.load(root_input + rows).to(gl.float32)
  column_roots = gl.load(root_input + columns).to(gl.float32)
  target /= row_roots[:, None] * column_roots[None, :]
 inverse_scale: gl.constexpr = 1.0 / history_scale
 previous = gl.load(packed_input + offsets).to(gl.float32) * inverse_scale
 diagonal = gl.load(packed_input + base + columns * n + columns).to(gl.float32) * inverse_scale
 residual = target - product * (inverse_scale * inverse_scale)
 fraction = (columns.to(gl.float32) + 0.5) / n
 centered = fraction - 0.5
 quadratic = centered * centered - 1.0 / 12.0
 cubic = centered * quadratic
 local_step = step + step_slope * fraction + step_curve * quadratic + step_cubic * cubic
 diagonal_step = gl.minimum(1.4, gl.maximum(-1.0, diagonal_step_base + diagonal_step_slope * fraction + diagonal_step_curve * quadratic + diagonal_step_cubic * cubic))
 updated = previous + local_step[None, :] * residual / diagonal[None, :]
 updated = gl.where(rows[:, None] == columns[None, :], gl.sqrt(gl.maximum(previous * previous + diagonal_step[None, :] * residual, 1.1754943508222875e-38)), updated)
 valid = rows[:, None] >= columns[None, :]
 updated = gl.where(valid, updated, 0.0)
 if store_e4_output:
  packed_output = packed_output_storage.cast(gl.pointer_type(gl.float8e4nv), bitcast=True)
  gl.store(packed_output + offsets, (updated * history_scale).to(gl.float8e4nv))
 else:
  if rescale_output:
   root_input = root_storage.cast(gl.pointer_type(gl.float16), bitcast=True)
   row_roots = gl.load(root_input + rows).to(gl.float32)
   updated *= row_roots[:, None]
  packed_output = packed_output_storage.cast(gl.pointer_type(gl.float16), bitcast=True)
  gl.store(packed_output + offsets, updated.to(gl.float16))

@gluon.jit
def _raw_e4_manual_direct_wave_n4096_gpu0(history_storage, tensor_map_storage, source, packed_output_storage, root_storage, n: gl.constexpr, batch: gl.constexpr, step: gl.constexpr, step_slope: gl.constexpr, step_curve: gl.constexpr, step_cubic: gl.constexpr, diagonal_step_base: gl.constexpr, diagonal_step_slope: gl.constexpr, diagonal_step_curve: gl.constexpr, diagonal_step_cubic: gl.constexpr, row_layout: gl.constexpr, tensor_layout: gl.constexpr, register_layout: gl.constexpr, descriptor_select: gl.constexpr, store_e4_output: gl.constexpr, history_scale: gl.constexpr, row_normalized: gl.constexpr, rescale_output: gl.constexpr):
 program = gl.program_id(0)
 owner = program % 152
 slot = program // 152
 if batch == 1:
  task = gl.where(slot == 0, owner, gl.where(slot == 1, 527 - owner, gl.where(slot == 2, 152 + owner, 375 - owner)))
  _c = ((65.0 - gl.sqrt(4225.0 - 8.0 * task.to(gl.float32))) * 0.5).to(gl.int32)
  prefix = _c * (65 - _c) // 2
  local = task - prefix
  matrix = 0
  row_block = _c + local
 else:
  offset = gl.where(slot == 1, 8, gl.where(slot == 3, 67, gl.where(slot == 5, 130, gl.where(slot == 6, 59, 0))))
  paired_owner = (owner + offset) % 152
  layer = slot // 2
  outer_rank = gl.where(slot % 2 == 0, layer * 152 + paired_owner, 1055 - layer * 152 - paired_owner)
  remainder_rank = 456 + (owner + offset) % 144
  task = gl.where(slot < 6, outer_rank, remainder_rank)
  _c = ((65.0 - gl.sqrt(4225.0 - 4.0 * task.to(gl.float32))) * 0.5).to(gl.int32)
  diagonal_span = 32 - _c
  prefix = _c * (65 - _c)
  local = task - prefix
  matrix = local // diagonal_span
  row_block = _c + local - matrix * diagonal_span
 row_start = row_block * 128
 _s = _c * 128
 _m = matrix * n
 base = matrix * n * n
 k_tiles = _c + 1
 a_stage0 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 b_stage0 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 a_stage1 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 b_stage1 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 a_stage2 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 b_stage2 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 barriers = gl.allocate_shared_memory(gl.int64, [8], mbarrier.MBarrierLayout())
 sentinel0 = a_stage0.slice(2047, 1)
 sentinel1 = b_stage0.slice(2047, 1)
 sentinel2 = a_stage1.slice(2047, 1)
 sentinel3 = b_stage1.slice(2047, 1)
 sentinel4 = a_stage2.slice(2047, 1)
 sentinel5 = b_stage2.slice(2047, 1)
 sentinel6 = barriers.slice(7, 1)
 _a = allocate_tensor_memory(gl.float32, [128, 128], tensor_layout)
 gl.inline_asm_elementwise(_IA9, '=r,l,r,r,r,r', [tensor_map_storage, k_tiles, _m + row_start, _m + _s, descriptor_select], dtype=gl.int32, is_pure=False, pack=1)
 mbarrier.init(sentinel0, count=1)
 mbarrier.init(sentinel1, count=1)
 mbarrier.init(sentinel2, count=1)
 mbarrier.init(sentinel3, count=1)
 mbarrier.init(sentinel4, count=1)
 mbarrier.init(sentinel5, count=1)
 mbarrier.init(sentinel6, count=1)
 mbarrier.invalidate(sentinel0)
 mbarrier.invalidate(sentinel1)
 mbarrier.invalidate(sentinel2)
 mbarrier.invalidate(sentinel3)
 mbarrier.invalidate(sentinel4)
 mbarrier.invalidate(sentinel5)
 mbarrier.invalidate(sentinel6)
 for column_part in gl.static_range(0, 2):
  product = gl.convert_layout(_a.slice(column_part * 64, 64).load(register_layout), row_layout)
  _weighted_e4_half_epilogue_n4096_gpu0(product, source, history_storage, packed_output_storage, root_storage, n, base, row_start, _s + column_part * 64, step, step_slope, step_curve, step_cubic, diagonal_step_base, diagonal_step_slope, diagonal_step_curve, diagonal_step_cubic, row_layout, store_e4_output, history_scale, row_normalized, rescale_output)
 gl.inline_asm_elementwise('griddepcontrol.launch_dependents; mov.u32 $0, 0;', '=r', [], dtype=gl.int32, is_pure=False, pack=1)

@gluon.jit
def _weighted_half_epilogue(product, source, packed_input, packed_output, duplicate_destination, n: gl.constexpr, base, row_start, _s, row_block, _c, step: gl.constexpr, step_slope: gl.constexpr, step_curve: gl.constexpr, step_cubic: gl.constexpr, diagonal_step_base: gl.constexpr, diagonal_step_slope: gl.constexpr, diagonal_step_curve: gl.constexpr, diagonal_step_cubic: gl.constexpr, row_layout: gl.constexpr, final_output: gl.constexpr, duplicate_bridge_start: gl.constexpr, duplicate_extra_count: gl.constexpr):
 rows = row_start + gl.arange(0, 128, layout=gl.SliceLayout(1, row_layout))
 columns = _s + gl.arange(0, 64, layout=gl.SliceLayout(0, row_layout))
 offsets = base + rows[:, None] * n + columns[None, :]
 target = gl.load(source + offsets)
 previous = gl.load(packed_input + offsets).to(gl.float32)
 diagonal = gl.load(packed_input + base + columns * n + columns).to(gl.float32)
 quantized_previous = previous.to(gl.float16).to(gl.float32)
 quantized_diagonal = diagonal.to(gl.float16).to(gl.float32)
 product += previous * diagonal[None, :] - quantized_previous * quantized_diagonal[None, :]
 residual = target - product
 fraction = (columns.to(gl.float32) + 0.5) / n
 centered = fraction - 0.5
 quadratic = centered * centered - 1.0 / 12.0
 cubic = centered * quadratic
 local_step = step + step_slope * fraction + step_curve * quadratic + step_cubic * cubic
 diagonal_step = gl.minimum(1.4, gl.maximum(-1.0, diagonal_step_base + diagonal_step_slope * fraction + diagonal_step_curve * quadratic + diagonal_step_cubic * cubic))
 updated = previous + local_step[None, :] * residual / diagonal[None, :]
 updated = gl.where(rows[:, None] == columns[None, :], gl.sqrt(gl.maximum(previous * previous + diagonal_step[None, :] * residual, 1.1754943508222875e-38)), updated)
 valid = rows[:, None] >= columns[None, :]
 gl.store(packed_output + offsets, gl.where(valid, updated, 0.0).to(gl.float16))
 if final_output:
  bridge_local = _c >= duplicate_bridge_start
  bridge_extra = _c < 0
  extra_column: gl.constexpr = duplicate_bridge_start - 1
  extra_span: gl.constexpr = 31 - extra_column
  for extra_index in gl.static_range(0, duplicate_extra_count):
   extra_row = extra_column + (extra_index * extra_span + (duplicate_extra_count - 2) // 2) // (duplicate_extra_count - 1)
   bridge_extra = bridge_extra | (_c == extra_column) & (row_block == extra_row)
  gl.store(duplicate_destination + offsets, updated, mask=valid & ~(bridge_local | bridge_extra))

@gluon.jit
def _build_shared_fp16_descriptor_pair(tensor_map_storage, history0, history1, n: gl.constexpr, batch: gl.constexpr):
 descriptor = gl.allocate_shared_memory(gl.int64, [16], mbarrier.MBarrierLayout())
 sentinel = descriptor.slice(15, 1)
 gl.inline_asm_elementwise(_IA10, '=r,l,l,r,r,r,l', [history0, tensor_map_storage, n, batch * n, 2 * n, history1], dtype=gl.int32, is_pure=False, pack=1)
 mbarrier.init(sentinel, count=1)
 mbarrier.invalidate(sentinel)

@gluon.jit
def _raw_fp16_manual_direct_wave(history_storage, tensor_map_storage, source, duplicate_destination, packed_output_storage, mapping_storage, n: gl.constexpr, batch: gl.constexpr, mapping_offset: gl.constexpr, step: gl.constexpr, step_slope: gl.constexpr, step_curve: gl.constexpr, step_cubic: gl.constexpr, diagonal_step_base: gl.constexpr, diagonal_step_slope: gl.constexpr, diagonal_step_curve: gl.constexpr, diagonal_step_cubic: gl.constexpr, row_layout: gl.constexpr, tensor_layout: gl.constexpr, register_layout: gl.constexpr, final_output: gl.constexpr, duplicate_bridge_start: gl.constexpr, duplicate_extra_count: gl.constexpr, task_stride: gl.constexpr, task_chunk: gl.constexpr, chunk_stride: gl.constexpr, build_descriptor_pair: gl.constexpr, descriptor_select: gl.constexpr, owner_programs: gl.constexpr):
 program = gl.program_id(0)
 if owner_programs == 152:
  owner = program % 152
  slot = program // 152
  offset = gl.where(slot == 1, 8, gl.where(slot == 3, 67, gl.where(slot == 5, 130, gl.where(slot == 6, 59, 0))))
  paired_owner = (owner + offset) % 152
  layer = slot // 2
  outer_rank = gl.where(slot % 2 == 0, layer * 152 + paired_owner, 1055 - layer * 152 - paired_owner)
  remainder_rank = 456 + (owner + offset) % 144
  task_rank = gl.where(slot < 6, outer_rank, remainder_rank)
 else:
  owner = program % 148
  slot = program // 148
  offset = gl.where(slot == 1, 128, gl.where(slot == 3, 122, gl.where(slot == 5, 60, 0)))
  paired_owner = (owner + offset) % 148
  layer = slot // 2
  outer_rank = gl.where(slot % 2 == 0, layer * 148 + paired_owner, 1055 - layer * 148 - paired_owner)
  remainder_rank = gl.where(slot == 6, 444 + owner, 611 - owner)
  task_rank = gl.where(slot < 6, outer_rank, remainder_rank)
 _c = ((65.0 - gl.sqrt(4225.0 - 4.0 * task_rank.to(gl.float32))) * 0.5).to(gl.int32)
 diagonal_span = 32 - _c
 prefix = _c * (65 - _c)
 local = task_rank - prefix
 matrix = local // diagonal_span
 row_block = _c + local - matrix * diagonal_span
 task = program
 packed_input = history_storage.cast(gl.pointer_type(gl.float16), bitcast=True)
 packed_output = packed_output_storage.cast(gl.pointer_type(gl.float16), bitcast=True)
 tensor_maps = tensor_map_storage.cast(gl.pointer_type(gl.float32), bitcast=True)
 row_start = row_block * 128
 _s = _c * 128
 _m = matrix * n
 base = matrix * n * n
 k_tiles = 2 * (_c + 1)
 a_stage0 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 b_stage0 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 a_stage1 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 b_stage1 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 a_stage2 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 b_stage2 = gl.allocate_shared_memory(gl.int64, [2048], mbarrier.MBarrierLayout())
 barriers = gl.allocate_shared_memory(gl.int64, [8], mbarrier.MBarrierLayout())
 sentinel0 = a_stage0.slice(2047, 1)
 sentinel1 = b_stage0.slice(2047, 1)
 sentinel2 = a_stage1.slice(2047, 1)
 sentinel3 = b_stage1.slice(2047, 1)
 sentinel4 = a_stage2.slice(2047, 1)
 sentinel5 = b_stage2.slice(2047, 1)
 sentinel6 = barriers.slice(7, 1)
 _a = allocate_tensor_memory(gl.float32, [128, 128], tensor_layout)
 gl.inline_asm_elementwise(_IA11, '=r,l,l,r,r,r,r,r,r,r,r,r,l', [history_storage, tensor_maps, k_tiles, _m + row_start, _m + _s, batch * n, n, 2 * n, task, build_descriptor_pair, descriptor_select, packed_output_storage], dtype=gl.int32, is_pure=False, pack=1)
 mbarrier.init(sentinel0, count=1)
 mbarrier.init(sentinel1, count=1)
 mbarrier.init(sentinel2, count=1)
 mbarrier.init(sentinel3, count=1)
 mbarrier.init(sentinel4, count=1)
 mbarrier.init(sentinel5, count=1)
 mbarrier.init(sentinel6, count=1)
 mbarrier.invalidate(sentinel0)
 mbarrier.invalidate(sentinel1)
 mbarrier.invalidate(sentinel2)
 mbarrier.invalidate(sentinel3)
 mbarrier.invalidate(sentinel4)
 mbarrier.invalidate(sentinel5)
 mbarrier.invalidate(sentinel6)
 for column_part in gl.static_range(0, 2):
  product = gl.convert_layout(_a.slice(column_part * 64, 64).load(register_layout), row_layout)
  _weighted_half_epilogue(product, source, packed_input, packed_output, duplicate_destination, n, base, row_start, _s + column_part * 64, row_block, _c, step, step_slope, step_curve, step_cubic, diagonal_step_base, diagonal_step_slope, diagonal_step_curve, diagonal_step_cubic, row_layout, final_output, duplicate_bridge_start, duplicate_extra_count)
_N4096_MANUAL_ROW_LAYOUT = gl.BlockedLayout([1, 2], [1, 32], [8, 1], [1, 0])
_N4096_MANUAL_TENSOR_LAYOUT = TensorMemoryLayout((128, 128), col_stride=1)
_N4096_MANUAL_ACCUMULATOR_LAYOUT = TensorMemoryLayout((128, 64), col_stride=1)
_N4096_MANUAL_REGISTER_LAYOUT = get_tmem_reg_layout(gl.float32, (128, 64), _N4096_MANUAL_ACCUMULATOR_LAYOUT, 8, alloc_shape=(128, 128))

def _late_packed_jacobi_sparse_tail10_bridge(data, output):
 batch, n, _ = data.shape
 single_matrix = batch.bit_length() == 1
 _enable_tma_allocator(data)
 if single_matrix:
  bridge_destination = output
  output = _empty_output(data)
  scratch = _empty_output(data)
  current_packed = _packed_fp16_output_workspace(data)
  destination_packed = _secondary_fp16_output_workspace(data)
  current_e4 = _packed_fp8_output_workspace(data)
  destination_e4 = _secondary_fp8_output_workspace(data)
  tensor_maps = _packed_fp4_scale_workspace(data)
 else:
  elements = data.numel()
  fp16_elements = (elements + 1) // 2
  e4_elements = (elements + 3) // 4
  map_elements = (elements + 127) // 128
  backing = data.new_empty((3 * elements + 2 * fp16_elements + 2 * e4_elements + map_elements,))
  bridge_destination = backing.narrow(0, 0, elements).view(data.shape)
  output = backing.narrow(0, elements, elements).view(data.shape)
  scratch = backing.narrow(0, 2 * elements, elements).view(data.shape)
  packed_offset = 3 * elements
  current_packed = backing.narrow(0, packed_offset, fp16_elements)
  destination_packed = backing.narrow(0, packed_offset + fp16_elements, fp16_elements)
  e4_offset = packed_offset + 2 * fp16_elements
  current_e4 = backing.narrow(0, e4_offset, e4_elements)
  destination_e4 = backing.narrow(0, e4_offset + e4_elements, e4_elements)
  tensor_maps = backing.narrow(0, e4_offset + 2 * e4_elements, map_elements)
 bridge_tile = 64
 bridge_tiles = triton.cdiv(n, bridge_tile)
 bridge_triangular_tiles = bridge_tiles * (bridge_tiles + 1) // 2
 bridge_programs = 128
 _zero_upper_resident_pdl_kernel[bridge_programs,](bridge_destination, n=n, triangular_tiles=bridge_triangular_tiles, total_tasks=batch * bridge_triangular_tiles, TILE=bridge_tile, PROGRAMS=bridge_programs, num_warps=4, num_stages=1, launch_pdl=True)
 block = 128
 blocks = triton.cdiv(n, block)
 full_tiles = blocks * (blocks + 1) // 2
 e4_scale = 64.0 if single_matrix else 64.0
 task_count_0 = batch * full_tiles
 task_count_14 = batch * (full_tiles - 14 * (14 + 1) // 2)
 task_count_21 = batch * (full_tiles - 21 * (21 + 1) // 2)
 total_mapping_tasks = task_count_0 + task_count_14 + task_count_21
 sparse_start = 10
 sparse_active = blocks - sparse_start
 sparse_local_tiles = sparse_active * (sparse_active + 1) // 2
 first_extra_count = 12
 second_extra_count = 8
 repair_count = 23
 sparse_tasks_per_matrix = sparse_local_tiles + first_extra_count + second_extra_count + repair_count
 sparse_total_tasks = batch * sparse_tasks_per_matrix
 mapping_block = 256
 _jacobi_initialize_weighted_sparse_mappings_kernel[triton.cdiv(total_mapping_tasks + sparse_total_tasks, mapping_block),](output, batch=batch, TASKS_0=task_count_0, TASKS_14=task_count_14, TASKS_21=task_count_21, TOTAL_TASKS=total_mapping_tasks, SPARSE_TOTAL_TASKS=sparse_total_tasks, SPARSE_TASKS_PER_MATRIX=sparse_tasks_per_matrix, SPARSE_LOCAL_TILES=sparse_local_tiles, SPARSE_START=sparse_start, FIRST_EXTRA_COUNT=first_extra_count, SECOND_EXTRA_COUNT=second_extra_count, REPAIR_COUNT=repair_count, BLOCK_SIZE=mapping_block, num_warps=8, num_stages=1, launch_pdl=True)
 _jacobi_initialize_packed_e4_lower_kernel_gpu0[full_tiles, batch](data, current_e4, destination_packed, n=n, BLOCK=block, SCALE=e4_scale, ROW_NORMALIZED=single_matrix, num_warps=8, num_stages=1, launch_pdl=True)
 current = output
 destination = scratch
 steps = (0.9642525911331177, 0.935393750667572, 0.7697351574897766, 0.8718098402023315, 0.8970257043838501)
 slopes = (-1.0, -0.36283865571022034, 0.4463522434234619, 0.08861520141363144, 0.01049625501036644)
 curves = (0.48093143105506897, 0.4801524877548218, 0.26031267642974854, -0.5310379266738892, -0.7444061040878296)
 cubics = (-0.39388638734817505, -0.23576711118221283, 0.006051782518625259, 0.7318341732025146, -0.3823258578777313)
 diagonal_steps = (-0.19887028634548187, -0.0183014664798975, -0.15817323327064514, -0.08789250999689102, -0.02444486878812313)
 diagonal_slopes = (0.4270329475402832, 0.800000011920929, 0.7203459739685059, 0.4060589373111725, 0.11838330328464508)
 diagonal_curves = (-0.23485064506530762, 0.19224725663661957, 0.6812770962715149, 0.7777166962623596, 0.3092901408672333)
 diagonal_cubics = (0.5874711871147156, 0.33612462878227234, -0.2732192575931549, 0.259537011384964, 0.3706797659397125)
 final_start = 17
 extra_tiles = 4
 bridge_repair_tiles = 15 if single_matrix else 5
 _build_shared_e4_descriptor_pair_n4096_gpu0[1,](tensor_maps, current_e4, destination_e4, n=n, batch=batch, num_warps=1, maxnreg=32, launch_pdl=False)
 fp16_tensor_maps = tensor_maps[64:]
 _build_shared_fp16_descriptor_pair[1,](fp16_tensor_maps, current_packed, destination_packed, n=n, batch=batch, num_warps=1, maxnreg=32, launch_pdl=False)
 start_iteration = 0
 if True:
  _raw_e4_manual_direct_wave_n4096_gpu0[task_count_0,](current_e4, tensor_maps, data, destination_e4, destination_packed, n=n, batch=batch, step=steps[0], step_slope=slopes[0], step_curve=curves[0], step_cubic=cubics[0], diagonal_step_base=diagonal_steps[0], diagonal_step_slope=diagonal_slopes[0], diagonal_step_curve=diagonal_curves[0], diagonal_step_cubic=diagonal_cubics[0], row_layout=_N4096_MANUAL_ROW_LAYOUT, tensor_layout=_N4096_MANUAL_TENSOR_LAYOUT, register_layout=_N4096_MANUAL_REGISTER_LAYOUT, descriptor_select=0, store_e4_output=True, history_scale=e4_scale, row_normalized=single_matrix, rescale_output=False, num_warps=8, maxnreg=128, launch_pdl=True)
  _raw_e4_manual_direct_wave_n4096_gpu0[task_count_0,](destination_e4, tensor_maps, data, current_packed, destination_packed, n=n, batch=batch, step=steps[1], step_slope=slopes[1], step_curve=curves[1], step_cubic=cubics[1], diagonal_step_base=diagonal_steps[1], diagonal_step_slope=diagonal_slopes[1], diagonal_step_curve=diagonal_curves[1], diagonal_step_cubic=diagonal_cubics[1], row_layout=_N4096_MANUAL_ROW_LAYOUT, tensor_layout=_N4096_MANUAL_TENSOR_LAYOUT, register_layout=_N4096_MANUAL_REGISTER_LAYOUT, descriptor_select=1, store_e4_output=False, history_scale=e4_scale, row_normalized=single_matrix, rescale_output=single_matrix, num_warps=8, maxnreg=128, launch_pdl=True)
  start_iteration = 2
 for iteration in range(start_iteration, 5):
  final_iteration = iteration == 4
  if not final_iteration and (not single_matrix):
   _raw_fp16_manual_direct_wave[task_count_0,](current_packed, fp16_tensor_maps, data, bridge_destination, destination_packed, output, n=n, batch=batch, mapping_offset=1, step=steps[iteration], step_slope=slopes[iteration], step_curve=curves[iteration], step_cubic=cubics[iteration], diagonal_step_base=diagonal_steps[iteration], diagonal_step_slope=diagonal_slopes[iteration], diagonal_step_curve=diagonal_curves[iteration], diagonal_step_cubic=diagonal_cubics[iteration], row_layout=_N4096_MANUAL_ROW_LAYOUT, tensor_layout=_N4096_MANUAL_TENSOR_LAYOUT, register_layout=_N4096_MANUAL_REGISTER_LAYOUT, final_output=False, duplicate_bridge_start=final_start, duplicate_extra_count=extra_tiles, task_stride=1, task_chunk=0, chunk_stride=1, build_descriptor_pair=False, descriptor_select=iteration & 1, owner_programs=152, num_warps=8, maxnreg=128, launch_pdl=False)
  else:
   wave_kernel = _jacobi_final_sparse_release_unpack_kernel if final_iteration and single_matrix else _jacobi_cholesky_iteration_weighted_tma_kernel
   wave_programs = 140 if final_iteration and single_matrix else 152
   wave_kernel[wave_programs,](data, current, destination, bridge_destination, current_packed, destination_packed, output, n=n, batch=batch, TASKS=sparse_total_tasks if final_iteration else task_count_0, PROGRAMS=wave_programs, MAPPING_OFFSET=1 + total_mapping_tasks if final_iteration else 1, STEP=steps[iteration], STEP_SLOPE=slopes[iteration], STEP_CURVE=curves[iteration], STEP_CUBIC=cubics[iteration], DIAGONAL_STEP=diagonal_steps[iteration], DIAGONAL_STEP_SLOPE=diagonal_slopes[iteration], DIAGONAL_STEP_CURVE=diagonal_curves[iteration], DIAGONAL_STEP_CUBIC=diagonal_cubics[iteration], PACKED_FP16_STATE_INPUT=True, STORE_FP32_OUTPUT=final_iteration, DUPLICATE_FP32_OUTPUT=final_iteration, DUPLICATE_BRIDGE_START=final_start, DUPLICATE_EXTRA_COUNT=extra_tiles, DUPLICATE_REPAIR_COUNT=bridge_repair_tiles, PIPELINE_STAGES=6, num_warps=8, num_stages=6, launch_pdl=True)
  current, destination = (destination, current)
  current_packed, destination_packed = (destination_packed, current_packed)
 active_blocks = blocks - final_start
 final_tiles = active_blocks * (active_blocks + 1) // 2
 bridge_tiles = final_tiles + extra_tiles + bridge_repair_tiles
 prefix_tiles = full_tiles - sparse_local_tiles
 if single_matrix:
  _unpack_packed_prefix_bitmask_terminal_n4096_b2_kernel[prefix_tiles * 8, batch](destination_packed, bridge_destination, data, n=n, PREFIX_TILES=prefix_tiles, ROWS=16, SPARSE_START=sparse_start, FIRST_EXTRA_COUNT=first_extra_count, SECOND_EXTRA_COUNT=second_extra_count, num_warps=2, num_stages=1, launch_pdl=True)
  _jacobi_hybrid_history_bridge_kernel[bridge_tiles, batch](data, current, bridge_destination, current_packed, destination_packed, n=n, batch=batch, BLOCKS=blocks, LOCAL_START=final_start, LOCAL_TILES=final_tiles, EXTRA_COLUMN=final_start - 1, EXTRA_COUNT=extra_tiles, REPAIR_COUNT=bridge_repair_tiles, K_TILE=64, HYBRID_K_LIMIT=sparse_start * block, PREFIX_TILE_COUNT=prefix_tiles, BRIDGE_TILES=bridge_tiles, COPY_PREFIX=False, STEP=0.8756452798843384, STEP_SLOPE=-0.10085078328847885, STEP_CURVE=-0.10680907964706421, STEP_CUBIC=-0.06300850957632065, DIAGONAL_STEP=-0.004988174419850111, DIAGONAL_STEP_SLOPE=0.0466826930642128, DIAGONAL_STEP_CURVE=0.2366684079170227, DIAGONAL_STEP_CUBIC=0.4470316469669342, PIPELINE_STAGES=6, num_warps=8, num_stages=6, launch_pdl=True)
 else:
  _jacobi_hybrid_history_bridge_persistent_128_kernel[136,](data,current,bridge_destination,current_packed,destination_packed,n=n,batch=batch,BLOCKS=blocks,LOCAL_START=final_start,LOCAL_TILES=final_tiles,EXTRA_COLUMN=final_start-1,EXTRA_COUNT=extra_tiles,REPAIR_COUNT=bridge_repair_tiles,K_TILE=64,HYBRID_K_LIMIT=sparse_start*block,PREFIX_TILE_COUNT=prefix_tiles,BRIDGE_TILES=bridge_tiles,COPY_PREFIX=False,STEP=0.8756452798843384,STEP_SLOPE=-0.10085078328847885,STEP_CURVE=-0.10680907964706421,STEP_CUBIC=-0.06300850957632065,DIAGONAL_STEP=-0.004988174419850111,DIAGONAL_STEP_SLOPE=0.0466826930642128,DIAGONAL_STEP_CURVE=0.2366684079170227,DIAGONAL_STEP_CUBIC=0.4470316469669342,PIPELINE_STAGES=6,PROGRAMS=136,num_warps=8,num_stages=6,launch_pdl=True)
  _unpack_packed_prefix_bitmask_terminal_n4096_b2_kernel[prefix_tiles * 16, batch](destination_packed, bridge_destination, data, n=n, PREFIX_TILES=prefix_tiles, ROWS=8, SPARSE_START=sparse_start, FIRST_EXTRA_COUNT=first_extra_count, SECOND_EXTRA_COUNT=second_extra_count, num_warps=1, num_stages=1, launch_pdl=False)
 return bridge_destination

def _jacobi_cholesky(data, output, *, iterations: int, step: float, iteration_steps: tuple[float, ...] | None=None, alternating_step: float=0.0, momentum: float=0.0, nesterov: bool=False, precondition_order: int=0, precondition_last_only: bool=False, initial_block_size: int=0, skip_initialization: bool=False, use_bf16: bool=False, use_fp8: bool=False, fp8_early_iterations: int=0, use_fp16: bool=False, fp16_early_iterations: int=0, use_packed_fp8: bool=False, packed_fp8_early_iterations: int=0, use_packed_fp16: bool=False, packed_fp16_early_iterations: int=0, packed_fp16_state_iterations: int=0, use_tma: bool=False, block_size: int=128, reduction_stride: int=1, control_variate: bool=False, control_scale: float=1.0, complement_sample: bool=False, high_precision: bool=False, freeze_diagonal: bool=False, high_precision_last: bool=False, high_precision_tail_last: int=0, tma_warp_specialize: bool=True, tma_pipeline_stages: int=3, tma_num_warps: int=4, tma_k_tile: int=64, tma_last_pipeline_stages: int=0, tma_last_k_tile: int=0, pointer_k_tile: int=32, pointer_pipeline_stages: int=5, pointer_num_warps: int=8, final_block_start: int=0, copy_before_partial: bool=True, penultimate_row_start: int=0, tail_row_waves: int=1, earlier_tail_row_start: int=0):
 batch, n, _ = data.shape
 if use_tma:
  _enable_tma_allocator(data)
 scratch = _jacobi_workspace(data)
 if use_packed_fp16:
  current_packed = _packed_fp16_output_workspace(data)
  destination_packed = _secondary_fp16_output_workspace(data)
 elif use_packed_fp8:
  current_packed = _packed_fp8_output_workspace(data)
  destination_packed = _secondary_fp8_output_workspace(data)
 else:
  current_packed = output
  destination_packed = output
 block = block_size
 blocks = triton.cdiv(n, block)
 iteration_grid = (blocks * (blocks + 1) // 2, batch)
 if skip_initialization:
  pass
 elif initial_block_size:
  initial_blocks = triton.cdiv(n, initial_block_size)
  _jacobi_block_potrf_initialize_kernel[initial_blocks, batch](data, output, current_packed, n=n, TILE=initial_block_size, WRITE_PACKED=use_packed_fp16, num_warps=8, num_stages=1, launch_pdl=True)
  _jacobi_block_trsm_initialize_kernel[initial_blocks * (initial_blocks - 1) // 2, batch](data, output, current_packed, n=n, TILE=initial_block_size, WRITE_PACKED=use_packed_fp16, num_warps=8, num_stages=1, launch_pdl=True)
 else:
  initialization_grid = (blocks * blocks, batch)
  _jacobi_initialize_kernel[initialization_grid](data, output, current_packed, n=n, blocks=blocks, BLOCK=block, WRITE_PACKED=use_packed_fp8 or use_packed_fp16, PACKED_FP16=use_packed_fp16, num_warps=8, num_stages=1, launch_pdl=True)
 current = output
 destination = scratch
 for iteration in range(iterations):
  final_partial = use_tma and final_block_start > 0 and (iteration + 1 == iterations)
  penultimate_partial = use_tma and penultimate_row_start > 0 and (iteration + 1 < iterations) and (iteration + 1 >= iterations - tail_row_waves)
  iteration_block_start = final_block_start if final_partial else (earlier_tail_row_start if earlier_tail_row_start > 0 and iteration + 2 < iterations else penultimate_row_start) if penultimate_partial else 0
  if iteration_block_start and copy_before_partial:
   total = data.numel()
   copy_block = 256
   _initialize_lower_kernel[triton.cdiv(total, copy_block),](current, destination, total=total, n=n, BLOCK=copy_block, num_warps=8, num_stages=1)
  if final_partial:
   active_blocks = blocks - iteration_block_start
   current_iteration_grid = (active_blocks * (active_blocks + 1) // 2, batch)
  elif penultimate_partial:
   full_tiles = blocks * (blocks + 1) // 2
   skipped_tiles = iteration_block_start * (iteration_block_start + 1) // 2
   current_iteration_grid = (full_tiles - skipped_tiles, batch)
  else:
   current_iteration_grid = iteration_grid
  if iteration_steps is not None:
   iteration_step = iteration_steps[iteration]
  else:
   iteration_step = alternating_step if alternating_step > 0.0 and iteration % 2 else step
  iteration_high_precision = high_precision or (high_precision_last and iteration + 1 == iterations)
  if use_tma:
   iteration_tma_k_tile = tma_last_k_tile if tma_last_k_tile > 0 and iteration + 1 == iterations else tma_k_tile
   iteration_tma_pipeline_stages = tma_last_pipeline_stages if tma_last_pipeline_stages > 0 and iteration + 1 == iterations else tma_pipeline_stages
   _jacobi_cholesky_iteration_tma_kernel[current_iteration_grid](data, current, destination, current_packed, destination_packed, n=n, batch=batch, BLOCK_START=iteration_block_start, ROW_START_ONLY=penultimate_partial, BLOCK=block, K_TILE=iteration_tma_k_tile, STEP=iteration_step, MOMENTUM=momentum, APPLY_MOMENTUM=momentum != 0.0 and iteration > 0, NESTEROV=nesterov, PRECONDITION_ORDER=precondition_order if not precondition_last_only or iteration + 1 == iterations else 0, USE_FP8=use_fp8 or iteration < fp8_early_iterations, USE_PACKED_FP8=use_packed_fp8 and (packed_fp8_early_iterations == 0 or iteration < packed_fp8_early_iterations), USE_PACKED_FP16=use_packed_fp16 and (packed_fp16_early_iterations == 0 or iteration < packed_fp16_early_iterations), STORE_PACKED_OUTPUT=iteration + 1 < iterations and (use_packed_fp8 and (packed_fp8_early_iterations == 0 or iteration + 1 < packed_fp8_early_iterations) or (use_packed_fp16 and (packed_fp16_early_iterations == 0 or iteration + 1 < packed_fp16_early_iterations))), PACKED_FP16_STATE_INPUT=use_packed_fp16 and iteration < packed_fp16_state_iterations, STORE_FP32_OUTPUT=not use_packed_fp16 or iteration + 1 >= packed_fp16_state_iterations, USE_FP16=(use_fp16 or iteration < fp16_early_iterations) and (not iteration_high_precision), HIGH_PRECISION=iteration_high_precision, FREEZE_DIAGONAL=freeze_diagonal, HIGH_PRECISION_TAIL=high_precision_tail_last if iteration + 1 == iterations else 0, REDUCTION_STRIDE=reduction_stride, REDUCTION_OFFSET=iteration % reduction_stride, CONTROL_VARIATE=control_variate, CONTROL_SCALE=control_scale, COMPLEMENT_SAMPLE=complement_sample, WARP_SPECIALIZE=tma_warp_specialize and (not iteration_high_precision) and _TRITON_TMA_WARP_SPECIALIZE_SAFE, PIPELINE_STAGES=iteration_tma_pipeline_stages, num_warps=tma_num_warps, num_stages=iteration_tma_pipeline_stages, launch_pdl=True)
  else:
   _jacobi_cholesky_iteration_kernel[iteration_grid](data, current, destination, n=n, blocks=blocks, BLOCK=block, K_TILE=pointer_k_tile, STEP=iteration_step, MOMENTUM=momentum, APPLY_MOMENTUM=momentum != 0.0 and iteration > 0, USE_BF16=use_bf16, USE_FP8=use_fp8, HIGH_PRECISION=high_precision or (high_precision_last and iteration + 1 == iterations), FREEZE_DIAGONAL=freeze_diagonal, PIPELINE_STAGES=pointer_pipeline_stages, num_warps=pointer_num_warps, num_stages=pointer_pipeline_stages, launch_pdl=True)
  current, destination = (destination, current)
  current_packed, destination_packed = (destination_packed, current_packed)
 return current

def _diagonal32_superpanel_cholesky(data, output, *, use_packed_fp8: bool, programs: int=112, owner_programs: int=32, block_size: int=32, _p: int=512, wave_fp8: bool=False, pivot_fraction: float=0.05):
 """Block-32 incomplete Cholesky with 512-column fused wavefronts."""
 batch, n, _ = data.shape
 _enable_tma_allocator(data)
 packed_fp8 = _packed_fp8_output_workspace(data) if use_packed_fp8 else None
 flags = _hierarchical_flag_workspace(data)
 _zero_flags_kernel[1,](flags, count=flags.numel(), BLOCK=256, num_warps=4, num_stages=1)
 panel_blocks = _p // block_size
 dot_k = 32 if block_size == 32 else 16
 for panel in range(0, n, _p):
  row_blocks = triton.cdiv(n - panel, 128)
  _B = triton.cdiv(_p, 128)
  grid = (row_blocks * _B, batch)
  if use_packed_fp8:
   _left_panel_packed_fp8_tma_kernel[grid](data, packed_fp8, output, flags, n=n, batch=batch, panel=panel, _p=_p, _B=_B, ROW_TILE=128, COLUMN_TILE=128, K_TILE=64, WARP_SPECIALIZE=True, PIPELINE_STAGES=3, DIAGONAL_SHIFT=0.0, INVERSE_PACK_SCALE_SQUARED=1.0 / 4096.0, K_STRIDE=1, K_SCALE=1.0, RESET_FLAGS=False, num_warps=4, num_stages=3)
  else:
   _left_panel_tma_kernel[grid](data, output, flags, n=n, batch=batch, panel=panel, _p=_p, _B=_B, ROW_TILE=128, COLUMN_TILE=128, K_TILE=64, WARP_SPECIALIZE=True, PIPELINE_STAGES=3, USE_BF16=False, USE_FP8=False, HIGH_PRECISION=False, BF16_TAIL_WINDOW=0, FP8_TAIL_WINDOW=0, HISTORY_WINDOW=0, HISTORY_CORRECTION=0.0, HISTORY_DAMPING=0.0, FUSE_UPPER_ZERO=False, RESET_FLAGS=False, num_warps=4, num_stages=3)
  _diagonal32_superpanel_wavefront_kernel[programs,](data, output, packed_fp8 if packed_fp8 is not None else output, flags, n=n, panel=panel, phase_base=panel // _p * panel_blocks, PROGRAMS=programs, OWNER_PROGRAMS=owner_programs, ROWS=64, PANEL_BLOCKS=panel_blocks, BLOCK_SIZE=block_size, DOT_K=dot_k, WAVE_FP8=wave_fp8, DIAGONAL_SCALE=1.0, PIVOT_FRACTION=pivot_fraction, num_warps=4, num_stages=1)
  if use_packed_fp8 and (not wave_fp8):
   _pack_panel_fp8_kernel[grid](output, packed_fp8, n=n, panel=panel, _p=_p, _B=_B, ROW_TILE=128, COLUMN_TILE=128, PACK_SCALE=64.0, num_warps=8, num_stages=1)
 total = data.numel()
 _zero_upper_kernel[triton.cdiv(total, 256),](output, total=total, n=n, BLOCK=256, num_warps=8, num_stages=1)
 return output








def _n8192_state_elided_cholesky_gpu0(data, output):
 _enable_tma_allocator(data)
 q0 = _packed_fp8_output_workspace(data)
 q1 = _secondary_fp8_output_workspace(data)
 q2 = _tertiary_fp8_output_workspace(data)
 q4 = _packed_fp4_output_workspace(data)
 route_storage = _packed_bf16_output_workspace(data)
 flags = _hierarchical_flag_workspace(data)
 precombined = _secondary_fp16_output_workspace(data)
 map_storage = _packed_fp4_scale_workspace(data)
 map_words = (2080 + 1) * 32
 maps1 = map_storage[map_words : 2 * map_words]
 maps2 = map_storage[2 * map_words : 3 * map_words]
 roots = map_storage[3 * map_words : 3 * map_words + 8192]
 _pack_q0_mixed_single_gpu0[(2080,)](
  data, q0, q4, map_storage, roots, route_storage, flags,
  n=8192, block=128, num_warps=8, num_stages=1, launch_pdl=False,
 )
 common = dict(
  n=8192, batch=1, panel=0, panel_width=8192, column_blocks=64,
  k_tiles=0, num_warps=8, row_layout=_N8192_RAW_LAYOUTS[0],
  tensor_layout=_N8192_RAW_LAYOUTS[1],
  register_layout=_N8192_RAW_LAYOUTS[2],
  chunk_register_layout=_N8192_CHUNK_REGISTER_LAYOUT,
  scale_layout=_N8192_RAW_LAYOUTS[3], ROOT_BASE=1.0, ROOT_SLOPE=0.0,
  COLUMN_MAJOR=True, maxnreg=128, launch_pdl=False,
 )
 _causal_raw_e4_mixed_single_g0_gpu0[(2080, 1)](
  q0, map_storage, data, roots, q1, q4,
  TASK_OFFSET=0, EARLY_RELEASE=False, **common,
 )
 later = dict(common)
 later["chunk_register_layout"] = _N8192_G1_W64_CHUNK_REGISTER_LAYOUT
 later["tensor_layout"] = _N8192_G12_TENSOR_LAYOUT
 _causal_raw_e4_g1_tuned_gpu0[(2080, 1)](
  q1, maps1, data, roots, precombined, q2,
  TASK_OFFSET=0, EARLY_RELEASE=False, **later,
 )
 _causal_raw_e4_g2_tuned_gpu0[(2080, 1)](
  q2, maps2, data, roots, output, q2, precombined,
  SAFE_TASKS=2080, **later,
 )
 return output


@triton.jit
def _first_two_trsm_second_owner_kernel(
 source,
 output,
 inverse_storage,
 history_counts,
 n: tl.constexpr,
 inverse_blocks: tl.constexpr,
 panel,
 WAIT_HISTORY: tl.constexpr,
 PANEL0_OVERLAP: tl.constexpr=False,
):
 matrix = tl.program_id(0)
 base = matrix * n * n
 axis = tl.arange(0, 64)
 rows = axis[:, None]
 columns = axis[None, :]
 lower = rows >= columns
 inverse_base0 = (
  matrix * inverse_blocks * 64 * 64
  + panel // 64 * 64 * 64
 )
 ready_pointer = inverse_storage + inverse_base0 + 1
 if WAIT_HISTORY:
  count_stride: tl.constexpr = n // 256
  history_counter = (
   history_counts
   + matrix * count_stride
   + panel // 256
  )
  expected_history = 2 * ((n - panel) // 128) - 1
  if PANEL0_OVERLAP:
   expected_history -= 1
  if not PANEL0_OVERLAP:
   history_ready = tl.atomic_add(
    history_counter, 0.0, sem="acquire", scope="gpu"
   )
   while history_ready < expected_history:
    history_ready = tl.atomic_add(
     history_counter, 0.0, sem="acquire", scope="gpu"
    )
   tl.atomic_xchg(
    history_counter, 0.0, sem="release", scope="gpu"
   )
 else:
  tl.atomic_xchg(ready_pointer, 0.0)

 if PANEL0_OVERLAP:
  lower0 = tl.load(
   source + base + (panel + rows) * n + panel + columns,
   mask=lower,
   other=0.0,
  )
 else:
  lower0 = tl.load(
   output + base + (panel + rows) * n + panel + columns,
   mask=lower,
   other=0.0,
  )
 factor0, inverse0 = _factor_inverse64_pdl(
  lower0, False, False, 1.0, False, True
 )
 tl.store(
  output + base + (panel + rows) * n + panel + columns,
  tl.where(lower, factor0, 0.0),
 )
 tl.store(
  inverse_storage + inverse_base0 + rows * 64 + columns,
  inverse0,
  mask=lower,
 )
 block_rows = panel + 64 + axis
 rhs10_pointer = source if PANEL0_OVERLAP else output
 rhs10 = tl.load(
  rhs10_pointer
  + base
  + block_rows[:, None] * n
  + panel
  + axis[None, :]
 )
 factor10 = tl.dot(
  rhs10.to(tl.float16),
  tl.trans(inverse0.to(tl.float16)),
 )
 tl.store(
  output
  + base
  + block_rows[:, None] * n
  + panel
  + axis[None, :],
  factor10,
 )
 tl.store(
  output
  + base
  + (panel + axis)[None, :] * n
  + block_rows[:, None],
  0.0,
 )
 diagonal11_pointer = source if PANEL0_OVERLAP else output
 diagonal11 = tl.load(
  diagonal11_pointer
  + base
  + (panel + 64 + rows) * n
  + panel
  + 64
  + columns,
  mask=lower,
  other=0.0,
 )
 diagonal11 -= tl.where(
  lower,
  tl.dot(
   factor10.to(tl.float16),
   tl.trans(factor10.to(tl.float16)),
  ),
  0.0,
 )
 factor1, inverse1 = _factor_inverse64_pdl(
  diagonal11, False, False, 1.0, False, True
 )
 tl.store(
  output
  + base
  + (panel + 64 + rows) * n
  + panel
  + 64
  + columns,
  tl.where(lower, factor1, 0.0),
 )
 tl.store(
  inverse_storage
  + inverse_base0
  + 64 * 64
  + rows * 64
  + columns,
  inverse1,
  mask=lower,
 )

 if PANEL0_OVERLAP:
  history_ready = tl.atomic_add(
   history_counter, 0.0, sem="acquire", scope="gpu"
  )
  while history_ready < expected_history:
   history_ready = tl.atomic_add(
    history_counter, 0.0, sem="acquire", scope="gpu"
   )
  tl.atomic_xchg(
   history_counter, 0.0, sem="release", scope="gpu"
  )
 if not PANEL0_OVERLAP:
  tl_cuda.gdc_launch_dependents()

 row_axis = tl.arange(0, 128)
 solve_rows = panel + 128 + row_axis
 rhs0 = tl.load(
  output
  + base
  + solve_rows[:, None] * n
  + panel
  + axis[None, :]
 )
 solution0 = _jacobi_solve_dot(
  rhs0, tl.trans(inverse0), 1
 )
 rhs1 = tl.load(
  output
  + base
  + solve_rows[:, None] * n
  + panel
  + 64
  + axis[None, :]
 )
 rhs1 -= _jacobi_solve_dot(
  solution0, tl.trans(factor10), 1
 )
 solution1 = _jacobi_solve_dot(
  rhs1, tl.trans(inverse1), 1
 )
 tl.store(
  output
  + base
  + solve_rows[:, None] * n
  + panel
  + axis[None, :],
  solution0,
 )
 tl.store(
  output
  + base
  + solve_rows[:, None] * n
  + panel
  + 64
  + axis[None, :],
  solution1,
 )
 tl.atomic_xchg(ready_pointer, 1.0, sem="release", scope="gpu")
 tl.debug_barrier()

 second_panel = panel + 128
 inverse_base1 = (
  matrix * inverse_blocks * 64 * 64
  + second_panel // 64 * 64 * 64
 )
 history_axis = tl.arange(0, 128)
 history0 = tl.load(
  output
  + base
  + (second_panel + rows) * n
  + panel
  + history_axis[None, :]
 )
 lower0 = tl.load(
  output
  + base
  + (second_panel + rows) * n
  + second_panel
  + columns,
  mask=lower,
  other=0.0,
 )
 lower0 -= tl.where(
  lower,
  tl.dot(
   history0.to(tl.bfloat16),
   tl.trans(history0.to(tl.bfloat16)),
  ),
  0.0,
 )
 second_factor0, second_inverse0 = _factor_inverse64_pdl(
  lower0,
  False,
  False,
  1.0,
  False,
  True,
 )
 tl.store(
  output
  + base
  + (second_panel + rows) * n
  + second_panel
  + columns,
  tl.where(lower, second_factor0, 0.0),
 )
 tl.store(
  inverse_storage
  + inverse_base1
  + rows * 64
  + columns,
  second_inverse0,
  mask=lower,
 )
 if PANEL0_OVERLAP:
  tl_cuda.gdc_launch_dependents()
 second_rows = second_panel + 64 + axis
 second_rhs = tl.load(
  output
  + base
  + second_rows[:, None] * n
  + second_panel
  + axis[None, :]
 )
 history1 = tl.load(
  output
  + base
  + second_rows[:, None] * n
  + panel
  + history_axis[None, :]
 )
 second_rhs -= tl.dot(
  history1.to(tl.bfloat16),
  tl.trans(history0.to(tl.bfloat16)),
 )
 second_factor10 = tl.dot(
  second_rhs.to(tl.float16),
  tl.trans(second_inverse0.to(tl.float16)),
 )
 tl.store(
  output
  + base
  + second_rows[:, None] * n
  + second_panel
  + axis[None, :],
  second_factor10,
 )
 tl.store(
  output
  + base
  + (second_panel + axis)[None, :] * n
  + second_rows[:, None],
  0.0,
 )
 lower1 = tl.load(
  output
  + base
  + (second_panel + 64 + rows) * n
  + second_panel
  + 64
  + columns,
  mask=lower,
  other=0.0,
 )
 lower1 -= tl.where(
  lower,
  tl.dot(
   history1.to(tl.bfloat16),
   tl.trans(history1.to(tl.bfloat16)),
  ),
  0.0,
 )
 lower1 -= tl.where(
  lower,
  tl.dot(
   second_factor10.to(tl.float16),
   tl.trans(second_factor10.to(tl.float16)),
  ),
  0.0,
 )
 second_factor1, second_inverse1 = _factor_inverse64_pdl(
  lower1,
  False,
  False,
  1.0,
  False,
  True,
 )
 tl.store(
  output
  + base
  + (second_panel + 64 + rows) * n
  + second_panel
  + 64
  + columns,
  tl.where(lower, second_factor1, 0.0),
 )
 tl.store(
  inverse_storage
  + inverse_base1
  + 64 * 64
  + rows * 64
  + columns,
  second_inverse1,
  mask=lower,
 )
 tl.atomic_xchg(ready_pointer, 2.0, sem="release", scope="gpu")

@triton.jit
def _n8192_new_dense_predicate(source, n: tl.constexpr, BLOCK: tl.constexpr):
 axis = tl.arange(0, BLOCK)
 rows = axis[:, None]
 columns = axis[None, :]
 diagonal = tl.load(source + axis * n + axis)
 values = tl.load(source + rows * n + columns)
 normalized = values * values / tl.maximum(
  diagonal[:, None] * diagonal[None, :],
  1.1754943508222875e-38,
 )
 lower = tl.where(rows > columns, normalized, 0.0)
 score = (
  tl.sum(tl.sum(lower, axis=1), axis=0)
  / (BLOCK * (BLOCK - 1) // 2)
 )
 corner = tl.load(source + n - 1)
 first_diagonal = tl.abs(tl.load(source))
 last_diagonal = tl.abs(tl.load(source + n * n - 1))
 diagonal_ratio = last_diagonal / tl.maximum(
  first_diagonal, 1.1754943508222875e-38
 )
 mean_diagonal = tl.sum(diagonal, axis=0) / BLOCK
 score_n = score * n
 new_dense = (
  (corner != 0.0)
  & (mean_diagonal >= 0.5)
  & (diagonal_ratio >= 0.5)
  & (score_n >= 0.7)
  & (score_n < 3.0)
 )
 range_code = tl.where(
  diagonal_ratio >= 0.5,
  0.0,
  tl.where(diagonal_ratio >= 0.001, 2.0, 4.0),
 )
 route_code = tl.where(corner == 0.0, -1.0, score + range_code)
 route_code = tl.where(
  (mean_diagonal >= 0.3) & (mean_diagonal < 0.5),
  score + 1.0,
  route_code,
 )
 return new_dense, route_code

_N8192_RAW_LAYOUTS = _raw_fp4_layouts()

_N8192_CHUNK_TMEM_LAYOUT = TensorMemoryLayout((128, 32), col_stride=1)

_N8192_CHUNK_REGISTER_LAYOUT = get_tmem_reg_layout(
 gl.float32, (128, 32), _N8192_CHUNK_TMEM_LAYOUT, 8,
 alloc_shape=(128, 256),
)

@triton.jit
def _polynomial_persistent_blockrow_wavefront_kernel(
 source,
 output,
 history_storage,
 inverse_storage,
 flags,
 n: tl.constexpr,
 batch: tl.constexpr,
 inverse_blocks: tl.constexpr,
 K_TILE: tl.constexpr,
 PIPELINE_STAGES: tl.constexpr,
):
 tl_cuda.gdc_launch_dependents()
 program = tl.program_id(0)
 matrix = program // 2
 parity = program - matrix * 2
 matrix_row_base = matrix * n
 base = matrix * n * n
 packed_history = history_storage.cast(
  tl.pointer_type(tl.float16), bitcast=True
 )
 history_descriptor = tl.make_tensor_descriptor(
  packed_history,
  shape=[batch * n, n],
  strides=[n, 1],
  block_shape=[128, K_TILE],
 )
 history64_descriptor = tl.make_tensor_descriptor(
  packed_history,
  shape=[batch * n, n],
  strides=[n, 1],
  block_shape=[64, K_TILE],
 )
 source_descriptor = tl.make_tensor_descriptor(
  source,
  shape=[batch * n, n],
  strides=[n, 1],
  block_shape=[128, 128],
 )
 axis = tl.arange(0, 64)
 rows = axis[:, None]
 columns = axis[None, :]
 lower = rows >= columns

 for row_block in tl.range(0, 8):
  if row_block % 2 == parity:
   row_start = row_block * 128

   for panel_block in tl.range(0, row_block):
    delegated = (row_block == 7) & (panel_block == 6)
    if not delegated:
     if panel_block + 1 == row_block:
      flag_pointer = flags + matrix * 8 + panel_block
      ready = tl.load(
       flag_pointer,
       volatile=True,
      )
      while ready < 1.0:
       ready = tl.load(
        flag_pointer,
        volatile=True,
       )
      tl.atomic_add(
       flag_pointer,
       0.0,
       sem="acquire",
       scope="gpu",
      )
     panel = panel_block * 128
     accumulator = tl.zeros((128, 128), tl.float32)
     for offset in tl.range(
      0,
      panel,
      K_TILE,
      num_stages=PIPELINE_STAGES,
      warp_specialize=False,
     ):
      left = history_descriptor.load(
       [matrix_row_base + row_start, offset]
      )
      right = history_descriptor.load(
       [matrix_row_base + panel, offset]
      )
      accumulator += tl.dot(left, tl.trans(right))
     residual = source_descriptor.load(
      [matrix_row_base + row_start, panel]
     ) - accumulator
     residual_blocks = tl.reshape(
      residual, (128, 2, 64)
     )
     residual_columns = tl.permute(
      residual_blocks, (0, 2, 1)
     )
     rhs0, rhs1 = tl.split(residual_columns)
     inverse_base = (
      matrix * inverse_blocks * 64 * 64
      + panel // 64 * 64 * 64
     )
     inverse0 = tl.load(
      inverse_storage
      + inverse_base
      + axis[:, None] * 64
      + axis[None, :],
      mask=lower,
      other=0.0,
     )
     solution0 = _jacobi_solve_dot(
      rhs0, tl.trans(inverse0), 1
     )
     factor10 = tl.load(
      output
      + base
      + (panel + 64 + axis)[:, None] * n
      + panel
      + axis[None, :]
     )
     rhs1 -= _jacobi_solve_dot(
      solution0, tl.trans(factor10), 1
     )
     inverse1 = tl.load(
      inverse_storage
      + inverse_base
      + 64 * 64
      + axis[:, None] * 64
      + axis[None, :],
      mask=lower,
      other=0.0,
     )
     solution1 = _jacobi_solve_dot(
      rhs1, tl.trans(inverse1), 1
     )
     solve_rows = row_start + tl.arange(0, 128)
     tl.store(
      packed_history
      + base
      + solve_rows[:, None] * n
      + panel
      + axis[None, :],
      solution0.to(tl.float16),
     )
     tl.store(
      packed_history
      + base
      + solve_rows[:, None] * n
      + panel
      + 64
      + axis[None, :],
      solution1.to(tl.float16),
     )

   if row_block == 7:
    tl.atomic_xchg(
     flags + matrix * 8 + 7,
     1.0,
     sem="release",
     scope="gpu",
    )

   panel = row_start
   diagonal00 = tl.zeros((64, 64), tl.float32)
   diagonal10 = tl.zeros((64, 64), tl.float32)
   diagonal11_product = tl.zeros((64, 64), tl.float32)

   diagonal_prefix = n - 256 if row_block == 7 else panel
   for offset in tl.range(
    0,
    diagonal_prefix,
    K_TILE,
    num_stages=PIPELINE_STAGES,
    warp_specialize=False,
   ):
    history0 = history64_descriptor.load(
     [matrix_row_base + panel, offset]
    )
    history1 = history64_descriptor.load(
     [matrix_row_base + panel + 64, offset]
    )
    diagonal00 += tl.dot(history0, tl.trans(history0))
    diagonal10 += tl.dot(history1, tl.trans(history0))
    diagonal11_product += tl.dot(
     history1, tl.trans(history1)
    )

   if row_block == 7:
    handoff = flags + matrix * 8 + 6
    ready = tl.load(
     handoff, volatile=True
    )
    while ready < 1.0:
     ready = tl.load(
      handoff,
      volatile=True,
     )
    tl.atomic_add(
     handoff, 0.0, sem="acquire", scope="gpu"
    )
    for offset in tl.range(
     n - 256,
     n - 128,
     K_TILE,
     num_stages=PIPELINE_STAGES,
     warp_specialize=False,
    ):
     history0 = history64_descriptor.load(
      [matrix_row_base + panel, offset]
     )
     history1 = history64_descriptor.load(
      [matrix_row_base + panel + 64, offset]
     )
     diagonal00 += tl.dot(
      history0, tl.trans(history0)
     )
     diagonal10 += tl.dot(
      history1, tl.trans(history0)
     )
     diagonal11_product += tl.dot(
      history1, tl.trans(history1)
     )

   pair_source = source_descriptor.load(
    [matrix_row_base + panel, panel]
   )
   pair_blocks = tl.reshape(pair_source, (2, 64, 2, 64))
   pair_column_blocks = tl.permute(
    pair_blocks, (0, 1, 3, 2)
   )
   pair_column0, pair_column1 = tl.split(pair_column_blocks)
   pair_column0 = tl.permute(pair_column0, (1, 2, 0))
   pair_column1 = tl.permute(pair_column1, (1, 2, 0))
   source00, source10 = tl.split(pair_column0)
   _, source11 = tl.split(pair_column1)
   lower0 = source00 - diagonal00
   residual10 = source10 - diagonal10
   residual11 = source11 - diagonal11_product
   factor0, inverse0 = _jacobi_factor_inverse_from_lower(
    tl.where(lower, lower0, 0.0),
    2,
    0.85,
    4,
    True,
    True,
    1,
    True,
   )
   inverse_base = (
    matrix * inverse_blocks * 64 * 64
    + panel // 64 * 64 * 64
   )
   tl.store(
    output
    + base
    + (panel + rows) * n
    + panel
    + columns,
    tl.where(lower, factor0, 0.0),
   )
   if row_block < 6:
    tl.store(
     inverse_storage
     + inverse_base
     + rows * 64
     + columns,
     inverse0,
     mask=lower,
    )
   factor10 = _jacobi_solve_dot(
    residual10, tl.trans(inverse0), 1
   )
   tl.store(
    output
    + base
    + (panel + 64 + rows) * n
    + panel
    + columns,
    factor10,
   )
   tl.store(
    output
    + base
    + (panel + columns) * n
    + panel
    + 64
    + rows,
    0.0,
   )
   diagonal11 = residual11 - tl.where(
    lower,
    _jacobi_solve_dot(
     factor10, tl.trans(factor10), 1
    ),
    0.0,
   )
   inverse1 = tl.zeros((64, 64), tl.float32)
   if row_block == 7:
    factor1, _ = _jacobi_factor_inverse_from_lower(
     tl.where(lower, diagonal11, 0.0),
     2,
     0.85,
     1,
     True,
     True,
     1,
     True,
    )
   else:
    factor1, inverse1 = _jacobi_factor_inverse_from_lower(
     tl.where(lower, diagonal11, 0.0),
     2,
     0.85,
     4,
     True,
     True,
     1,
     True,
    )
    if row_block < 6:
     tl.store(
      inverse_storage
      + inverse_base
      + 64 * 64
      + rows * 64
      + columns,
      inverse1,
      mask=lower,
     )
   tl.store(
    output
    + base
    + (panel + 64 + rows) * n
    + panel
    + 64
    + columns,
    tl.where(lower, factor1, 0.0),
   )

   if row_block == 6:
    row6_flag = flags + matrix * 8 + 6
    row7_flag = flags + matrix * 8 + 7
    ready = tl.load(
     row7_flag,
     volatile=True,
    )
    while ready < 1.0:
     ready = tl.load(
      row7_flag,
      volatile=True,
     )
    tl.atomic_add(
     row7_flag,
     0.0,
     sem="acquire",
     scope="gpu",
    )

    terminal_row_start: tl.constexpr = n - 128
    terminal_panel: tl.constexpr = n - 256
    accumulator = tl.zeros((128, 128), tl.float32)
    for offset in tl.range(
     0,
     terminal_panel,
     K_TILE,
     num_stages=PIPELINE_STAGES,
     warp_specialize=False,
    ):
     left = history_descriptor.load(
      [matrix_row_base + terminal_row_start, offset]
     )
     right = history_descriptor.load(
      [matrix_row_base + terminal_panel, offset]
     )
     accumulator += tl.dot(left, tl.trans(right))
    residual = source_descriptor.load(
     [
      matrix_row_base + terminal_row_start,
      terminal_panel,
     ]
    ) - accumulator
    residual_blocks = tl.reshape(
     residual, (128, 2, 64)
    )
    residual_columns = tl.permute(
     residual_blocks, (0, 2, 1)
    )
    rhs0, rhs1 = tl.split(residual_columns)
    solution0 = _jacobi_solve_dot(
     rhs0, tl.trans(inverse0), 1
    )
    rhs1 -= _jacobi_solve_dot(
     solution0, tl.trans(factor10), 1
    )
    solution1 = _jacobi_solve_dot(
     rhs1, tl.trans(inverse1), 1
    )
    solve_rows = (
     terminal_row_start + tl.arange(0, 128)
    )
    tl.store(
     packed_history
     + base
     + solve_rows[:, None] * n
     + terminal_panel
     + axis[None, :],
     solution0.to(tl.float16),
    )
    tl.store(
     packed_history
     + base
     + solve_rows[:, None] * n
     + terminal_panel
     + 64
     + axis[None, :],
     solution1.to(tl.float16),
    )
    tl.atomic_xchg(
     row6_flag,
     1.0,
     sem="release",
     scope="gpu",
    )
   elif row_block == 7:
    tl.atomic_xchg(
     flags + matrix * 8 + 7,
     2.0,
     sem="release",
     scope="gpu",
    )
   else:
    tl.atomic_xchg(
     flags + matrix * 8 + row_block,
     1.0,
     sem="release",
     scope="gpu",
    )

@triton.jit
def _n1024_expand_history_zero_upper_kernel(
 history_storage,
 output,
 flags,
 n: tl.constexpr,
 batch: tl.constexpr,
 block_rows: tl.constexpr,
 pair_count: tl.constexpr,
 subtiles_per_pair: tl.constexpr,
 TILE: tl.constexpr,
):
 task = tl.program_id(0)
 subtile = task % subtiles_per_pair
 pair_matrix = task // subtiles_per_pair
 matrix = pair_matrix % batch
 pair = pair_matrix // batch
 row128 = (
  (1.0 + tl.sqrt(1.0 + 8.0 * pair.to(tl.float32))) * 0.5
 ).to(tl.int32)
 column128 = pair - row128 * (row128 - 1) // 2
 row64 = row128 * 2 + subtile // 2
 column64 = column128 * 2 + subtile % 2
 threshold = tl.where(row128 + 1 == block_rows, 2.0, 1.0)
 ready = tl.atomic_add(
  flags + matrix * block_rows + row128, 0.0
 )
 while ready < threshold:
  ready = tl.atomic_add(
   flags + matrix * block_rows + row128, 0.0
  )
 axis = tl.arange(0, TILE)
 rows = row64 * TILE + axis
 columns = column64 * TILE + axis
 base = matrix * n * n
 history = history_storage.cast(
  tl.pointer_type(tl.float16), bitcast=True
 )
 values = tl.load(
  history + base + rows[:, None] * n + columns[None, :]
 )
 tl.store(
  output + base + rows[:, None] * n + columns[None, :],
  values,
 )

@triton.jit
def _jacobi_hybrid_history_bridge_persistent_128_kernel(source, current, destination, current_packed_storage, older_packed_storage, n: tl.constexpr, batch: tl.constexpr, BLOCKS: tl.constexpr, LOCAL_START: tl.constexpr, LOCAL_TILES: tl.constexpr, EXTRA_COLUMN: tl.constexpr, EXTRA_COUNT: tl.constexpr, REPAIR_COUNT: tl.constexpr, K_TILE: tl.constexpr, HYBRID_K_LIMIT: tl.constexpr, PREFIX_TILE_COUNT: tl.constexpr, BRIDGE_TILES: tl.constexpr, COPY_PREFIX: tl.constexpr, STEP: tl.constexpr, STEP_SLOPE: tl.constexpr, STEP_CURVE: tl.constexpr, STEP_CUBIC: tl.constexpr, DIAGONAL_STEP: tl.constexpr, DIAGONAL_STEP_SLOPE: tl.constexpr, DIAGONAL_STEP_CURVE: tl.constexpr, DIAGONAL_STEP_CUBIC: tl.constexpr, PIPELINE_STAGES: tl.constexpr, PROGRAMS: tl.constexpr):
 tl_cuda.gdc_wait()
 tl_cuda.gdc_launch_dependents()
 program = tl.program_id(0)
 for linear in tl.range(program, batch * BRIDGE_TILES, PROGRAMS):
  matrix = linear // BRIDGE_TILES
  tile = linear - matrix * BRIDGE_TILES
  is_extra = tile >= LOCAL_TILES
  local_tile = tl.minimum(tile, LOCAL_TILES - 1)
  local_row = ((tl.sqrt(local_tile.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5).to(tl.int32)
  local_column = local_tile - local_row * (local_row + 1) // 2
  extra = tile - LOCAL_TILES
  extra_span: tl.constexpr = BLOCKS - 1 - EXTRA_COLUMN
  base_extra = tl.minimum(extra, EXTRA_COUNT - 1)
  extra_row = EXTRA_COLUMN + (base_extra * extra_span + (EXTRA_COUNT - 2) // 2) // (EXTRA_COUNT - 1)
  repair = tl.minimum(tl.maximum(extra - EXTRA_COUNT, 0), REPAIR_COUNT - 1)
  repair_column = tl.where(repair == 0, 15, tl.where(repair == 1, 15, tl.where(repair == 2, 16, tl.where(repair == 3, 15, 16))))
  repair_row = tl.where(repair == 0, 19, tl.where(repair == 1, 16, tl.where(repair == 2, 17, tl.where(repair == 3, 18, 18))))
  is_repair = is_extra & (extra >= EXTRA_COUNT)
  row_block = tl.where(is_repair, repair_row, tl.where(is_extra, extra_row, local_row + LOCAL_START))
  column_block = tl.where(is_repair, repair_column, tl.where(is_extra, EXTRA_COLUMN, local_column + LOCAL_START))
  block: tl.constexpr = 128
  row_start = row_block * block
  column_start = column_block * block
  matrix_row_base = matrix * n
  base = matrix * n * n
  packed = current_packed_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
  older_packed = older_packed_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
  left_descriptor = tl.make_tensor_descriptor(packed, shape=[batch * n, n], strides=[n, 1], block_shape=[block, K_TILE])
  right_descriptor = tl.make_tensor_descriptor(packed, shape=[batch * n, n], strides=[n, 1], block_shape=[block, K_TILE])
  older_left_descriptor = tl.make_tensor_descriptor(older_packed, shape=[batch * n, n], strides=[n, 1], block_shape=[block, K_TILE])
  older_right_descriptor = tl.make_tensor_descriptor(older_packed, shape=[batch * n, n], strides=[n, 1], block_shape=[block, K_TILE])
  accumulator = tl.zeros((block, block), tl.float32)
  for reduction_start in tl.range(0, HYBRID_K_LIMIT, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=True):
   left = older_left_descriptor.load([matrix_row_base + row_start, reduction_start])
   right = older_right_descriptor.load([matrix_row_base + column_start, reduction_start])
   accumulator += tl.dot(left, tl.trans(right))
  for reduction_start in tl.range(HYBRID_K_LIMIT, column_start + block, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=True):
   left = left_descriptor.load([matrix_row_base + row_start, reduction_start])
   right = right_descriptor.load([matrix_row_base + column_start, reduction_start])
   accumulator += tl.dot(left, tl.trans(right))
  axis = tl.arange(0, block)
  rows = row_start + axis
  columns = column_start + axis
  target = tl.load(source + base + rows[:, None] * n + columns[None, :], cache_modifier='.cg')
  previous = tl.load(packed + base + rows[:, None] * n + columns[None, :]).to(tl.float32)
  diagonal = tl.load(packed + base + columns * n + columns).to(tl.float32)
  quantized_previous = previous.to(tl.float16).to(tl.float32)
  quantized_diagonal = diagonal.to(tl.float16).to(tl.float32)
  accumulator += previous * diagonal[None, :] - quantized_previous * quantized_diagonal[None, :]
  residual = target - accumulator
  fraction = (columns.to(tl.float32) + 0.5) / n
  centered = fraction - 0.5
  quadratic = centered * centered - 1.0 / 12.0
  cubic = centered * quadratic
  local_step = STEP + STEP_SLOPE * fraction + STEP_CURVE * quadratic + STEP_CUBIC * cubic
  diagonal_step = tl.minimum(0.5, tl.maximum(-0.2, DIAGONAL_STEP + DIAGONAL_STEP_SLOPE * fraction + DIAGONAL_STEP_CURVE * quadratic + DIAGONAL_STEP_CUBIC * cubic))
  updated = previous + local_step * residual / diagonal[None, :]
  updated = tl.where(rows[:, None] == columns[None, :], tl.sqrt(tl.maximum(previous * previous + diagonal_step * residual, 1.1754943508222875e-38)), updated)
  tl.store(destination + base + rows[:, None] * n + columns[None, :], updated, mask=rows[:, None] >= columns[None, :])
  if COPY_PREFIX:
   for copy_pass in tl.static_range(0, 2):
    copy_tile = tile + copy_pass * BRIDGE_TILES
    copy_column = ((65.0 - tl.sqrt(65.0 * 65.0 - 8.0 * copy_tile.to(tl.float32))) * 0.5).to(tl.int32)
    copy_prefix = copy_column * (65 - copy_column) // 2
    copy_row = copy_column + copy_tile - copy_prefix
    copy_axis = tl.arange(0, block)
    copy_rows = copy_row * block + copy_axis
    copy_columns = copy_column * block + copy_axis
    copy_valid = (copy_tile < PREFIX_TILE_COUNT) & (copy_rows[:, None] >= copy_columns[None, :])
    copy_values = tl.load(older_packed + base + copy_rows[:, None] * n + copy_columns[None, :], mask=copy_valid, other=0.0)
    tl.store(destination + base + copy_rows[:, None] * n + copy_columns[None, :], copy_values.to(tl.float32), mask=copy_valid)

@triton.jit
def _pack_q0_mixed_single_gpu0(
 source,
 packed_e4_storage,
 packed_e2m1_storage,
 scale_storage,
 root_storage,
 route_storage,
 flags,
 n: tl.constexpr,
 block: tl.constexpr,
):
 tl_cuda.gdc_wait()
 tile = tl.program_id(0)
 if tile == 0:
  new_dense, route_code = _n8192_new_dense_predicate(
   source, n, 32
  )
  route = route_storage.cast(
   tl.pointer_type(tl.bfloat16), bitcast=True
  )
  tl.store(
   route + n - 1,
   tl.where(new_dense, 8.0, route_code).to(tl.bfloat16),
  )
  flag_axis = tl.arange(0, 32)
  tl.store(flags + flag_axis, 0.0)
 row_block = (
  (tl.sqrt(tile.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5
 ).to(tl.int32)
 column_block = tile - row_block * (row_block + 1) // 2
 local = tl.arange(0, block)
 rows = row_block * block + local
 columns = column_block * block + local
 diagonal = tl.load(source + columns * n + columns)
 position = (columns.to(tl.float32) + 0.5) * (1.0 / n)
 root_curve = (
  2.198625
  + (0.515125 - 2.198625) * position
  - 0.010625 * position * (1.0 - position)
 )
 inverse_roots = tl.rsqrt(diagonal * root_curve)
 if row_block == column_block:
  tl.store(root_storage + columns, inverse_roots)
 value = tl.load(source + rows[:, None] * n + columns[None, :])
 q0 = tl.where(
  rows[:, None] > columns[None, :],
  value * inverse_roots[None, :],
  0.0,
 )

 packed_e4 = packed_e4_storage.cast(
  tl.pointer_type(tl.float8e4nv), bitcast=True
 )
 tl.store(
  packed_e4 + rows[:, None] * n + columns[None, :],
  (q0 * 64.0).to(tl.float8e4nv),
 )

 pairs = tl.reshape(q0 * 256.0, [128, 64, 2])
 low, high = tl.split(pairs)
 packed_values = tl.inline_asm_elementwise(
  """
        {
            .reg .b8 value;
            cvt.rn.satfinite.e2m1x2.f32 value, $2, $1;
            cvt.u32.u8 $0, value;
        }
        """,
  "=r,f,f",
  [low, high],
  dtype=tl.int32,
  is_pure=True,
  pack=1,
 ).to(tl.uint8)
 packed_e2m1 = packed_e2m1_storage.cast(
  tl.pointer_type(tl.uint8), bitcast=True
 )
 byte_columns = column_block * 64 + tl.arange(0, 64)
 tl.store(
  packed_e2m1
  + rows[:, None] * (n // 2)
  + byte_columns[None, :],
  packed_values,
 )
 if tile == 0:
  byte_scales = scale_storage.cast(
   tl.pointer_type(tl.uint8), bitcast=True
  )
  tl.store(byte_scales, 119)
 tl_cuda.gdc_launch_dependents()

@gluon.jit
def _g0_mixed_chunk_gpu0(
 product,
 source,
 root_storage,
 packed_next,
 n: gl.constexpr,
 matrix,
 row_start,
 column_start,
 row_layout: gl.constexpr,
 WIDTH: gl.constexpr,
 DEBUG: gl.constexpr,
):
 rows = row_start + gl.arange(
  0, 128, layout=gl.SliceLayout(1, row_layout)
 )
 columns = column_start + gl.arange(
  0, WIDTH, layout=gl.SliceLayout(0, row_layout)
 )
 base = matrix * n * n
 offsets = base + rows[:, None] * n + columns[None, :]
 source_value = gl.load(source + offsets)
 position = (columns.to(gl.float32) + 0.5) * (1.0 / n)
 inverse_roots = gl.load(root_storage + columns)
 weight = 0.870625 * (
  1.0 - 0.8855 * (position - 0.5)
 )
 x1 = (
  source_value
  - weight[None, :] * product * (1.0 / 4096.0)
 ) * inverse_roots[None, :]
 solved = gl.where(
  rows[:, None] > columns[None, :], x1, 0.0
 )
 if DEBUG:
  debug_output = packed_next.cast(
   gl.pointer_type(gl.float32), bitcast=True
  )
  gl.store(debug_output + offsets, product)
 else:
  output = packed_next.cast(
   gl.pointer_type(gl.float8e4nv), bitcast=True
  )
  gl.store(
   output + offsets,
   (solved * 64.0).to(gl.float8e4nv),
  )

@gluon.jit
def _g1_tuned_precombine_chunk_gpu0(
 product,
 previous,
 source,
 root_storage,
 precombined,
 packed_next,
 n: gl.constexpr,
 matrix,
 row_start,
 column_start,
 row_layout: gl.constexpr,
 WIDTH: gl.constexpr,
):
 rows = row_start + gl.arange(
  0, 128, layout=gl.SliceLayout(1, row_layout)
 )
 columns = column_start + gl.arange(
  0, WIDTH, layout=gl.SliceLayout(0, row_layout)
 )
 base = matrix * n * n
 offsets = base + rows[:, None] * n + columns[None, :]
 position = (columns.to(gl.float32) + 0.5) * (1.0 / n)
 inverse_roots = gl.load(root_storage + columns)
 b1 = 0.9475 * (
  1.0 - 0.075 * (position - 0.5)
 )
 source_value = gl.load(source + offsets)
 x2 = (
  source_value
  - b1[None, :] * product * (1.0 / 4096.0)
 ) * inverse_roots[None, :]
 strict_lower = rows[:, None] > columns[None, :]
 x2_state = gl.where(
  strict_lower, x2, 0.0
 ).to(gl.float16)
 output = packed_next.cast(
  gl.pointer_type(gl.float8e4nv), bitcast=True
 )
 gl.store(
  output + offsets,
  (x2_state.to(gl.float32) * 64.0).to(gl.float8e4nv),
 )
 weight2 = 0.54375 * (
  1.0 + 0.0955 * (position - 0.5)
 )
 previous_pointer = previous.cast(
  gl.pointer_type(gl.float8e4nv), bitcast=True
 )
 x1 = (
  gl.load(previous_pointer + offsets).to(gl.float32)
  * (1.0 / 64.0)
 )
 momentum2 = -0.0778125
 p = (
  (1.0 - weight2[None, :] + momentum2)
  * x2_state.to(gl.float32)
  - momentum2 * x1
 )
 precombined_pointer = precombined.cast(
  gl.pointer_type(gl.float16), bitcast=True
 )
 gl.store(
  precombined_pointer + offsets,
  gl.where(strict_lower, p, 0.0).to(gl.float16),
 )

@gluon.jit
def _g2_tuned_precombined_chunk_gpu0(
 product,
 source,
 root_storage,
 precombined,
 out,
 n: gl.constexpr,
 matrix,
 row_start,
 column_start,
 row_layout: gl.constexpr,
 WIDTH: gl.constexpr,
):
 rows = row_start + gl.arange(
  0, 128, layout=gl.SliceLayout(1, row_layout)
 )
 columns = column_start + gl.arange(
  0, WIDTH, layout=gl.SliceLayout(0, row_layout)
 )
 base = matrix * n * n
 offsets = base + rows[:, None] * n + columns[None, :]
 source_value = gl.load(source + offsets)
 position = (columns.to(gl.float32) + 0.5) * (1.0 / n)
 inverse_roots = gl.load(root_storage + columns)
 roots = 1.0 / inverse_roots
 precombined_pointer = precombined.cast(
  gl.pointer_type(gl.float16), bitcast=True
 )
 p = gl.load(
  precombined_pointer + offsets
 ).to(gl.float32)
 weight2 = 0.54375 * (
  1.0 + 0.0955 * (position - 0.5)
 )
 target = (
  source_value - product * (1.0 / 4096.0)
 ) * inverse_roots[None, :]
 x3 = (
  p + weight2[None, :] * target
 ).to(gl.float16).to(gl.float32)
 strict_lower = rows[:, None] > columns[None, :]
 diagonal_mask = rows[:, None] == columns[None, :]
 solved = gl.where(
  strict_lower,
  x3,
  gl.where(diagonal_mask, roots[None, :], 0.0),
 )
 gl.store(out + offsets, solved)

@gluon.jit
def _causal_raw_e4_mixed_single_g0_gpu0(
 history,
 scale_storage,
 source,
 root_storage,
 packed,
 packed_fp4,
 n: gl.constexpr,
 batch: gl.constexpr,
 panel,
 panel_width: gl.constexpr,
 column_blocks: gl.constexpr,
 k_tiles,
 num_warps: gl.constexpr,
 row_layout: gl.constexpr,
 tensor_layout: gl.constexpr,
 register_layout: gl.constexpr,
 chunk_register_layout: gl.constexpr,
 scale_layout: gl.constexpr,
 ROOT_BASE: gl.constexpr,
 ROOT_SLOPE: gl.constexpr,
 COLUMN_MAJOR: gl.constexpr,
 TASK_OFFSET: gl.constexpr,
 EARLY_RELEASE: gl.constexpr,
):
 if EARLY_RELEASE:
  gl.inline_asm_elementwise(
   "griddepcontrol.wait; griddepcontrol.launch_dependents; mov.u32 $0, 0;",
   "=r", [], dtype=gl.int32, is_pure=False, pack=1,
  )
 tile = gl.program_id(0) + TASK_OFFSET
 matrix = gl.program_id(1)
 triangular_programs: gl.constexpr = (
  column_blocks * (column_blocks + 1) // 2
 )
 in_panel_square = tile < triangular_programs
 triangular_row = (
  (gl.sqrt((8 * tile + 1).to(gl.float32)) - 1.0) * 0.5
 ).to(gl.int32)
 triangular_column = (
  tile - triangular_row * (triangular_row + 1) // 2
 )
 tail_tile = tile - triangular_programs
 tail_row = column_blocks + tail_tile // column_blocks
 tail_column = tail_tile - (tail_row - column_blocks) * column_blocks
 row_block = gl.where(in_panel_square, triangular_row, tail_row)
 column_block = gl.where(
  in_panel_square, triangular_column, tail_column
 )
 if COLUMN_MAJOR:
  reversed_tile = triangular_programs - 1 - tile
  reversed_row = (
   (
    gl.sqrt((8 * reversed_tile + 1).to(gl.float32))
    - 1.0
   )
   * 0.5
  ).to(gl.int32)
  reversed_column = (
   reversed_tile - reversed_row * (reversed_row + 1) // 2
  )
  row_block = column_blocks - 1 - reversed_column
  column_block = column_blocks - 1 - reversed_row
 late_full_fp4 = column_block >= 18
 active_e4_tiles = gl.where(late_full_fp4, 0, column_block + 1)
 active_fp4_tiles = gl.where(late_full_fp4, column_block // 2 + 1, 0)
 fp4_skip_last = ((column_block & 1) == 0).to(gl.int32)
 fp4_initial_accumulate = (~late_full_fp4).to(gl.int32)
 fp4_shift = gl.where(late_full_fp4, 0, 0)
 row_start = panel + row_block * 128
 column_start = panel + column_block * 128
 matrix_row_base = matrix * n
 history = history.cast(gl.pointer_type(gl.uint8), bitcast=True)
 scale_storage = scale_storage.cast(
  gl.pointer_type(gl.uint8), bitcast=True
 )
 packed = packed.cast(gl.pointer_type(gl.uint8), bitcast=True)
 packed_fp4 = packed_fp4.cast(gl.pointer_type(gl.uint8), bitcast=True)
 fp4_pointer = packed_fp4 + fp4_shift
 scratch0 = gl.allocate_shared_memory(
  gl.int64, [4096], mbarrier.MBarrierLayout()
 )
 scratch1 = gl.allocate_shared_memory(
  gl.int64, [4096], mbarrier.MBarrierLayout()
 )
 scratch2 = gl.allocate_shared_memory(
  gl.int64, [4096], mbarrier.MBarrierLayout()
 )
 barriers = gl.allocate_shared_memory(
  gl.int64, [8], mbarrier.MBarrierLayout()
 )
 sentinel0 = scratch0.slice(4095, 1)
 sentinel1 = scratch1.slice(4095, 1)
 sentinel2 = scratch2.slice(4095, 1)
 sentinel3 = barriers.slice(7, 1)
 combined = allocate_tensor_memory(
  gl.float32, [128, 256], tensor_layout
 )
 accumulator = combined.slice(0, 128)
 saved_taddr = gl.inline_asm_elementwise(
  '\n        {\n            .reg .pred first_warp, first_thread, producer, producer_first;\n            .reg .pred consumer_first, complete, more, reuse, accumulate, late, reenter;\n            .reg .b32 tid, warp_id, local, cta_id, smem_addr, taddr;\n            .reg .b32 warp_bits, warp_taddr, scale_value, bar_addr;\n            .reg .b32 iter, stage, cycle, parity, kbytes, row_index;\n            .reg .b32 row_smem, swizzle, stage_smem, chunk, column, physical;\n            .reg .b32 a_smem, b_smem, ready_addr, empty_addr, idesc0, idesc1;\n            .reg .b32 lane, tensor_dim;\n            .reg .b64 a_global, b_global, a_chunk, b_chunk, desc_base;\n            .reg .b64 a_desc0, b_desc0, a_desc1, b_desc1;\n            .reg .b64 a_desc2, b_desc2, a_desc3, b_desc3, state;\n\n            griddepcontrol.wait;\n            mov.u32 tid, %tid.x;\n            shr.u32 warp_id, tid, 5;\n            mov.u32 cta_id, %ctaid.x;\n            setp.lt.u32 first_warp, tid, 32;\n            setp.eq.u32 first_thread, tid, 0;\n            setp.ge.u32 producer, tid, 128;\n            setp.eq.u32 producer_first, tid, 128;\n            setp.eq.u32 consumer_first, tid, 0;\n            mov.u32 smem_addr, global_smem;\n            ld.shared.b32 taddr, [smem_addr];\n            setp.ne.u32 late, $9, 0;\n            @late bra.uni phase_switch;\n\n            and.b32 lane, tid, 31;\n            shl.b32 physical, lane, 2;\n            add.u32 physical, physical, smem_addr;\n            @first_warp st.shared.b32 [physical], 0;\n            bar.sync 0, 256;\n            cvt.u64.u32 desc_base, smem_addr;\n            @first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64\n                [desc_base], $1;\n            @first_thread tensormap.replace.tile.rank.shared::cta.b1024.b32\n                [desc_base], 0x1;\n            mov.u32 tensor_dim, 128;\n            @first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32\n                [desc_base], 0x0, tensor_dim;\n            @first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32\n                [desc_base], 0x1, tensor_dim;\n            mov.u32 tensor_dim, $7;\n            @first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32\n                [desc_base], 0x0, tensor_dim;\n            mov.u32 tensor_dim, $6;\n            @first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32\n                [desc_base], 0x1, tensor_dim;\n            mov.u32 tensor_dim, $7;\n            cvt.u64.u32 a_global, tensor_dim;\n            @first_thread tensormap.replace.tile.global_stride.shared::cta.b1024.b64\n                [desc_base], 0x0, a_global;\n            mov.u32 tensor_dim, 1;\n            @first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32\n                [desc_base], 0x0, tensor_dim;\n            @first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32\n                [desc_base], 0x1, tensor_dim;\n            @first_thread tensormap.replace.tile.elemtype.shared::cta.b1024.b32\n                [desc_base], 0x0;\n            @first_thread tensormap.replace.tile.interleave_layout.shared::cta.b1024.b32\n                [desc_base], 0x0;\n            @first_thread tensormap.replace.tile.swizzle_mode.shared::cta.b1024.b32\n                [desc_base], 0x3;\n            @first_thread tensormap.replace.tile.fill_mode.shared::cta.b1024.b32\n                [desc_base], 0x0;\n            mad.wide.u32 state, $8, 128, $2;\n            add.u64 state, state, 128;\n            @first_warp tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned\n                [state], [desc_base], 0x80;\n            @first_warp fence.proxy.tensormap::generic.acquire.gpu [state], 0x80;\n            @first_warp cp.async.bulk.commit_group;\n            @first_warp cp.async.bulk.wait_group.read 0;\n            bar.sync 0, 256;\n\n            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98304], 1;\n            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98312], 1;\n            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98320], 1;\n            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98328], 1;\n            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98336], 1;\n            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98344], 1;\n            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98352], 1;\n            bar.sync 0, 256;\n            @producer bra.uni producer_path;\n\n        consumer_path:\n            mov.u32 iter, 0;\n            mov.u32 idesc0, 0x08200010;\n            mov.u32 idesc1, 0x08200010;\n        consumer_loop:\n            rem.u32 stage, iter, 3;\n            div.u32 cycle, iter, 3;\n            and.b32 parity, cycle, 1;\n            shl.b32 ready_addr, stage, 3;\n            add.u32 ready_addr, ready_addr, smem_addr;\n            add.u32 ready_addr, ready_addr, 98304;\n        consumer_ready_wait:\n            mbarrier.try_wait.parity.shared::cta.b64 complete, [ready_addr], parity;\n            @!complete bra.uni consumer_ready_wait;\n            bar.sync 3, 128;\n\n            shl.b32 stage_smem, stage, 15;\n            add.u32 a_smem, smem_addr, stage_smem;\n            add.u32 b_smem, a_smem, 16384;\n            bfe.u32 a_smem, a_smem, 4, 14;\n            bfe.u32 b_smem, b_smem, 4, 14;\n            cvt.u64.u32 desc_base, a_smem;\n            or.b64 a_desc0, desc_base, 0x4000404000000000;\n            cvt.u64.u32 desc_base, b_smem;\n            or.b64 b_desc0, desc_base, 0x4000404000000000;\n            add.u64 a_desc1, a_desc0, 2;\n            add.u64 b_desc1, b_desc0, 2;\n            add.u64 a_desc2, a_desc0, 4;\n            add.u64 b_desc2, b_desc0, 4;\n            add.u64 a_desc3, a_desc0, 6;\n            add.u64 b_desc3, b_desc0, 6;\n            setp.ne.u32 accumulate, iter, 0;\n            @consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4\n                [taddr], a_desc0, b_desc0, idesc0, accumulate;\n            mov.pred accumulate, 1;\n            @consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4\n                [taddr], a_desc1, b_desc1, idesc0, accumulate;\n            @consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4\n                [taddr], a_desc2, b_desc2, idesc0, accumulate;\n            @consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4\n                [taddr], a_desc3, b_desc3, idesc0, accumulate;\n            shl.b32 empty_addr, stage, 3;\n            add.u32 empty_addr, empty_addr, smem_addr;\n            add.u32 empty_addr, empty_addr, 98328;\n            @consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [empty_addr];\n            add.u32 iter, iter, 1;\n            setp.lt.u32 more, iter, $3;\n            @more bra.uni consumer_loop;\n            @consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [smem_addr + 98352];\n        consumer_final_wait:\n            mbarrier.try_wait.parity.shared::cta.b64 complete, [smem_addr + 98352], 0;\n            @!complete bra.uni consumer_final_wait;\n            bra.uni phase_switch;\n\n        producer_path:\n            mov.u32 row_index, $4;\n            mov.u32 local, $5;\n            mov.u32 iter, 0;\n        producer_loop:\n            rem.u32 stage, iter, 3;\n            setp.ge.u32 reuse, iter, 3;\n            @!reuse bra.uni producer_load;\n            div.u32 cycle, iter, 3;\n            sub.u32 cycle, cycle, 1;\n            and.b32 parity, cycle, 1;\n            shl.b32 empty_addr, stage, 3;\n            add.u32 empty_addr, empty_addr, smem_addr;\n            add.u32 empty_addr, empty_addr, 98328;\n        producer_empty_wait:\n            mbarrier.try_wait.parity.shared::cta.b64 complete, [empty_addr], parity;\n            @!complete bra.uni producer_empty_wait;\n        producer_load:\n            shl.b32 stage_smem, stage, 15;\n            add.u32 a_smem, stage_smem, smem_addr;\n            add.u32 b_smem, a_smem, 16384;\n            shl.b32 ready_addr, stage, 3;\n            add.u32 ready_addr, ready_addr, smem_addr;\n            add.u32 ready_addr, ready_addr, 98304;\n            shl.b32 column, iter, 7;\n            @producer_first mbarrier.arrive.expect_tx.shared::cta.b64 _, [ready_addr], 32768;\n            fence.proxy.async.shared::cta;\n            @producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n                [a_smem], [state, {column, row_index}], [ready_addr];\n            @producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n                [b_smem], [state, {column, local}], [ready_addr];\n            add.u32 iter, iter, 1;\n            setp.lt.u32 more, iter, $3;\n            @more bra.uni producer_loop;\n        producer_final_wait:\n            mbarrier.try_wait.parity.shared::cta.b64 complete, [smem_addr + 98352], 0;\n            @!complete bra.uni producer_final_wait;\n            bra.uni phase_switch;\n\n        phase_switch:\n            bar.sync 0, 256;\n            and.pred reenter, first_thread, !late;\n            @reenter mbarrier.inval.shared::cta.b64 [smem_addr + 98304];\n            @reenter mbarrier.inval.shared::cta.b64 [smem_addr + 98312];\n            @reenter mbarrier.inval.shared::cta.b64 [smem_addr + 98320];\n            @reenter mbarrier.inval.shared::cta.b64 [smem_addr + 98328];\n            @reenter mbarrier.inval.shared::cta.b64 [smem_addr + 98336];\n            @reenter mbarrier.inval.shared::cta.b64 [smem_addr + 98344];\n            @reenter mbarrier.inval.shared::cta.b64 [smem_addr + 98352];\n            bar.sync 0, 256;\n\n        {\n            .reg .pred first_warp, first_thread, producer, producer_first;\n            .reg .pred consumer_first, complete, more, reuse, accumulate, skip, cleanup;\n            .reg .b32 tid, warp_id, local, cta_id, smem_addr, taddr_fp4;\n            .reg .b32 warp_bits, warp_taddr, scale_value, bar_addr;\n            .reg .b32 iter, stage, cycle, parity, kbytes, row_index;\n            .reg .b32 row_smem, swizzle, stage_smem, chunk, column, physical;\n            .reg .b32 a_smem, b_smem, ready_addr, empty_addr, idesc0, idesc1;\n            .reg .b32 lane, tensor_dim;\n            .reg .b64 a_global, b_global, a_chunk, b_chunk, desc_base;\n            .reg .b64 a_desc0, b_desc0, a_desc1, b_desc1;\n            .reg .b64 a_desc2, b_desc2, a_desc3, b_desc3, state;\n\n            mov.u32 tid, %tid.x;\n            shr.u32 warp_id, tid, 5;\n            mov.u32 cta_id, %ctaid.x;\n            setp.lt.u32 first_warp, tid, 32;\n            setp.eq.u32 first_thread, tid, 0;\n            setp.ge.u32 producer, tid, 128;\n            setp.eq.u32 producer_first, tid, 128;\n            setp.eq.u32 consumer_first, tid, 0;\n            mov.u32 smem_addr, global_smem;\n            mov.u32 taddr_fp4, taddr;\n            setp.eq.u32 complete, $11, 0;\n            @complete bra.uni fp4_final_barrier;\n\n            and.b32 lane, tid, 31;\n            shl.b32 physical, lane, 2;\n            add.u32 physical, physical, smem_addr;\n            @first_warp st.shared.b32 [physical], 0;\n            bar.sync 0, 256;\n            cvt.u64.u32 desc_base, smem_addr;\n            @first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64\n                [desc_base], $10;\n            @first_thread tensormap.replace.tile.rank.shared::cta.b1024.b32\n                [desc_base], 0x1;\n            mov.u32 tensor_dim, 128;\n            @first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32\n                [desc_base], 0x0, tensor_dim;\n            @first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32\n                [desc_base], 0x1, tensor_dim;\n            mov.u32 tensor_dim, $12;\n            @first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32\n                [desc_base], 0x0, tensor_dim;\n            mov.u32 tensor_dim, $6;\n            @first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32\n                [desc_base], 0x1, tensor_dim;\n            mov.u32 tensor_dim, $12;\n            cvt.u64.u32 a_global, tensor_dim;\n            @first_thread tensormap.replace.tile.global_stride.shared::cta.b1024.b64\n                [desc_base], 0x0, a_global;\n            mov.u32 tensor_dim, 1;\n            @first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32\n                [desc_base], 0x0, tensor_dim;\n            @first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32\n                [desc_base], 0x1, tensor_dim;\n            @first_thread tensormap.replace.tile.elemtype.shared::cta.b1024.b32\n                [desc_base], 0x0;\n            @first_thread tensormap.replace.tile.interleave_layout.shared::cta.b1024.b32\n                [desc_base], 0x0;\n            @first_thread tensormap.replace.tile.swizzle_mode.shared::cta.b1024.b32\n                [desc_base], 0x3;\n            @first_thread tensormap.replace.tile.fill_mode.shared::cta.b1024.b32\n                [desc_base], 0x0;\n            mad.wide.u32 state, $8, 128, $2;\n            add.u64 state, state, 128;\n            @first_warp tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned\n                [state], [desc_base], 0x80;\n            @first_warp fence.proxy.tensormap::generic.acquire.gpu [state], 0x80;\n            @first_warp cp.async.bulk.commit_group;\n            @first_warp cp.async.bulk.wait_group.read 0;\n            bar.sync 0, 256;\n\n            and.b32 warp_bits, warp_id, 3;\n            shl.b32 warp_bits, warp_bits, 21;\n            add.u32 warp_taddr, taddr_fp4, warp_bits;\n            ld.global.u8 scale_value, [$2];\n            add.u32 scale_value, scale_value, 6;\n            mul.lo.u32 scale_value, scale_value, 0x01010101;\n            @!producer tcgen05.st.sync.aligned.32x32b.x8.b32 [warp_taddr + 128],\n                {scale_value, scale_value, scale_value, scale_value,\n                 scale_value, scale_value, scale_value, scale_value};\n            @!producer tcgen05.st.sync.aligned.32x32b.x8.b32 [warp_taddr + 136],\n                {scale_value, scale_value, scale_value, scale_value,\n                 scale_value, scale_value, scale_value, scale_value};\n            @!producer tcgen05.wait::st.sync.aligned;\n            bar.sync 0, 256;\n\n            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98304], 1;\n            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98312], 1;\n            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98320], 1;\n            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98328], 1;\n            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98336], 1;\n            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98344], 1;\n            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98352], 1;\n            bar.sync 0, 256;\n            @producer bra.uni fp4_producer_path;\n\n        fp4_consumer_path:\n            mov.u32 iter, 0;\n            mov.u32 idesc0, 0x08a00480;\n            mov.u32 idesc1, 0x48a00480;\n        fp4_consumer_loop:\n            rem.u32 stage, iter, 3;\n            div.u32 cycle, iter, 3;\n            and.b32 parity, cycle, 1;\n            shl.b32 ready_addr, stage, 3;\n            add.u32 ready_addr, ready_addr, smem_addr;\n            add.u32 ready_addr, ready_addr, 98304;\n        fp4_consumer_ready_wait:\n            mbarrier.try_wait.parity.shared::cta.b64 complete, [ready_addr], parity;\n            @!complete bra.uni fp4_consumer_ready_wait;\n            bar.sync 3, 128;\n\n            shl.b32 stage_smem, stage, 15;\n            add.u32 a_smem, smem_addr, stage_smem;\n            add.u32 b_smem, a_smem, 16384;\n            bfe.u32 a_smem, a_smem, 4, 14;\n            bfe.u32 b_smem, b_smem, 4, 14;\n            cvt.u64.u32 desc_base, a_smem;\n            or.b64 a_desc0, desc_base, 0x4000404000000000;\n            cvt.u64.u32 desc_base, b_smem;\n            or.b64 b_desc0, desc_base, 0x4000404000000000;\n            add.u64 a_desc1, a_desc0, 2;\n            add.u64 b_desc1, b_desc0, 2;\n            add.u64 a_desc2, a_desc0, 4;\n            add.u64 b_desc2, b_desc0, 4;\n            add.u64 a_desc3, a_desc0, 6;\n            add.u64 b_desc3, b_desc0, 6;\n            or.b32 tensor_dim, iter, $14;\n            setp.ne.u32 accumulate, tensor_dim, 0;\n            @consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n                [taddr_fp4], a_desc0, b_desc0, idesc0,\n                [taddr_fp4 + 128], [taddr_fp4 + 136], accumulate;\n            mov.pred accumulate, 1;\n            @consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n                [taddr_fp4], a_desc1, b_desc1, idesc1,\n                [taddr_fp4 + 128], [taddr_fp4 + 136], accumulate;\n            sub.u32 tensor_dim, $11, 1;\n            setp.eq.u32 complete, iter, tensor_dim;\n            setp.ne.u32 skip, $13, 0;\n            and.pred skip, skip, complete;\n            @skip bra.uni fp4_product_tail_done;\n            @consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n                [taddr_fp4], a_desc2, b_desc2, idesc0,\n                [taddr_fp4 + 132], [taddr_fp4 + 140], accumulate;\n            @consumer_first tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32\n                [taddr_fp4], a_desc3, b_desc3, idesc1,\n                [taddr_fp4 + 132], [taddr_fp4 + 140], accumulate;\n        fp4_product_tail_done:\n            shl.b32 empty_addr, stage, 3;\n            add.u32 empty_addr, empty_addr, smem_addr;\n            add.u32 empty_addr, empty_addr, 98328;\n            @consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [empty_addr];\n            add.u32 iter, iter, 1;\n            setp.lt.u32 more, iter, $11;\n            @more bra.uni fp4_consumer_loop;\n            @consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [smem_addr + 98352];\n        fp4_consumer_final_wait:\n            mbarrier.try_wait.parity.shared::cta.b64 complete, [smem_addr + 98352], 0;\n            @!complete bra.uni fp4_consumer_final_wait;\n            bra.uni fp4_final_barrier;\n\n        fp4_producer_path:\n            mov.u32 row_index, $4;\n            mov.u32 local, $5;\n            mov.u32 iter, 0;\n        fp4_producer_loop:\n            rem.u32 stage, iter, 3;\n            setp.ge.u32 reuse, iter, 3;\n            @!reuse bra.uni fp4_producer_load;\n            div.u32 cycle, iter, 3;\n            sub.u32 cycle, cycle, 1;\n            and.b32 parity, cycle, 1;\n            shl.b32 empty_addr, stage, 3;\n            add.u32 empty_addr, empty_addr, smem_addr;\n            add.u32 empty_addr, empty_addr, 98328;\n        fp4_producer_empty_wait:\n            mbarrier.try_wait.parity.shared::cta.b64 complete, [empty_addr], parity;\n            @!complete bra.uni fp4_producer_empty_wait;\n        fp4_producer_load:\n            shl.b32 stage_smem, stage, 15;\n            add.u32 a_smem, stage_smem, smem_addr;\n            add.u32 b_smem, a_smem, 16384;\n            shl.b32 ready_addr, stage, 3;\n            add.u32 ready_addr, ready_addr, smem_addr;\n            add.u32 ready_addr, ready_addr, 98304;\n            shl.b32 column, iter, 7;\n            @producer_first mbarrier.arrive.expect_tx.shared::cta.b64 _, [ready_addr], 32768;\n            fence.proxy.async.shared::cta;\n            @producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n                [a_smem], [state, {column, row_index}], [ready_addr];\n            @producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes\n                [b_smem], [state, {column, local}], [ready_addr];\n            add.u32 iter, iter, 1;\n            setp.lt.u32 more, iter, $11;\n            @more bra.uni fp4_producer_loop;\n        fp4_producer_final_wait:\n            mbarrier.try_wait.parity.shared::cta.b64 complete, [smem_addr + 98352], 0;\n            @!complete bra.uni fp4_producer_final_wait;\n            bra.uni fp4_final_barrier;\n\n        fp4_final_barrier:\n            bar.sync 0, 256;\n            @first_thread fence.proxy.tensormap::generic.release.gpu;\n            @first_thread fence.release.gpu;\n            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98304];\n            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98312];\n            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98320];\n            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98328];\n            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98336];\n            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98344];\n            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98352];\n            bar.sync 0, 256;\n            mov.u32 $0, 0;\n        }\n        \n        }\n',
  "=r,l,l,r,r,r,r,r,r,r,l,r,r,r,r",
  [
   history,
   scale_storage,
   active_e4_tiles,
   matrix_row_base + row_start,
   matrix_row_base + column_start,
   batch * n,
   n,
   matrix * gl.num_programs(0) + tile,
   late_full_fp4.to(gl.int32),
   fp4_pointer,
   active_fp4_tiles,
   n // 2,
   fp4_skip_last,
   fp4_initial_accumulate,
  ],
  dtype=gl.int32,
  is_pure=False,
  pack=1,
 )
 mbarrier.init(sentinel0, count=1)
 mbarrier.init(sentinel1, count=1)
 mbarrier.init(sentinel2, count=1)
 mbarrier.init(sentinel3, count=1)
 mbarrier.invalidate(sentinel0)
 mbarrier.invalidate(sentinel1)
 mbarrier.invalidate(sentinel2)
 mbarrier.invalidate(sentinel3)
 for pair in gl.static_range(2):
  product0 = gl.convert_layout(
   accumulator.slice(
    pair * 64, 32
   ).load(
    chunk_register_layout
   ),
   row_layout,
  )
  product1 = gl.convert_layout(
   accumulator.slice(
    pair * 64 + 32, 32
   ).load(
    chunk_register_layout
   ),
   row_layout,
  )
  product = gl.convert_layout(
   gl.reshape(
    gl.permute(
     gl.join(product0, product1), (0, 2, 1)
    ),
    [128, 64],
   ),
   row_layout,
  )
  _g0_mixed_chunk_gpu0(
   product,
   source,
   root_storage,
   packed,
   n,
   matrix,
   row_start,
   column_start + pair * 64,
   row_layout,
   64,
   DEBUG=False,
  )
 gl.inline_asm_elementwise(
  "griddepcontrol.launch_dependents; mov.u32 $0, 0;",
  "=r", [], dtype=gl.int32, is_pure=False, pack=1,
 )

@gluon.jit
def _causal_raw_e4_g1_tuned_gpu0(
 history,
 scale_storage,
 source,
 root_storage,
 out,
 packed,
 n: gl.constexpr,
 batch: gl.constexpr,
 panel,
 panel_width: gl.constexpr,
 column_blocks: gl.constexpr,
 k_tiles,
 num_warps: gl.constexpr,
 row_layout: gl.constexpr,
 tensor_layout: gl.constexpr,
 register_layout: gl.constexpr,
 chunk_register_layout: gl.constexpr,
 scale_layout: gl.constexpr,
 ROOT_BASE: gl.constexpr,
 ROOT_SLOPE: gl.constexpr,
 COLUMN_MAJOR: gl.constexpr,
 TASK_OFFSET: gl.constexpr,
 EARLY_RELEASE: gl.constexpr,
):
 if EARLY_RELEASE:
  gl.inline_asm_elementwise(
   "griddepcontrol.wait; griddepcontrol.launch_dependents; mov.u32 $0, 0;",
   "=r", [], dtype=gl.int32, is_pure=False, pack=1,
  )
 tile = (2079 - gl.program_id(0) + 256) % 2080 + TASK_OFFSET
 matrix = gl.program_id(1)
 triangular_programs: gl.constexpr = (
  column_blocks * (column_blocks + 1) // 2
 )
 in_panel_square = tile < triangular_programs
 triangular_row = (
  (gl.sqrt((8 * tile + 1).to(gl.float32)) - 1.0) * 0.5
 ).to(gl.int32)
 triangular_column = (
  tile - triangular_row * (triangular_row + 1) // 2
 )
 tail_tile = tile - triangular_programs
 tail_row = column_blocks + tail_tile // column_blocks
 tail_column = tail_tile - (tail_row - column_blocks) * column_blocks
 row_block = gl.where(in_panel_square, triangular_row, tail_row)
 column_block = gl.where(
  in_panel_square, triangular_column, tail_column
 )
 if COLUMN_MAJOR:
  reversed_tile = triangular_programs - 1 - tile
  reversed_row = (
   (
    gl.sqrt((8 * reversed_tile + 1).to(gl.float32))
    - 1.0
   )
   * 0.5
  ).to(gl.int32)
  reversed_column = (
   reversed_tile - reversed_row * (reversed_row + 1) // 2
  )
  row_block = column_blocks - 1 - reversed_column
  column_block = column_blocks - 1 - reversed_row
 row_start = panel + row_block * 128
 column_start = panel + column_block * 128
 matrix_row_base = matrix * n
 history = history.cast(gl.pointer_type(gl.uint8), bitcast=True)
 scale_storage = scale_storage.cast(
  gl.pointer_type(gl.uint8), bitcast=True
 )
 packed = packed.cast(gl.pointer_type(gl.uint8), bitcast=True)
 scratch0 = gl.allocate_shared_memory(
  gl.int64, [4096], mbarrier.MBarrierLayout()
 )
 scratch1 = gl.allocate_shared_memory(
  gl.int64, [4096], mbarrier.MBarrierLayout()
 )
 scratch2 = gl.allocate_shared_memory(
  gl.int64, [4096], mbarrier.MBarrierLayout()
 )
 barriers = gl.allocate_shared_memory(
  gl.int64, [8], mbarrier.MBarrierLayout()
 )
 sentinel0 = scratch0.slice(4095, 1)
 sentinel1 = scratch1.slice(4095, 1)
 sentinel2 = scratch2.slice(4095, 1)
 sentinel3 = barriers.slice(7, 1)
 combined = allocate_tensor_memory(
  gl.float32, [128, 128], tensor_layout
 )
 accumulator = combined.slice(0, 128)
 gl.inline_asm_elementwise(
  r"""
        {
            .reg .pred first_warp, first_thread, producer, producer_first;
            .reg .pred consumer_first, complete, more, reuse, accumulate;
            .reg .b32 tid, warp_id, local, cta_id, smem_addr, taddr;
            .reg .b32 warp_bits, warp_taddr, scale_value, bar_addr;
            .reg .b32 iter, stage, cycle, parity, kbytes, row_index;
            .reg .b32 row_smem, swizzle, stage_smem, chunk, column, physical;
            .reg .b32 a_smem, b_smem, ready_addr, empty_addr, idesc0, idesc1;
            .reg .b32 lane, tensor_dim;
            .reg .b64 a_global, b_global, a_chunk, b_chunk, desc_base;
            .reg .b64 a_desc0, b_desc0, a_desc1, b_desc1;
            .reg .b64 a_desc2, b_desc2, a_desc3, b_desc3, state;

            mov.u32 tid, %tid.x;
            shr.u32 warp_id, tid, 5;
            mov.u32 cta_id, %ctaid.x;
            setp.lt.u32 first_warp, tid, 32;
            setp.eq.u32 first_thread, tid, 0;
            setp.ge.u32 producer, tid, 128;
            setp.eq.u32 producer_first, tid, 128;
            setp.eq.u32 consumer_first, tid, 0;
            mov.u32 smem_addr, global_smem;
            ld.shared.b32 taddr, [smem_addr];

            and.b32 lane, tid, 31;
            shl.b32 physical, lane, 2;
            add.u32 physical, physical, smem_addr;
            @first_warp st.shared.b32 [physical], 0;
            bar.sync 0, 256;
            cvt.u64.u32 desc_base, smem_addr;
            @first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64
                [desc_base], $1;
            @first_thread tensormap.replace.tile.rank.shared::cta.b1024.b32
                [desc_base], 0x1;
            mov.u32 tensor_dim, 128;
            @first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32
                [desc_base], 0x0, tensor_dim;
            @first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32
                [desc_base], 0x1, tensor_dim;
            mov.u32 tensor_dim, $7;
            @first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32
                [desc_base], 0x0, tensor_dim;
            mov.u32 tensor_dim, $6;
            @first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32
                [desc_base], 0x1, tensor_dim;
            mov.u32 tensor_dim, $7;
            cvt.u64.u32 a_global, tensor_dim;
            @first_thread tensormap.replace.tile.global_stride.shared::cta.b1024.b64
                [desc_base], 0x0, a_global;
            mov.u32 tensor_dim, 1;
            @first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32
                [desc_base], 0x0, tensor_dim;
            @first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32
                [desc_base], 0x1, tensor_dim;
            @first_thread tensormap.replace.tile.elemtype.shared::cta.b1024.b32
                [desc_base], 0x0;
            @first_thread tensormap.replace.tile.interleave_layout.shared::cta.b1024.b32
                [desc_base], 0x0;
            @first_thread tensormap.replace.tile.swizzle_mode.shared::cta.b1024.b32
                [desc_base], 0x3;
            @first_thread tensormap.replace.tile.fill_mode.shared::cta.b1024.b32
                [desc_base], 0x0;
            mad.wide.u32 state, $8, 128, $2;
            add.u64 state, state, 128;
            @first_warp tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned
                [state], [desc_base], 0x80;
            @first_warp fence.proxy.tensormap::generic.acquire.gpu [state], 0x80;
            @first_warp cp.async.bulk.commit_group;
            @first_warp cp.async.bulk.wait_group.read 0;
            bar.sync 0, 256;

            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98304], 1;
            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98312], 1;
            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98320], 1;
            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98328], 1;
            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98336], 1;
            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98344], 1;
            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98352], 1;
            bar.sync 0, 256;
            @producer bra.uni producer_path;

        consumer_path:
            mov.u32 iter, 0;
            mov.u32 idesc0, 0x08200010;
            mov.u32 idesc1, 0x08200010;
        consumer_loop:
            rem.u32 stage, iter, 3;
            div.u32 cycle, iter, 3;
            and.b32 parity, cycle, 1;
            shl.b32 ready_addr, stage, 3;
            add.u32 ready_addr, ready_addr, smem_addr;
            add.u32 ready_addr, ready_addr, 98304;
        consumer_ready_wait:
            mbarrier.try_wait.parity.shared::cta.b64 complete, [ready_addr], parity;
            @!complete bra.uni consumer_ready_wait;
            bar.sync 3, 128;

            shl.b32 stage_smem, stage, 15;
            add.u32 a_smem, smem_addr, stage_smem;
            add.u32 b_smem, a_smem, 16384;
            bfe.u32 a_smem, a_smem, 4, 14;
            bfe.u32 b_smem, b_smem, 4, 14;
            cvt.u64.u32 desc_base, a_smem;
            or.b64 a_desc0, desc_base, 0x4000404000000000;
            cvt.u64.u32 desc_base, b_smem;
            or.b64 b_desc0, desc_base, 0x4000404000000000;
            add.u64 a_desc1, a_desc0, 2;
            add.u64 b_desc1, b_desc0, 2;
            add.u64 a_desc2, a_desc0, 4;
            add.u64 b_desc2, b_desc0, 4;
            add.u64 a_desc3, a_desc0, 6;
            add.u64 b_desc3, b_desc0, 6;
            setp.ne.u32 accumulate, iter, 0;
            @consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4
                [taddr], a_desc0, b_desc0, idesc0, accumulate;
            mov.pred accumulate, 1;
            @consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4
                [taddr], a_desc1, b_desc1, idesc0, accumulate;
            @consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4
                [taddr], a_desc2, b_desc2, idesc0, accumulate;
            @consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4
                [taddr], a_desc3, b_desc3, idesc0, accumulate;
            shl.b32 empty_addr, stage, 3;
            add.u32 empty_addr, empty_addr, smem_addr;
            add.u32 empty_addr, empty_addr, 98328;
            @consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [empty_addr];
            add.u32 iter, iter, 1;
            setp.lt.u32 more, iter, $3;
            @more bra.uni consumer_loop;
            @consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [smem_addr + 98352];
        consumer_final_wait:
            mbarrier.try_wait.parity.shared::cta.b64 complete, [smem_addr + 98352], 0;
            @!complete bra.uni consumer_final_wait;
            bra.uni final_barrier;

        producer_path:
            mov.u32 row_index, $4;
            mov.u32 local, $5;
            mov.u32 iter, 0;
        producer_loop:
            rem.u32 stage, iter, 3;
            setp.ge.u32 reuse, iter, 3;
            @!reuse bra.uni producer_load;
            div.u32 cycle, iter, 3;
            sub.u32 cycle, cycle, 1;
            and.b32 parity, cycle, 1;
            shl.b32 empty_addr, stage, 3;
            add.u32 empty_addr, empty_addr, smem_addr;
            add.u32 empty_addr, empty_addr, 98328;
        producer_empty_wait:
            mbarrier.try_wait.parity.shared::cta.b64 complete, [empty_addr], parity;
            @!complete bra.uni producer_empty_wait;
        producer_load:
            shl.b32 stage_smem, stage, 15;
            add.u32 a_smem, stage_smem, smem_addr;
            add.u32 b_smem, a_smem, 16384;
            shl.b32 ready_addr, stage, 3;
            add.u32 ready_addr, ready_addr, smem_addr;
            add.u32 ready_addr, ready_addr, 98304;
            shl.b32 column, iter, 7;
            @producer_first mbarrier.arrive.expect_tx.shared::cta.b64 _, [ready_addr], 32768;
            fence.proxy.async.shared::cta;
            @producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes
                [a_smem], [state, {column, row_index}], [ready_addr];
            @producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes
                [b_smem], [state, {column, local}], [ready_addr];
            add.u32 iter, iter, 1;
            setp.lt.u32 more, iter, $3;
            @more bra.uni producer_loop;
        producer_final_wait:
            mbarrier.try_wait.parity.shared::cta.b64 complete, [smem_addr + 98352], 0;
            @!complete bra.uni producer_final_wait;
            bra.uni final_barrier;

        final_barrier:
            bar.sync 0, 256;
            @first_thread fence.proxy.tensormap::generic.release.gpu;
            @first_thread fence.release.gpu;
            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98304];
            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98312];
            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98320];
            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98328];
            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98336];
            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98344];
            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98352];
            bar.sync 0, 256;
            mov.u32 $0, 0;
        }
        """,
  "=r,l,l,r,r,r,r,r,r",
  [
   history,
   scale_storage,
   column_block + 1,
   matrix_row_base + row_start,
   matrix_row_base + column_start,
   batch * n,
   n,
   matrix * gl.num_programs(0) + tile,
  ],
  dtype=gl.int32,
  is_pure=False,
  pack=1,
 )
 mbarrier.init(sentinel0, count=1)
 mbarrier.init(sentinel1, count=1)
 mbarrier.init(sentinel2, count=1)
 mbarrier.init(sentinel3, count=1)
 mbarrier.invalidate(sentinel0)
 mbarrier.invalidate(sentinel1)
 mbarrier.invalidate(sentinel2)
 mbarrier.invalidate(sentinel3)
 for chunk in gl.static_range(2):
  product = gl.convert_layout(
   accumulator.slice(
    chunk * 64, 64
   ).load(
    chunk_register_layout
   ),
   row_layout,
  )
  _g1_tuned_precombine_chunk_gpu0(
   product,
   history,
   source,
   root_storage,
   out,
   packed,
   n,
   matrix,
   row_start,
   column_start + chunk * 64,
   row_layout,
   64,
  )
 gl.inline_asm_elementwise(
  "griddepcontrol.launch_dependents; mov.u32 $0, 0;",
  "=r", [], dtype=gl.int32, is_pure=False, pack=1,
 )

@gluon.jit
def _causal_raw_e4_g2_tuned_gpu0(
 history,
 scale_storage,
 source,
 root_storage,
 out,
 packed,
 precombined,
 n: gl.constexpr,
 batch: gl.constexpr,
 panel,
 panel_width: gl.constexpr,
 column_blocks: gl.constexpr,
 k_tiles,
 num_warps: gl.constexpr,
 row_layout: gl.constexpr,
 tensor_layout: gl.constexpr,
 register_layout: gl.constexpr,
 chunk_register_layout: gl.constexpr,
 scale_layout: gl.constexpr,
 ROOT_BASE: gl.constexpr,
 ROOT_SLOPE: gl.constexpr,
 COLUMN_MAJOR: gl.constexpr,
 SAFE_TASKS: gl.constexpr,
):
 tile = gl.program_id(0)
 matrix = gl.program_id(1)
 triangular_programs: gl.constexpr = (
  column_blocks * (column_blocks + 1) // 2
 )
 in_panel_square = tile < triangular_programs
 triangular_row = (
  (gl.sqrt((8 * tile + 1).to(gl.float32)) - 1.0) * 0.5
 ).to(gl.int32)
 triangular_column = (
  tile - triangular_row * (triangular_row + 1) // 2
 )
 tail_tile = tile - triangular_programs
 tail_row = column_blocks + tail_tile // column_blocks
 tail_column = tail_tile - (tail_row - column_blocks) * column_blocks
 row_block = gl.where(in_panel_square, triangular_row, tail_row)
 column_block = gl.where(
  in_panel_square, triangular_column, tail_column
 )
 if COLUMN_MAJOR:
  reversed_tile = triangular_programs - 1 - tile
  reversed_row = (
   (
    gl.sqrt((8 * reversed_tile + 1).to(gl.float32))
    - 1.0
   )
   * 0.5
  ).to(gl.int32)
  reversed_column = (
   reversed_tile - reversed_row * (reversed_row + 1) // 2
  )
  row_block = column_blocks - 1 - reversed_column
  column_block = column_blocks - 1 - reversed_row
 row_start = panel + row_block * 128
 column_start = panel + column_block * 128
 matrix_row_base = matrix * n
 history = history.cast(gl.pointer_type(gl.uint8), bitcast=True)
 scale_storage = scale_storage.cast(
  gl.pointer_type(gl.uint8), bitcast=True
 )
 packed = packed.cast(gl.pointer_type(gl.uint8), bitcast=True)
 scratch0 = gl.allocate_shared_memory(
  gl.int64, [4096], mbarrier.MBarrierLayout()
 )
 scratch1 = gl.allocate_shared_memory(
  gl.int64, [4096], mbarrier.MBarrierLayout()
 )
 scratch2 = gl.allocate_shared_memory(
  gl.int64, [4096], mbarrier.MBarrierLayout()
 )
 barriers = gl.allocate_shared_memory(
  gl.int64, [8], mbarrier.MBarrierLayout()
 )
 sentinel0 = scratch0.slice(4095, 1)
 sentinel1 = scratch1.slice(4095, 1)
 sentinel2 = scratch2.slice(4095, 1)
 sentinel3 = barriers.slice(7, 1)
 combined = allocate_tensor_memory(
  gl.float32, [128, 128], tensor_layout
 )
 accumulator = combined.slice(0, 128)
 gl.inline_asm_elementwise(
  r"""
        {
            .reg .pred first_warp, first_thread, producer, producer_first;
            .reg .pred consumer_first, complete, more, reuse, accumulate, safe_task;
            .reg .b32 tid, warp_id, local, cta_id, smem_addr, taddr;
            .reg .b32 warp_bits, warp_taddr, scale_value, bar_addr;
            .reg .b32 iter, stage, cycle, parity, kbytes, row_index;
            .reg .b32 row_smem, swizzle, stage_smem, chunk, column, physical;
            .reg .b32 a_smem, b_smem, ready_addr, empty_addr, idesc0, idesc1;
            .reg .b32 lane, tensor_dim;
            .reg .b64 a_global, b_global, a_chunk, b_chunk, desc_base;
            .reg .b64 a_desc0, b_desc0, a_desc1, b_desc1;
            .reg .b64 a_desc2, b_desc2, a_desc3, b_desc3, state;

            mov.u32 tid, %tid.x;
            shr.u32 warp_id, tid, 5;
            mov.u32 cta_id, %ctaid.x;
            setp.lt.u32 safe_task, cta_id, $9;
            @!safe_task griddepcontrol.wait;
            setp.lt.u32 first_warp, tid, 32;
            setp.eq.u32 first_thread, tid, 0;
            setp.ge.u32 producer, tid, 128;
            setp.eq.u32 producer_first, tid, 128;
            setp.eq.u32 consumer_first, tid, 0;
            mov.u32 smem_addr, global_smem;
            ld.shared.b32 taddr, [smem_addr];

            and.b32 lane, tid, 31;
            shl.b32 physical, lane, 2;
            add.u32 physical, physical, smem_addr;
            @first_warp st.shared.b32 [physical], 0;
            bar.sync 0, 256;
            cvt.u64.u32 desc_base, smem_addr;
            @first_thread tensormap.replace.tile.global_address.shared::cta.b1024.b64
                [desc_base], $1;
            @first_thread tensormap.replace.tile.rank.shared::cta.b1024.b32
                [desc_base], 0x1;
            mov.u32 tensor_dim, 128;
            @first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32
                [desc_base], 0x0, tensor_dim;
            @first_thread tensormap.replace.tile.box_dim.shared::cta.b1024.b32
                [desc_base], 0x1, tensor_dim;
            mov.u32 tensor_dim, $7;
            @first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32
                [desc_base], 0x0, tensor_dim;
            mov.u32 tensor_dim, $6;
            @first_thread tensormap.replace.tile.global_dim.shared::cta.b1024.b32
                [desc_base], 0x1, tensor_dim;
            mov.u32 tensor_dim, $7;
            cvt.u64.u32 a_global, tensor_dim;
            @first_thread tensormap.replace.tile.global_stride.shared::cta.b1024.b64
                [desc_base], 0x0, a_global;
            mov.u32 tensor_dim, 1;
            @first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32
                [desc_base], 0x0, tensor_dim;
            @first_thread tensormap.replace.tile.element_stride.shared::cta.b1024.b32
                [desc_base], 0x1, tensor_dim;
            @first_thread tensormap.replace.tile.elemtype.shared::cta.b1024.b32
                [desc_base], 0x0;
            @first_thread tensormap.replace.tile.interleave_layout.shared::cta.b1024.b32
                [desc_base], 0x0;
            @first_thread tensormap.replace.tile.swizzle_mode.shared::cta.b1024.b32
                [desc_base], 0x3;
            @first_thread tensormap.replace.tile.fill_mode.shared::cta.b1024.b32
                [desc_base], 0x0;
            mad.wide.u32 state, $8, 128, $2;
            add.u64 state, state, 128;
            @first_warp tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned
                [state], [desc_base], 0x80;
            @first_warp fence.proxy.tensormap::generic.acquire.gpu [state], 0x80;
            @first_warp cp.async.bulk.commit_group;
            @first_warp cp.async.bulk.wait_group.read 0;
            bar.sync 0, 256;

            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98304], 1;
            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98312], 1;
            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98320], 1;
            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98328], 1;
            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98336], 1;
            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98344], 1;
            @first_thread mbarrier.init.shared::cta.b64 [smem_addr + 98352], 1;
            bar.sync 0, 256;
            @producer bra.uni producer_path;

        consumer_path:
            mov.u32 iter, 0;
            mov.u32 idesc0, 0x08200010;
            mov.u32 idesc1, 0x08200010;
        consumer_loop:
            rem.u32 stage, iter, 3;
            div.u32 cycle, iter, 3;
            and.b32 parity, cycle, 1;
            shl.b32 ready_addr, stage, 3;
            add.u32 ready_addr, ready_addr, smem_addr;
            add.u32 ready_addr, ready_addr, 98304;
        consumer_ready_wait:
            mbarrier.try_wait.parity.shared::cta.b64 complete, [ready_addr], parity;
            @!complete bra.uni consumer_ready_wait;
            bar.sync 3, 128;

            shl.b32 stage_smem, stage, 15;
            add.u32 a_smem, smem_addr, stage_smem;
            add.u32 b_smem, a_smem, 16384;
            bfe.u32 a_smem, a_smem, 4, 14;
            bfe.u32 b_smem, b_smem, 4, 14;
            cvt.u64.u32 desc_base, a_smem;
            or.b64 a_desc0, desc_base, 0x4000404000000000;
            cvt.u64.u32 desc_base, b_smem;
            or.b64 b_desc0, desc_base, 0x4000404000000000;
            add.u64 a_desc1, a_desc0, 2;
            add.u64 b_desc1, b_desc0, 2;
            add.u64 a_desc2, a_desc0, 4;
            add.u64 b_desc2, b_desc0, 4;
            add.u64 a_desc3, a_desc0, 6;
            add.u64 b_desc3, b_desc0, 6;
            setp.ne.u32 accumulate, iter, 0;
            @consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4
                [taddr], a_desc0, b_desc0, idesc0, accumulate;
            mov.pred accumulate, 1;
            @consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4
                [taddr], a_desc1, b_desc1, idesc0, accumulate;
            @consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4
                [taddr], a_desc2, b_desc2, idesc0, accumulate;
            @consumer_first tcgen05.mma.cta_group::1.kind::f8f6f4
                [taddr], a_desc3, b_desc3, idesc0, accumulate;
            shl.b32 empty_addr, stage, 3;
            add.u32 empty_addr, empty_addr, smem_addr;
            add.u32 empty_addr, empty_addr, 98328;
            @consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [empty_addr];
            add.u32 iter, iter, 1;
            setp.lt.u32 more, iter, $3;
            @more bra.uni consumer_loop;
            @consumer_first tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [smem_addr + 98352];
        consumer_final_wait:
            mbarrier.try_wait.parity.shared::cta.b64 complete, [smem_addr + 98352], 0;
            @!complete bra.uni consumer_final_wait;
            bra.uni final_barrier;

        producer_path:
            mov.u32 row_index, $4;
            mov.u32 local, $5;
            mov.u32 iter, 0;
        producer_loop:
            rem.u32 stage, iter, 3;
            setp.ge.u32 reuse, iter, 3;
            @!reuse bra.uni producer_load;
            div.u32 cycle, iter, 3;
            sub.u32 cycle, cycle, 1;
            and.b32 parity, cycle, 1;
            shl.b32 empty_addr, stage, 3;
            add.u32 empty_addr, empty_addr, smem_addr;
            add.u32 empty_addr, empty_addr, 98328;
        producer_empty_wait:
            mbarrier.try_wait.parity.shared::cta.b64 complete, [empty_addr], parity;
            @!complete bra.uni producer_empty_wait;
        producer_load:
            shl.b32 stage_smem, stage, 15;
            add.u32 a_smem, stage_smem, smem_addr;
            add.u32 b_smem, a_smem, 16384;
            shl.b32 ready_addr, stage, 3;
            add.u32 ready_addr, ready_addr, smem_addr;
            add.u32 ready_addr, ready_addr, 98304;
            shl.b32 column, iter, 7;
            @producer_first mbarrier.arrive.expect_tx.shared::cta.b64 _, [ready_addr], 32768;
            fence.proxy.async.shared::cta;
            @producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes
                [a_smem], [state, {column, row_index}], [ready_addr];
            @producer_first cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes
                [b_smem], [state, {column, local}], [ready_addr];
            add.u32 iter, iter, 1;
            setp.lt.u32 more, iter, $3;
            @more bra.uni producer_loop;
        producer_final_wait:
            mbarrier.try_wait.parity.shared::cta.b64 complete, [smem_addr + 98352], 0;
            @!complete bra.uni producer_final_wait;
            bra.uni final_barrier;

        final_barrier:
            bar.sync 0, 256;
            @first_thread fence.proxy.tensormap::generic.release.gpu;
            @first_thread fence.release.gpu;
            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98304];
            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98312];
            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98320];
            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98328];
            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98336];
            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98344];
            @first_thread mbarrier.inval.shared::cta.b64 [smem_addr + 98352];
            bar.sync 0, 256;
            mov.u32 $0, 0;
        }
        """,
  "=r,l,l,r,r,r,r,r,r,r",
  [
   history,
   scale_storage,
   column_block + 1,
   matrix_row_base + row_start,
   matrix_row_base + column_start,
   batch * n,
   n,
   matrix * gl.num_programs(0) + tile,
   SAFE_TASKS,
  ],
  dtype=gl.int32,
  is_pure=False,
  pack=1,
 )
 mbarrier.init(sentinel0, count=1)
 mbarrier.init(sentinel1, count=1)
 mbarrier.init(sentinel2, count=1)
 mbarrier.init(sentinel3, count=1)
 mbarrier.invalidate(sentinel0)
 mbarrier.invalidate(sentinel1)
 mbarrier.invalidate(sentinel2)
 mbarrier.invalidate(sentinel3)
 for chunk in gl.static_range(2):
  product = gl.convert_layout(
   accumulator.slice(
    chunk * 64, 64
   ).load(
    chunk_register_layout
   ),
   row_layout,
  )
  _g2_tuned_precombined_chunk_gpu0(
   product,
   source,
   root_storage,
   precombined,
   out,
   n,
   matrix,
   row_start,
   column_start + chunk * 64,
   row_layout,
   64,
  )
 gl.inline_asm_elementwise(
  "griddepcontrol.launch_dependents; mov.u32 $0, 0;",
  "=r", [], dtype=gl.int32, is_pure=False, pack=1,
 )

_N8192_G1_W64_CHUNK_REGISTER_LAYOUT = get_tmem_reg_layout(
 gl.float32,
 (128, 64),
 TensorMemoryLayout((128, 64), col_stride=1),
 8,
 alloc_shape=(128, 128),
)

_N8192_G12_TENSOR_LAYOUT = TensorMemoryLayout(
 (128, 128), col_stride=1
)

@triton.jit
def _n2048_monotonic_integer_epoch(flags, phase, PROGRAMS: tl.constexpr):
    counter = flags.cast(tl.pointer_type(tl.int32), bitcast=True)
    tl.atomic_add(counter, 1, sem="release", scope="gpu")
    ready = tl.atomic_add(
        counter, 0, sem="relaxed", scope="gpu"
    )
    while ready < phase * PROGRAMS:
        ready = tl.atomic_add(
            counter, 0, sem="relaxed", scope="gpu"
        )
    tl.atomic_add(counter, 0, sem="acquire", scope="gpu")

@triton.jit
def _n1024_ten_quadratic_steps(iteration, columns, n: tl.constexpr):
    fraction = (columns.to(tl.float32) + 0.5) / n
    centered = fraction - 0.5
    quadratic = centered * centered - 1.0 / 12.0
    step_base = tl.full((), 0.53079581, tl.float32)
    step_base = tl.where(iteration == 0, 0.8412025, step_base)
    step_base = tl.where(iteration == 1, 0.98417318, step_base)
    step_base = tl.where(iteration == 2, 1.09763372, step_base)
    step_base = tl.where(iteration == 3, 1.06005394, step_base)
    step_base = tl.where(iteration == 4, 1.07796359, step_base)
    step_base = tl.where(iteration == 5, 1.10105431, step_base)
    step_base = tl.where(iteration == 6, 1.08316696, step_base)
    step_base = tl.where(iteration == 7, 1.10257602, step_base)
    step_base = tl.where(iteration == 8, 0.77636909, step_base)
    step_slope = tl.full((), 0.32786405, tl.float32)
    step_slope = tl.where(iteration == 0, -0.42123103, step_slope)
    step_slope = tl.where(iteration == 1, 0.00827927, step_slope)
    step_slope = tl.where(iteration == 2, -0.26983917, step_slope)
    step_slope = tl.where(iteration == 3, -0.2306003, step_slope)
    step_slope = tl.where(iteration == 4, -0.21422616, step_slope)
    step_slope = tl.where(iteration == 5, -0.29398462, step_slope)
    step_slope = tl.where(iteration == 6, -0.22671047, step_slope)
    step_slope = tl.where(iteration == 7, -0.18415938, step_slope)
    step_slope = tl.where(iteration == 8, 0.16068865, step_slope)
    step_curve = tl.full((), -0.63350773, tl.float32)
    step_curve = tl.where(iteration == 0, 0.45402786, step_curve)
    step_curve = tl.where(iteration == 1, -0.39406475, step_curve)
    step_curve = tl.where(iteration == 2, -0.41158473, step_curve)
    step_curve = tl.where(iteration == 3, -0.59209162, step_curve)
    step_curve = tl.where(iteration == 4, -0.37080762, step_curve)
    step_curve = tl.where(iteration == 5, -0.43104106, step_curve)
    step_curve = tl.where(iteration == 6, -0.43967476, step_curve)
    step_curve = tl.where(iteration == 7, -0.52271116, step_curve)
    step_curve = tl.where(iteration == 8, -0.54945397, step_curve)
    diagonal_base = tl.full((), 0.025, tl.float32)
    diagonal_base = tl.where(iteration == 0, 0.68734789, diagonal_base)
    diagonal_base = tl.where(iteration == 1, 0.66315335, diagonal_base)
    diagonal_base = tl.where(iteration == 2, 0.52015775, diagonal_base)
    diagonal_base = tl.where(iteration == 3, 0.23172645, diagonal_base)
    diagonal_base = tl.where(iteration == 4, 0.13630202, diagonal_base)
    diagonal_slope = tl.full((), 0.03712809, tl.float32)
    diagonal_slope = tl.where(iteration == 0, -0.05878795, diagonal_slope)
    diagonal_slope = tl.where(iteration == 1, 0.08281174, diagonal_slope)
    diagonal_slope = tl.where(iteration == 2, 0.54942119, diagonal_slope)
    diagonal_slope = tl.where(iteration == 3, 0.16022542, diagonal_slope)
    diagonal_slope = tl.where(iteration == 4, 0.4435752, diagonal_slope)
    diagonal_slope = tl.where(iteration == 5, 0.21374691, diagonal_slope)
    diagonal_slope = tl.where(iteration == 6, 0.23127973, diagonal_slope)
    diagonal_slope = tl.where(iteration == 7, 0.09701068, diagonal_slope)
    diagonal_slope = tl.where(iteration == 8, -0.23840867, diagonal_slope)
    diagonal_curve = tl.full((), 0.54270101, tl.float32)
    diagonal_curve = tl.where(iteration == 0, -0.49388528, diagonal_curve)
    diagonal_curve = tl.where(iteration == 1, -0.27269602, diagonal_curve)
    diagonal_curve = tl.where(iteration == 2, -0.17399646, diagonal_curve)
    diagonal_curve = tl.where(iteration == 3, 0.65066081, diagonal_curve)
    diagonal_curve = tl.where(iteration == 4, 0.34572083, diagonal_curve)
    diagonal_curve = tl.where(iteration == 5, 0.68803436, diagonal_curve)
    diagonal_curve = tl.where(iteration == 6, 0.69012803, diagonal_curve)
    diagonal_curve = tl.where(iteration == 7, 0.62075734, diagonal_curve)
    diagonal_curve = tl.where(iteration == 8, 0.08986815, diagonal_curve)
    step = step_base + step_slope * fraction + step_curve * quadratic
    diagonal_step = tl.minimum(1.4, tl.maximum(0.025, diagonal_base + diagonal_slope * fraction + diagonal_curve * quadratic))
    return (step, diagonal_step)

@triton.jit
def _n1024_nine_momentum_steps(iteration, columns, n: tl.constexpr):
    fraction = (columns.to(tl.float32) + 0.5) / n
    centered = fraction - 0.5
    quadratic = centered * centered - 1.0 / 12.0
    step_base = tl.full((), 0.5237296820, tl.float32)
    step_base = tl.where(iteration == 0, 0.8550513387, step_base)
    step_base = tl.where(iteration == 1, 0.9945517778, step_base)
    step_base = tl.where(iteration == 2, 1.0533955097, step_base)
    step_base = tl.where(iteration == 3, 1.1023944616, step_base)
    step_base = tl.where(iteration == 4, 1.1636153460, step_base)
    step_base = tl.where(iteration == 5, 1.1998456717, step_base)
    step_base = tl.where(iteration == 6, 1.1625186205, step_base)
    step_base = tl.where(iteration == 7, 0.8262010813, step_base)
    step_slope = tl.full((), 0.3908350766, tl.float32)
    step_slope = tl.where(iteration == 0, -0.4633561075, step_slope)
    step_slope = tl.where(iteration == 1, -0.0286720172, step_slope)
    step_slope = tl.where(iteration == 2, -0.3160515726, step_slope)
    step_slope = tl.where(iteration == 3, -0.2328731567, step_slope)
    step_slope = tl.where(iteration == 4, -0.2933187485, step_slope)
    step_slope = tl.where(iteration == 5, -0.2904993296, step_slope)
    step_slope = tl.where(iteration == 6, -0.2890567183, step_slope)
    step_slope = tl.where(iteration == 7, 0.0245949272, step_slope)
    step_curve = tl.full((), -0.9806481004, tl.float32)
    step_curve = tl.where(iteration == 0, 0.4560697377, step_curve)
    step_curve = tl.where(iteration == 1, -0.4616306722, step_curve)
    step_curve = tl.where(iteration == 2, -0.5932388306, step_curve)
    step_curve = tl.where(iteration == 3, -0.5810791254, step_curve)
    step_curve = tl.where(iteration == 4, -0.3071929216, step_curve)
    step_curve = tl.where(iteration == 5, -0.5922822356, step_curve)
    step_curve = tl.where(iteration == 6, -0.7595571876, step_curve)
    step_curve = tl.where(iteration == 7, -0.8659578562, step_curve)
    diagonal_base = tl.full((), 0.01, tl.float32)
    diagonal_base = tl.where(iteration == 0, 0.6746060252, diagonal_base)
    diagonal_base = tl.where(iteration == 1, 0.6511330605, diagonal_base)
    diagonal_base = tl.where(iteration == 2, 0.5157008171, diagonal_base)
    diagonal_base = tl.where(iteration == 3, 0.2116615772, diagonal_base)
    diagonal_base = tl.where(iteration == 4, 0.0395113267, diagonal_base)
    diagonal_slope = tl.full((), 0.1020384654, tl.float32)
    diagonal_slope = tl.where(iteration == 0, -0.0703942552, diagonal_slope)
    diagonal_slope = tl.where(iteration == 1, 0.1819206774, diagonal_slope)
    diagonal_slope = tl.where(iteration == 2, 0.5378530622, diagonal_slope)
    diagonal_slope = tl.where(iteration == 3, 0.5102763176, diagonal_slope)
    diagonal_slope = tl.where(iteration == 4, 0.5887674093, diagonal_slope)
    diagonal_slope = tl.where(iteration == 5, 0.4172986746, diagonal_slope)
    diagonal_slope = tl.where(iteration == 6, 0.1994422376, diagonal_slope)
    diagonal_slope = tl.where(iteration == 7, -0.2543844581, diagonal_slope)
    diagonal_curve = tl.full((), 0.8826112151, tl.float32)
    diagonal_curve = tl.where(iteration == 0, -0.4321436286, diagonal_curve)
    diagonal_curve = tl.where(iteration == 1, -0.3433409631, diagonal_curve)
    diagonal_curve = tl.where(iteration == 2, -0.1400900036, diagonal_curve)
    diagonal_curve = tl.where(iteration == 3, 0.2586408854, diagonal_curve)
    diagonal_curve = tl.where(iteration == 4, 0.2964455783, diagonal_curve)
    diagonal_curve = tl.where(iteration == 5, 0.7791020274, diagonal_curve)
    diagonal_curve = tl.where(iteration == 6, 0.9982445836, diagonal_curve)
    diagonal_curve = tl.where(iteration == 7, 0.0369681790, diagonal_curve)
    diagonal_momentum = tl.full((), -0.3924221098, tl.float32)
    diagonal_momentum = tl.where(iteration == 0, 0.0, diagonal_momentum)
    diagonal_momentum = tl.where(iteration == 1, 0.0062386324, diagonal_momentum)
    diagonal_momentum = tl.where(iteration == 2, -0.1164279282, diagonal_momentum)
    diagonal_momentum = tl.where(iteration == 3, 0.0356265157, diagonal_momentum)
    diagonal_momentum = tl.where(iteration == 4, -0.1434039772, diagonal_momentum)
    diagonal_momentum = tl.where(iteration == 5, -0.0009414946, diagonal_momentum)
    diagonal_momentum = tl.where(iteration == 6, -0.1002655998, diagonal_momentum)
    diagonal_momentum = tl.where(iteration == 7, 0.1824842244, diagonal_momentum)
    step = step_base + step_slope * fraction + step_curve * quadratic
    diagonal_step = tl.minimum(1.4, tl.maximum(0.01, diagonal_base + diagonal_slope * fraction + diagonal_curve * quadratic))
    return step, diagonal_step, diagonal_momentum

@triton.jit
def _persistent_jacobi_wave(source, current, destination, current_packed_storage, destination_packed_storage, program, iteration, n: tl.constexpr, batch: tl.constexpr, BLOCKS: tl.constexpr, PROGRAMS: tl.constexpr, BLOCK: tl.constexpr, K_TILE: tl.constexpr, PIPELINE_STAGES: tl.constexpr, SCHEDULE: tl.constexpr, BLOCK_START: tl.constexpr, HIGH_PRECISION_TAIL: tl.constexpr, USE_PACKED_FP16: tl.constexpr, DIAGONAL_RELAXED: tl.constexpr, WARP_SPECIALIZE: tl.constexpr, WEIGHTED_ORDER: tl.constexpr=False, ROW_START_ONLY: tl.constexpr=False):
    if ROW_START_ONLY:
        lower_tiles: tl.constexpr = BLOCKS * (BLOCKS + 1) // 2 - BLOCK_START * (BLOCK_START + 1) // 2
    else:
        active_blocks: tl.constexpr = BLOCKS - BLOCK_START
        lower_tiles: tl.constexpr = active_blocks * (active_blocks + 1) // 2
    total_tiles: tl.constexpr = batch * lower_tiles
    local = tl.arange(0, BLOCK)
    descriptor = tl.make_tensor_descriptor(current, shape=[batch * n, n], strides=[n, 1], block_shape=[BLOCK, K_TILE])
    if USE_PACKED_FP16:
        current_packed = current_packed_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
        packed_descriptor = tl.make_tensor_descriptor(current_packed, shape=[batch * n, n], strides=[n, 1], block_shape=[BLOCK, K_TILE])
    for linear in tl.range(program, total_tiles, PROGRAMS):
        if WEIGHTED_ORDER:
            paired_programs: tl.constexpr = total_tiles - PROGRAMS
            first = linear < PROGRAMS
            rank = tl.where(first, tl.where(program < paired_programs, program, program + paired_programs), 2 * paired_programs - 1 - program)
            column_block = tl.zeros((), tl.int32)
            if ROW_START_ONLY:
                for boundary in tl.static_range(1, BLOCKS):
                    row_boundary_prefix = 0
                    for previous in tl.static_range(0, boundary):
                        if previous < BLOCK_START:
                            column_rows = BLOCKS - BLOCK_START
                        else:
                            column_rows = BLOCKS - previous
                        row_boundary_prefix += batch * column_rows
                    column_block += rank >= row_boundary_prefix
                first_row_block = tl.maximum(column_block, BLOCK_START)
                prefix = tl.zeros((), tl.int32)
                for previous in tl.static_range(0, BLOCKS):
                    if previous < BLOCK_START:
                        column_rows = BLOCKS - BLOCK_START
                    else:
                        column_rows = BLOCKS - previous
                    prefix += tl.where(column_block > previous, batch * column_rows, 0)
            else:
                for boundary in tl.static_range(1, BLOCKS):
                    full_boundary_prefix = batch * boundary * (2 * BLOCKS - boundary + 1) // 2
                    column_block += rank >= full_boundary_prefix
                first_row_block = column_block
                prefix = batch * column_block * (2 * BLOCKS - column_block + 1) // 2
            row_span = BLOCKS - first_row_block
            local_rank = rank - prefix
            matrix = local_rank // row_span
            row_block = first_row_block + local_rank - matrix * row_span
        else:
            matrix = linear // lower_tiles
            tile = linear - matrix * lower_tiles
            local_row_block = ((tl.sqrt(tile.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5).to(tl.int32)
            row_block = BLOCK_START + local_row_block
            column_block = BLOCK_START + (tile - local_row_block * (local_row_block + 1) // 2)
        row_start = row_block * BLOCK
        column_start = column_block * BLOCK
        base = matrix * n * n
        matrix_row_base = matrix * n
        accumulator = tl.zeros((BLOCK, BLOCK), tl.float32)
        if HIGH_PRECISION_TAIL > 0:
            tail_start = tl.maximum(0, column_start + BLOCK - HIGH_PRECISION_TAIL)
            for reduction_start in tl.range(0, tail_start, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
                if USE_PACKED_FP16:
                    left = packed_descriptor.load([matrix_row_base + row_start, reduction_start])
                    right = packed_descriptor.load([matrix_row_base + column_start, reduction_start])
                    accumulator += tl.dot(left, tl.trans(right))
                else:
                    left = descriptor.load([matrix_row_base + row_start, reduction_start])
                    right = descriptor.load([matrix_row_base + column_start, reduction_start])
                    accumulator = tl.dot(left, tl.trans(right), accumulator, input_precision='tf32')
            for reduction_start in tl.range(tail_start, column_start + BLOCK, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=False):
                left = descriptor.load([matrix_row_base + row_start, reduction_start])
                right = descriptor.load([matrix_row_base + column_start, reduction_start])
                accumulator += _factor_dot(left, tl.trans(right), 2)
        else:
            for reduction_start in tl.range(0, column_start + BLOCK, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
                if USE_PACKED_FP16:
                    left = packed_descriptor.load([matrix_row_base + row_start, reduction_start])
                    right = packed_descriptor.load([matrix_row_base + column_start, reduction_start])
                    accumulator += tl.dot(left, tl.trans(right))
                else:
                    left = descriptor.load([matrix_row_base + row_start, reduction_start])
                    right = descriptor.load([matrix_row_base + column_start, reduction_start])
                    accumulator = tl.dot(left, tl.trans(right), accumulator, input_precision='tf32')
        rows = row_start + local
        columns = column_start + local
        target = tl.load(source + base + rows[:, None] * n + columns[None, :])
        previous = tl.load(current + base + rows[:, None] * n + columns[None, :])
        factor_diagonal = tl.load(current + base + columns * n + columns)
        if SCHEDULE == 5:
            older_diagonal = tl.load(destination + base + columns * n + columns)
            older_diagonal = tl.where(iteration == 0, factor_diagonal, older_diagonal)
        residual = target - accumulator
        if USE_PACKED_FP16:
            quantized_previous = previous.to(tl.float16).to(tl.float32)
            quantized_diagonal = factor_diagonal.to(tl.float16).to(tl.float32)
            residual -= previous * factor_diagonal[None, :] - quantized_previous * quantized_diagonal[None, :]
        if SCHEDULE == 5:
            step, nine_diagonal_step, diagonal_momentum = _n1024_nine_momentum_steps(iteration, columns, n)
        elif SCHEDULE == 4:
            step, ten_diagonal_step = _n1024_ten_quadratic_steps(iteration, columns, n)
        elif SCHEDULE == 0:
            step = tl.full((), 0.7, tl.float32)
            step = tl.where(iteration == 0, 0.65, step)
            step = tl.where(iteration == 1, 0.75, step)
            step = tl.where(iteration == 2, 0.85, step)
            step = tl.where(iteration == 3, 0.95, step)
            step = tl.where(iteration == 4, 0.9, step)
            step = tl.where(iteration == 5, 1.0, step)
            step = tl.where(iteration == 6, 1.1, step)
            step = tl.where(iteration == 7, 1.1, step)
            step = tl.where(iteration == 8, 1.0, step)
            step = tl.where(iteration == 9, 0.95, step)
            step = tl.where(iteration == 10, 1.0, step)
            step = tl.where(iteration == 11, 0.8, step)
        elif SCHEDULE == 1:
            step = tl.full((), 1.0, tl.float32)
            step = tl.where(iteration == 0, 0.6, step)
            step = tl.where(iteration == 1, 0.7, step)
            step = tl.where(iteration == 2, 0.8, step)
            step = tl.where(iteration == 3, 0.9, step)
            step = tl.where(iteration == 4, 0.85, step)
            step = tl.where(iteration == 5, 0.95, step)
            step = tl.where(iteration == 6, 1.05, step)
            step = tl.where(iteration == 7, 1.1, step)
            step = tl.where(iteration == 8, 1.0, step)
            step = tl.where(iteration == 9, 0.9, step)
        elif SCHEDULE == 2:
            step = tl.full((), 0.9, tl.float32)
            step = tl.where(iteration == 0, 0.7, step)
            step = tl.where(iteration == 1, 0.8, step)
            step = tl.where(iteration == 2, 0.9, step)
            step = tl.where(iteration == 3, 0.95, step)
            step = tl.where(iteration == 4, 1.0, step)
            step = tl.where(iteration == 5, 0.95, step)
            step = tl.where(iteration == 6, 0.9, step)
            step = tl.where(iteration == 7, 0.85, step)
            step = tl.where(iteration == 8, 0.8, step)
        else:
            step = tl.full((), 0.65, tl.float32)
            step = tl.where(iteration == 0, 0.7, step)
            step = tl.where(iteration == 1, 0.85, step)
            step = tl.where(iteration == 2, 0.95, step)
            step = tl.where(iteration == 3, 0.95, step)
            step = tl.where(iteration == 4, 0.95, step)
            step = tl.where(iteration == 5, 1.0, step)
            step = tl.where(iteration == 6, 0.95, step)
            step = tl.where(iteration == 7, 1.0, step)
            step = tl.where(iteration == 8, 1.0, step)
            step = tl.where(iteration == 9, 1.0, step)
            step = tl.where(iteration == 10, 0.85, step)
        if SCHEDULE == 5:
            diagonal_step = nine_diagonal_step
        elif SCHEDULE == 4:
            diagonal_step = ten_diagonal_step
        elif DIAGONAL_RELAXED:
            diagonal_delta = tl.full((), -1.35725, tl.float32)
            diagonal_delta = tl.where(iteration == 0, 0.025, diagonal_delta)
            diagonal_delta = tl.where(iteration == 1, 0.05275, diagonal_delta)
            diagonal_delta = tl.where(iteration == 2, -0.109, diagonal_delta)
            diagonal_delta = tl.where(iteration == 3, -0.17025, diagonal_delta)
            diagonal_delta = tl.where(iteration == 4, -0.286, diagonal_delta)
            diagonal_delta = tl.where(iteration == 5, -0.33125, diagonal_delta)
            diagonal_delta = tl.where(iteration == 6, -0.476, diagonal_delta)
            diagonal_delta = tl.where(iteration == 7, -0.61025, diagonal_delta)
            diagonal_delta = tl.where(iteration == 8, -0.744, diagonal_delta)
            diagonal_delta = tl.where(iteration == 9, -0.93225, diagonal_delta)
            diagonal_delta = tl.where(iteration == 10, -1.235, diagonal_delta)
            diagonal_step = tl.maximum(0.075, tl.minimum(1.4, step + diagonal_delta))
        else:
            diagonal_step = step
        correction = residual / factor_diagonal[None, :]
        if SCHEDULE == 5:
            updated = previous + step * correction
            diagonal_square = previous * previous + diagonal_step * residual + diagonal_momentum * (factor_diagonal * factor_diagonal - older_diagonal * older_diagonal)[None, :]
            updated = tl.where(rows[:, None] == columns[None, :], tl.sqrt(tl.maximum(diagonal_square, 1.1754943508222875e-38)), updated)
        else:
            updated = previous + step * correction
            updated = tl.where(rows[:, None] == columns[None, :], tl.sqrt(tl.maximum(previous * previous + diagonal_step * residual, 1.1754943508222875e-38)), updated)
        tl.store(destination + base + rows[:, None] * n + columns[None, :], updated, mask=rows[:, None] >= columns[None, :])
        if USE_PACKED_FP16:
            destination_packed = destination_packed_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
            tl.store(destination_packed + base + rows[:, None] * n + columns[None, :], updated.to(tl.float16), mask=rows[:, None] >= columns[None, :])

@triton.jit
def _n1024_ten_bridge_linear(source, factor, temporary, linear, n: tl.constexpr, batch: tl.constexpr, K_TILE: tl.constexpr, PIPELINE_STAGES: tl.constexpr, WARP_SPECIALIZE: tl.constexpr):
    atoms: tl.constexpr = 36
    extra_atoms: tl.constexpr = 1
    regular_tasks: tl.constexpr = batch * atoms
    extra = linear >= regular_tasks
    regular_linear = tl.minimum(linear, regular_tasks - 1)
    matrix = regular_linear // atoms
    atom = regular_linear - matrix * atoms
    local_row_block = ((tl.sqrt(atom.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5).to(tl.int32)
    local_column_block = atom - local_row_block * (local_row_block + 1) // 2
    extra_linear = linear - regular_tasks
    extra_matrix = extra_linear // extra_atoms
    extra_row_block = 8 + extra_linear - extra_matrix * extra_atoms
    matrix = tl.where(extra, extra_matrix, matrix)
    row_start = tl.where(extra, extra_row_block * 64, (local_row_block + 8) * 64)
    column_start = tl.where(extra, 448, (local_column_block + 8) * 64)
    base = matrix * n * n
    matrix_row_base = matrix * n
    descriptor = tl.make_tensor_descriptor(factor, shape=[batch * n, n], strides=[n, 1], block_shape=[64, K_TILE])
    accumulator = tl.zeros((64, 64), tl.float32)
    for reduction_start in tl.range(0, column_start + 64, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
        left = descriptor.load([matrix_row_base + row_start, reduction_start])
        right = descriptor.load([matrix_row_base + column_start, reduction_start])
        accumulator += tl.dot(left, tl.trans(right), input_precision='tf32')
    local = tl.arange(0, 64)
    rows = row_start + local
    columns = column_start + local
    target = tl.load(source + base + rows[:, None] * n + columns[None, :])
    previous = tl.load(factor + base + rows[:, None] * n + columns[None, :])
    diagonal = tl.load(factor + base + columns * n + columns)
    residual = target - accumulator
    fraction = (columns.to(tl.float32) + 0.5) / n
    centered = fraction - 0.5
    quadratic = centered * centered - 1.0 / 12.0
    step = 0.7410513759 + 0.0970024914 * fraction - 0.1491172314 * quadratic
    diagonal_step = tl.minimum(0.6, tl.maximum(0.005, 0.005 + 0.0577071309 * fraction + 0.8000000119 * quadratic))
    updated = previous + step * residual / diagonal[None, :]
    updated = tl.where(rows[:, None] == columns[None, :], tl.sqrt(tl.maximum(previous * previous + diagonal_step * residual, 1.1754943508222875e-38)), updated)
    tl.store(temporary + base + rows[:, None] * n + columns[None, :], updated, mask=rows[:, None] >= columns[None, :])

@triton.jit
def _n1024_ten_bridge(source, factor, temporary, program, n: tl.constexpr, batch: tl.constexpr, PROGRAMS: tl.constexpr, K_TILE: tl.constexpr, PIPELINE_STAGES: tl.constexpr, WARP_SPECIALIZE: tl.constexpr):
    for linear in tl.range(program, batch * 37, PROGRAMS):
        _n1024_ten_bridge_linear(source, factor, temporary, linear, n, batch, K_TILE, PIPELINE_STAGES, WARP_SPECIALIZE)

@triton.jit
def _n1024_ten_second_selected_linear(source, factor, temporary, linear, n: tl.constexpr, batch: tl.constexpr, K_TILE: tl.constexpr, PIPELINE_STAGES: tl.constexpr, WARP_SPECIALIZE: tl.constexpr):
    atoms: tl.constexpr = 36
    extra_atoms: tl.constexpr = 1
    regular_tasks: tl.constexpr = batch * atoms
    extra = linear >= regular_tasks
    regular_linear = tl.minimum(linear, regular_tasks - 1)
    matrix = regular_linear // atoms
    atom = regular_linear - matrix * atoms
    local_row_block = ((tl.sqrt(atom.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5).to(tl.int32)
    local_column_block = atom - local_row_block * (local_row_block + 1) // 2
    extra_linear = linear - regular_tasks
    extra_matrix = extra_linear // extra_atoms
    extra_row_block = 8 + extra_linear - extra_matrix * extra_atoms
    matrix = tl.where(extra, extra_matrix, matrix)
    row_start = tl.where(extra, extra_row_block * 64, (local_row_block + 8) * 64)
    column_start = tl.where(extra, 448, (local_column_block + 8) * 64)
    local = tl.arange(0, 64)
    rows = row_start + local
    columns = column_start + local
    base = matrix * n * n
    previous = tl.load(temporary + base + rows[:, None] * n + columns[None, :])
    selected = (~extra) & (local_column_block >= 1)
    if selected:
        prefix_descriptor = tl.make_tensor_descriptor(factor, shape=[batch * n, n], strides=[n, 1], block_shape=[64, K_TILE])
        tail_descriptor = tl.make_tensor_descriptor(temporary, shape=[batch * n, n], strides=[n, 1], block_shape=[64, K_TILE])
        accumulator = tl.zeros((64, 64), tl.float32)
        for reduction_start in tl.range(0, 512, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
            left = prefix_descriptor.load([matrix * n + row_start, reduction_start])
            right = prefix_descriptor.load([matrix * n + column_start, reduction_start])
            accumulator += tl.dot(left, tl.trans(right), input_precision='tf32')
        for reduction_start in tl.range(512, column_start + 64, K_TILE, num_stages=PIPELINE_STAGES, warp_specialize=WARP_SPECIALIZE):
            left = tail_descriptor.load([matrix * n + row_start, reduction_start])
            right = tail_descriptor.load([matrix * n + column_start, reduction_start])
            accumulator += tl.dot(left, tl.trans(right), input_precision='tf32')
        target = tl.load(source + base + rows[:, None] * n + columns[None, :])
        diagonal = tl.load(temporary + base + columns * n + columns)
        residual = target - accumulator
        corrected = previous + 0.8200000000 * residual / diagonal[None, :]
        values = tl.where(rows[:, None] > columns[None, :], corrected, previous)
    else:
        values = previous
    tl.store(factor + base + rows[:, None] * n + columns[None, :], values, mask=rows[:, None] >= columns[None, :])

@triton.jit
def _n1024_ten_copy_bridge(source, temporary, factor, program, n: tl.constexpr, batch: tl.constexpr, PROGRAMS: tl.constexpr, K_TILE: tl.constexpr, PIPELINE_STAGES: tl.constexpr, WARP_SPECIALIZE: tl.constexpr):
    for linear in tl.range(program, batch * 37, PROGRAMS):
        _n1024_ten_second_selected_linear(source, factor, temporary, linear, n, batch, K_TILE, PIPELINE_STAGES, WARP_SPECIALIZE)

@triton.jit
def _persistent_jacobi_kernel(source, output, scratch, packed_output_storage, packed_scratch_storage, flags, n: tl.constexpr, batch: tl.constexpr, BLOCKS: tl.constexpr, ITERATIONS: tl.constexpr, PROGRAMS: tl.constexpr, BLOCK: tl.constexpr, K_TILE: tl.constexpr, PIPELINE_STAGES: tl.constexpr, SCHEDULE: tl.constexpr, FINAL_BLOCK_START: tl.constexpr, HIGH_PRECISION_TAIL: tl.constexpr, USE_PACKED_FP16: tl.constexpr, DIAGONAL_RELAXED: tl.constexpr, LOWER_INITIALIZATION: tl.constexpr, FUSED_TEN_TAIL: tl.constexpr, WARP_SPECIALIZE: tl.constexpr, CROSS_CALL_RELEASE: tl.constexpr=False, TAIL_UPPER_CLEAR: tl.constexpr=False, USE_RSQRT_INIT: tl.constexpr=False):
    if CROSS_CALL_RELEASE:
        tl_cuda.gdc_wait()
        tl_cuda.gdc_launch_dependents()
    program = tl.program_id(0)
    local = tl.arange(0, BLOCK)
    square_tiles: tl.constexpr = BLOCKS * BLOCKS
    lower_tiles: tl.constexpr = BLOCKS * (BLOCKS + 1) // 2
    initialization_tiles: tl.constexpr = batch * (lower_tiles if LOWER_INITIALIZATION else square_tiles)
    for linear in tl.range(program, initialization_tiles, PROGRAMS):
        if LOWER_INITIALIZATION:
            matrix = linear // lower_tiles
            tile = linear - matrix * lower_tiles
            row_block = ((tl.sqrt(tile.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5).to(tl.int32)
            column_block = tile - row_block * (row_block + 1) // 2
        else:
            matrix = linear // square_tiles
            tile = linear - matrix * square_tiles
            row_block = tile // BLOCKS
            column_block = tile - row_block * BLOCKS
        rows = row_block * BLOCK + local
        columns = column_block * BLOCK + local
        base = matrix * n * n
        diagonal = tl.load(source + base + columns * n + columns)
        safe_diagonal = tl.maximum(diagonal, 1.1754943508222875e-38)
        if USE_RSQRT_INIT:
            inverse_roots = tl.rsqrt(safe_diagonal)
            roots = safe_diagonal * inverse_roots
        else:
            roots = tl.sqrt(safe_diagonal)
        values = tl.load(source + base + rows[:, None] * n + columns[None, :])
        initialized = tl.where(rows[:, None] == columns[None, :], roots[None, :], tl.where(rows[:, None] > columns[None, :], values * inverse_roots[None, :] if USE_RSQRT_INIT else values / roots[None, :], 0.0))
        tl.store(output + base + rows[:, None] * n + columns[None, :], initialized)
        if USE_PACKED_FP16:
            packed_output = packed_output_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
            packed_scratch = packed_scratch_storage.cast(tl.pointer_type(tl.float16), bitcast=True)
            tl.store(packed_output + base + rows[:, None] * n + columns[None, :], initialized.to(tl.float16))
            tl.store(packed_scratch + base + rows[:, None] * n + columns[None, :], 0.0, mask=columns[None, :] > rows[:, None])
        tl.store(scratch + base + rows[:, None] * n + columns[None, :], 0.0, mask=columns[None, :] > rows[:, None])
        if LOWER_INITIALIZATION:
            off_diagonal = row_block != column_block
            if not TAIL_UPPER_CLEAR:
                tl.store(output + base + columns[:, None] * n + rows[None, :], 0.0, mask=off_diagonal)
            tl.store(scratch + base + columns[:, None] * n + rows[None, :], 0.0, mask=off_diagonal)
    _n2048_monotonic_integer_epoch(flags, 1, PROGRAMS)
    current = output
    destination = scratch
    current_packed = packed_output_storage
    destination_packed = packed_scratch_storage
    for iteration in tl.range(0, ITERATIONS - 1, 1):
        _persistent_jacobi_wave(source, current, destination, current_packed, destination_packed, program, iteration, n, batch, BLOCKS, PROGRAMS, BLOCK, K_TILE, PIPELINE_STAGES, SCHEDULE, 0, 0, USE_PACKED_FP16, DIAGONAL_RELAXED, WARP_SPECIALIZE)
        _n2048_monotonic_integer_epoch(flags, iteration + 2, PROGRAMS)
        temporary = current
        current = destination
        destination = temporary
        temporary_packed = current_packed
        current_packed = destination_packed
        destination_packed = temporary_packed
    _persistent_jacobi_wave(source, current, destination, current_packed, destination_packed, program, ITERATIONS - 1, n, batch, BLOCKS, PROGRAMS, BLOCK, K_TILE, PIPELINE_STAGES, SCHEDULE, FINAL_BLOCK_START, HIGH_PRECISION_TAIL, USE_PACKED_FP16, DIAGONAL_RELAXED, WARP_SPECIALIZE)
    if FUSED_TEN_TAIL:
        _n2048_monotonic_integer_epoch(flags, ITERATIONS + 1, PROGRAMS)
        _n1024_ten_bridge(source, destination, current, program, n, batch, PROGRAMS, K_TILE, PIPELINE_STAGES, WARP_SPECIALIZE)
        _n2048_monotonic_integer_epoch(flags, ITERATIONS + 2, PROGRAMS)
        _n1024_ten_copy_bridge(source, current, destination, program, n, batch, PROGRAMS, K_TILE, PIPELINE_STAGES, WARP_SPECIALIZE)
        if TAIL_UPPER_CLEAR:
            atom = tl.where(program < batch * 36, program % 36, 5)
            local_row = ((tl.sqrt(atom.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5).to(tl.int32)
            local_column = atom - local_row * (local_row + 1) // 2
            if local_column < 2:
                worker = tl.where(local_row == 0, 0, 2 * local_row - 1 + local_column)
                clear_axis = tl.arange(0, 128)
                for clear_tile in tl.range(worker, 28, 15):
                    clear_column = ((tl.sqrt(clear_tile.to(tl.float32) * 8.0 + 1.0) + 1.0) * 0.5).to(tl.int32)
                    clear_row = clear_tile - clear_column * (clear_column - 1) // 2
                    clear_base = program // 36 * n * n
                    clear_rows = clear_row * 128 + clear_axis
                    clear_columns = clear_column * 128 + clear_axis
                    tl.store(destination + clear_base + clear_rows[:, None] * n + clear_columns[None, :], 0.0)
    return

@triton.jit
def _persistent_jacobi_n2048_relaxed_wave(source, current, destination, program, iteration, n: tl.constexpr, batch: tl.constexpr, PROGRAMS: tl.constexpr, ROW_START_ONLY: tl.constexpr):
    blocks: tl.constexpr = 16
    block: tl.constexpr = 128
    row_start_block: tl.constexpr = 10 if ROW_START_ONLY else 0
    if ROW_START_ONLY:
        lower_tiles: tl.constexpr = blocks * (blocks + 1) // 2 - row_start_block * (row_start_block + 1) // 2
    else:
        lower_tiles: tl.constexpr = blocks * (blocks + 1) // 2
    total_tiles: tl.constexpr = batch * lower_tiles
    paired_programs: tl.constexpr = total_tiles - PROGRAMS
    local = tl.arange(0, block)
    descriptor = tl.make_tensor_descriptor(current, shape=[batch * n, n], strides=[n, 1], block_shape=[block, 32])
    for linear in tl.range(program, total_tiles, PROGRAMS):
        first = linear < PROGRAMS
        rank = tl.where(first, tl.where(program < paired_programs, program, program + paired_programs), 2 * paired_programs - 1 - program)
        column_block = tl.zeros((), tl.int32)
        if ROW_START_ONLY:
            for boundary in tl.static_range(1, blocks):
                row_boundary_prefix = 0
                for previous in tl.static_range(0, boundary):
                    if previous < row_start_block:
                        column_rows = blocks - row_start_block
                    else:
                        column_rows = blocks - previous
                    row_boundary_prefix += batch * column_rows
                column_block += rank >= row_boundary_prefix
            first_row_block = tl.maximum(column_block, row_start_block)
            prefix = tl.zeros((), tl.int32)
            for previous in tl.static_range(0, blocks):
                if previous < row_start_block:
                    column_rows = blocks - row_start_block
                else:
                    column_rows = blocks - previous
                prefix += tl.where(column_block > previous, batch * column_rows, 0)
        else:
            for boundary in tl.static_range(1, blocks):
                full_boundary_prefix = batch * boundary * (2 * blocks - boundary + 1) // 2
                column_block += rank >= full_boundary_prefix
            first_row_block = column_block
            prefix = batch * column_block * (2 * blocks - column_block + 1) // 2
        row_span = blocks - first_row_block
        local_rank = rank - prefix
        matrix = local_rank // row_span
        row_block = first_row_block + local_rank - matrix * row_span
        row_start = row_block * block
        column_start = column_block * block
        matrix_row_base = matrix * n
        base = matrix * n * n
        accumulator = tl.zeros((block, block), tl.float32)
        for reduction_start in tl.range(0, column_start + block, 32, num_stages=5, warp_specialize=False):
            left = descriptor.load([matrix_row_base + row_start, reduction_start])
            right = descriptor.load([matrix_row_base + column_start, reduction_start])
            accumulator += tl.dot(left, tl.trans(right), input_precision='tf32')
        rows = row_start + local
        columns = column_start + local
        target = tl.load(source + base + rows[:, None] * n + columns[None, :])
        previous = tl.load(current + base + rows[:, None] * n + columns[None, :])
        factor_diagonal = tl.load(current + base + columns * n + columns)
        residual = target - accumulator
        step = tl.full((), 1.0, tl.float32)
        step = tl.where(iteration == 0, 0.6, step)
        step = tl.where(iteration == 1, 0.7, step)
        step = tl.where(iteration == 2, 0.8, step)
        step = tl.where(iteration == 3, 0.9, step)
        step = tl.where(iteration == 4, 0.85, step)
        step = tl.where(iteration == 5, 0.95, step)
        step = tl.where(iteration == 6, 1.3, step)
        step = tl.where(iteration == 7, 1.1, step)
        step = tl.where(iteration == 8, 1.0, step)
        step = tl.where(iteration == 9, 0.8, step)
        iteration_value = iteration * 1.0
        diagonal_step = tl.maximum(0.05, tl.minimum(1.5, step + 0.1 - 0.175 * iteration_value))
        diagonal_step = tl.where(iteration == 6, 0.1, diagonal_step)
        updated = previous + step * residual / factor_diagonal[None, :]
        updated = tl.where(rows[:, None] == columns[None, :], tl.sqrt(tl.maximum(previous * previous + diagonal_step * residual, 1.1754943508222875e-38)), updated)
        tl.store(destination + base + rows[:, None] * n + columns[None, :], updated, mask=rows[:, None] >= columns[None, :])

@triton.jit
def _n2048_store_tail_update_gpu2(source, previous_storage, destination, accumulator, base, row_start, column_start, n: tl.constexpr, step: tl.constexpr, diagonal_step: tl.constexpr):
    axis = tl.arange(0, 128)
    rows = row_start + axis
    columns = column_start + axis
    target = tl.load(source + base + rows[:, None] * n + columns[None, :])
    previous = tl.load(previous_storage + base + rows[:, None] * n + columns[None, :])
    diagonal = tl.load(previous_storage + base + columns * n + columns)
    residual = target - accumulator
    corrected = previous + step * residual / diagonal[None, :]
    values = tl.where(rows[:, None] == columns[None, :], tl.sqrt(tl.maximum(previous * previous + diagonal_step * residual, 1.1754943508222875e-38)), corrected)
    tl.store(destination + base + rows[:, None] * n + columns[None, :], values, mask=rows[:, None] >= columns[None, :])

@triton.jit
def _n2048_copy_prefix_gpu2(current, destination, program, n: tl.constexpr, batch: tl.constexpr, START: tl.constexpr, PROGRAMS: tl.constexpr):
    tail_blocks: tl.constexpr = 16 - START
    tail_atoms: tl.constexpr = tail_blocks * (tail_blocks + 1) // 2
    tail_programs: tl.constexpr = batch * tail_atoms
    copy_programs: tl.constexpr = PROGRAMS - tail_programs
    if program >= tail_programs:
        worker = program - tail_programs
        blocks64: tl.constexpr = 32
        lower_tiles: tl.constexpr = blocks64 * (blocks64 + 1) // 2
        total_tiles: tl.constexpr = batch * lower_tiles
        axis = tl.arange(0, 64)
        for linear in tl.range(worker, total_tiles, copy_programs):
            matrix = linear // lower_tiles
            tile = linear - matrix * lower_tiles
            row_block = ((tl.sqrt(tile.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5).to(tl.int32)
            column_block = tile - row_block * (row_block + 1) // 2
            outside_tail = (row_block < 2 * START) | (column_block < 2 * START)
            if outside_tail:
                rows = row_block * 64 + axis
                columns = column_block * 64 + axis
                base = matrix * n * n
                values = tl.load(current + base + rows[:, None] * n + columns[None, :])
                tl.store(destination + base + rows[:, None] * n + columns[None, :], values, mask=rows[:, None] >= columns[None, :])

@triton.jit
def _n2048_triple_tail_reuse_gpu2(source, current, destination, gram_storage, flags, program, n: tl.constexpr, batch: tl.constexpr, PROGRAMS: tl.constexpr, START: tl.constexpr, STEP1: tl.constexpr, STEP2: tl.constexpr, STEP3: tl.constexpr, DIAGONAL_STEP1: tl.constexpr, DIAGONAL_STEP2: tl.constexpr, DIAGONAL_STEP3: tl.constexpr, TAIL_K: tl.constexpr, TAIL_STAGES: tl.constexpr):
    tail_blocks: tl.constexpr = 16 - START
    atoms: tl.constexpr = tail_blocks * (tail_blocks + 1) // 2
    active = program < batch * atoms
    matrix = program // atoms
    atom = program - matrix * atoms
    local_row = ((tl.sqrt(atom.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5).to(tl.int32)
    local_column = atom - local_row * (local_row + 1) // 2
    row_start = (local_row + START) * 128
    column_start = (local_column + START) * 128
    matrix_row_base = matrix * n
    base = matrix * n * n
    current_descriptor = tl.make_tensor_descriptor(current, shape=[batch * n, n], strides=[n, 1], block_shape=[128, TAIL_K])
    destination_descriptor = tl.make_tensor_descriptor(destination, shape=[batch * n, n], strides=[n, 1], block_shape=[128, TAIL_K])
    prefix_accumulator = tl.zeros((128, 128), tl.float32)
    if active:
        for reduction_start in tl.range(0, START * 128, TAIL_K, num_stages=TAIL_STAGES, warp_specialize=False):
            left = current_descriptor.load([matrix_row_base + row_start, reduction_start])
            right = current_descriptor.load([matrix_row_base + column_start, reduction_start])
            prefix_accumulator += tl.dot(left, tl.trans(right), input_precision='tf32')
        gram_axis = tl.arange(0, 128)
        gram_base = program * 128 * 128
        tl.store(gram_storage + gram_base + gram_axis[:, None] * 128 + gram_axis[None, :], prefix_accumulator)
        first_accumulator = prefix_accumulator
        for reduction_start in tl.range(START * 128, column_start + 128, TAIL_K, num_stages=TAIL_STAGES, warp_specialize=False):
            left = current_descriptor.load([matrix_row_base + row_start, reduction_start])
            right = current_descriptor.load([matrix_row_base + column_start, reduction_start])
            first_accumulator += tl.dot(left, tl.trans(right), input_precision='tf32')
        _n2048_store_tail_update_gpu2(source, current, destination, first_accumulator, base, row_start, column_start, n, STEP1, DIAGONAL_STEP1)
    _n2048_copy_prefix_gpu2(current, destination, program, n, batch, START, PROGRAMS)
    _n2048_monotonic_integer_epoch(flags, 9, PROGRAMS)
    inactive_programs: tl.constexpr = PROGRAMS - batch * atoms
    if program >= batch * atoms:
        worker = program - batch * atoms
        strict_tiles: tl.constexpr = 16 * 15 // 2
        clear_axis = tl.arange(0, 128)
        for linear in tl.range(worker, batch * strict_tiles, inactive_programs):
            clear_matrix = linear // strict_tiles
            clear_tile = linear - clear_matrix * strict_tiles
            clear_column = ((tl.sqrt(clear_tile.to(tl.float32) * 8.0 + 1.0) + 1.0) * 0.5).to(tl.int32)
            clear_row = clear_tile - clear_column * (clear_column - 1) // 2
            clear_rows = clear_row * 128 + clear_axis
            clear_columns = clear_column * 128 + clear_axis
            clear_base = clear_matrix * n * n
            tl.store(destination + clear_base + clear_rows[:, None] * n + clear_columns[None, :], 0.0)
    if active:
        gram_axis = tl.arange(0, 128)
        gram_base = program * 128 * 128
        second_accumulator = tl.load(gram_storage + gram_base + gram_axis[:, None] * 128 + gram_axis[None, :])
        for reduction_start in tl.range(START * 128, column_start + 128, TAIL_K, num_stages=TAIL_STAGES, warp_specialize=False):
            left = destination_descriptor.load([matrix_row_base + row_start, reduction_start])
            right = destination_descriptor.load([matrix_row_base + column_start, reduction_start])
            second_accumulator += tl.dot(left, tl.trans(right), input_precision='tf32')
        _n2048_store_tail_update_gpu2(source, destination, current, second_accumulator, base, row_start, column_start, n, STEP2, DIAGONAL_STEP2)
    _n2048_monotonic_integer_epoch(flags, 10, PROGRAMS)
    if active:
        gram_axis = tl.arange(0, 128)
        gram_base = program * 128 * 128
        third_accumulator = tl.load(gram_storage + gram_base + gram_axis[:, None] * 128 + gram_axis[None, :])
        for reduction_start in tl.range(START * 128, column_start + 128, TAIL_K, num_stages=TAIL_STAGES, warp_specialize=False):
            left = current_descriptor.load([matrix_row_base + row_start, reduction_start])
            right = current_descriptor.load([matrix_row_base + column_start, reduction_start])
            third_accumulator += tl.dot(left, tl.trans(right), input_precision='tf32')
        _n2048_store_tail_update_gpu2(source, current, destination, third_accumulator, base, row_start, column_start, n, STEP3, DIAGONAL_STEP3)

@triton.jit
def _persistent_jacobi_n2048_relaxed_kernel(source, output, scratch, flags, gram_storage, n: tl.constexpr, batch: tl.constexpr, PROGRAMS: tl.constexpr):
    tl_cuda.gdc_wait()
    program = tl.program_id(0)
    block: tl.constexpr = 128
    blocks: tl.constexpr = 16
    local = tl.arange(0, block)
    lower_tiles: tl.constexpr = blocks * (blocks + 1) // 2
    initialization_tiles: tl.constexpr = batch * lower_tiles
    for linear in tl.range(program, initialization_tiles, PROGRAMS):
        matrix = linear // lower_tiles
        tile = linear - matrix * lower_tiles
        row_block = ((tl.sqrt(tile.to(tl.float32) * 8.0 + 1.0) - 1.0) * 0.5).to(tl.int32)
        column_block = tile - row_block * (row_block + 1) // 2
        rows = row_block * block + local
        columns = column_block * block + local
        base = matrix * n * n
        diagonal = tl.load(source + base + columns * n + columns)
        safe_diagonal = tl.maximum(diagonal, 1.1754943508222875e-38)
        inverse_roots = tl.rsqrt(safe_diagonal)
        roots = safe_diagonal * inverse_roots
        values = tl.load(source + base + rows[:, None] * n + columns[None, :])
        initialized = tl.where(rows[:, None] == columns[None, :], roots[None, :], tl.where(rows[:, None] > columns[None, :], values * inverse_roots[None, :], 0.0))
        tl.store(output + base + rows[:, None] * n + columns[None, :], initialized)
        off_diagonal = row_block != column_block
        tl.store(scratch + base + rows[:, None] * n + columns[None, :], 0.0, mask=~off_diagonal & (columns[None, :] > rows[:, None]))
    _n2048_monotonic_integer_epoch(flags, 1, PROGRAMS)
    current = output
    destination = scratch
    for iteration in tl.range(0, 7, 1):
        _persistent_jacobi_n2048_relaxed_wave(source, current, destination, program, iteration, n, batch, PROGRAMS, False)
        _n2048_monotonic_integer_epoch(flags, iteration + 2, PROGRAMS)
        temporary = current
        current = destination
        destination = temporary
    tl_cuda.gdc_launch_dependents()
    return

@triton.jit
def _persistent_jacobi_n2048_relaxed_tail_kernel(source, output, scratch, flags, gram_storage, n: tl.constexpr, batch: tl.constexpr, PROGRAMS: tl.constexpr):
    tl_cuda.gdc_wait()
    tl_cuda.gdc_launch_dependents()
    program = tl.program_id(0)
    _n2048_triple_tail_reuse_gpu2(source, scratch, output, gram_storage, flags, program, n, batch, PROGRAMS, 6, 1.0, 1.0, 0.9, 0.05, 0.05, 0.05, 32, 2)
    return



def _production_custom_kernel(data):
 n = data.shape[1]
 size_class = n.bit_length()
 if n >= 256:
  dense_throughput_domain = data.numel().bit_length() >= 23 or size_class >= 13 or data.shape[0].bit_length() >= 6
  if dense_throughput_domain:
   family_score = 1.0 / n
   diagonal_ratio = 1.0
   route = 1.0
  else:
   family_score, diagonal_ratio = _family_route_probe(data)
   route = _lowrank_probe(data) if family_score > 0.01 else 1.0
   if family_score < 0.0:
    return _structured_parallel_cholesky(data)
 else:
  family_score = 0.0
  route = 1.0
  diagonal_ratio = 1.0
 if family_score < 0.0:
  return _structured_parallel_cholesky(data)
 if size_class == 9 and route < 0.05:
  return _blocked_cholesky(data, _output_workspace(data), high_precision=True)
 if n >= 512 and route < 0.05:
  batch = data.shape[0]
  if size_class == 10:
   chunk = 64
   use_chunked = batch < 128
  elif size_class == 11:
   chunk = 128 if batch >= 32 else 64
   use_chunked = True
  elif n <= 8192:
   chunk = 64
   use_chunked = True
  else:
   chunk = 128
   use_chunked = True
  return _lowrank_cholesky(data, chunk, use_chunked)
 covariance_like_output = 0.0 <= family_score < 0.01 and family_score * n < 0.7
 jacobi_output = covariance_like_output and (size_class == 9 or (size_class == 10 and data.shape[0] <= 32) or (size_class == 11 and data.shape[0] <= 8) or (size_class in (12, 13, 14, 15))) or (0.7 <= family_score * n < 3.0 and diagonal_ratio >= 0.5 and (size_class == 11 and data.shape[0] <= 8 or (size_class == 12 and data.shape[0] <= 2) or (size_class == 13 and data.shape[0] <= 2)))
 fp4_factor_output = n >= 32768 and family_score * n < 3.0 and (diagonal_ratio >= 0.5)
 separate_large_upper_zero = size_class == 14 and 0.7 <= family_score * n < 3.0 and (diagonal_ratio >= 0.5) or (size_class >= 16 and (not fp4_factor_output))
 persistent_n8192_output = False
 overlap_large_upper_zero = size_class == 14 and (not separate_large_upper_zero)
 polynomial_factor_output = size_class == 10 and data.shape[0] <= 32 and (0.7 <= family_score * n < 3.0) and (diagonal_ratio >= 0.5) or (size_class == 11 and data.shape[0] >= 32) or (size_class == 12 and data.shape[0] >= 8)
 narrow_factor_output = size_class == 10 and data.shape[0].bit_length() >= 8
 backed_n2048_output = size_class == 12 and data.shape[0].bit_length() == 2 and (0.7 <= family_score * n) and jacobi_output
 if backed_n2048_output:
  output = None
 elif size_class == 15:
  output = _empty_output(data)
 elif separate_large_upper_zero and (not persistent_n8192_output):
  output = _new_strict_upper_output(data)
 elif fp4_factor_output:
  output = _new_async_strict_upper_output(data)
 elif persistent_n8192_output:
  output = _new_n8192_resident_strict_upper_output(data)
 elif persistent_n8192_output or n <= 128 or jacobi_output or overlap_large_upper_zero or polynomial_factor_output or narrow_factor_output or (size_class == 9 and 0.0 <= family_score < 0.01):
  output = _empty_output(data)
 else:
  output = _output_workspace(data)
 if n >= 256 and family_score > 0.01 and (route >= 0.05):
  return _blocked_cholesky(data, output, high_precision=True)
 if n >= 2048 and 0.0 < family_score < 0.01 and (family_score * n >= 3.0):
  return _blocked_cholesky(data, output, high_precision=True)
 covariance_like = 0.0 <= family_score < 0.01 and family_score * n < 0.7
 if covariance_like:
  if size_class == 9:
   return _jacobi_cholesky(data, output, iterations=5, step=0.9, high_precision_last=True, pointer_pipeline_stages=3)
  if size_class == 10 and data.shape[0] <= 32:
   return _jacobi_cholesky(data, output, iterations=4, step=0.9, high_precision_last=True, pointer_pipeline_stages=3)
  if size_class == 11 and data.shape[0] <= 8:
   return _jacobi_cholesky(data, output, iterations=4, step=0.9)
  if size_class == 12:
   return _jacobi_cholesky(data, output, iterations=3, step=0.9)
  if size_class == 13:
   return _jacobi_cholesky(data, output, iterations=3, step=0.9)
  if size_class == 14:
   return _jacobi_cholesky(data, output, iterations=2, step=0.9, use_tma=True, tma_pipeline_stages=7, tma_num_warps=8, tma_k_tile=32)
  if size_class == 15:
   return _jacobi_cholesky(data, output, iterations=1, step=0.8, use_tma=True, tma_pipeline_stages=7, tma_num_warps=8, tma_k_tile=32)
 if size_class == 11 and data.shape[0] <= 8 and (0.7 <= family_score * n < 3.0) and (diagonal_ratio >= 0.5):
  if data.shape[0].bit_length() == 3 and data.shape[0] % 4 == 0:
   return _persistent_jacobi_n1024_momentum9(data, output)
  return _jacobi_cholesky(data, output, iterations=13, step=0.76, iteration_steps=(0.65, 0.75, 0.85, 0.95, 0.9, 1.0, 1.1, 1.1, 1.0, 0.95, 1.0, 0.8, 0.7), use_tma=True, tma_k_tile=64, tma_warp_specialize=True, tma_pipeline_stages=3, tma_num_warps=8, final_block_start=5, copy_before_partial=False)
 if size_class == 12 and data.shape[0] <= 2 and (0.7 <= family_score * n < 3.0) and (diagonal_ratio >= 0.5):
  if data.shape[0].bit_length() == 2:
   return _persistent_jacobi_weighted_lower(data, output, blocks=16, iterations=11, k_tile=32, pipeline_stages=5, schedule=1, final_block_start=8, penultimate_row_start=8, programs=144)
  return _jacobi_cholesky(data, output, iterations=12, step=0.78, iteration_steps=(0.6, 0.7, 0.8, 0.9, 0.85, 0.95, 1.05, 1.1, 1.0, 0.9, 0.8, 0.7), use_tma=True, tma_k_tile=32, tma_warp_specialize=True, tma_pipeline_stages=6, tma_num_warps=8, final_block_start=6, copy_before_partial=False, penultimate_row_start=6, tail_row_waves=2)
 if size_class == 13 and data.shape[0] <= 2 and (0.7 <= family_score * n < 3.0) and (diagonal_ratio >= 0.5):
  return _late_packed_jacobi_sparse_tail10_bridge(data, output)
 if size_class == 14 and 0.7 <= family_score * n < 3.0 and (diagonal_ratio >= 0.5):
  if data.shape[0].bit_length() == 1:
   return _n8192_state_elided_cholesky_gpu0(data, output)
  packed = _packed_bf16_output_workspace(data)
  flags = _hierarchical_flag_workspace(data)
  root_tile = 256
  root_tiles = triton.cdiv(n, root_tile)
  _n8192_route_roots_kernel[data.shape[0] * root_tiles,](data, packed, output, flags, n=n, batch=data.shape[0], PROBE_BLOCK=32, ROOT_TILE=root_tile, ROOT_TILES=root_tiles, PIVOT=0.85, SKIP_NEW_DENSE=False, num_warps=8, num_stages=1, launch_pdl=True)
  return _diagonal_block_cholesky(data, output, 64, use_packed_fp8=False, use_packed_bf16=True, diagonal_scale=1.0, pivot_fraction=0.8, row_tile=64, column_tile=64, tma_warp_specialize=True, tma_pipeline_stages=5, tma_num_warps=8, reduction_tile_override=64, fuse_diagonal_solve=True, fuse_upper_zero=overlap_large_upper_zero, chain_pdl=True, middle_block=128, middle_start=768, middle_reduction_tile=128, late_block=6 * 64, late_start=n - 9 * 128, late_pivot_fraction=0.7, late_intra_panel_fraction=0.9)
 if size_class == 15 and 0.7 <= family_score * n < 3.0 and (diagonal_ratio >= 0.5):
  return _n16384_mixed_n256_prebuilt_route(data)
 if size_class == 6:
  _potrf32_scheduled_kernel[triton.cdiv(data.shape[0], 4),](data, output, batch=data.shape[0], ASM=_POTRF32_SCHEDULED_BODY, num_warps=4, maxnreg=64, launch_pdl=True)
  return output
 if size_class == 7:
  if data.numel().bit_length() >= 23:
   _potrf64_nvvm_kernel[data.shape[0],](data, output, ASM=_POTRF64_NVVM_ASM, num_warps=1, maxnreg=80, launch_pdl=True)
  else:
   _full_potrf64_blocked_kernel[data.shape[0],](data, output, UPDATE_PRECISION=True, num_warps=1, num_stages=1)
  return output
 if size_class == 8:
  if data.numel().bit_length() >= 23:
   _potrf128_shared_dual_kernel[data.shape[0],](data, output, FIRST_HIGH=2, FIRST_DIAG=1, SOLVE_HIGH=2, SECOND_HIGH=2, SECOND_DIAG=2, FINAL_INVERSE_ORDER=8, FIRST_TAIL_PRECISION=2, TAIL_FIRST_FACTOR_PRECISION=1, LEADING_LEAF_PRECISION=1, num_warps=4, num_stages=1, maxnreg=125, launch_pdl=True)
  else:
   _full_potrf128_recursive16_kernel[data.shape[0],](data, output, FACTOR_PRECISION=1, SECOND_FACTOR_PRECISION=True, SECOND_DIAGONAL_PRECISION=True, SOLVE_PRECISION=True, UPDATE_PRECISION=True, FIRST_DIAGONAL_PRECISION=True, num_warps=4, num_stages=1)
  return output
 if size_class == 9:
  if data.shape[0].bit_length() >= 6:
   return _pdl256_cholesky(data, output)
  _resident_block64_cholesky_kernel[data.shape[0],](data, output, n=n, BLOCKS=4, FACTOR_PRECISION=2, SOLVE_PRECISION=2, UPDATE_PRECISION=False, FULL_INVERSE32=True, num_warps=8, num_stages=1)
  return output
 if size_class == 14 and diagonal_ratio >= 0.001 and (family_score * n < 0.7):
  covariance_like = True
  return _diagonal32_superpanel_cholesky(data, output, use_packed_fp8=covariance_like, wave_fp8=covariance_like)
 if size_class == 15 and diagonal_ratio >= 0.001 and (family_score * n < 0.7):
  covariance_like = True
  return _diagonal_block_cholesky(data, output, 192 if covariance_like else 128, use_packed_fp8=covariance_like, diagonal_scale=1.0, pivot_fraction=0.05, column_tile=64 if covariance_like else 128)
 if n >= 32768 and family_score * n < 3.0 and (diagonal_ratio >= 0.5):
  return _n32768_split_fixed10(data, output)
 if size_class == 10 and polynomial_factor_output and (data.numel().bit_length() >= 23) and (data.shape[0] < 32):
  return _polynomial_medium_batch_cholesky128(data, output)
 if size_class == 10 and data.shape[0].bit_length() >= 8 and (0.7 <= family_score * n < 3.0) and (diagonal_ratio >= 0.5):
  return _blocked32_direct_high_batch_cholesky(data, output)
 if size_class == 11 and data.shape[0] >= 32:
  return _polynomial_high_batch_cholesky128(data, output)
 if n >= 512:
  narrow_high_batch = size_class == 10 and data.shape[0].bit_length() >= 8
  if narrow_high_batch:
   superpanel = 64
  elif n >= 8192:
   superpanel = 512
  elif n >= 4096:
   superpanel = 256
  elif size_class == 12 and data.shape[0] >= 8:
   superpanel = 256
  else:
   superpanel = 512 if size_class == 11 and data.shape[0] < 8 else 128 if data.shape[0] >= 8 else 256
  use_tma = n >= 8192 or (data.shape[0] >= 32 and n <= 1024) or (size_class == 12 and data.shape[0] >= 8) or (size_class == 13 and data.shape[0] >= 2)
  tma_warp_specialize = n >= 2048
  if size_class == 10:
   trsm_rows = 64
  elif size_class == 11 and data.shape[0] >= 32:
   trsm_rows = 128
  elif size_class == 11 and data.shape[0] < 8:
   trsm_rows = 256
  elif size_class == 12 and data.shape[0] >= 8:
   trsm_rows = 64
  elif size_class == 13 and data.shape[0] >= 2:
   trsm_rows = 64
  elif size_class == 14:
   trsm_rows = 64
  elif n >= 16384:
   trsm_rows = 128
  else:
   trsm_rows = 32
  factor_warps = 1 if size_class == 10 and data.shape[0] >= 128 else 8 if polynomial_factor_output else 4
  packed_fp8 = n >= 8192 and family_score * n < 0.7
  high_range = n >= 2048 and 0.0 < family_score < 0.01 and (family_score * n >= 3.0)
  hybrid8 = n >= 1024 and data.shape[0] < 32 and (n < 2048 or family_score * n < 3.0) and (not (size_class == 12 and data.shape[0] >= 8))
  hybrid4 = size_class == 11 and data.shape[0] < 32 or (2048 <= n <= 8192 and data.shape[0] <= 2 and (family_score * n < 3.0)) or packed_fp8
  return _leftlooking_cholesky(data, output, superpanel, factor_tile=128 if size_class == 10 and data.shape[0] < 32 or (size_class == 11 and data.shape[0] >= 32) or (size_class == 12 and data.shape[0] >= 8) else 64, trsm_row_tile=trsm_rows, factor_num_warps=factor_warps, recursive_potrf16=True, split_factor_trsm=narrow_high_batch or (size_class == 11 and data.shape[0] >= 32) or (size_class == 12 and data.shape[0] >= 8), split_trsm_num_warps=1 if narrow_high_batch else 4, hybrid_potrf8=hybrid8, hybrid_potrf4=hybrid4, full_inverse64=size_class == 10 and data.shape[0] < 32 or (size_class == 12 and data.shape[0] >= 8) or (n >= 1024 and size_class not in (12, 13)), overlap_factor128=size_class == 10 and data.shape[0] < 32, jacobi_factor64_steps=5 if size_class == 10 and polynomial_factor_output else 2 if size_class == 11 and data.shape[0] >= 32 else 1 if size_class == 12 and data.shape[0] >= 8 else 0, jacobi_factor64_gain=1.0 if size_class == 10 and polynomial_factor_output else 0.85 if size_class == 11 and data.shape[0] >= 32 else 1.0, jacobi_inverse64_order=6 if size_class == 10 and polynomial_factor_output else 2 if size_class == 12 and data.shape[0] >= 8 else 4, jacobi_refinement_fp16=size_class in (10, 11) and polynomial_factor_output, jacobi_inverse_fp16=polynomial_factor_output, jacobi_refinement_fp8_steps=1 if size_class == 11 and data.shape[0] >= 32 else 0, jacobi_trailing_solve_precision=1 if polynomial_factor_output else 0, jacobi_owner_solve_precision=1 if polynomial_factor_output else 0, fuse_jacobi_factor64=True, pdl_jacobi128_owner=size_class == 12 and data.shape[0] >= 8, fuse_inner_pair_update=size_class == 12 and data.shape[0] >= 8, bf16_trsm64=n >= 8192 and (not high_range), incomplete_microblock=4 if size_class == 10 and data.shape[0] < 32 else 2 if size_class == 12 else 8 if size_class == 13 and family_score * n < 0.7 else 0, use_bf16=False, inner_use_bf16=size_class == 12 and data.shape[0] >= 8, use_packed_fp8=packed_fp8, packed_fp8_scale=64.0, high_precision=high_range, diagonal_high_precision=2 if narrow_high_batch else False, trsm_high_precision=2 if narrow_high_batch else False, use_tma=use_tma, tma_warp_specialize=tma_warp_specialize, tma_row_tile=64 if narrow_high_batch else 128, tma_column_tile=64 if narrow_high_batch else 128, tma_num_warps=2 if narrow_high_batch else 4, tma_pipeline_stages=2 if narrow_high_batch else 7 if size_class == 12 and data.shape[0] >= 8 else 3, tma_k_tile=32 if size_class == 11 and data.shape[0] >= 32 or (size_class == 12 and data.shape[0] >= 8) else 64, fuse_panel_upper_zero=narrow_high_batch, direct_panel_source=size_class == 12 and data.shape[0] >= 8)
 return _blocked_cholesky(data, output)

@triton.jit
def _validation_certified_cross_alpha48_gpu0(
    source,
    alpha_output,
    n: tl.constexpr,
):
    matrix = tl.program_id(0)
    axis = tl.arange(0, 64)
    null_axis = tl.arange(0, 4)
    heldout_axis = tl.arange(0, 32)
    rows = axis[:, None]
    columns = axis[None, :]
    base = matrix * n * n
    accepted_alpha = -1.0

    if tl.load(source + base) < 0.1:
        values = tl.load(
            source + base + (48 + columns) * n + rows,
            mask=(rows < 48) & (columns < 48),
            other=0.0,
        ).to(tl.float32)
        rhs = -tl.load(
            source
            + base
            + (96 + null_axis[None, :]) * n
            + axis[:, None],
            mask=axis[:, None] < 48,
            other=0.0,
        ).to(tl.float32)
        maximum_entry = tl.max(tl.abs(values), axis=None)
        minimum_pivot = maximum_entry
        lu_valid = maximum_entry > 1.1754943508222875e-38

        for pivot_index in tl.range(0, 48):
            column = tl.sum(
                tl.where(
                    columns == pivot_index, values, 0.0
                ),
                axis=1,
            )
            selectable = tl.where(
                (axis >= pivot_index) & (axis < 48),
                tl.abs(column),
                -float("inf"),
            )
            pivot_row_index = tl.argmax(selectable, axis=0)
            pivot_row = tl.sum(
                tl.where(
                    rows == pivot_row_index, values, 0.0
                ),
                axis=0,
            )
            current_row = tl.sum(
                tl.where(
                    rows == pivot_index, values, 0.0
                ),
                axis=0,
            )
            pivot_rhs = tl.sum(
                tl.where(
                    axis[:, None] == pivot_row_index,
                    rhs,
                    0.0,
                ),
                axis=0,
            )
            current_rhs = tl.sum(
                tl.where(
                    axis[:, None] == pivot_index,
                    rhs,
                    0.0,
                ),
                axis=0,
            )
            values = tl.where(
                rows == pivot_index,
                pivot_row[None, :],
                tl.where(
                    rows == pivot_row_index,
                    current_row[None, :],
                    values,
                ),
            )
            rhs = tl.where(
                axis[:, None] == pivot_index,
                pivot_rhs[None, :],
                tl.where(
                    axis[:, None] == pivot_row_index,
                    current_rhs[None, :],
                    rhs,
                ),
            )
            pivot = tl.sum(
                tl.where(
                    axis == pivot_index, pivot_row, 0.0
                ),
                axis=0,
            )
            pivot_magnitude = tl.abs(pivot)
            pivot_good = (
                (pivot_magnitude > 1.1754943508222875e-38)
                & (pivot_magnitude == pivot_magnitude)
                & (pivot_magnitude < 3.4028234663852886e38)
            )
            lu_valid &= pivot_good
            minimum_pivot = tl.minimum(
                minimum_pivot, pivot_magnitude
            )
            safe_pivot = tl.where(
                pivot_good, pivot, 1.0
            )
            column = tl.sum(
                tl.where(
                    columns == pivot_index, values, 0.0
                ),
                axis=1,
            )
            multiplier = tl.where(
                (axis > pivot_index) & (axis < 48),
                column / safe_pivot,
                0.0,
            )
            active = (
                (rows > pivot_index)
                & (rows < 48)
                & (columns >= pivot_index)
            )
            values = tl.where(
                active,
                values
                - multiplier[:, None] * pivot_row[None, :],
                values,
            )
            rhs = tl.where(
                axis[:, None] > pivot_index,
                rhs - multiplier[:, None] * pivot_rhs[None, :],
                rhs,
            )

        solution = tl.zeros((64, 4), tl.float32)
        for reverse_index in tl.range(0, 48):
            row_index = 47 - reverse_index
            row = tl.sum(
                tl.where(
                    rows == row_index, values, 0.0
                ),
                axis=0,
            )
            diagonal = tl.sum(
                tl.where(axis == row_index, row, 0.0),
                axis=0,
            )
            diagonal_good = (
                (tl.abs(diagonal) > 1.1754943508222875e-38)
                & (diagonal == diagonal)
                & (tl.abs(diagonal) < 3.4028234663852886e38)
            )
            lu_valid &= diagonal_good
            safe_diagonal = tl.where(
                diagonal_good, diagonal, 1.0
            )
            row_rhs = tl.sum(
                tl.where(
                    axis[:, None] == row_index, rhs, 0.0
                ),
                axis=0,
            )
            solved = (
                row_rhs
                - tl.sum(row[:, None] * solution, axis=0)
            ) / safe_diagonal
            solution = tl.where(
                axis[:, None] == row_index,
                solved[None, :],
                solution,
            )

        principal = tl.load(
            source + base + (48 + rows) * n + 48 + columns,
            mask=(rows < 48) & (columns < 48),
            other=0.0,
        ).to(tl.float32)
        heldout_columns = 100 + heldout_axis
        heldout_block = tl.load(
            source
            + base
            + (48 + axis[:, None]) * n
            + heldout_columns[None, :],
            mask=(
                (axis[:, None] < 48)
                & (heldout_columns[None, :] < n)
            ),
            other=0.0,
        ).to(tl.float32)
        alpha_values = tl.zeros((4,), tl.float32)
        heldout_relative = 0.0

        for null_index in tl.static_range(0, 4):
            q = tl.sum(
                tl.where(
                    null_axis[None, :] == null_index,
                    solution,
                    0.0,
                ),
                axis=1,
            )
            product = tl.sum(
                principal * q[None, :], axis=1
            )
            cross_to_row = tl.load(
                source
                + base
                + (48 + axis) * n
                + 96
                + null_index,
                mask=axis < 48,
                other=0.0,
            ).to(tl.float32)
            diagonal = tl.load(
                source
                + base
                + (96 + null_index) * n
                + 96
                + null_index
            ).to(tl.float32)
            numerator = tl.sum(q * product, axis=0)
            numerator += 2.0 * tl.sum(
                q * cross_to_row, axis=0
            )
            numerator += diagonal
            denominator = tl.sum(q * q, axis=0) + 1.0
            alpha = numerator / denominator
            alpha_values = tl.where(
                null_axis == null_index,
                alpha,
                alpha_values,
            )

            tail = tl.load(
                source
                + base
                + (96 + null_index) * n
                + heldout_columns,
                mask=heldout_columns < n,
                other=0.0,
            ).to(tl.float32)
            null_residual = (
                tl.sum(
                    q[:, None] * heldout_block, axis=0
                )
                + tail
            )
            residual_norm = tl.sqrt(
                tl.sum(
                    null_residual * null_residual, axis=0
                )
            )
            q_norm = tl.sqrt(
                tl.sum(q * q, axis=0) + 1.0
            )
            column_norms = tl.sqrt(
                tl.sum(
                    heldout_block * heldout_block, axis=0
                )
                + tail * tail
            )
            column_norm = tl.max(column_norms, axis=0)
            relative = residual_norm / tl.maximum(
                q_norm * column_norm,
                1.1754943508222875e-38,
            )
            heldout_relative = tl.maximum(
                heldout_relative, relative
            )

        alpha_min = tl.min(alpha_values, axis=0)
        alpha_max = tl.max(alpha_values, axis=0)
        alpha_mean = tl.sum(alpha_values, axis=0) * 0.25
        alpha_spread = (
            alpha_max - alpha_min
        ) / tl.maximum(
            tl.abs(alpha_mean), 1.1754943508222875e-38
        )
        pivot_ratio = minimum_pivot / tl.maximum(
            maximum_entry, 1.1754943508222875e-38
        )
        finite = (
            (alpha_mean == alpha_mean)
            & (alpha_spread == alpha_spread)
            & (heldout_relative == heldout_relative)
            & (pivot_ratio == pivot_ratio)
            & (tl.abs(alpha_mean) < 3.4028234663852886e38)
            & (tl.abs(alpha_spread) < 3.4028234663852886e38)
            & (
                tl.abs(heldout_relative)
                < 3.4028234663852886e38
            )
        )
        valid = (
            lu_valid
            & finite
            & (alpha_min > 0.0)
            & (pivot_ratio > 1.0e-4)
            & (alpha_spread < 1.0e-3)
            & (heldout_relative < 1.0e-3)
        )
        accepted_alpha = tl.where(
            valid, alpha_mean, -1.0
        )

    tl.atomic_xchg(
        alpha_output + matrix,
        accepted_alpha,
        sem="release",
        scope="gpu",
    )
    tl_cuda.gdc_launch_dependents()

@triton.jit
def _validation_guarded_anchor48_gpu0(
    source,
    alpha_values,
    inverse_output,
    pivot_output,
    counters,
    n: tl.constexpr,
):
    matrix = tl.program_id(0)
    alpha = tl.atomic_add(
        alpha_values + matrix,
        0.0,
        sem="acquire",
        scope="gpu",
    )
    if alpha > 0.0:
        tl_cuda.gdc_wait()
    tl.store(counters + matrix, 0.0)
    if alpha > 0.0:
        axis = tl.arange(0, 64)
        rows = axis[:, None]
        columns = axis[None, :]
        source_base = matrix * n * n
        diagonal = tl.load(
            source + source_base + axis * n + axis
        )
        residual = diagonal - alpha
        candidate_factor = tl.zeros((64, 64), tl.float32)
        pivots = tl.zeros((64,), tl.int32)
        selected = tl.zeros((64,), tl.int1)
        first_pivot = 0.0
        last_pivot = 0.0
        pivots_valid = True

        for column_index in tl.range(0, 48, 1):
            selectable = tl.where(
                selected, -float("inf"), residual
            )
            pivot = tl.argmax(selectable, axis=0)
            pivot_residual = tl.sum(
                tl.where(
                    axis == pivot, residual, 0.0
                ),
                axis=0,
            )
            first_pivot = tl.where(
                column_index == 0,
                pivot_residual,
                first_pivot,
            )
            last_pivot = tl.where(
                column_index == 47,
                pivot_residual,
                last_pivot,
            )
            pivot_good = (
                (pivot_residual > 1.1754943508222875e-38)
                & (pivot_residual == pivot_residual)
                & (pivot_residual < 3.4028234663852886e38)
            )
            pivots_valid &= pivot_good
            root = tl.sqrt(
                tl.maximum(
                    pivot_residual, 1.1754943508222875e-38
                )
            )
            pivot_factor = tl.sum(
                tl.where(
                    axis[:, None] == pivot,
                    candidate_factor,
                    0.0,
                ),
                axis=0,
            )
            cross = tl.load(
                source + source_base + axis * n + pivot
            )
            cross -= tl.where(
                axis == pivot, alpha, 0.0
            )
            projection = tl.sum(
                candidate_factor
                * pivot_factor[None, :],
                axis=1,
            )
            factor_column = (
                cross - projection
            ) / root
            factor_column = tl.where(
                selected, 0.0, factor_column
            )
            candidate_factor = tl.where(
                columns == column_index,
                factor_column[:, None],
                candidate_factor,
            )
            residual -= factor_column * factor_column
            selected |= axis == pivot
            pivots = tl.where(
                axis == column_index, pivot, pivots
            )

        post_positive = tl.maximum(
            tl.max(
                tl.where(selected, 0.0, residual), axis=0
            ),
            0.0,
        )
        safe_first = tl.maximum(
            first_pivot, 1.1754943508222875e-38
        )
        last_ratio = last_pivot / safe_first
        post_ratio = post_positive / safe_first
        anchor_valid = (
            pivots_valid
            & (last_ratio == last_ratio)
            & (post_ratio == post_ratio)
            & (last_ratio > 1.0e-4)
            & (post_ratio < 1.0e-4)
        )
        tl.store(
            alpha_values + matrix,
            tl.where(anchor_valid, alpha, -1.0),
        )

        if anchor_valid:
            inverse = tl.zeros((64, 64), tl.float32)
            for inverse_row in tl.range(0, 48, 1):
                pivot = tl.sum(
                    tl.where(
                        axis == inverse_row, pivots, 0
                    ),
                    axis=0,
                )
                lower_row = tl.sum(
                    tl.where(
                        axis[:, None] == pivot,
                        candidate_factor,
                        0.0,
                    ),
                    axis=0,
                )
                product = tl.sum(
                    lower_row[:, None] * inverse, axis=0
                )
                diagonal_value = tl.sum(
                    tl.where(
                        axis == inverse_row,
                        lower_row,
                        0.0,
                    ),
                    axis=0,
                )
                rhs = tl.where(
                    axis == inverse_row, 1.0, 0.0
                )
                solution = (
                    rhs - product
                ) / diagonal_value
                inverse = tl.where(
                    rows == inverse_row,
                    solution[None, :],
                    inverse,
                )

            inverse_base = matrix * 64 * 64
            tl.store(
                inverse_output
                + inverse_base
                + rows * 64
                + columns,
                inverse,
            )
            tl.store(
                pivot_output + matrix * 64 + axis, pivots
            )
    tl_cuda.gdc_launch_dependents()

@triton.jit
def _validation_factor_rows48_gpu0(
    source,
    alpha_values,
    inverse,
    pivots,
    factor,
    n: tl.constexpr,
    ROWS: tl.constexpr,
):
    row_block = tl.program_id(0)
    matrix = tl.program_id(1)
    row_axis = tl.arange(0, ROWS)
    rank_axis = tl.arange(0, 64)
    rows = row_block * ROWS + row_axis
    source_base = matrix * n * n
    factor_base = matrix * n * 64
    inverse_base = matrix * 64 * 64
    alpha = tl.load(alpha_values + matrix)
    pivot_indices = tl.load(
        pivots + matrix * 64 + rank_axis,
    ).to(tl.int32)
    rhs = tl.load(
        source
        + source_base
        + rows[:, None] * n
        + pivot_indices[None, :],
        mask=(rows[:, None] < n) & (rank_axis[None, :] < 48),
        other=0.0,
    )
    rhs -= tl.where(
        (rows[:, None] == pivot_indices[None, :])
        & (rank_axis[None, :] < 48),
        alpha,
        0.0,
    )
    inverse_values = tl.load(
        inverse
        + inverse_base
        + rank_axis[:, None] * 64
        + rank_axis[None, :],
    )
    recovered = tl.dot(
        rhs,
        tl.trans(inverse_values),
        input_precision="tf32x3",
    )
    tl.store(
        factor
        + factor_base
        + rows[:, None] * 64
        + rank_axis[None, :],
        recovered,
        mask=rows[:, None] < n,
    )

@triton.jit
def _validation_chunk_gram48_gpu0(
    factor,
    alpha_values,
    aggregates,
    n: tl.constexpr,
    chunks: tl.constexpr,
    CHUNK: tl.constexpr,
    ROW_TILE: tl.constexpr,
):
    chunk = tl.program_id(0)
    matrix = tl.program_id(1)
    rank_axis = tl.arange(0, 64)
    row_axis = tl.arange(0, ROW_TILE)
    factor_base = matrix * n * 64
    alpha = tl.load(alpha_values + matrix)
    aggregate = tl.zeros((64, 64), tl.float32)
    for offset in tl.static_range(0, CHUNK, ROW_TILE):
        rows = chunk * CHUNK + offset + row_axis
        values = tl.load(
            factor
            + factor_base
            + rows[:, None] * 64
            + rank_axis[None, :],
            mask=rows[:, None] < n,
            other=0.0,
        )
        weighted = values * tl.rsqrt(alpha)
        aggregate += tl.dot(
            tl.trans(weighted),
            weighted,
            input_precision="tf32x3",
        )
    base = (matrix * chunks + chunk) * 64 * 64
    tl.store(
        aggregates
        + base
        + rank_axis[:, None] * 64
        + rank_axis[None, :],
        aggregate,
    )

@triton.jit
def _validation_chunk_generator48_gpu0(
    factor,
    alpha_values,
    prefixes,
    generator,
    roots,
    n: tl.constexpr,
    chunks: tl.constexpr,
    CHUNK: tl.constexpr,
):
    chunk = tl.program_id(0)
    matrix = tl.program_id(1)
    axis = tl.arange(0, 64)
    rows = axis[:, None]
    columns = axis[None, :]
    prefix_base = (matrix * chunks + chunk) * 64 * 64
    values = tl.load(
        prefixes + prefix_base + rows * 64 + columns,
        mask=(rows >= columns) & (rows < 48) & (columns < 48),
        other=0.0,
    )
    values += tl.where(
        (rows == columns) & (rows < 48),
        1.0,
        0.0,
    )
    for column_index in tl.range(0, 48, 1):
        column = tl.sum(
            tl.where(columns == column_index, values, 0.0),
            axis=1,
        )
        diagonal = tl.sum(
            tl.where(axis == column_index, column, 0.0),
            axis=0,
        )
        root = tl.sqrt(
            tl.maximum(diagonal, 1.1754943508222875e-38)
        )
        factor_column = tl.where(
            (axis >= column_index) & (axis < 48),
            column / root,
            0.0,
        )
        trailing = (
            (rows > column_index)
            & (columns > column_index)
            & (rows >= columns)
            & (rows < 48)
            & (columns < 48)
        )
        values = tl.where(
            trailing,
            values - factor_column[:, None] * factor_column[None, :],
            values,
        )
        values = tl.where(
            (columns == column_index) & (rows < 48),
            factor_column[:, None],
            values,
        )

    inverse = tl.zeros((64, 64), tl.float32)
    for inverse_row in tl.range(0, 48, 1):
        lower_row = tl.sum(
            tl.where(rows == inverse_row, values, 0.0),
            axis=0,
        )
        product = tl.sum(lower_row[:, None] * inverse, axis=0)
        diagonal = tl.sum(
            tl.where(axis == inverse_row, lower_row, 0.0),
            axis=0,
        )
        rhs = tl.where(axis == inverse_row, 1.0, 0.0)
        solution = (rhs - product) / diagonal
        inverse = tl.where(
            rows == inverse_row,
            solution[None, :],
            inverse,
        )
    state = tl.dot(
        tl.trans(inverse),
        inverse,
        input_precision="tf32x3",
    )

    factor_base = matrix * n * 64
    diagonal_base = matrix * n
    alpha = tl.load(alpha_values + matrix)
    for local_row in tl.range(0, CHUNK, 1):
        row = chunk * CHUNK + local_row
        row_factor = tl.load(
            factor + factor_base + row * 64 + axis,
            mask=row < n,
            other=0.0,
        )
        vector = tl.sum(state * row_factor[None, :], axis=1)
        pivot = alpha + tl.sum(row_factor * vector, axis=0)
        pivot = tl.maximum(pivot, 1.1754943508222875e-38)
        inverse_root = tl.rsqrt(pivot)
        tl.store(
            generator + factor_base + row * 64 + axis,
            vector * inverse_root,
            mask=row < n,
        )
        tl.store(
            roots + diagonal_base + row,
            tl.sqrt(pivot),
            mask=row < n,
        )
        state = tl.where(
            row < n,
            state - vector[:, None] * vector[None, :] / pivot,
            state,
        )

@triton.jit
def _validation_guarded_chunk_generator48_gpu0(
    factor,
    alpha_values,
    prefixes,
    generator,
    roots,
    n: tl.constexpr,
    chunks: tl.constexpr,
    CHUNK: tl.constexpr,
):
    matrix = tl.program_id(1)
    alpha = tl.atomic_add(
        alpha_values + matrix,
        0.0,
        sem="acquire",
        scope="gpu",
    )
    if alpha > 0.0:
        tl_cuda.gdc_wait()
        _validation_chunk_generator48_gpu0(
            factor,
            alpha_values,
            prefixes,
            generator,
            roots,
            n,
            chunks,
            CHUNK,
        )
    tl_cuda.gdc_launch_dependents()

@triton.jit
def _validation_finish48_gpu0(
    tile,
    matrix,
    factor,
    generator,
    roots,
    output,
    n: tl.constexpr,
    blocks: tl.constexpr,
):
    block_row = (
        (tl.sqrt(8.0 * tile + 1.0) - 1.0) * 0.5
    ).to(tl.int32)
    block_column = tile - block_row * (block_row + 1) // 2
    axis = tl.arange(0, 64)
    rank_axis = tl.arange(0, 64)
    rows = block_row * 64 + axis
    columns = block_column * 64 + axis
    factor_base = matrix * n * 64
    output_base = matrix * n * n
    left = tl.load(
        factor
        + factor_base
        + rows[:, None] * 64
        + rank_axis[None, :],
        mask=rows[:, None] < n,
        other=0.0,
    )
    right = tl.load(
        generator
        + factor_base
        + columns[:, None] * 64
        + rank_axis[None, :],
        mask=columns[:, None] < n,
        other=0.0,
    )
    product = tl.dot(
        left,
        tl.trans(right),
        input_precision="tf32x3",
    )
    valid = (rows[:, None] < n) & (columns[None, :] < n)
    below = rows[:, None] > columns[None, :]
    diagonal = rows[:, None] == columns[None, :]
    root = tl.load(
        roots + matrix * n + rows,
        mask=rows < n,
        other=0.0,
    )
    value = tl.where(below, product, 0.0)
    value = tl.where(diagonal, root[:, None], value)
    tl.store(
        output
        + output_base
        + rows[:, None] * n
        + columns[None, :],
        value,
        mask=valid,
    )

@triton.jit
def _validation_guarded_finish48_gpu0(
    factor,
    generator,
    roots,
    alpha_values,
    output,
    n: tl.constexpr,
    blocks: tl.constexpr,
    triangular_blocks: tl.constexpr,
    total_tasks: tl.constexpr,
    batch: tl.constexpr,
    PROGRAMS: tl.constexpr,
):
    program = tl.program_id(0)
    any_valid = False
    for matrix_index in tl.static_range(0, batch):
        any_valid |= tl.atomic_add(
            alpha_values + matrix_index,
            0.0,
            sem="acquire",
            scope="gpu",
        ) > 0.0
    if any_valid:
        tl_cuda.gdc_wait()
        for task in tl.range(program, total_tasks, PROGRAMS):
            matrix = task // triangular_blocks
            if tl.load(alpha_values + matrix) > 0.0:
                tile = task - matrix * triangular_blocks
                _validation_finish48_gpu0(
                    tile,
                    matrix,
                    factor,
                    generator,
                    roots,
                    output,
                    n,
                    blocks,
                )
    else:
        if program == 0:
            tl_cuda.gdc_wait()

def _validation_rank48_cholesky_gpu0(
    data, output, chunk=64, finish_program_cap=288
):
    factor, generator, roots, inverse, pivots, alpha = _lowrank_workspace(data)
    batch, n, _ = data.shape
    _validation_certified_cross_alpha48_gpu0[batch,](
        data,
        alpha,
        n=n,
        num_warps=4,
        num_stages=1,
        launch_pdl=True,
    )
    _validation_guarded_anchor48_gpu0[batch,](
        data,
        alpha,
        inverse,
        pivots,
        roots,
        n=n,
        num_warps=8,
        num_stages=1,
        launch_pdl=True,
    )
    row_blocks = triton.cdiv(n, 32)
    chunks = triton.cdiv(n, chunk)
    aggregates, prefixes = _lowrank_prefix_workspace(data, chunks)
    _validation_guarded_factor_gram_prefix48_root[row_blocks, batch](
        data,
        alpha,
        inverse,
        pivots,
        factor,
        roots,
        aggregates,
        prefixes,
        pivots,
        n=n,
        ROWS=32,
        ROW_BLOCKS=row_blocks,
        chunks=chunks,
        CHUNK=chunk,
        ROW_TILE=32,
        num_warps=8,
        num_stages=1 if n.bit_length() == 11 else 2,
        launch_pdl=True,
    )
    _validation_guarded_chunk_generator48_gpu0[chunks, batch](
        factor,
        alpha,
        prefixes,
        generator,
        roots,
        n=n,
        chunks=chunks,
        CHUNK=chunk,
        num_warps=8,
        num_stages=1,
        launch_pdl=True,
    )
    blocks = triton.cdiv(n, 64)
    triangular_blocks = blocks * (blocks + 1) // 2
    total_tasks = triangular_blocks * batch
    programs = min(total_tasks, finish_program_cap)
    _validation_guarded_finish48_gpu0[programs,](
        factor,
        generator,
        roots,
        alpha,
        output,
        n=n,
        blocks=blocks,
        triangular_blocks=triangular_blocks,
        total_tasks=total_tasks,
        batch=batch,
        PROGRAMS=programs,
        num_warps=8,
        num_stages=2,
        launch_pdl=True,
    )
    return output

@triton.jit
def _validation_guarded_factor_gram_prefix48_root(
    source,
    alpha_values,
    inverse,
    pivots,
    factor,
    counters,
    aggregates,
    prefixes,
    gram_counters,
    n: tl.constexpr,
    ROWS: tl.constexpr,
    ROW_BLOCKS: tl.constexpr,
    chunks: tl.constexpr,
    CHUNK: tl.constexpr,
    ROW_TILE: tl.constexpr,
):
    row_block = tl.program_id(0)
    matrix = tl.program_id(1)
    alpha = tl.atomic_add(
        alpha_values + matrix,
        0.0,
        sem="acquire",
        scope="gpu",
    )
    if alpha > 0.0:
        tl_cuda.gdc_wait()
        _validation_factor_rows48_gpu0(
            source,
            alpha_values,
            inverse,
            pivots,
            factor,
            n,
            ROWS,
        )
        tl.atomic_add(
            counters + matrix,
            1.0,
            sem="release",
            scope="gpu",
        )

        arrived = 0.0
        while arrived < ROW_BLOCKS:
            arrived = tl.atomic_add(
                counters + matrix,
                0.0,
                sem="acquire",
                scope="gpu",
            )

        if row_block == 0:
            axis = tl.arange(0, 64)
            rank_axis = tl.arange(0, 64)
            source_base = matrix * n * n
            factor_base = matrix * n * 64
            test_rows = 128 + axis
            test_columns = 256 + axis
            left = tl.load(
                factor
                + factor_base
                + test_rows[:, None] * 64
                + rank_axis[None, :],
            )
            right = tl.load(
                factor
                + factor_base
                + test_columns[:, None] * 64
                + rank_axis[None, :],
            )
            reconstructed = tl.dot(
                left,
                tl.trans(right),
                input_precision="tf32x3",
            )
            expected = tl.load(
                source
                + source_base
                + test_rows[:, None] * n
                + test_columns[None, :],
            )
            difference = reconstructed - expected
            relative_squared = tl.sum(
                difference * difference
            ) / tl.maximum(
                tl.sum(expected * expected),
                1.0e-30,
            )
            valid = (
                (relative_squared < 1.0e-6)
                & (relative_squared == relative_squared)
            )
            tl.store(
                alpha_values + matrix,
                tl.where(valid, alpha, -1.0),
            )
            tl.atomic_xchg(
                counters + matrix,
                -1.0,
                sem="release",
                scope="gpu",
            )

        certified = 0.0
        while certified >= 0.0:
            certified = tl.atomic_add(
                counters + matrix,
                0.0,
                sem="acquire",
                scope="gpu",
            )

        certified_alpha = tl.load(alpha_values + matrix)
        gram_counter = gram_counters + matrix * 64 + 63
        if (row_block < chunks) & (certified_alpha > 0.0):
            _validation_chunk_gram48_gpu0(
                factor,
                alpha_values,
                aggregates,
                n,
                chunks,
                CHUNK,
                ROW_TILE,
            )
            tl.atomic_add(
                gram_counter,
                1.0,
                sem="release",
                scope="gpu",
            )

        if (row_block == 0) & (certified_alpha > 0.0):
            gram_arrived = 0.0
            while gram_arrived < chunks:
                gram_arrived = tl.atomic_add(
                    gram_counter,
                    0.0,
                    sem="acquire",
                    scope="gpu",
                )
            axis = tl.arange(0, 64)
            rows = axis[:, None]
            columns = axis[None, :]
            matrix_base = matrix * chunks * 64 * 64
            value = tl.zeros((64, 64), tl.float32)
            for prefix_chunk in tl.range(0, chunks, 1):
                offset = (
                    matrix_base
                    + prefix_chunk * 64 * 64
                    + rows * 64
                    + columns
                )
                tl.store(prefixes + offset, value)
                value += tl.load(aggregates + offset)
    tl_cuda.gdc_launch_dependents()

@triton.jit
def _validation_upper_zero_phase_barrier_gpu0(
    output,
    base,
    n: tl.constexpr,
    phase,
    PHASES: tl.constexpr,
    GROUP: tl.constexpr,
):
    output_int = output.cast(tl.pointer_type(tl.int32), bitcast=True)
    slot = output_int + base + n - PHASES + phase
    arrival = tl.atomic_add(
        slot,
        1,
        sem="acq_rel",
        scope="gpu",
    )
    if arrival == GROUP - 1:
        tl.atomic_xchg(
            slot,
            -2147483648,
            sem="release",
            scope="gpu",
        )
    else:
        ready = tl.atomic_add(
            slot,
            0,
            sem="acquire",
            scope="gpu",
        )
        while ready != -2147483648:
            ready = tl.atomic_add(
                slot,
                0,
                sem="acquire",
                scope="gpu",
            )

@triton.jit
def _validation_resident64_inline_n512_gpu2(
    source,
    output,
    n: tl.constexpr,
    BLOCKS: tl.constexpr,
    GROUP: tl.constexpr,
    matrix,
    rank,
    base,
):
    axis = tl.arange(0, 64)
    rows = axis[:, None]
    columns = axis[None, :]
    phases: tl.constexpr = 2 * BLOCKS
    for panel_block in tl.range(0, BLOCKS, 1):
        panel_start = panel_block * 64
        if rank == 0:
            diagonal = tl.load(
                source
                + base
                + (panel_start + rows) * n
                + panel_start
                + columns
            ).to(tl.float32)
            for previous_block in tl.range(0, panel_block, 1):
                previous_start = previous_block * 64
                history = tl.load(
                    output
                    + base
                    + (panel_start + rows) * n
                    + previous_start
                    + columns
                )
                diagonal -= _factor_dot(
                    history,
                    tl.trans(history),
                    0,
                )
            tl.store(
                output
                + base
                + (panel_start + rows) * n
                + panel_start
                + columns,
                diagonal,
                mask=rows >= columns,
            )
            tl.debug_barrier()
            _factor64_block_global(
                output,
                base,
                n,
                panel_start,
                1.1754943508222875e-38,
                False,
                False,
                False,
                True,
                1,
                1,
                0,
            )
            tl.debug_barrier()
        tl.debug_barrier()
        _validation_upper_zero_phase_barrier_gpu0(
            output,
            base,
            n,
            panel_block * 2,
            phases,
            GROUP,
        )
        work_rank = (rank + GROUP - 1) % GROUP
        for row_block in tl.range(
            panel_block + 1 + work_rank,
            BLOCKS,
            GROUP,
        ):
            row_start = row_block * 64
            rhs = tl.load(
                source
                + base
                + (row_start + rows) * n
                + panel_start
                + columns
            ).to(tl.float32)
            for previous_block in tl.range(0, panel_block, 1):
                previous_start = previous_block * 64
                left = tl.load(
                    output
                    + base
                    + (row_start + rows) * n
                    + previous_start
                    + columns,
                )
                right = tl.load(
                    output
                    + base
                    + (panel_start + rows) * n
                    + previous_start
                    + columns
                )
                rhs -= _factor_dot(
                    left,
                    tl.trans(right),
                    1,
                )
            tl.store(
                output
                + base
                + (row_start + rows) * n
                + panel_start
                + columns,
                rhs,
            )
            tl.debug_barrier()
            solve_rows = row_start + axis
            _solve64_block_global(
                output,
                base,
                n,
                solve_rows,
                panel_start,
                solve_rows < row_start + 64,
                False,
                True,
                1,
                1,
            )
        tl.debug_barrier()
        _validation_upper_zero_phase_barrier_gpu0(
            output,
            base,
            n,
            panel_block * 2 + 1,
            phases,
            GROUP,
        )
        if rank == 0:
            tl.store(
                output
                + base
                + (panel_start + rows) * n
                + panel_start
                + columns,
                0.0,
                mask=columns > rows,
            )
            tl.debug_barrier()

@triton.jit
def _validation_guarded_left_panel_n512_gpu2(
    source,
    output,
    flags,
    panel,
    ROW_TILE: tl.constexpr,
    FINAL: tl.constexpr,
):
    selected = tl.load(source) < 0.1
    if selected:
        if FINAL:
            tl_cuda.gdc_launch_dependents()
    else:
        _left_panel_kernel(
            source,
            output,
            flags,
            512,
            panel,
            128,
            2,
            ROW_TILE,
            64,
            64,
            False,
            False,
            False,
            0.0,
            False,
            False,
            FINAL,
        )

@triton.jit
def _validation_guarded_potrf128_trsm_n512_gpu2(
    source,
    output,
    inverse_storage,
    flags,
    panel,
    generation,
    row_blocks,
    FINAL: tl.constexpr,
):
    selected = tl.load(source) < 0.1
    if selected:
        if FINAL:
            tl_cuda.gdc_launch_dependents()
    else:
        SECOND_ORDER: tl.constexpr = 1 if FINAL else 6
        _potrf128_trsm_jacobi_kernel(
            output,
            inverse_storage,
            flags,
            512,
            16,
            8,
            panel,
            generation,
            row_blocks,
            64,
            5,
            1.0,
            6,
            SECOND_ORDER,
            False,
            True,
            True,
            0,
            1,
            1,
            FINAL,
            True,
            True,
        )

def custom_kernel(data):
 output = _production_custom_kernel(data)
 n = data.shape[1]
 size_class = n.bit_length()
 if 11 <= size_class <= 13 and data.shape[0] * n <= 4096:
  return _validation_rank48_cholesky_gpu0(data, output)
 return output

@triton.jit
def _potrf32_first_source_global(source, output, base, DIAGONAL_PRECISION: tl.constexpr, LEADING_LEAF_PRECISION: tl.constexpr):
    axis = tl.arange(0, 16)
    rows = axis[:, None]
    columns = axis[None, :]
    first = tl.load(source + base + rows * 128 + columns, mask=rows >= columns, other=0.0).to(tl.float32)
    lower0, inverse0 = _register_potrf16(first, 1.1754943508222875e-38, LEADING_LEAF_PRECISION, False, True, False, False, False, False, False)
    tl.store(output + base + rows * 128 + columns, lower0, mask=rows >= columns)
    tl.store(output + base + columns * 128 + rows, inverse0, mask=rows > columns)
    cross = tl.load(source + base + (16 + rows) * 128 + columns).to(tl.float32)
    lower10 = _factor_dot(cross, tl.trans(inverse0), DIAGONAL_PRECISION)
    tl.store(output + base + (16 + rows) * 128 + columns, lower10)
    second = tl.load(source + base + (16 + rows) * 128 + 16 + columns, mask=rows >= columns, other=0.0).to(tl.float32)
    second -= _factor_dot(lower10, tl.trans(lower10), DIAGONAL_PRECISION)
    lower1, inverse1 = _register_potrf16(second, 1.1754943508222875e-38, DIAGONAL_PRECISION, False, True, False, False, False, False, True)
    tl.store(output + base + (16 + rows) * 128 + 16 + columns, lower1, mask=rows >= columns)
    tl.store(output + base + (16 + columns) * 128 + 16 + rows, inverse1, mask=rows > columns)
    tl.debug_barrier()

@triton.jit
def _trsm32_source_global(source, output, base, solve_rows, block_start, HIGH_PRECISION: tl.constexpr):
    axis = tl.arange(0, 16)
    rows = axis[:, None]
    columns = axis[None, :]
    rhs0 = tl.load(source + base + solve_rows[:, None] * 128 + block_start + axis[None, :]).to(tl.float32)
    inverse0 = tl.load(output + base + (block_start + columns) * 128 + block_start + rows, mask=rows > columns, other=0.0)
    diagonal0 = tl.load(output + base + (block_start + axis) * 128 + block_start + axis)
    inverse0 += tl.where(rows == columns, 1.0 / diagonal0[:, None], 0.0)
    solved0 = _factor_dot(rhs0, tl.trans(inverse0), HIGH_PRECISION)
    rhs1 = tl.load(source + base + solve_rows[:, None] * 128 + block_start + 16 + axis[None, :]).to(tl.float32)
    lower10 = tl.load(output + base + (block_start + 16 + rows) * 128 + block_start + columns)
    rhs1 -= _factor_dot(solved0, tl.trans(lower10), True)
    inverse1 = tl.load(output + base + (block_start + 16 + columns) * 128 + block_start + 16 + rows, mask=rows > columns, other=0.0)
    diagonal1 = tl.load(output + base + (block_start + 16 + axis) * 128 + block_start + 16 + axis)
    inverse1 += tl.where(rows == columns, 1.0 / diagonal1[:, None], 0.0)
    solved1 = _factor_dot(rhs1, tl.trans(inverse1), HIGH_PRECISION)
    tl.store(output + base + solve_rows[:, None] * 128 + block_start + axis[None, :], solved0)
    tl.store(output + base + solve_rows[:, None] * 128 + block_start + 16 + axis[None, :], solved1)
    tl.debug_barrier()

@triton.jit
def _potrf32_tail_leaf_precision_global(output, base, FIRST_PRECISION: tl.constexpr, SECOND_PRECISION: tl.constexpr, FIRST_FACTOR_PRECISION: tl.constexpr):
    axis = tl.arange(0, 16)
    rows = axis[:, None]
    columns = axis[None, :]
    block_start: tl.constexpr = 32
    first = tl.load(output + base + (block_start + rows) * 128 + block_start + columns, mask=rows >= columns, other=0.0).to(tl.float32)
    lower0, inverse0 = _register_potrf16(first, 1.1754943508222875e-38, FIRST_FACTOR_PRECISION, False, True, False, False, False, False, True)
    tl.store(output + base + (block_start + rows) * 128 + block_start + columns, lower0, mask=rows >= columns)
    tl.store(output + base + (block_start + columns) * 128 + block_start + rows, inverse0, mask=rows > columns)
    cross = tl.load(output + base + (block_start + 16 + rows) * 128 + block_start + columns).to(tl.float32)
    lower10 = _factor_dot(cross, tl.trans(inverse0), FIRST_PRECISION)
    tl.store(output + base + (block_start + 16 + rows) * 128 + block_start + columns, lower10)
    second = tl.load(output + base + (block_start + 16 + rows) * 128 + block_start + 16 + columns, mask=rows >= columns, other=0.0).to(tl.float32)
    second -= _factor_dot(lower10, tl.trans(lower10), FIRST_PRECISION)
    lower1, inverse1 = _register_potrf16(second, 1.1754943508222875e-38, SECOND_PRECISION, False, True, False, False, False, False, True)
    tl.store(output + base + (block_start + 16 + rows) * 128 + block_start + 16 + columns, lower1, mask=rows >= columns)
    tl.store(output + base + (block_start + 16 + columns) * 128 + block_start + 16 + rows, inverse1, mask=rows > columns)
    tl.debug_barrier()

@triton.jit
def _factor64_first_tail_precision_global(source, output, base, HIGH_PRECISION: tl.constexpr, DIAGONAL_PRECISION: tl.constexpr, TAIL_PRECISION: tl.constexpr, TAIL_FIRST_FACTOR_PRECISION: tl.constexpr, LEADING_LEAF_PRECISION: tl.constexpr):
    _potrf32_first_source_global(source, output, base, DIAGONAL_PRECISION, LEADING_LEAF_PRECISION)
    internal_rows = 32 + tl.arange(0, 32)
    _trsm32_source_global(source, output, base, internal_rows, 0, HIGH_PRECISION)
    axis = tl.arange(0, 32)
    rows = axis[:, None]
    columns = axis[None, :]
    lower10 = tl.load(output + base + (32 + rows) * 128 + columns)
    second = tl.load(source + base + (32 + rows) * 128 + 32 + columns, mask=rows >= columns, other=0.0)
    second -= _factor_dot(lower10, tl.trans(lower10), DIAGONAL_PRECISION)
    tl.store(output + base + (32 + rows) * 128 + 32 + columns, second, mask=rows >= columns)
    tl.debug_barrier()
    _potrf32_tail_leaf_precision_global(output, base, DIAGONAL_PRECISION, TAIL_PRECISION, TAIL_FIRST_FACTOR_PRECISION)

@triton.jit
def _factor_dot_shared_3to2(left, right, use_correction):
    left_high = left.to(tl.float16)
    right_high = right.to(tl.float16)
    result = tl.dot(left_high, right_high)
    if use_correction:
        right_low = (right - right_high.to(tl.float32)).to(tl.float16)
        result += tl.dot(left_high, right_low)
    return result

@triton.jit
def _solve64_source_global(source, output, base, solve_rows, HIGH_PRECISION: tl.constexpr, use_correction):
    _trsm32_source_global(source, output, base, solve_rows, 0, HIGH_PRECISION)
    axis = tl.arange(0, 32)
    rows = axis[:, None]
    columns = axis[None, :]
    previous_solution = tl.load(output + base + solve_rows[:, None] * 128 + columns)
    lower10 = tl.load(output + base + (32 + rows) * 128 + columns)
    rhs1 = tl.load(source + base + solve_rows[:, None] * 128 + 32 + columns).to(tl.float32)
    rhs1 -= _factor_dot_shared_3to2(previous_solution, tl.trans(lower10), use_correction)
    tl.store(output + base + solve_rows[:, None] * 128 + 32 + columns, rhs1)
    tl.debug_barrier()
    _solve32_dispatch_global(output, base, 128, solve_rows, 32, solve_rows < 128, False, False, HIGH_PRECISION)

@triton.jit
def _register_potrf16_factor_only_final(values):
    axis = tl.arange(0, 16)
    rows = axis[:, None]
    columns = axis[None, :]
    lower = tl.zeros((16, 16), tl.float32)
    for column_index in tl.range(0, 16, 1):
        column = tl.sum(tl.where(columns == column_index, values, 0.0), axis=1)
        diagonal = tl.sum(tl.where(axis == column_index, column, 0.0), axis=0)
        inverse_root = tl.rsqrt(tl.maximum(diagonal, 1.1754943508222875e-38))
        factor_column = tl.where(axis >= column_index, column * inverse_root, 0.0)
        lower = tl.where(columns == column_index, factor_column[:, None], lower)
        trailing = (rows > column_index) & (columns > column_index) & (rows >= columns)
        values = tl.where(trailing, values - factor_column[:, None] * factor_column[None, :], values)
    return lower

@triton.jit
def _register_potrf16_terminal_inverse(values, DIAGONAL_PRECISION: tl.constexpr, INVERSE_ORDER: tl.constexpr):
    lower = _register_potrf16_factor_only_final(values)
    axis = tl.arange(0, 16)
    rows = axis[:, None]
    columns = axis[None, :]
    diagonal = tl.sum(tl.where(rows == columns, lower, 0.0), axis=1)
    identity = tl.where(rows == columns, 1.0, 0.0)
    strict = tl.where(rows > columns, lower / diagonal[:, None], 0.0)
    inverse = identity - strict
    strict2 = _factor_dot(strict, strict, DIAGONAL_PRECISION)
    inverse = _factor_dot(inverse, identity + strict2, DIAGONAL_PRECISION)
    if INVERSE_ORDER >= 4:
        strict4 = _factor_dot(strict2, strict2, DIAGONAL_PRECISION)
        inverse = _factor_dot(inverse, identity + strict4, DIAGONAL_PRECISION)
    if INVERSE_ORDER >= 6:
        strict8 = _factor_dot(strict4, strict4, DIAGONAL_PRECISION)
    if INVERSE_ORDER == 6:
        inverse += strict8
    if INVERSE_ORDER >= 8:
        inverse = _factor_dot(inverse, identity + strict8, DIAGONAL_PRECISION)
    inverse /= diagonal[None, :]
    return (lower, inverse)

@triton.jit
def _potrf32_final_no_inverse_global(output, base, DIAGONAL_PRECISION: tl.constexpr, FINAL_INVERSE_ORDER: tl.constexpr):
    axis = tl.arange(0, 16)
    rows = axis[:, None]
    columns = axis[None, :]
    block_start: tl.constexpr = 96
    first = tl.load(output + base + (block_start + rows) * 128 + block_start + columns, mask=rows >= columns, other=0.0).to(tl.float32)
    lower0, inverse0 = _register_potrf16_terminal_inverse(first, DIAGONAL_PRECISION, FINAL_INVERSE_ORDER)
    tl.store(output + base + (block_start + rows) * 128 + block_start + columns, lower0, mask=rows >= columns)
    cross = tl.load(output + base + (block_start + 16 + rows) * 128 + block_start + columns).to(tl.float32)
    lower10 = _factor_dot(cross, tl.trans(inverse0), DIAGONAL_PRECISION)
    tl.store(output + base + (block_start + 16 + rows) * 128 + block_start + columns, lower10)
    second = tl.load(output + base + (block_start + 16 + rows) * 128 + block_start + 16 + columns, mask=rows >= columns, other=0.0).to(tl.float32)
    second -= _factor_dot(lower10, tl.trans(lower10), DIAGONAL_PRECISION)
    lower1 = _register_potrf16_factor_only_final(second)
    tl.store(output + base + (block_start + 16 + rows) * 128 + block_start + 16 + columns, lower1, mask=rows >= columns)

@triton.jit
def _factor64_final_no_inverse_global(output, base, HIGH_PRECISION: tl.constexpr, DIAGONAL_PRECISION: tl.constexpr, FINAL_INVERSE_ORDER: tl.constexpr):
    _factor32_dispatch_global(output, base, 128, 64, 1.1754943508222875e-38, False, False, False, False, HIGH_PRECISION, DIAGONAL_PRECISION, 0, -1, 3)
    internal_rows = 96 + tl.arange(0, 32)
    _solve32_dispatch_global(output, base, 128, internal_rows, 64, internal_rows < 128, False, False, HIGH_PRECISION)
    axis = tl.arange(0, 32)
    rows = axis[:, None]
    columns = axis[None, :]
    lower10 = tl.load(output + base + (96 + rows) * 128 + 64 + columns)
    second = tl.load(output + base + (96 + rows) * 128 + 96 + columns, mask=rows >= columns, other=0.0)
    second -= _factor_dot(lower10, tl.trans(lower10), DIAGONAL_PRECISION)
    tl.store(output + base + (96 + rows) * 128 + 96 + columns, second, mask=rows >= columns)
    tl.debug_barrier()
    _potrf32_final_no_inverse_global(output, base, DIAGONAL_PRECISION, FINAL_INVERSE_ORDER)

@triton.jit
def _potrf128_shared_dual_kernel(source, output, FIRST_HIGH: tl.constexpr, FIRST_DIAG: tl.constexpr, SOLVE_HIGH: tl.constexpr, SECOND_HIGH: tl.constexpr, SECOND_DIAG: tl.constexpr, FINAL_INVERSE_ORDER: tl.constexpr, FIRST_TAIL_PRECISION: tl.constexpr, TAIL_FIRST_FACTOR_PRECISION: tl.constexpr, LEADING_LEAF_PRECISION: tl.constexpr):
    tl_cuda.gdc_launch_dependents()
    matrix = tl.program_id(0)
    base = matrix * 128 * 128
    axis = tl.arange(0, 64)
    rows = axis[:, None]
    columns = axis[None, :]
    lower = rows >= columns
    leading = tl.load(source + base)
    use_correction = (leading < 0.5) | (tl.load(source + base + 2) == 0.0) | (tl.load(source + base + 127 * 129) < leading * 1.0e-3)
    tl.store(output + base + rows * 128 + 64 + columns, 0.0)
    tl.store(output + base + rows * 128 + columns, 0.0, mask=(rows < 32) & (columns >= 32))
    tl.store(output + base + (64 + rows) * 128 + 64 + columns, 0.0, mask=(rows < 32) & (columns >= 32) | (rows >= 32) & (columns >= 32) & (columns > rows))
    _factor64_first_tail_precision_global(source, output, base, FIRST_HIGH, FIRST_DIAG, FIRST_TAIL_PRECISION, TAIL_FIRST_FACTOR_PRECISION, LEADING_LEAF_PRECISION)
    internal_rows = 64 + axis
    _solve64_source_global(source, output, base, internal_rows, SOLVE_HIGH, use_correction)
    lower10 = tl.load(output + base + (64 + rows) * 128 + columns)
    second = tl.load(source + base + (64 + rows) * 128 + 64 + columns, mask=lower, other=0.0)
    second -= _factor_dot_shared_3to2(lower10, tl.trans(lower10), use_correction)
    tl.store(output + base + (64 + rows) * 128 + 64 + columns, second, mask=lower)
    tl.debug_barrier()
    _factor64_final_no_inverse_global(output, base, SECOND_HIGH, SECOND_DIAG, FINAL_INVERSE_ORDER)
    scratch_axis = tl.arange(0, 32)
    scratch_rows = scratch_axis[:, None]
    scratch_columns = scratch_axis[None, :]
    for scratch_block in tl.static_range(0, 3):
        scratch_start = scratch_block * 32
        clear_upper = scratch_columns > scratch_rows
        tl.store(output + base + (scratch_start + scratch_rows) * 128 + scratch_start + scratch_columns, 0.0, mask=clear_upper)