submission 273060

#!POPCORN leaderboard nvfp4_dual_gemm
#!POPCORN gpu NVIDIA

import torch
from task import input_t, output_t
from torch.utils.cpp_extension import load_inline


# -------------------------------------------------------------------------
# Fallback Reference Implementation (for non-optimized shapes)
# -------------------------------------------------------------------------
def ceil_div(a, b):
    return (a + b - 1) // b


def to_blocked(input_matrix):
    rows, cols = input_matrix.shape
    n_row_blocks = ceil_div(rows, 128)
    n_col_blocks = ceil_div(cols, 4)
    padded = input_matrix
    blocks = padded.view(n_row_blocks, 128, n_col_blocks, 4).permute(0, 2, 1, 3)
    rearranged = blocks.reshape(-1, 4, 32, 4).transpose(1, 2).reshape(-1, 32, 16)
    return rearranged.flatten()


def fallback_kernel(data: input_t) -> output_t:
    """
    Robust PyTorch fallback for shapes not handled by the optimized kernel.
    """
    a_ref, b1_ref, b2_ref, sfa_ref_cpu, sfb1_ref_cpu, sfb2_ref_cpu, _, _, _, c_ref = (
        data
    )
    m, n, l = c_ref.shape

    ref1 = torch.empty((l, m, n), dtype=torch.float32, device="cuda").permute(1, 2, 0)
    ref2 = torch.empty((l, m, n), dtype=torch.float32, device="cuda").permute(1, 2, 0)

    for l_idx in range(l):
        scale_a = to_blocked(sfa_ref_cpu[:, :, l_idx])
        scale_b1 = to_blocked(sfb1_ref_cpu[:, :, l_idx])
        scale_b2 = to_blocked(sfb2_ref_cpu[:, :, l_idx])

        # Note: PyTorch _scaled_mm expects row-major inputs usually,
        # but reference implies transposing B.
        res1 = torch._scaled_mm(
            a_ref[:, :, l_idx],
            b1_ref[:, :, l_idx].transpose(0, 1),
            scale_a.cuda(),
            scale_b1.cuda(),
            bias=None,
            out_dtype=torch.float32,
        )
        ref1[:, :, l_idx] = res1

        res2 = torch._scaled_mm(
            a_ref[:, :, l_idx],
            b2_ref[:, :, l_idx].transpose(0, 1),
            scale_a.cuda(),
            scale_b2.cuda(),
            bias=None,
            out_dtype=torch.float32,
        )
        ref2[:, :, l_idx] = res2

    c_out = (torch.nn.functional.silu(ref1) * ref2).to(torch.float16)
    return c_out


# -------------------------------------------------------------------------
# CUDA / PTX Implementation
# -------------------------------------------------------------------------
CUDA_SRC_COMMON = r"""
#include <cudaTypedefs.h>
#include <cuda_fp16.h>
#include <torch/library.h>
#include <ATen/core/Tensor.h>

constexpr int WARP_SIZE = 32;
constexpr int MMA_K = 64; 

constexpr uint64_t EVICT_NORMAL = 0x1000000000000000;
constexpr uint64_t EVICT_FIRST = 0x12F0000000000000;
constexpr uint64_t EVICT_LAST = 0x14F0000000000000;

__device__ inline constexpr uint64_t desc_encode(uint64_t x) { return (x & 0x3'FFFFULL) >> 4ULL; };

__device__ uint32_t elect_sync() {
  uint32_t pred = 0;
  asm volatile(
    "{\n\t"
    ".reg .pred %%px;\n\t"
    "elect.sync _|%%px, %1;\n\t"
    "@%%px mov.s32 %0, 1;\n\t"
    "}"
    : "+r"(pred) : "r"(0xFFFFFFFF)
  );
  return pred;
}

__device__ inline void mbarrier_init(int mbar_addr, int count) {
  asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;" :: "r"(mbar_addr), "r"(count));
}

__device__ void mbarrier_wait(int mbar_addr, int phase) {
  uint32_t ticks = 0x989680; 
  asm volatile(
    "{\n\t"
    ".reg .pred P1;\n\t"
    "LAB_WAIT:\n\t"
    "mbarrier.try_wait.parity.acquire.cta.shared::cta.b64 P1, [%0], %1, %2;\n\t"
    "@P1 bra.uni DONE;\n\t"
    "bra.uni LAB_WAIT;\n\t"
    "DONE:\n\t"
    "}"
    :: "r"(mbar_addr), "r"(phase), "r"(ticks)
  );
}

__device__ inline void tma_gmem2smem(int dst, const void *src, int size, int mbar_addr, uint64_t cache_policy) {
  asm volatile("cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint [%0], [%1], %2, [%3], %4;"
              :: "r"(dst), "l"(src), "r"(size), "r"(mbar_addr), "l"(cache_policy));
}

__device__ inline void tma_3d_gmem2smem(int dst, const void *tmap_ptr, int x, int y, int z, int mbar_addr, uint64_t cache_policy) {
  asm volatile("cp.async.bulk.tensor.3d.shared::cta.global.mbarrier::complete_tx::bytes.cta_group::1.L2::cache_hint "
              "[%0], [%1, {%2, %3, %4}], [%5], %6;"
              :: "r"(dst), "l"(tmap_ptr), "r"(x), "r"(y), "r"(z), "r"(mbar_addr), "l"(cache_policy)
              : "memory");
}

__device__ inline void tcgen05_cp_nvfp4(int taddr, uint64_t s_desc) {
  asm volatile("tcgen05.cp.cta_group::1.32x128b.warpx4 [%0], %1;" :: "r"(taddr), "l"(s_desc));
}

__device__ inline void tcgen05_mma_nvfp4(
  uint64_t a_desc,
  uint64_t b_desc,
  uint32_t i_desc,
  int scale_A_tmem,
  int scale_B_tmem,
  int enable_input_d,
  int d_tmem_offset) 
{
  asm volatile(
    "{\n\t"
    ".reg .pred p;\n\t"
    "setp.ne.b32 p, %6, 0;\n\t"
    "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16 [%0], %1, %2, %3, [%4], [%5], p;\n\t"
    "}"
    :: "r"(d_tmem_offset), "l"(a_desc), "l"(b_desc), "r"(i_desc),
       "r"(scale_A_tmem), "r"(scale_B_tmem), "r"(enable_input_d)
  );
}

// -------------------------------------------------------------------------
// Register Loading Templates (Epilogue)
// -------------------------------------------------------------------------
struct SHAPE {
  static constexpr char _32x32b[]  = ".32x32b"; 
  static constexpr char _16x256b[] = ".16x256b";
};
struct NUM {
  static constexpr char x4[]  = ".x4";
  static constexpr char x8[]  = ".x8";
  static constexpr char x16[] = ".x16";
  static constexpr char x32[] = ".x32";
  static constexpr char x64[] = ".x64";
  static constexpr char x128[] = ".x128";
};

// Instantiate specific load routines needed for our block sizes
__device__ inline void tcgen05_ld_32x32bx64(float *tmp, int row, int col) {
  asm volatile("tcgen05.ld.sync.aligned%65%66.b32 "
              "{ %0,  %1,  %2,  %3,  %4,  %5,  %6,  %7, %8,  %9, %10, %11, %12, %13, %14, %15, "
              " %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, "
              " %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, "
              " %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, [%64];"
              : "=f"(tmp[ 0]), "=f"(tmp[ 1]), "=f"(tmp[ 2]), "=f"(tmp[ 3]), "=f"(tmp[ 4]), "=f"(tmp[ 5]), "=f"(tmp[ 6]), "=f"(tmp[ 7]),
                "=f"(tmp[ 8]), "=f"(tmp[ 9]), "=f"(tmp[10]), "=f"(tmp[11]), "=f"(tmp[12]), "=f"(tmp[13]), "=f"(tmp[14]), "=f"(tmp[15]),
                "=f"(tmp[16]), "=f"(tmp[17]), "=f"(tmp[18]), "=f"(tmp[19]), "=f"(tmp[20]), "=f"(tmp[21]), "=f"(tmp[22]), "=f"(tmp[23]),
                "=f"(tmp[24]), "=f"(tmp[25]), "=f"(tmp[26]), "=f"(tmp[27]), "=f"(tmp[28]), "=f"(tmp[29]), "=f"(tmp[30]), "=f"(tmp[31]),
                "=f"(tmp[32]), "=f"(tmp[33]), "=f"(tmp[34]), "=f"(tmp[35]), "=f"(tmp[36]), "=f"(tmp[37]), "=f"(tmp[38]), "=f"(tmp[39]),
                "=f"(tmp[40]), "=f"(tmp[41]), "=f"(tmp[42]), "=f"(tmp[43]), "=f"(tmp[44]), "=f"(tmp[45]), "=f"(tmp[46]), "=f"(tmp[47]),
                "=f"(tmp[48]), "=f"(tmp[49]), "=f"(tmp[50]), "=f"(tmp[51]), "=f"(tmp[52]), "=f"(tmp[53]), "=f"(tmp[54]), "=f"(tmp[55]),
                "=f"(tmp[56]), "=f"(tmp[57]), "=f"(tmp[58]), "=f"(tmp[59]), "=f"(tmp[60]), "=f"(tmp[61]), "=f"(tmp[62]), "=f"(tmp[63])
              : "r"((row << 16) | col), "C"(SHAPE::_32x32b), "C"(NUM::x64));
}

// -------------------------------------------------------------------------
// Helper Functions
// -------------------------------------------------------------------------
void check_cu(CUresult err) {
  if (err == CUDA_SUCCESS) return;
  TORCH_CHECK(false, "cuTensorMapEncodeTiled error");
}

void init_AB_tmap(
  CUtensorMap *tmap,
  const char *ptr,
  uint64_t global_height, uint64_t global_width,
  uint32_t shared_height, uint32_t shared_width) {
  constexpr uint32_t rank = 3;
  uint64_t globalDim[rank]       = {256, global_height, global_width / 256};
  uint64_t globalStrides[rank-1] = {global_width / 2, 128}; 
  uint32_t boxDim[rank]          = {256, shared_height, shared_width / 256};
  uint32_t elementStrides[rank]  = {1, 1, 1};
  auto err = cuTensorMapEncodeTiled(
    tmap,
    CUtensorMapDataType::CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN8B,
    rank,
    (void *)ptr,
    globalDim,
    globalStrides,
    boxDim,
    elementStrides,
    CUtensorMapInterleave::CU_TENSOR_MAP_INTERLEAVE_NONE,
    CUtensorMapSwizzle::CU_TENSOR_MAP_SWIZZLE_128B,
    CUtensorMapL2promotion::CU_TENSOR_MAP_L2_PROMOTION_NONE,
    CUtensorMapFloatOOBfill::CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE
  );
  check_cu(err);
}
"""

CUDA_SRC_DUAL = r"""
template <
  int K,
  int BLOCK_M,
  int BLOCK_N,
  int BLOCK_K,
  int NUM_STAGES>
__global__ __launch_bounds__(BLOCK_M + 2 * WARP_SIZE)
void dual_kernel(
  const __grid_constant__ CUtensorMap A_tmap,
  const __grid_constant__ CUtensorMap B1_tmap,
  const __grid_constant__ CUtensorMap B2_tmap,
  const char *SFA_ptr,
  const char *SFB1_ptr,
  const char *SFB2_ptr,
  half *C_ptr,
  int M, int N) 
{
  const int tid = threadIdx.x;
  const int bid = blockIdx.x;
  const int warp_id = tid / WARP_SIZE;
  
  // Grid Logic
  const int grid_n = N / BLOCK_N;
  const int bid_m = bid / grid_n;
  const int bid_n = bid % grid_n;
  const int off_m = bid_m * BLOCK_M;
  const int off_n = bid_n * BLOCK_N;
  
  constexpr int NUM_WARPS = BLOCK_M / WARP_SIZE + 2;

  // Shared Memory Layout
  extern __shared__ __align__(1024) char smem_ptr[];
  const int smem = static_cast<int>(__cvta_generic_to_shared(smem_ptr));
  
  constexpr int A_size = BLOCK_M * BLOCK_K / 2;
  constexpr int B_size = BLOCK_N * BLOCK_K / 2;
  constexpr int SFA_size = 128 * BLOCK_K / 16;
  constexpr int SFB_size = 128 * BLOCK_K / 16;
  
  constexpr int STAGE_SIZE = A_size + (2 * B_size) + SFA_size + (2 * SFB_size);

  #pragma nv_diag_suppress static_var_with_dynamic_init
  __shared__ int64_t mbars[NUM_STAGES * 2 + 1];
  const int tma_mbar_addr = static_cast<int>(__cvta_generic_to_shared(mbars));
  const int mma_mbar_addr = tma_mbar_addr + NUM_STAGES * 8;
  const int mainloop_mbar_addr = mma_mbar_addr + NUM_STAGES * 8;

  constexpr int C1_tmem = 0;
  constexpr int C2_tmem = BLOCK_N; 
  constexpr int SFA_tmem = 2 * BLOCK_N;
  constexpr int SFB1_tmem = SFA_tmem + 4 * (BLOCK_K / MMA_K);
  constexpr int SFB2_tmem = SFB1_tmem + 4 * (BLOCK_K / MMA_K);

  // INIT Phase
  if (warp_id == 0 && elect_sync()) {
    for (int i = 0; i < NUM_STAGES * 2 + 1; i++)
      mbarrier_init(tma_mbar_addr + i * 8, 1);
    asm volatile("fence.mbarrier_init.release.cluster;");
  }
  else if (warp_id == 1) {
    asm volatile("tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" 
                 :: "r"(smem), "r"(BLOCK_N * 4)); 
  }
  __syncthreads();

  const int num_iters = K / BLOCK_K;

  // -----------------------------------------------------------------------
  // TMA WARP (Producer)
  // -----------------------------------------------------------------------
  if (warp_id == NUM_WARPS - 2 && elect_sync()) {
    uint64_t cache_A = (M > N) ? EVICT_FIRST : EVICT_LAST;
    uint64_t cache_B = (M > N) ? EVICT_LAST : EVICT_FIRST;

    auto issue_tma = [&](int iter_k, int stage_id) {
      const int mbar_addr = tma_mbar_addr + stage_id * 8;
      
      int curr_smem = smem + stage_id * STAGE_SIZE;
      const int A_smem = curr_smem; curr_smem += A_size;
      const int B1_smem = curr_smem; curr_smem += B_size;
      const int B2_smem = curr_smem; curr_smem += B_size;
      const int SFA_smem = curr_smem; curr_smem += SFA_size;
      const int SFB1_smem = curr_smem; curr_smem += SFB_size;
      const int SFB2_smem = curr_smem;

      const int off_k = iter_k * BLOCK_K;
      
      tma_3d_gmem2smem(A_smem, &A_tmap, 0, off_m, off_k / 256, mbar_addr, cache_A);
      tma_3d_gmem2smem(B1_smem, &B1_tmap, 0, off_n, off_k / 256, mbar_addr, cache_B);
      tma_3d_gmem2smem(B2_smem, &B2_tmap, 0, off_n, off_k / 256, mbar_addr, cache_B);

      const int rest_k = K / 16 / 4;
      int sfa_offset = ((off_m / 128) * rest_k + off_k / (16 * 4)) * 512;
      int sfb_offset = ((off_n / 128) * rest_k + off_k / (16 * 4)) * 512;

      tma_gmem2smem(SFA_smem, SFA_ptr + sfa_offset, SFA_size, mbar_addr, cache_A);
      tma_gmem2smem(SFB1_smem, SFB1_ptr + sfb_offset, SFB_size, mbar_addr, cache_B);
      tma_gmem2smem(SFB2_smem, SFB2_ptr + sfb_offset, SFB_size, mbar_addr, cache_B);

      asm volatile("mbarrier.arrive.expect_tx.release.cta.shared::cta.b64 _, [%0], %1;"
                  :: "r"(mbar_addr), "r"(STAGE_SIZE) : "memory");
    };

    for (int iter_k = 0; iter_k < NUM_STAGES; iter_k++) issue_tma(iter_k, iter_k);
    
    for (int iter_k = NUM_STAGES; iter_k < num_iters; iter_k++) {
      const int stage_id = iter_k % NUM_STAGES;
      const int mma_phase = (iter_k / NUM_STAGES - 1) % 2;
      mbarrier_wait(mma_mbar_addr + stage_id * 8, mma_phase);
      issue_tma(iter_k, stage_id);
    }
  }

  // -----------------------------------------------------------------------
  // MMA WARP (Consumer)
  // -----------------------------------------------------------------------
  else if (warp_id == NUM_WARPS - 1 && elect_sync()) {
    constexpr int MMA_N = BLOCK_N;
    constexpr int MMA_M = 128;
    constexpr uint32_t i_desc = (1U << 7U) | (1U << 10U) | ((uint32_t)MMA_N >> 3U << 17U) | ((uint32_t)MMA_M >> 7U << 27U);

    for (int iter_k = 0; iter_k < num_iters; iter_k++) {
      const int stage_id = iter_k % NUM_STAGES;
      const int tma_phase = (iter_k / NUM_STAGES) % 2;
      mbarrier_wait(tma_mbar_addr + stage_id * 8, tma_phase);

      int curr_smem = smem + stage_id * STAGE_SIZE;
      const int A_smem = curr_smem; curr_smem += A_size;
      const int B1_smem = curr_smem; curr_smem += B_size;
      const int B2_smem = curr_smem; curr_smem += B_size;
      const int SFA_smem = curr_smem; curr_smem += SFA_size;
      const int SFB1_smem = curr_smem; curr_smem += SFB_size;
      const int SFB2_smem = curr_smem;

      auto make_desc_AB = [](int addr) -> uint64_t {
        const int SBO = 8 * 128;
        return desc_encode(addr) | (desc_encode(SBO) << 32ULL) | (1ULL << 46ULL) | (2ULL << 61ULL);
      };
      auto make_desc_SF = [](int addr) -> uint64_t {
        const int SBO = 8 * 16;
        return desc_encode(addr) | (desc_encode(SBO) << 32ULL) | (1ULL << 46ULL);
      };

      constexpr uint64_t SF_desc_base = make_desc_SF(0);
      const uint64_t SFA_desc = SF_desc_base + ((uint64_t)SFA_smem >> 4ULL);
      const uint64_t SFB1_desc = SF_desc_base + ((uint64_t)SFB1_smem >> 4ULL);
      const uint64_t SFB2_desc = SF_desc_base + ((uint64_t)SFB2_smem >> 4ULL);

      for (int k = 0; k < BLOCK_K / MMA_K; k++) {
        uint64_t k_off = (uint64_t)k * (512ULL >> 4ULL);
        tcgen05_cp_nvfp4(SFA_tmem + k * 4, SFA_desc + k_off);
        tcgen05_cp_nvfp4(SFB1_tmem + k * 4, SFB1_desc + k_off);
        tcgen05_cp_nvfp4(SFB2_tmem + k * 4, SFB2_desc + k_off);
      }

      for (int k1 = 0; k1 < BLOCK_K / 256; k1++) {
        for (int k2 = 0; k2 < 256 / MMA_K; k2++) {
          uint64_t a_desc = make_desc_AB(A_smem + k1 * BLOCK_M * 128 + k2 * 32);
          uint64_t b1_desc = make_desc_AB(B1_smem + k1 * BLOCK_N * 128 + k2 * 32);
          uint64_t b2_desc = make_desc_AB(B2_smem + k1 * BLOCK_N * 128 + k2 * 32);
          
          int k_sf = k1 * 4 + k2;
          const int sa_tmem = SFA_tmem + k_sf * 4 + (bid_m % (128 / BLOCK_M)) * (BLOCK_M / 32);
          const int sb1_tmem = SFB1_tmem + k_sf * 4 + (bid_n % (128 / BLOCK_N)) * (BLOCK_N / 32);
          const int sb2_tmem = SFB2_tmem + k_sf * 4 + (bid_n % (128 / BLOCK_N)) * (BLOCK_N / 32);
          
          const int enable_d = (k1 == 0 && k2 == 0) ? iter_k : 1;
          
          tcgen05_mma_nvfp4(a_desc, b1_desc, i_desc, sa_tmem, sb1_tmem, enable_d, C1_tmem);
          tcgen05_mma_nvfp4(a_desc, b2_desc, i_desc, sa_tmem, sb2_tmem, enable_d, C2_tmem);
        }
      }

      asm volatile("tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%0];"
                  :: "r"(mma_mbar_addr + stage_id * 8) : "memory");
    }
    asm volatile("tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%0];"
                :: "r"(mainloop_mbar_addr) : "memory");
  }

  // -----------------------------------------------------------------------
  // EPILOGUE WARPS
  // -----------------------------------------------------------------------
  else if (tid < BLOCK_M) {
    mbarrier_wait(mainloop_mbar_addr, 0);
    asm volatile("tcgen05.fence::after_thread_sync;");

    constexpr int WIDTH = BLOCK_N; 
    
    float tmp1[WIDTH];
    float tmp2[WIDTH];

    for (int n = 0; n < BLOCK_N / WIDTH; n++) {
        if constexpr (WIDTH == 64) tcgen05_ld_32x32bx64(tmp1, warp_id * 32, n * WIDTH + C1_tmem);
        if constexpr (WIDTH == 64) tcgen05_ld_32x32bx64(tmp2, warp_id * 32, n * WIDTH + C2_tmem);
        
        asm volatile("tcgen05.wait::ld.sync.aligned;");

        for (int i = 0; i < WIDTH; i++) {
            float val1 = tmp1[i];
            float val2 = tmp2[i];
            
            float silu_val = val1 * (1.0f / (1.0f + expf(-val1)));
            float res = silu_val * val2;

            int row = off_m + tid;
            int col = off_n + n * WIDTH + i;
            
            // Write M-Major (coalesced)
            if (row < M && col < N) {
                C_ptr[col * M + row] = __float2half(res); 
            }
        }
    }
    
    asm volatile("bar.sync 1, %0;" :: "r"(BLOCK_M) : "memory");
    if (warp_id == 0)
      asm volatile("tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" :: "r"(0), "r"(BLOCK_N * 4));
  }
}

template <int K, int BLOCK_M, int BLOCK_N, int BLOCK_K, int NUM_STAGES>
at::Tensor dual_gemm_launch(
  const at::Tensor& A,
  const at::Tensor& B1,
  const at::Tensor& B2,
  const at::Tensor& SFA,
  const at::Tensor& SFB1,
  const at::Tensor& SFB2,
  at::Tensor& C) 
{
  const int M = A.size(0);
  const int N = B1.size(0);
  
  auto A_ptr   = reinterpret_cast<const char *>(A.data_ptr());
  auto B1_ptr  = reinterpret_cast<const char *>(B1.data_ptr());
  auto B2_ptr  = reinterpret_cast<const char *>(B2.data_ptr());
  auto SFA_ptr = reinterpret_cast<const char *>(SFA.data_ptr());
  auto SFB1_ptr = reinterpret_cast<const char *>(SFB1.data_ptr());
  auto SFB2_ptr = reinterpret_cast<const char *>(SFB2.data_ptr());
  auto C_ptr    = reinterpret_cast<half *>(C.data_ptr());

  CUtensorMap A_tmap, B1_tmap, B2_tmap;
  init_AB_tmap(&A_tmap, A_ptr, M, K, BLOCK_M, BLOCK_K);
  init_AB_tmap(&B1_tmap, B1_ptr, N, K, BLOCK_N, BLOCK_K);
  init_AB_tmap(&B2_tmap, B2_ptr, N, K, BLOCK_N, BLOCK_K);

  int grid = (M / BLOCK_M) * (N / BLOCK_N);
  int tb_size = BLOCK_M + 2 * WARP_SIZE;
  
  int A_size = BLOCK_M * BLOCK_K / 2;
  int B_size = BLOCK_N * BLOCK_K / 2;
  int SF_size = 128 * (BLOCK_K / 16);
  int stage_size = A_size + (2 * B_size) + SF_size + (2 * SF_size);
  int smem_size = stage_size * NUM_STAGES;

  auto kernel_func = dual_kernel<K, BLOCK_M, BLOCK_N, BLOCK_K, NUM_STAGES>;
  
  if (smem_size > 48000)
    cudaFuncSetAttribute(kernel_func, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size);
    
  kernel_func<<<grid, tb_size, smem_size>>>(
      A_tmap, B1_tmap, B2_tmap, SFA_ptr, SFB1_ptr, SFB2_ptr, C_ptr, M, N
  );
  
  return C;
}

at::Tensor dual_gemm_dispatch(
  const at::Tensor& A,
  const at::Tensor& B1,
  const at::Tensor& B2,
  const at::Tensor& SFA,
  const at::Tensor& SFB1,
  const at::Tensor& SFB2,
  at::Tensor& C) 
{
  const int K = A.size(1) * 2;
  // Specific optimized shapes
  if (K == 7168) {
      // High SMEM usage, slightly fewer stages might be safer if occupancy is an issue, 
      // but 5 stages should fit in ~180KB which is fine for Blackwell.
      dual_gemm_launch<7168, 128, 64, 256, 5>(A, B1, B2, SFA, SFB1, SFB2, C);
  } else if (K == 4096) {
      dual_gemm_launch<4096, 128, 64, 256, 5>(A, B1, B2, SFA, SFB1, SFB2, C);
  } else {
      // Optimized Fallback for other large powers of 2
      dual_gemm_launch<2048, 128, 64, 256, 5>(A, B1, B2, SFA, SFB1, SFB2, C);
  }
  return C;
}

TORCH_LIBRARY(dual_gemm_mod, m) {
  m.def("dual_gemm_dispatch(Tensor A, Tensor B1, Tensor B2, Tensor SFA, Tensor SFB1, Tensor SFB2, Tensor(a!) C) -> Tensor");
  m.impl("dual_gemm_dispatch", &dual_gemm_dispatch);
}
"""

load_inline(
    "dual_gemm_mod",
    cpp_sources="",
    cuda_sources=CUDA_SRC_COMMON + CUDA_SRC_DUAL,
    verbose=True,
    is_python_module=False,
    no_implicit_headers=True,
    extra_cuda_cflags=[
        "-O3",
        "-gencode=arch=compute_100a,code=sm_100a",
        "--use_fast_math",
        "--expt-relaxed-constexpr",
        "-lineinfo",
    ],
    extra_ldflags=["-lcuda"],
)

dual_gemm_op = torch.ops.dual_gemm_mod.dual_gemm_dispatch


def custom_kernel(data: input_t) -> output_t:
    # Unpack the 10-element tuple.
    # indices 0,1,2: A, B1, B2
    # indices 3,4,5: SFA, SFB1, SFB2 (Ref CPU/CUDA copies)
    # indices 6,7,8: SFA, SFB1, SFB2 (Permuted for Kernel)
    # index 9: C
    a, b1, b2 = data[0], data[1], data[2]
    c = data[9]

    # Check for optimized path eligibility
    K = a.shape[1] * 2

    # We only run the optimized kernel for the specific test shapes we tuned for
    # to avoid correctness issues on edge cases (weird padding, small K).
    if K in [7168, 4096, 2048]:
        sfa_perm, sfb1_perm, sfb2_perm = data[6], data[7], data[8]
        # Kernel writes column-major C into the buffer.
        # To get the correct row-major PyTorch tensor C, we view the buffer as (N, M) and transpose.
        # This is a zero-copy metadata operation.
        M, N = c.shape[0], c.shape[1]

        # We need a buffer that matches C's underlying storage to write into
        # The kernel expects C_ptr to be M*N
        dual_gemm_op(a, b1, b2, sfa_perm, sfb1_perm, sfb2_perm, c)

        # Since the kernel wrote M-major data:
        # data[col * M + row].
        # If we view this as (N, M), then element (n, m) is at n*M + m.
        # This matches exactly. So we view as (N, M) and transpose to get (M, N).
        return c.view(N, M, 1).transpose(0, 1).reshape(M, N, 1)
    else:
        return fallback_kernel(data)