GPU portability for the world's most demanding AI

  from openai import OpenAI
  
  client = OpenAI(
      base_url="http://localhost:8000/v1",
      api_key="EMPTY",
  )
  
  completion = client.chat.completions.create(
      model="google/gemma-3-27b-it",
      messages=[
          {
            "role": "user",
            "content": "Who won the world series in 2020?"
          },
      ],
  )
  
  print(completion.choices[0].message.content)
  
  

  # total file size - 500MB.  This docker container deploys
  across GPUs and CPUs, all without vendor libraries like CUDA
  
  docker run --gpus all \
      -v ~/.cache/huggingface:/root/.cache/huggingface \
      --env "HF_HUB_ENABLE_HF_TRANSFER=1" \
      --env "HF_TOKEN=" \
      -p 8000:8000 \
      docker.modular.com/modular/max-nvidia-base:nightly \
      --model-path meta-llama/Llama-3.3-70B-Instruct

  alias M = C.shape[0]()
  alias N = C.shape[1]()
  alias K = A.shape[1]()
  
  var warp_id = thread_idx.x // WARP_SIZE
  
  warp_y = warp_id // (BN // WN)
  warp_x = warp_id % (BN // WN)
  
  C_warp_tile = C.tile[BM, BN](block_idx.y, block_idx.x).tile[WM, WN](warp_y, warp_x)
  
  mma_op = TensorCore[A.dtype, C.dtype, Index(MMA_M, MMA_N, MMA_K)]()
  
  A_sram_tile = tb[A.dtype]().row_major[BM, BK]().shared().alloc()
  B_sram_tile = tb[B.dtype]().row_major[BK, BN]().shared().alloc()
  
  c_reg = (
      tb[C.dtype]()
      .row_major[WM // MMA_M, (WN * 4) // MMA_N]()
      .local()
      .alloc()
      .fill(0)
  )
  
  for k_i in range(K // BK):
      barrier()
  
      A_dram_tile = A.tile[BM, BK](block_idx.y, k_i)
      B_dram_tile = B.tile[BK, BN](k_i, block_idx.x)
  
      copy_dram_to_sram_async[thread_layout = Layout.row_major(4, 8)](
          A_sram_tile.vectorize[1, 4](), A_dram_tile.vectorize[1, 4]()
      )
      copy_dram_to_sram_async[thread_layout = Layout.row_major(4, 8)](
          B_sram_tile.vectorize[1, 4](), B_dram_tile.vectorize[1, 4]()
      )
  
      async_copy_wait_all()
      barrier()
  
      A_warp_tile = A_sram_tile.tile[WM, BK](warp_y, 0)
      B_warp_tile = B_sram_tile.tile[BK, WN](0, warp_x)
  
      @parameter
      for mma_k in range(BK // MMA_K):
  
          @parameter
          for mma_m in range(WM // MMA_M):
  
              @parameter
              for mma_n in range(WN // MMA_N):
                  c_reg_m_n = c_reg.tile[1, 4](mma_m, mma_n)
  
                  A_mma_tile = A_warp_tile.tile[MMA_M, MMA_K](mma_m, mma_k)
                  B_mma_tile = B_warp_tile.tile[MMA_K, MMA_N](mma_k, mma_n)
  
                  a_reg = mma_op.load_a(A_mma_tile)
                  b_reg = mma_op.load_b(B_mma_tile)
  
                  var d_reg_m_n = mma_op.mma_op(
                      a_reg,
                      b_reg,
                      c_reg_m_n,
                  )
  
                  c_reg_m_n.copy_from(d_reg_m_n)
  
  @parameter
  for mma_m in range(WM // MMA_M):
  
      @parameter
      for mma_n in range(WN // MMA_N):
          var C_mma_tile = C_warp_tile.tile[MMA_M, MMA_N](mma_m, mma_n)
          var c_reg_m_n = c_reg.tile[1, 4](mma_m, mma_n)
          mma_op.store_d(C_mma_tile, c_reg_m_n)