diff --git a/CMakeLists.txt b/CMakeLists.txt
index e6681317..235556fa 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -365,6 +365,9 @@ if (SD_CUDA)
 endif()
 set(SD_BACKEND_REGISTRATIONS)
 if (NOT SD_USE_UPSTREAM_GGML AND (SD_CUDA OR SD_HIPBLAS OR SD_MUSA))
+    if(SD_HIPBLAS)
+        include(cmake/hip.cmake)
+    endif()
     add_subdirectory(src/core/cuda)
     foreach(VENDOR CUDA HIP MUSA)
         if(SD_${VENDOR} OR (VENDOR STREQUAL "HIP" AND SD_HIPBLAS))
diff --git a/cmake/hip.cmake b/cmake/hip.cmake
new file mode 100644
index 00000000..e0d81334
--- /dev/null
+++ b/cmake/hip.cmake
@@ -0,0 +1,12 @@
+if(NOT WIN32 AND NOT CMAKE_CXX_COMPILER MATCHES "hipcc")
+    if(NOT CMAKE_HIP_ARCHITECTURES)
+        if(GPU_TARGETS)
+            set(CMAKE_HIP_ARCHITECTURES ${GPU_TARGETS})
+        elseif(AMDGPU_TARGETS)
+            set(CMAKE_HIP_ARCHITECTURES ${AMDGPU_TARGETS})
+        endif()
+    endif()
+    # HIP objects are linked by the parent target, so initialize its link rules here.
+    enable_language(HIP)
+    set_property(TARGET ${SD_LIB} PROPERTY HIP_ARCHITECTURES "${CMAKE_HIP_ARCHITECTURES}")
+endif()
diff --git a/src/core/cuda/CMakeLists.txt b/src/core/cuda/CMakeLists.txt
index a1258d94..7bca6aa3 100644
--- a/src/core/cuda/CMakeLists.txt
+++ b/src/core/cuda/CMakeLists.txt
@@ -44,14 +44,6 @@ if(SD_HIPBLAS)
         sd_cuda_sources(hip CXX)
         target_link_libraries(${SD_CUDA_TARGET} PRIVATE hip::device)
     else()
-        if(NOT CMAKE_HIP_ARCHITECTURES)
-            if(GPU_TARGETS)
-                set(CMAKE_HIP_ARCHITECTURES ${GPU_TARGETS})
-            elseif(AMDGPU_TARGETS)
-                set(CMAKE_HIP_ARCHITECTURES ${AMDGPU_TARGETS})
-            endif()
-        endif()
-        enable_language(HIP)
         sd_cuda_sources(hip HIP)
         set_target_properties(${SD_CUDA_TARGET} PROPERTIES HIP_STANDARD 17 HIP_STANDARD_REQUIRED ON)
     endif()