API tools faq
paste
Guest User

Untitled

a guest
Aug 9th, 2026
141
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 18.21 KB | None | 0 0
raw download clone embed print report
  1. diff --git a/common/arg.cpp b/common/arg.cpp
  2. index b75f4f0..0140dcc 100644
  3. --- a/common/arg.cpp
  4. +++ b/common/arg.cpp
  5. @@ -1713,6 +1713,21 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
  6. string_format("error: unknown value for --flash-attn: '%s'\n", value.c_str()));
  7. }
  8. }).set_env("LLAMA_ARG_FLASH_ATTN"));
  9. + add_opt(common_arg({ "--pipeline-parallel" }, "[on|off|auto]",
  10. + string_format("set pipeline parallelism ('on', 'off', or 'auto', default: '%s')",
  11. + llama_pipeline_parallel_type_name(params.pipeline_parallel_type)),
  12. + [](common_params & params, const std::string & value) {
  13. + if (is_truthy(value)) {
  14. + params.pipeline_parallel_type = LLAMA_PIPELINE_PARALLEL_TYPE_ENABLED;
  15. + } else if (is_falsey(value)) {
  16. + params.pipeline_parallel_type = LLAMA_PIPELINE_PARALLEL_TYPE_DISABLED;
  17. + } else if (is_autoy(value)) {
  18. + params.pipeline_parallel_type = LLAMA_PIPELINE_PARALLEL_TYPE_AUTO;
  19. + } else {
  20. + throw std::runtime_error(
  21. + string_format("error: unknown value for --pipeline-parallel: '%s'\n", value.c_str()));
  22. + }
  23. + }).set_env("LLAMA_ARG_PIPELINE_PARALLEL"));
  24. add_opt(common_arg(
  25. {"-p", "--prompt"}, "PROMPT",
  26. "prompt to start generation with; for system message, use -sys",
  27. diff --git a/common/common.cpp b/common/common.cpp
  28. index d9ce575..2a45d38 100644
  29. --- a/common/common.cpp
  30. +++ b/common/common.cpp
  31. @@ -1667,6 +1667,7 @@ struct llama_context_params common_context_params_to_llama(const common_params &
  32. cparams.pooling_type = params.pooling_type;
  33. cparams.attention_type = params.attention_type;
  34. cparams.flash_attn_type = params.flash_attn_type;
  35. + cparams.pipeline_parallel_type = params.pipeline_parallel_type;
  36. cparams.cb_eval = params.cb_eval;
  37. cparams.cb_eval_user_data = params.cb_eval_user_data;
  38. cparams.offload_kqv = !params.no_kv_offload;
  39. diff --git a/common/common.h b/common/common.h
  40. index 78d0877..3718997 100644
  41. --- a/common/common.h
  42. +++ b/common/common.h
  43. @@ -494,6 +494,7 @@ struct common_params {
  44. enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED; // pooling type for embeddings
  45. enum llama_attention_type attention_type = LLAMA_ATTENTION_TYPE_UNSPECIFIED; // attention type for embeddings
  46. enum llama_flash_attn_type flash_attn_type = LLAMA_FLASH_ATTN_TYPE_AUTO; // whether to use Flash Attention
  47. + enum llama_pipeline_parallel_type pipeline_parallel_type = LLAMA_PIPELINE_PARALLEL_TYPE_AUTO; // whether to use pipeline parallelism
  48.  
  49. struct common_params_sampling sampling;
  50. struct common_params_speculative speculative;
  51. diff --git a/common/fit.cpp b/common/fit.cpp
  52. index c82d066..bb1a547 100644
  53. --- a/common/fit.cpp
  54. +++ b/common/fit.cpp
  55. @@ -182,6 +182,7 @@ static void common_params_fit_impl(
  56. constexpr int64_t MiB = 1024*1024;
  57. typedef std::vector<llama_device_memory_data> dmds_t;
  58. const llama_model_params default_mparams = llama_model_default_params();
  59. + const bool fixed_gpu_layers = mparams->n_gpu_layers != default_mparams.n_gpu_layers;
  60.  
  61. std::vector<ggml_backend_dev_t> devs;
  62. uint32_t hp_ngl = 0; // hparams.n_gpu_layers
  63. @@ -314,7 +315,7 @@ static void common_params_fit_impl(
  64. sum_used_target -= margins[id];
  65. }
  66. }
  67. - if (nd > 1) {
  68. + if (nd > 1 && !fixed_gpu_layers) {
  69. // for multiple devices we need to be more conservative in terms of how much context we think can fit:
  70. // - for dense models only whole layers can be assigned to devices
  71. // - for MoE models only whole tensors can be assigned to devices, which we estimate to be <= 1/3 of a layer
  72. @@ -334,6 +335,30 @@ static void common_params_fit_impl(
  73. sum_projected_used_min_ctx += dmds_min_ctx[id].mb.total();
  74. }
  75. }
  76. + if (nd > 1 && fixed_gpu_layers) {
  77. + uint32_t n_ctx_fit = hp_nct;
  78. + bool fits = true;
  79. + for (size_t id = 0; id < nd; id++) {
  80. + const int64_t used_full = dmds_full[id].mb.total();
  81. + const int64_t used_min = dmds_min_ctx[id].mb.total();
  82. + const int64_t target = dmds_full[id].free - margins[id];
  83. + if (used_full <= target) {
  84. + continue;
  85. + }
  86. + uint32_t n_ctx_device = n_ctx_min;
  87. + fits = fits && target >= used_min;
  88. + if (target > used_min && used_full > used_min) {
  89. + n_ctx_device += (hp_nct - n_ctx_min) * (target - used_min) / (used_full - used_min);
  90. + }
  91. + n_ctx_fit = std::min(n_ctx_fit, n_ctx_device);
  92. + }
  93. + cparams->n_ctx = std::max(n_ctx_fit - n_ctx_fit % 256, n_ctx_min);
  94. + LOG_TRC("%s: context size reduced from %" PRIu32 " to %" PRIu32 " using per-device limits\n",
  95. + __func__, hp_nct, cparams->n_ctx);
  96. + if (fits) {
  97. + return;
  98. + }
  99. + }
  100. if (sum_used_target > sum_projected_used_min_ctx) {
  101. // linear interpolation between minimum and maximum context size:
  102. cparams->n_ctx += (hp_nct - n_ctx_min) * (sum_used_target - sum_projected_used_min_ctx)
  103. @@ -371,7 +396,7 @@ static void common_params_fit_impl(
  104. throw common_params_fit_exception("was unable to fit model into system memory by reducing context, abort");
  105. }
  106.  
  107. - if (mparams->n_gpu_layers != default_mparams.n_gpu_layers) {
  108. + if (fixed_gpu_layers) {
  109. throw common_params_fit_exception("n_gpu_layers already set by user to " + std::to_string(mparams->n_gpu_layers) + ", abort");
  110. }
  111. if (nd > 1) {
  112. diff --git a/common/speculative.cpp b/common/speculative.cpp
  113. index 70dc0ac..5c22d87 100644
  114. --- a/common/speculative.cpp
  115. +++ b/common/speculative.cpp
  116. @@ -2321,6 +2321,10 @@ common_speculative_init_result::common_speculative_init_result(
  117.  
  118. if (spec_mtp) {
  119. cparams.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
  120. + cparams.pipeline_parallel_type = LLAMA_PIPELINE_PARALLEL_TYPE_DISABLED;
  121. + // use the same (possibly fit-reduced) context size as the target
  122. + // avoids MTP draft reserving flash attention workspace for full n_ctx
  123. + cparams.n_ctx = llama_n_ctx(ctx_tgt);
  124. }
  125.  
  126. // note: for small models maybe we can set this to the maximum possible draft from all speculative types
  127. diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu
  128. index ab7a3b2..980f21e 100644
  129. --- a/ggml/src/ggml-cuda/fattn.cu
  130. +++ b/ggml/src/ggml-cuda/fattn.cu
  131. @@ -457,6 +457,14 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
  132. // 192 satisfies % 64 == 0 but has no vec instance (DKQ != DV); force it onto the MMA path.
  133. const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % 64 == 0 && Q->ne[0] != 192 && K->ne[1] % FATTN_KQ_STRIDE == 0;
  134.  
  135. +#ifdef GGML_USE_HIP
  136. + // HIP quantized-KV TILE/MMA paths materialize large F16 temporary buffers;
  137. + // VEC dequantizes in-register and is also the safe path on RDNA2.
  138. + if ((ggml_is_quantized(K->type) || ggml_is_quantized(V->type)) && can_use_vector_kernel) {
  139. + return BEST_FATTN_KERNEL_VEC;
  140. + }
  141. +#endif
  142. +
  143. // If Turing tensor cores are available, use them:
  144. if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) {
  145. if (can_use_vector_kernel) {
  146. diff --git a/include/llama.h b/include/llama.h
  147. index fb2ca38..9ab521d 100644
  148. --- a/include/llama.h
  149. +++ b/include/llama.h
  150. @@ -195,6 +195,14 @@ extern "C" {
  151.  
  152. LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type);
  153.  
  154. + enum llama_pipeline_parallel_type {
  155. + LLAMA_PIPELINE_PARALLEL_TYPE_AUTO = -1,
  156. + LLAMA_PIPELINE_PARALLEL_TYPE_DISABLED = 0,
  157. + LLAMA_PIPELINE_PARALLEL_TYPE_ENABLED = 1,
  158. + };
  159. +
  160. + LLAMA_API const char * llama_pipeline_parallel_type_name(enum llama_pipeline_parallel_type pipeline_parallel_type);
  161. +
  162. enum llama_split_mode {
  163. LLAMA_SPLIT_MODE_NONE = 0, // single GPU
  164. LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs
  165. @@ -362,6 +370,7 @@ extern "C" {
  166. enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id
  167. enum llama_attention_type attention_type; // attention type to use for embeddings
  168. enum llama_flash_attn_type flash_attn_type; // when to enable Flash Attention
  169. + enum llama_pipeline_parallel_type pipeline_parallel_type; // when to enable pipeline parallelism
  170.  
  171. // ref: https://github.com/ggml-org/llama.cpp/pull/2054
  172. float rope_freq_base; // RoPE base frequency, 0 = from model
  173. diff --git a/src/llama-context.cpp b/src/llama-context.cpp
  174. index 19cca7d..3a17395 100644
  175. --- a/src/llama-context.cpp
  176. +++ b/src/llama-context.cpp
  177. @@ -307,6 +307,7 @@ llama_context::llama_context(
  178. LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch);
  179. LLAMA_LOG_INFO("%s: causal_attn = %d\n", __func__, cparams.causal_attn);
  180. LLAMA_LOG_INFO("%s: flash_attn = %s\n", __func__, llama_flash_attn_type_name(params.flash_attn_type));
  181. + LLAMA_LOG_INFO("%s: pipeline mode = %s\n", __func__, llama_pipeline_parallel_type_name(params.pipeline_parallel_type));
  182. LLAMA_LOG_INFO("%s: kv_unified = %s\n", __func__, cparams.kv_unified ? "true" : "false");
  183. LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base);
  184. LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale);
  185. @@ -422,6 +423,8 @@ llama_context::llama_context(
  186. // TODO: move these checks to ggml_backend_sched
  187. // enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary
  188. bool pipeline_parallel =
  189. + cparams.ctx_type != LLAMA_CONTEXT_TYPE_MTP &&
  190. + params.pipeline_parallel_type != LLAMA_PIPELINE_PARALLEL_TYPE_DISABLED &&
  191. model.n_devices() > 1 &&
  192. model.n_gpu_layers() > model.hparams.n_layer_all &&
  193. model.split_mode() == LLAMA_SPLIT_MODE_LAYER &&
  194. @@ -448,6 +451,11 @@ llama_context::llama_context(
  195. }
  196. }
  197.  
  198. + if (cparams.ctx_type != LLAMA_CONTEXT_TYPE_MTP &&
  199. + params.pipeline_parallel_type == LLAMA_PIPELINE_PARALLEL_TYPE_ENABLED && !pipeline_parallel) {
  200. + LLAMA_LOG_WARN("%s: pipeline parallelism requested but unavailable\n", __func__);
  201. + }
  202. +
  203. cparams.pipeline_parallel = pipeline_parallel;
  204.  
  205. if (cparams.pipeline_parallel) {
  206. @@ -3495,6 +3503,7 @@ llama_context_params llama_context_default_params() {
  207. /*.pooling_type =*/ LLAMA_POOLING_TYPE_UNSPECIFIED,
  208. /*.attention_type =*/ LLAMA_ATTENTION_TYPE_UNSPECIFIED,
  209. /*.flash_attn_type =*/ LLAMA_FLASH_ATTN_TYPE_AUTO,
  210. + /*.pipeline_parallel_type =*/ LLAMA_PIPELINE_PARALLEL_TYPE_AUTO,
  211. /*.rope_freq_base =*/ 0.0f,
  212. /*.rope_freq_scale =*/ 0.0f,
  213. /*.yarn_ext_factor =*/ -1.0f,
  214. diff --git a/src/llama.cpp b/src/llama.cpp
  215. index d6e0bbf..0ade8f2 100644
  216. --- a/src/llama.cpp
  217. +++ b/src/llama.cpp
  218. @@ -46,6 +46,18 @@ const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_ty
  219. GGML_ABORT("fatal error");
  220. }
  221.  
  222. +const char * llama_pipeline_parallel_type_name(enum llama_pipeline_parallel_type pipeline_parallel_type) {
  223. + switch (pipeline_parallel_type) {
  224. + case LLAMA_PIPELINE_PARALLEL_TYPE_AUTO:
  225. + return "auto";
  226. + case LLAMA_PIPELINE_PARALLEL_TYPE_DISABLED:
  227. + return "disabled";
  228. + case LLAMA_PIPELINE_PARALLEL_TYPE_ENABLED:
  229. + return "enabled";
  230. + }
  231. + GGML_ABORT("fatal error");
  232. +}
  233. +
  234. const char * llama_load_mode_name(enum llama_load_mode load_mode) {
  235. switch (load_mode) {
  236. case LLAMA_LOAD_MODE_NONE:
  237. @@ -603,4 +615,3 @@ const char * llama_print_system_info(void) {
  238.  
  239. return s.c_str();
  240. }
  241. -
  242. diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
  243. index 5d2798c..d2a0966 100644
  244. --- a/tools/server/server-context.cpp
  245. +++ b/tools/server/server-context.cpp
  246. @@ -1133,6 +1133,20 @@ private:
  247. auto cparams_dft = common_context_params_to_llama(params_dft);
  248. if (spec_mtp) {
  249. cparams_dft.ctx_type = LLAMA_CONTEXT_TYPE_MTP;
  250. + cparams_dft.pipeline_parallel_type = LLAMA_PIPELINE_PARALLEL_TYPE_DISABLED;
  251. + // MTP draft uses the target's (possibly fit-reduced) n_ctx at runtime.
  252. + // Estimate with the same n_ctx to avoid over-reserving fit_params_target.
  253. + {
  254. + auto mparams_tgt = common_model_params_to_llama(params_base);
  255. + auto cparams_tgt = common_context_params_to_llama(params_base);
  256. + common_fit_params(params_dft.model.path.c_str(), &mparams_tgt, &cparams_tgt,
  257. + params_base.tensor_split,
  258. + params_base.tensor_buft_overrides.data(),
  259. + params_base.fit_params_target.data(),
  260. + params_base.fit_params_min_ctx,
  261. + GGML_LOG_LEVEL_ERROR);
  262. + cparams_dft.n_ctx = cparams_tgt.n_ctx;
  263. + }
  264. }
  265. cparams_dft.n_rs_seq = 0;
  266.  
  267. @@ -1156,6 +1170,7 @@ private:
  268. }
  269. }
  270.  
  271. + std::vector<size_t> measured_bytes(tgt_devices.size(), 0);
  272. for (size_t j = 0; j < devs.size(); ++j) {
  273. const size_t bytes = (measure_model_bytes ? dmd[j].model : 0) + dmd[j].context + dmd[j].compute;
  274. total += bytes;
  275. @@ -1164,6 +1179,7 @@ private:
  276. SRV_DBG("[spec] adding %.2f MiB to fit_params_target for device %s\n",
  277. bytes / (1024.0 * 1024.0), ggml_backend_dev_name(devs[j]));
  278. params_base.fit_params_target[i] += bytes;
  279. + measured_bytes[i] += bytes;
  280. break;
  281. }
  282. }
  283. @@ -1171,6 +1187,47 @@ private:
  284. SRV_TRC("[spec] estimated memory usage of %s is %.2f MiB\n",
  285. has_draft ? "draft model" : "MTP context",
  286. total / (1024.0 * 1024.0));
  287. +
  288. + if (spec_mtp) {
  289. + for (int pass = 0; pass < 2; ++pass) {
  290. + auto mparams_tgt = common_model_params_to_llama(params_base);
  291. + auto cparams_tgt = common_context_params_to_llama(params_base);
  292. + common_fit_params(params_dft.model.path.c_str(), &mparams_tgt, &cparams_tgt,
  293. + params_base.tensor_split,
  294. + params_base.tensor_buft_overrides.data(),
  295. + params_base.fit_params_target.data(),
  296. + params_base.fit_params_min_ctx,
  297. + GGML_LOG_LEVEL_ERROR);
  298. + if (cparams_dft.n_ctx == cparams_tgt.n_ctx) {
  299. + break;
  300. + }
  301. +
  302. + cparams_dft.n_ctx = cparams_tgt.n_ctx;
  303. + auto dmd_refined = common_get_device_memory_data(
  304. + params_dft.model.path.c_str(), &mparams_dft, &cparams_dft,
  305. + devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
  306. +
  307. + std::vector<size_t> refined_bytes(tgt_devices.size(), 0);
  308. + total = 0;
  309. + for (size_t j = 0; j < devs.size(); ++j) {
  310. + const size_t bytes = (measure_model_bytes ? dmd_refined[j].model : 0) + dmd_refined[j].context + dmd_refined[j].compute;
  311. + total += bytes;
  312. + for (size_t i = 0; i < tgt_devices.size(); ++i) {
  313. + if (tgt_devices[i] == devs[j]) {
  314. + refined_bytes[i] += bytes;
  315. + break;
  316. + }
  317. + }
  318. + }
  319. + for (size_t i = 0; i < tgt_devices.size(); ++i) {
  320. + params_base.fit_params_target[i] -= measured_bytes[i];
  321. + params_base.fit_params_target[i] += refined_bytes[i];
  322. + }
  323. + measured_bytes = std::move(refined_bytes);
  324. + SRV_TRC("[spec] refined MTP memory estimate at n_ctx=%" PRIu32 " is %.2f MiB\n",
  325. + cparams_dft.n_ctx, total / (1024.0 * 1024.0));
  326. + }
  327. + }
  328. } catch (const std::exception & e) {
  329. SRV_WRN("[spec] failed to measure %s memory: %s\n",
  330. has_draft ? "draft model" : "MTP context", e.what());
  331.  
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!