API tools faq
paste
cichosz

doc

cichosz
Nov 7th, 2026 (edited)
139
0
22 min
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 14.62 KB | Food | 0 0
raw download clone embed print report
  1. # vim: filetype=dockerfile
  2.  
  3.  
  4. ARG ROCMVERSION=7.2.1
  5. ARG JETPACK5VERSION=r35.4.1
  6. ARG JETPACK6VERSION=r36.4.0
  7. ARG CMAKEVERSION=3.31.2
  8. ARG NINJAVERSION=1.12.1
  9. ARG VULKANVERSION=1.4.321.1
  10.  
  11. # Default empty stages for local MLX source overrides.
  12. # Override with: docker build --build-context local-mlx=../mlx --build-context local-mlx-c=../mlx-c
  13. FROM scratch AS local-mlx
  14. FROM scratch AS local-mlx-c
  15.  
  16. FROM quay.io/almalinuxorg/almalinux:8 AS init
  17. #RUN dnf update -y
  18.  
  19.  
  20. FROM init AS init-conf
  21. # TODO: add this repo to fs file
  22. RUN --mount=type=cache,target=/root/.ccache \
  23. dnf install -y \
  24. git \
  25. unzip \
  26. epel-release\
  27. yum-utils \
  28. && yum-config-manager --add-repo "https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo"
  29.  
  30. FROM init-conf AS base-amd64
  31.  
  32. RUN --mount=type=cache,target=/root/.ccache \
  33. dnf install -y \
  34. ccache \
  35. gcc-toolset-13-gcc \
  36. gcc-toolset-13-gcc-c++ \
  37. gcc-toolset-13-binutils
  38.  
  39. RUN --mount=type=cache,target=/root/.ccache \
  40. dnf install -y \
  41. cuda-toolkit-13-4.x86_64 \
  42. openblas.x86_64 lapack.x86_64 \
  43. libcudnn9-cuda-13.x86_64 \
  44. libcudnn9-devel-cuda-13.x86_64 \
  45. libnccl.x86_64 \
  46. libnccl-devel.x86_64 \
  47. && dnf clean all
  48.  
  49.  
  50. FROM base-amd64 AS base
  51.  
  52. # TODO move Cmake and ninja to "offlice" version (download and move to dir)
  53. RUN curl --retry 3 -fsSL https://github.com/Kitware/CMake/releases/download/v3.31.2/cmake-3.31.2-linux-x86_64.tar.gz -o /tmp/cmake.tar.gz \
  54. && tar xzf /tmp/cmake.tar.gz -C /usr/local --strip-components 1 \
  55. && rm /tmp/cmake.tar.gz
  56. RUN curl --retry 3 -fsSL -o /tmp/ninja.zip https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux.zip \
  57. && unzip /tmp/ninja.zip -d /usr/local/bin \
  58. && rm /tmp/ninja.zip
  59.  
  60. ENV CMAKE_GENERATOR=Ninja
  61. ENV LDFLAGS=-s
  62. ENV PATH=/usr/local/cuda-13/bin:/opt/rh/gcc-toolset-13/root/usr/bin:$PATH
  63. ENV BLAS_INCLUDE_DIRS=/usr/include/openblas
  64. ENV LAPACK_INCLUDE_DIRS=/usr/include/openblas
  65. ENV CGO_LDFLAGS="-L/usr/local/cuda-13/lib64 -L/usr/local/cuda-13/targets/x86_64-linux/lib/stubs"
  66.  
  67.  
  68. #
  69. # Go build
  70. #
  71.  
  72. FROM base AS build
  73. WORKDIR /go/src/github.com/ollama/ollama
  74. COPY go.mod go.sum ./
  75. RUN curl --retry 3 -fsSL https://go.dev/dl/go1.26.0.linux-amd64.tar.gz -o /tmp/go.tar.gz \
  76. && tar xzf /tmp/go.tar.gz -C /usr/local \
  77. && rm /tmp/go.tar.gz
  78. ENV PATH=/usr/local/go/bin:$PATH
  79. RUN go mod download
  80.  
  81.  
  82.  
  83. # # #
  84. # # # GPU toolchain stages — provide compilers for llama-server GPU builds
  85. # # #
  86.  
  87.  
  88. # # FROM base AS cuda-12-deps
  89. # # ARG CUDA12VERSION=12.8
  90. # # RUN dnf install -y cuda-toolkit-${CUDA12VERSION//./-}
  91. # # ENV PATH=/usr/local/cuda-12/bin:$PATH
  92.  
  93.  
  94. # # FROM base AS rocm-7-deps
  95. # # ENV PATH=/opt/rocm/llvm/bin:/opt/rocm/hcc/bin:/opt/rocm/hip/bin:/opt/rocm/bin:$PATH
  96.  
  97. # # FROM base AS vulkan-deps
  98. # # ARG VULKANVERSION
  99. # # RUN ln -s /usr/bin/python3 /usr/bin/python \
  100. # # && curl --retry 3 -fsSL https://sdk.lunarg.com/sdk/download/${VULKANVERSION}/linux/vulkansdk-linux-x86_64-${VULKANVERSION}.tar.xz -o /tmp/vulkansdk.tar.xz \
  101. # # && tar xvf /tmp/vulkansdk.tar.xz -C /tmp \
  102. # # && /tmp/${VULKANVERSION}/vulkansdk -j 8 vulkan-headers \
  103. # # && /tmp/${VULKANVERSION}/vulkansdk -j 8 spirv-headers \
  104. # # && /tmp/${VULKANVERSION}/vulkansdk -j 8 shaderc \
  105. # # && cp -r /tmp/${VULKANVERSION}/x86_64/include/* /usr/local/include/ \
  106. # # && cp -r /tmp/${VULKANVERSION}/x86_64/lib/* /usr/local/lib \
  107. # # && cp -r /tmp/${VULKANVERSION}/x86_64/share/* /usr/local/share/ \
  108. # # && cp -r /tmp/${VULKANVERSION}/x86_64/bin/* /usr/local/bin/ \
  109. # # && rm -rf /tmp/${VULKANVERSION} /tmp/vulkansdk.tar.xz
  110. # # ENV VULKAN_SDK=/usr/local
  111.  
  112. # # #
  113. # # # llama-server stages — rebuild when LLAMA_CPP_VERSION, llama/server/, llama/compat/, or cmake/ changes.
  114. # # #
  115. # # # CPU stage: llama-server + ggml-base + ggml-cpu variants → lib/ollama/
  116. # # # GPU stages: GPU backend .so only → lib/ollama/<variant>/
  117. # # #
  118. #
  119. # MLX stage
  120. #
  121.  
  122. FROM build AS mlx-base
  123. # ARG OLLAMA_MLX_BUILD_JOBS=
  124. ARG OLLAMA_MLX_NVCC_THREADS=2
  125. # ARG MLX_CUDA_RAM_MB=
  126.  
  127. WORKDIR /go/src/github.com/ollama/ollama
  128. COPY CMakeLists.txt CMakePresets.json ./
  129. COPY cmake cmake
  130. COPY mlx mlx
  131. COPY mlxrunner/xgrammar/native mlxrunner/xgrammar/native
  132. COPY go.mod go.sum ./
  133. COPY MLX_VERSION MLX_C_VERSION ./
  134.  
  135. RUN curl --retry 3 -fsSL https://go.dev/dl/go1.26.0.linux-amd64.tar.gz -o /tmp/go.tar.gz \
  136. && tar xzf /tmp/go.tar.gz -C /usr/local \
  137. && rm /tmp/go.tar.gz
  138.  
  139.  
  140. RUN --mount=type=cache,target=/root/.ccache go mod download
  141. # RUN --mount=type=cache,target=/root/.ccache \
  142.  
  143. FROM mlx-base AS mlx
  144.  
  145. RUN --mount=type=bind,from=local-mlx,target=/tmp/local-mlx,rw \
  146. --mount=type=bind,from=local-mlx-c,target=/tmp/local-mlx-c,rw \
  147. if [ -f /tmp/local-mlx/CMakeLists.txt ]; then \
  148. export OLLAMA_MLX_SOURCE=/tmp/local-mlx; \
  149. fi \
  150. && if [ -f /tmp/local-mlx-c/CMakeLists.txt ]; then \
  151. export OLLAMA_MLX_C_SOURCE=/tmp/local-mlx-c; \
  152. fi \
  153. && cmake -S . \
  154. -B build/mlx_cuda_v13 \
  155. -DOLLAMA_MLX_BACKENDS=cuda_v13 \
  156. -DBLAS_INCLUDE_DIRS=/usr/include/openblas \
  157. -DLAPACK_INCLUDE_DIRS=/usr/include/openblas \
  158. -DCMAKE_CUDA_FLAGS="-t ${OLLAMA_MLX_NVCC_THREADS}" ${MLX_CUDA_RAM_MB:+-DMLX_CUDA_RAM_MB=${MLX_CUDA_RAM_MB}} \
  159. -DOLLAMA_PAYLOAD_INSTALL_PREFIX=/go/src/github.com/ollama/ollama/dist \
  160. && cmake --build build/mlx_cuda_v13 --target ollama-mlx-cuda_v13 -- --parallel 14
  161.  
  162.  
  163. FROM scratch AS publish-mlx
  164. COPY --from=mlx /go/src/github.com/ollama/ollama/dist/lib/ollama /lib/ollama/
  165.  
  166. FROM build AS llama-server-cpu
  167. COPY LLAMA_CPP_VERSION .
  168. COPY llama/server llama/server
  169. COPY llama/compat llama/compat
  170. COPY cmake cmake
  171.  
  172. # hacw a look no dile sate checked in loop and change loop to static
  173. # RUN --mount=type=cache,target=/root/.ccache \
  174. RUN --mount=type=cache,target=/root/.ccache \
  175. cmake -S llama/server --preset cpu
  176. RUN --mount=type=cache,target=/root/.ccache cmake --build build/llama-server-cpu -- -j 12 \
  177. && cmake --install build/llama-server-cpu --component llama-server --strip \
  178. && for lib in \
  179. /usr/lib64/libgomp.so* \
  180. /usr/lib64/libomp.so* \
  181. /opt/rh/gcc-toolset-13/root/usr/lib64/libgomp.so* \
  182. /opt/rh/gcc-toolset-13/root/usr/lib64/libomp.so*; do \
  183. [ -e "$lib" ] && cp -a "$lib" dist/lib/ollama/ || true; \
  184. done
  185. # CMD [ "/bin/sh" ]
  186.  
  187. # FROM build AS publish-llama
  188. # COPY --from=llama-server-cpu /go/src/github.com/ollama/ollama/dist/lib/ollama /lib/ollama/
  189.  
  190. # # # FROM cuda-12-deps AS llama-server-cuda_v12
  191. # # # COPY LLAMA_CPP_VERSION .
  192. # # # COPY llama/server llama/server
  193. # # # COPY llama/compat llama/compat
  194. # # # COPY cmake cmake
  195. # # # RUN --mount=type=cache,target=/root/.ccache \
  196. # # # cmake -S llama/server --preset llama_cuda_v12_linux \
  197. # # # && cmake --build build/llama-server-cuda_v12 -- -l $(nproc) \
  198. # # # && cmake --install build/llama-server-cuda_v12 --component llama-server --strip
  199.  
  200. # # # FROM scratch AS publish-llama-server-cuda_v12
  201. # # # COPY --from=llama-server-cuda_v12 dist/lib/ollama /lib/ollama/
  202.  
  203. FROM llama-server-cpu AS llama-server-cuda_v13
  204. COPY LLAMA_CPP_VERSION .
  205. COPY llama/server llama/server
  206. COPY llama/compat llama/compat
  207. COPY cmake cmake
  208. # RUN --mount=type=cache,target=/root/.ccache \
  209. RUN --mount=type=cache,target=/root/.ccache \
  210. cmake -S llama/server \
  211. -DCMAKE_C_COMPILER_LAUNCHER=ccache \
  212. -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
  213. --preset llama_cuda_v13_linux
  214.  
  215. RUN --mount=type=cache,target=/root/.ccache \
  216. cmake --build build/llama-server-cuda_v13 -j 12 \
  217. && cmake --install build/llama-server-cuda_v13 --component llama-server --strip
  218.  
  219. ENTRYPOINT [ "/bin/bash" ]
  220.  
  221. # FROM scratch AS publish-llama-server-cuda_v13
  222. # COPY --from=llama-server-cuda_v13 /go/src/github.com/ollama/ollama/dist/lib/ollama /lib/ollama/
  223.  
  224. # # FROM rocm-7-deps AS llama-server-rocm_v7_2
  225. # # ENV CC=clang CXX=clang++ CXXFLAGS=--gcc-toolchain=/opt/rh/gcc-toolset-13/root/usr
  226. # # COPY LLAMA_CPP_VERSION .
  227. # # COPY llama/server llama/server
  228. # # COPY llama/compat llama/compat
  229. # # COPY cmake cmake
  230. # # RUN --mount=type=cache,target=/root/.ccache \
  231. # # cmake -S llama/server --preset rocm_v7_2_linux \
  232. # # && cmake --build build/llama-server-rocm_v7_2 -- -l $(nproc) \
  233. # # && cmake --install build/llama-server-rocm_v7_2 --component llama-server --strip
  234. # # RUN rm -f dist/lib/ollama/rocm_v7_2/rocblas/library/*gfx90[06]*
  235.  
  236. # # FROM scratch AS publish-llama-server-rocm_v7_2
  237. # # COPY --from=llama-server-rocm_v7_2 dist/lib/ollama /lib/ollama/
  238.  
  239. # # FROM vulkan-deps AS llama-server-vulkan
  240. # # COPY LLAMA_CPP_VERSION .
  241. # # COPY llama/server llama/server
  242. # # COPY llama/compat llama/compat
  243. # # COPY cmake cmake
  244. # # RUN --mount=type=cache,target=/root/.ccache \
  245. # # cmake -S llama/server --preset vulkan \
  246. # # && cmake --build build/llama-server-vulkan -- -l $(nproc) \
  247. # # && cmake --install build/llama-server-vulkan --component llama-server --strip
  248.  
  249. # # FROM scratch AS publish-llama-server-vulkan
  250. # # COPY --from=llama-server-vulkan dist/lib/ollama /lib/ollama/
  251.  
  252. # # #
  253. # # # JetPack stages — self-contained with their own base images
  254. # # #
  255.  
  256. # # FROM --platform=linux/arm64 nvcr.io/nvidia/l4t-jetpack:${JETPACK5VERSION} AS jetpack-5
  257. # # ARG CMAKEVERSION
  258. # # ARG NINJAVERSION
  259. # # RUN apt-get update && apt-get install -y curl ccache git unzip \
  260. # # && curl --retry 3 -fsSL https://github.com/Kitware/CMake/releases/download/v${CMAKEVERSION}/cmake-${CMAKEVERSION}-linux-$(uname -m).tar.gz -o /tmp/cmake.tar.gz \
  261. # # && tar xzf /tmp/cmake.tar.gz -C /usr/local --strip-components 1 \
  262. # # && rm /tmp/cmake.tar.gz \
  263. # # && curl --retry 3 -fsSL -o /tmp/ninja.zip https://github.com/ninja-build/ninja/releases/download/v${NINJAVERSION}/ninja-linux-aarch64.zip \
  264. # # && unzip /tmp/ninja.zip -d /usr/local/bin \
  265. # # && rm /tmp/ninja.zip
  266. # # ENV CMAKE_GENERATOR=Ninja
  267. # # COPY LLAMA_CPP_VERSION .
  268. # # COPY llama/server llama/server
  269. # # COPY llama/compat llama/compat
  270. # # COPY cmake cmake
  271. # # RUN --mount=type=cache,target=/root/.ccache \
  272. # # cmake -S llama/server --preset llama_cuda_jetpack5 \
  273. # # && cmake --build build/llama-server-cuda_jetpack5 -- -l $(nproc) \
  274. # # && cmake --install build/llama-server-cuda_jetpack5 --component llama-server --strip
  275.  
  276. # # FROM scratch AS publish-llama-server-cuda_jetpack5
  277. # # COPY --from=jetpack-5 dist/lib/ollama /lib/ollama/
  278.  
  279. # # FROM --platform=linux/arm64 nvcr.io/nvidia/l4t-jetpack:${JETPACK6VERSION} AS jetpack-6
  280. # # ARG CMAKEVERSION
  281. # # ARG NINJAVERSION
  282. # # RUN apt-get update && apt-get install -y curl ccache git unzip \
  283. # # && curl --retry 3 -fsSL https://github.com/Kitware/CMake/releases/download/v${CMAKEVERSION}/cmake-${CMAKEVERSION}-linux-$(uname -m).tar.gz -o /tmp/cmake.tar.gz \
  284. # # && tar xzf /tmp/cmake.tar.gz -C /usr/local --strip-components 1 \
  285. # # && rm /tmp/cmake.tar.gz \
  286. # # && curl --retry 3 -fsSL -o /tmp/ninja.zip https://github.com/ninja-build/ninja/releases/download/v${NINJAVERSION}/ninja-linux-aarch64.zip \
  287. # # && unzip /tmp/ninja.zip -d /usr/local/bin \
  288. # # && rm /tmp/ninja.zip
  289. # # ENV CMAKE_GENERATOR=Ninja
  290. # # COPY LLAMA_CPP_VERSION .
  291. # # COPY llama/server llama/server
  292. # # COPY llama/compat llama/compat
  293. # # COPY cmake cmake
  294. # # RUN --mount=type=cache,target=/root/.ccache \
  295. # # cmake -S llama/server --preset llama_cuda_jetpack6 \
  296. # # && cmake --build build/llama-server-cuda_jetpack6 -- -l $(nproc) \
  297. # # && cmake --install build/llama-server-cuda_jetpack6 --component llama-server --strip
  298.  
  299. # # FROM scratch AS publish-llama-server-cuda_jetpack6
  300. # # COPY --from=jetpack-6 dist/lib/ollama /lib/ollama/
  301.  
  302.  
  303. # FROM build AS ollama-build
  304. # ARG GOFLAGS="'-ldflags=-w -s'"
  305. # ENV CGO_ENABLED=1
  306. # ARG CGO_CFLAGS
  307. # ARG CGO_CXXFLAGS
  308. # ENV CGO_CFLAGS="${CGO_CFLAGS}"
  309. # ENV CGO_CXXFLAGS="${CGO_CXXFLAGS}"
  310.  
  311. # COPY . .
  312. # COPY --from=llama-server-cuda_v13 /go/src/github.com/ollama/ollama/dist/lib/ollama /lib/ollama/
  313.  
  314. # # RUN --mount=type=cache,target=/root/.cache/go-build \
  315. # RUN --mount=type=cache,target=/root/.cache/go-build \
  316. # go build -p 1 -trimpath -buildmode=pie -o /bin/ollama .
  317. # RUN cmake -S . -B build/go-license \
  318. # -DOLLAMA_LLAMA_BACKENDS="cuda_v13" \
  319. # -DOLLAMA_MLX_BACKENDS="cuda_v13"
  320. # RUN cmake --build build/go-license --target ollama-go-license
  321.  
  322. # # FROM scratch AS publish-go
  323. # # COPY --from=ollama-build /bin/ollama /bin/ollama
  324. # # COPY --from=ollama-build /go/src/github.com/ollama/ollama/build/go-license/lib/ollama/GO_LICENSE /lib/ollama/GO_LICENSE
  325. # #
  326. # # Assembly stages — combine llama-server variants + GPU runtime libs
  327. # #
  328.  
  329. # # CMD [ "/bin/sh" ]
  330. # # FROM scratch AS amd64
  331. # # COPY --from=llama-server-cpu dist/lib/ollama /lib/ollama/
  332. # # COPY --from=llama-server-cuda_v12 dist/lib/ollama /lib/ollama/
  333. # # COPY --from=llama-server-cuda_v13 dist/lib/ollama /lib/ollama/
  334. # # COPY --from=llama-server-vulkan dist/lib/ollama /lib/ollama/
  335.  
  336. # # FROM scratch AS rocm
  337. # # COPY --from=llama-server-cpu dist/lib/ollama /lib/ollama
  338. # # COPY --from=llama-server-rocm_v7_2 dist/lib/ollama /lib/ollama
  339.  
  340. # # FROM --platform=linux/amd64 scratch AS amd64-archive
  341. # # COPY --from=amd64 /lib/ollama /lib/ollama/
  342. # # COPY --from=llama-server-rocm_v7_2 dist/lib/ollama /lib/ollama/
  343.  
  344. # # FROM --platform=linux/arm64 scratch AS arm64-archive
  345. # # COPY --from=arm64 /lib/ollama /lib/ollama/
  346.  
  347. # # FROM ${TARGETARCH}-archive AS archive
  348. # # COPY --from=build /bin/ollama /bin/ollama
  349. # # COPY --from=build /go/src/github.com/ollama/ollama/build/go-license/lib/ollama/GO_LICENSE /lib/ollama/GO_LICENSE
  350.  
  351. # FROM ubuntu:24.04 AS image-archive
  352. # COPY --from=ollama-build /bin/ollama /bin/ollama
  353. # COPY --from=ollama-build /go/src/github.com/ollama/ollama/build/go-license/lib/ollama/GO_LICENSE /lib/ollama/GO_LICENSE
  354.  
  355.  
  356. # FROM image-archive
  357. # # # SHELL [ "/bin/bash" ]
  358.  
  359. # WORKDIR /usr/bin
  360. # # COPY --from=image-archive "/bin/ollama" ./
  361.  
  362. # ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
  363. # ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64
  364.  
  365. # ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
  366. # ENV NVIDIA_VISIBLE_DEVICES=all
  367. # ENV OLLAMA_HOST=0.0.0.0:11434
  368. # ENV OLLAMA_DEBUG=1
  369. # ENV OLLAMA_KEEP_ALIVE=120m
  370. # ENV OLLAMA_KV_CACHE_TYPE=q8_0
  371. # ENV OLLAMA_CONTEXT_LENGTH=8192
  372. # #OLLAMA_FLASH_ATTENTION=1
  373. # ENV OLLAMA_MAX_LOADED_MODELS=1
  374. # ENV OLLAMA_NUM_PARALLEL=1
  375. # # for img generation
  376. # ENV OLLAMA_GPU_OVERHEAD=1073741824
  377. # #shared memory between vram and ram 1=on (enable when OOM)
  378. # ENV GGML_CUDA_ENABLE_UNIFIED_MEMORY=1
  379. # #GGML_CUDA_ENABLE_UNIFIED_MEMORY=0
  380.  
  381.  
  382.  
  383. # EXPOSE 11434
  384. # # # ENTRYPOINT ["/bin/ollama"]
  385. # # # # CMD ["serve"]
  386. # # ENTRYPOINT [ "/bin/sh" ]
  387. # CMD [ "/bin/sh" ]
  388.  
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!