ExLlamaV3 config

ENV VARS:

      - NVIDIA_VISIBLE_DEVICES=all
      - EXL3_MOE_PINNED_ARENA=0
      - EXL3_HOST_MEM_RESERVE_MB=0
      - EXL3_AUTOSPLIT_MARGIN_MB=0
      - EXL3_AUTOSPLIT_WORSTCASE=1
      #- EXL3_MOE_CPU_WSLOT_MB=384
      #- EXL3_MOE_CPU_STAGE_THREADS=8
      # Optional exllamav3 tuning knobs (see moe-inference skill):
      # - EXL3_MOE_CPU_SLOTS=8
      # - EXL3_MOE_CPU_THREADS=6
      # - EXL3_MOE_STREAM_T=16
      # - EXL3_MOE_ARENA_HUGEPAGE=1



TABBYAPI config.yaml:

# Sample YAML file for configuration.
# Comment and uncomment values as needed.
# Every value has a default within the application.
# This file serves to be a drop in for config.yml

# Unless specified in the comments, DO NOT put these options in quotes!
# You can use https://www.yamllint.com/ if you want to check your YAML formatting.

# Options for networking
network:
  # The IP to host on (default: 127.0.0.1).
  # Use 0.0.0.0 to expose on all network adapters.
  host: 0.0.0.0

  # The port to host on (default: 5000).
  port: 5000

  # Disable HTTP token authentication with requests.
  # WARNING: This will make your instance vulnerable!
  # Only turn this on if nothing but trusted local clients can reach the API.
  # Note that web pages open in a browser on this machine also count as local
  # callers; restrict allowed_origins below if you disable auth.
  disable_auth: true

  # Origins allowed to call the API from a browser (default: ["*"]).
  # This is a CORS allowlist, not authentication: "*" lets any site open in your
  # browser send requests to this instance, which matters most when disable_auth
  # is on. Restrict to your own frontends (e.g. ["http://localhost:8000"]) or use
  # an empty list [] to block all browser (cross-origin) callers.
  allowed_origins: ["*"]

  # Disable fetching external content in response to requests,such as images from URLs.
  disable_fetch_requests: false

  # Send tracebacks over the API (default: False).
  # NOTE: Only enable this for debug purposes.
  send_tracebacks: false

  # Select API servers to enable (default: ["OAI"]).
  # Possible values: OAI, Kobold.
  api_servers: ["OAI"]

  # Seconds between SSE keep-alive pings on streaming responses (default: 15).
  # Pings are SSE comments, ignored by compliant clients, and prevent
  # connections from dropping during long prefills. Set to 0 to disable.
  sse_ping_interval: 15

  # Log every HTTP request with client address, method, path and status (default: False).
  # Generation requests are already logged in detail; this adds the rest, such as model list and health polls.
  access_log: false

# Options for logging
logging:
  # Enable prompt logging (default: False).
  log_prompt: false

  # Enable generation parameter logging (default: False).
  log_generation_params: false

  # Enable request logging (default: False).
  # NOTE: Only use this for debugging!
  log_requests: false

  # Show a live status line below the log with cache usage and in-flight jobs (default: True).
  # Only shown on an interactive terminal.
  log_live_status: true

  # Prefix console log lines with the time of day (default: True).
  # The log files under logs/ always carry full timestamps.
  log_timestamps: true

  # Write every /v1/chat/completions request to logs/debug/ as JSON (default: False).
  # Also saves the fully templated prompt (the exact text sent to the tokenizer)
  # as a .txt file with the same basename.
  # PRIVACY WARNING: Enabling this creates a comprehensive request log, including the
  # full message history and generation parameters. API keys are redacted, but prompts
  # and user-provided content are preserved for bug-report reproduction.
  log_chat_completion_requests: false

# Options for model overrides and loading
# Please read the comments to understand how arguments are handled
# between initial and API loads
model:
  # Directory to look for models (default: models).
  # Windows users, do NOT put this path in quotes!
  model_dir: models

  # Allow direct loading of models from a completion or chat completion request (default: False).
  # This method of loading is strict by default.
  # Enable dummy models to add exceptions for invalid model names.
  inline_model_loading: false

  # Sends dummy model names when the models endpoint is queried. (default: False)
  # Enable this if the client is looking for specific OAI models.
  use_dummy_models: false

  # A list of fake model names that are sent via the /v1/models endpoint. (default: ["gpt-3.5-turbo"])
  # Also used as bypasses for strict mode if inline_model_loading is true.
  dummy_model_names: ["gpt-3.5-turbo"]

  # An initial model to load.
  # Make sure the model is located in the model directory!
  # REQUIRED: This must be filled out to load a model on startup.
  model_name: GLM-5.3-Flash-Turboderp-EXL3-4bpw

  # Names of args to use as a fallback for API load requests (default: []).
  # For example, if you always want cache_mode to be Q4 instead of on the inital model load, add "cache_mode" to this array.
  # Example: ['max_seq_len', 'cache_mode'].
  use_as_default: []

  # Backend to use for this model (auto-detect if not specified)
  # Options: exllamav3
  #backend:

  # Max sequence length (default: min(max_position_embeddings, cache_size)).
  # Set to -1 to fetch from the model's config.json
  max_seq_len: -1

  # Size of the key/value cache to allocate, in tokens (default: 4096).
  # Must be a multiple of 256.
  cache_size: 262144

  # Enable different cache modes for VRAM savings (default: FP16).
  # Specify the pair k_bits,v_bits where k_bits and v_bits are integers from 2-8 (i.e. 8,8).
  # The legacy values 'FP16', 'Q8', 'Q6', 'Q4' are also accepted.
  cache_mode: FP16

  # Load model with tensor parallelism.
  # Falls back to autosplit if GPU split isn't provided.
  # This ignores the gpu_split_auto value.
  tensor_parallel: false

  # Sets a backend type for tensor parallelism. (default: native).
  # Options: native, nccl
  # Native is recommended for PCIe GPUs
  # NCCL is recommended for NVLink.
  tensor_parallel_backend: nccl

  # Automatically allocate resources to GPUs (default: True).
  # Not parsed for single GPU users.
  gpu_split_auto: true

  # Reserve VRAM used for autosplit loading (default: 96 MB on GPU 0).
  # Represented as an array of MB per GPU.
  autosplit_reserve: [96, 96, 96]

  # Array of VRAM sizes to split between GPUs, in GB (default: []).
  # Used both with and without tensor parallelism.
  #gpu_split: [24576, 12288, 12288]

  # Number of mixture-of-expert layers to offload to CPU inference (default: 0)
  # Only affects MoE models. Set a large value such as 999 to offload all layers
  # Mutually exclusive with cpu_moe_split_experts.
  #cpu_moe_offload_layers: 34

  # Number of routed experts per MoE layer to offload to CPU inference (default: 0).
  # Unlike cpu_moe_offload_layers, this splits every MoE layer instead of offloading whole
  # layers: the coldest experts are kept in system RAM and computed on the CPU, overlapping
  # each layer's own GPU compute, with dynamic placement keeping hot experts in VRAM.
  # Mutually exclusive with cpu_moe_offload_layers; not supported with tensor parallelism.
  # GLM 5.3 flash has 288 experts per layer / 232 with env vars
  cpu_moe_split_experts: 234

  # Worker thread count for CPU MoE inference (default: None).
  # Applies to both cpu_moe_offload_layers and cpu_moe_split_experts. When unset,
  # defers to the EXL3_MOE_CPU_THREADS environment variable, then half the CPU core count.
  cpu_moe_threads: 12

  # Load a model's n-gram embedding table fully into system RAM (default: False).
  # Only affects PLE models with n-gram embeddings (e.g. Qwen3.8-Flash-Next). By default
  # the table is streamed from disk during inference; loading it into RAM avoids
  # per-token disk reads at the cost of tens of GB of system memory.
  ngram_ram: false

  # NOTE: If a model has YaRN rope scaling, it will automatically be enabled by ExLlama.
  # rope_scale and rope_alpha settings won't apply in this case.

  # Rope scale (default: 1.0).
  # Same as compress_pos_emb.
  # Use if the model was trained on long context with rope.
  # Leave blank to pull the value from the model.
  #rope_scale: 1.0

  # Rope alpha (default: None).
  # Same as alpha_value. Set to "auto" to auto-calculate.
  # Leaving this value blank will either pull from the model or auto-calculate.
  #rope_alpha:

  # Chunk size for prompt ingestion (default: 2048).
  # A lower value reduces VRAM usage but decreases ingestion speed.
  # NOTE: Effects vary depending on the model.
  # An ideal value is between 512 and 4096.
  chunk_size: 4096

  # Use output chunking (default: True)
  # Instead of allocating cache space for the entire completion at once, allocate in chunks as needed.
  # Used by EXL3 models only.
  output_chunking: true

  # Set the maximum number of generation jobs that can run concurrently
  # The default maximum batch size for transformer architectures is 32. Recurrent
  # models with linear or sliding attention use more VRAM to support larger batches,
  # so the default value is reduced to 4. If you do not require concurrency at all, you
  # can reduce it further to minimize VRAM overhead.
  max_batch_size: 1

  # Set the prompt template for this model. (default: None)
  # If empty, attempts to look for the model's chat template.
  # If a model contains multiple templates in its tokenizer_config.json,
  # set prompt_template to the name of the template you want to use.
  # NOTE: Only works with chat completion message lists!
  #prompt_template:

  # Enables vision support if the model supports it. (default: False)
  vision: true

  # Keep the vision model's weights in system RAM instead of VRAM (default: False).
  # Weights are stored in pinned host memory and streamed to the GPU during inference,
  # trading vision speed for VRAM. Only applies when vision is enabled.
  vision_offload: true

  # Default chat template variables (default: {}).
  # Merged into the template variables of every chat completion request; values
  # sent by the client (template_vars / chat_template_kwargs, or the top-level
  # reasoning_effort field) take precedence. Use for model-specific reasoning
  # knobs, e.g. {enable_thinking: true} or {reasoning_effort: high}.
  template_vars_default: {}

  # Forced chat template variables (default: {}).
  # Like template_vars_default, but these override any values sent by the client.
  # Replaces the deprecated force_enable_thinking option, which is still accepted
  # as an alias for {enable_thinking: true}.
  template_vars_force: {}

  # Enable reasoning parser (default: False).
  # Do NOT enable this if the model is not a reasoning model (e.g. deepseek-r1 series)
  reasoning: true

  # The start token for reasoning content (default: "<think>")
  #reasoning_start_token: "<think>"

  # The end token for reasoning content (default: "</think>")
  #reasoning_end_token: "</think>"

  # Whether generation starts inside a reasoning block (default: auto).
  # Options: auto, always, never
  # auto guesses by scanning the end of the templated prompt for an unclosed reasoning start token.
  start_in_reasoning: auto

  # Parse tool calls that occur inside reasoning content (default: True).
  # If False, tool call tags inside a reasoning block are treated as plain reasoning text.
  tool_calls_in_reasoning: true

  # Default reasoning token budget (default: None).
  # When a request's reasoning content exceeds the budget, the server forces the end of
  # the reasoning phase by injecting reasoning_budget_message followed by the model's
  # end-of-reasoning tokens. 0 ends reasoning as soon as it starts; None or a negative
  # value disables the budget. Overridable per request via reasoning_budget_tokens
  # (aliases: reasoning_budget, thinking_budget, thinking_token_budget) or
  # reasoning.max_tokens. Requires a reasoning format: reasoning tags, Harmony or Muse Glimmer.
  reasoning_budget_tokens:

  # Text injected before the end-of-reasoning tokens when the reasoning budget is
  # exhausted (default: no text, only the end-of-reasoning tokens are forced). Also overridable
  # per request via reasoning_budget_message.
  reasoning_budget_message:

  # Tool format, e.g. 'qwen3_coder'. See docs for supported formats. If left blank,
  # tool calls from the model will not be parsed by the server.
  tool_format: glm4_5

  # Parse responses in the Harmony message format (gpt-oss models).
  # Auto-detected from the model's special tokens by default; set to true or false
  # to override. Setting 'tool_format: harmony' is equivalent to setting this to true.
  # When active, supersedes the reasoning and tool format settings.
  harmony: false

  # Parse responses in the Muse Glimmer message format.
  # Auto-detected from the model's special tokens by default; set to true or false
  # to override. Setting 'tool_format: muse_glimmer' is equivalent to setting this to true.
  # When active, supersedes the reasoning and tool format settings.
  muse_glimmer: false

# Options for draft models (speculative decoding)
# This will use more VRAM!
draft_model:
  # Drafting mode for exllamav3 (default: model).
  # Options: model, disabled, mtp, ngram.
  # In `model` mode, drafting is disabled if no draft_model_name is provided.
  draft_mode: disabled

  # Directory to look for draft models (default: models)
  draft_model_dir: models

  # An initial draft model to load.
  # Ensure the model is in the model directory.
  #draft_model_name:

  # Rope scale for draft models (default: 1.0).
  # Same as compress_pos_emb.
  # Use if the draft model was trained on long context with rope.
  #draft_rope_scale: 1.0

  # Rope alpha for draft models (default: None).
  # Same as alpha_value. Set to "auto" to auto-calculate.
  # Leaving this value blank will either pull from the model or auto-calculate.
  #draft_rope_alpha:

  # Cache mode for draft models to save VRAM (default: FP16).
  # Specify the pair k_bits,v_bits where k_bits and v_bits are integers from 2-8 (i.e. 8,8).
  # The legacy values 'FP16', 'Q8', 'Q6', 'Q4' are also accepted.
  draft_cache_mode: Q4

  # Array of VRAM sizes to split between GPUs, in GB (default: []).
  # If this isn't filled in, the draft model is autosplit.
  #draft_gpu_split: []

  # Number of tokens to draft per iteration (default: draft model default)
  # Recurrent (linear or sliding attention) models use more VRAM for longer drafts.
  # This overhead multiplies with the max batch size, so for models with long drafts
  # (e.g. DFlash with 15 tokens by default) shorter drafts may be preferable.
  draft_num_tokens: 1

  # Adjust number of draft tokens dynamically based on observed acceptance rates (default: False)
  # Ceiling is given by num_draft_tokens.
  #dynamic_draft:

  # Minimum match length for exllamav3 n-gram drafting (default: 2).
  # Only used when draft_mode is ngram.
  #ngram_match_min: 2

# Options for Sampling
sampling:
  # Select a sampler override preset (default: None).
  # Find this in the sampler-overrides folder.
  # This overrides default fallbacks for sampler values that are passed to the API.
  # NOTE: safe_defaults is noob friendly and provides fallbacks for frontends that don't send sampling parameters.
  # for frontends that don't send sampling parameters. Leaving this blank means no fallbacks at all.
  override_preset: safe_defaults

# Options for Loras
lora:
  # Directory to look for LoRAs (default: loras).
  lora_dir: loras

  # List of LoRAs to load and associated scaling factors (default scale: 1.0).
  # For the YAML file, add each entry as a YAML list:
  # - name: lora1
  #   scaling: 1.0
  loras:

# Options for embedding models and loading.
# NOTE: Embeddings requires the "extras" feature to be installed
# Install it via "pip install .[extras]"
embeddings:
  # Directory to look for embedding models (default: models).
  embedding_model_dir: models

  # Device to load embedding models on (default: cpu).
  # Possible values: cpu, auto, cuda.
  # NOTE: It's recommended to load embedding models on the CPU.
  # If using an AMD GPU, set this value to 'cuda'.
  embeddings_device: cpu

  # An initial embedding model to load on the infinity backend.
  embedding_model_name:

# Global memory settings
memory:

  # Max size of recurrent cache in system memory, in MB (default: 4096)
  sysmem_recurrent_cache: 4096

  # Size of system memory second-tier K/V cache, in MB (default: 0)
  sysmem_kv_cache: 0

  # Size of the image embedding cache in system memory, in MB (default: 1024).
  # Encoded images are kept so repeated turns of a conversation don't re-run the vision model.
  # Images already in use by a request are never evicted; a context whose images exceed the
  # budget is cached only partially, with a warning. Only applies when vision is enabled.
  sysmem_multimodal_cache: 1024

  # Use the cudaMallocAsync allocator backend in Torch (default: False).
  # When False, the allocator is left to the environment: unless PYTORCH_CUDA_ALLOC_CONF is set,
  # ExLlamaV3 enables expandable segments in Torch's native allocator, which performs better
  # than cudaMallocAsync. Enable this to force the cudaMallocAsync backend instead.
  cuda_malloc_async: false

# Options for development and experimentation
developer:
  # Skip Exllamav3 version check (default: False).
  # WARNING: It's highly recommended to update your dependencies rather than enabling this flag.
  unsafe_launch: false

  # Disable API request streaming (default: False).
  disable_request_streaming: false

  # Set process to use a higher priority.
  # For realtime process priority, run as administrator or sudo.
  # Otherwise, the priority will be set to high.
  realtime_process_priority: false

  # Enable extremely verbose seqlog logging, requires a running Seq server
  seqlog: false

  # Seq server url:port
  seqlog_server_url: http://localhost:5341

  # Seq server API key (default: None)
  seqlog_api_key: