nixos/hosts/architect/llm.nix

{ pkgs, ... }:

let
  backendDomain = "ollama.giugl.io";
  ollamaHost = "127.0.0.1";
  ollamaPort = 11434;
  ollamaPkg = pkgs.unstablePkgs.ollama-cuda;
in
{
  environment = {
    systemPackages = [ ollamaPkg ];
  };

  services.ollama = {
    enable = true;

    package = ollamaPkg;
    host = ollamaHost;
    port = ollamaPort;
    acceleration = "cuda";
    environmentVariables = {
      OLLAMA_FLASH_ATTENTION = "1";
      OLLAMA_NUM_PARALLEL = "2";
      OLLAMA_KV_CACHE_TYPE = "q8_0";
    };
  };

  architect.vhost.${backendDomain} = {
    dnsInterfaces = [ "tailscale" "lan" ];

    locations."/" = {
      host = ollamaHost;
      port = ollamaPort;
      allowLan = true;
      allowWAN = true;
      recommendedProxySettings = false;
      extraConfig = ''
        proxy_buffering off;
        proxy_read_timeout 600s;
        proxy_set_header Host localhost:${toString ollamaPort};
      '';
    };
  };
}