diff --git a/hosts/architect/llm.nix b/hosts/architect/llm.nix index dd7f608..fe8e03e 100644 --- a/hosts/architect/llm.nix +++ b/hosts/architect/llm.nix @@ -11,18 +11,18 @@ in { environment = { systemPackages = [ ollamaPkg pkgs.aichat ]; - variables = { - OLLAMA_ORIGINS = "*"; - }; + # variables = { + # OLLAMA_ORIGINS = "*"; + # }; }; - services.ollama = { - inherit listenAddress; + # services.ollama = { + # inherit listenAddress; - enable = true; - acceleration = "cuda"; - package = ollamaPkg; - }; + # enable = true; + # acceleration = "cuda"; + # package = ollamaPkg; + # }; architect.vhost.${frontendDomain} = { dnsInterfaces = [ "tailscale" ]; @@ -49,13 +49,32 @@ in extraConfig = '' proxy_buffering off; proxy_read_timeout 600s; - proxy_set_header Host localhost:11434; + proxy_set_header Host localhost:${toString ollamaPort}; ''; }; }; virtualisation.oci-containers = { containers = { + ollama = { + image = "ollama/ollama:latest"; + autoStart = true; + extraOptions = [ + "--pull=always" + "--gpus=all" + ]; + environment = { + OLLAMA_ORIGINS = "*"; + }; + volumes = [ + "/ollama:/root/.ollama" + ]; + ports = [ + "${listenAddress}:${toString ollamaPort}" + "172.17.0.1:${toString ollamaPort}:${toString ollamaPort}" + ]; + }; + ollama-webui = { image = "ghcr.io/open-webui/open-webui:main"; autoStart = true;