peperunas/nixos
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

ollama: use native ollama

Browse Source
This commit is contained in:
Giulio De Pasquale 2024-03-14 11:55:56 +00:00
parent b785fe04aa
commit 32dc2af002
1 changed files with 26 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
hosts/architect/llm.nix
View File

@ -1,38 +1,51 @@
{ config, pkgs, ... }:
let
domain = "pino.giugl.io";
backendPort = 8080;
frontendDomain = "pino.giugl.io";
backendDomain = "ollama.giugl.io";
frontendPort = 3030;
# llama-cpp = pkgs.unstablePkgs.llama-cpp.override { cudaSupport = true; };
# ollama = pkgs.unstablePkgs.ollama.override { inherit llama-cpp; };
ollamaPort = 11434;
listenAddress = "127.0.0.1:${toString ollamaPort}";
ollamaPkg = pkgs.unstablePkgs.ollama;
in
{
# environment.systemPackages = [ ollama ];
architect.vhost.${domain} = {
environment = {
systemPackages = [ ollamaPkg ];
variables = {
OLLAMA_ORIGINS = "*";
};
};
services.ollama = {
inherit listenAddress;
enable = true;
acceleration = "cuda";
package = ollamaPkg;
};
architect.vhost.${frontendDomain} = {
dnsInterfaces = [ "tailscale" ];
locations."/" = {
host = "172.17.0.1";
host = "127.0.0.1";
port = frontendPort;
allowLan = true;
allowWAN = true;
# allow = [ config.architect.networks."tailscale".net ];
extraConfig = ''
proxy_read_timeout 600s;
'';
};
};
architect.vhost."ollama.giugl.io" = {
architect.vhost.${backendDomain} = {
dnsInterfaces = [ "tailscale" ];
locations."/" = {
host = "172.17.0.1";
host = "127.0.0.1";
port = 11434;
allowLan = true;
allowWAN = true;
# allow = [ config.architect.networks."tailscale".net ];
extraConfig = ''
proxy_read_timeout 600s;
'';
@ -46,12 +59,11 @@ in
autoStart = true;
ports = [
"172.17.0.1:${toString frontendPort}:${toString backendPort}"
"127.0.0.1:${toString frontendPort}:8080"
];
environment = {
PORT = "${toString backendPort}";
OLLAMA_API_BASE_URL = "http://172.17.0.1:11434/api";
OLLAMA_API_BASE_URL = "https://${backendDomain}/api";
};
extraOptions = [
@ -61,25 +73,6 @@ in
"/var/lib/ollama-webui:/app/backend/data"
];
};
ollama = {
image = "ollama/ollama:latest";
autoStart = true;
extraOptions = [
"--pull=always"
"--gpus=all"
];
environment = {
OLLAMA_ORIGINS = "*";
};
volumes = [
"/ollama:/root/.ollama"
];
ports = [
"127.0.0.1:11434:11434"
"172.17.0.1:11434:11434"
];
};
};
};
}