peperunas/nixos
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

ollama: use native ollama

Browse Source
This commit is contained in:
Giulio De Pasquale 2024-03-14 11:55:56 +00:00
parent b785fe04aa
commit 32dc2af002
1 changed files with 26 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
hosts/architect/llm.nix
View File

@ -1,38 +1,51 @@
{ config, pkgs, ... }: { config, pkgs, ... }:
let let
domain = "pino.giugl.io"; frontendDomain = "pino.giugl.io";
backendPort = 8080; backendDomain = "ollama.giugl.io";
frontendPort = 3030; frontendPort = 3030;
# llama-cpp = pkgs.unstablePkgs.llama-cpp.override { cudaSupport = true; }; ollamaPort = 11434;
# ollama = pkgs.unstablePkgs.ollama.override { inherit llama-cpp; }; listenAddress = "127.0.0.1:${toString ollamaPort}";
ollamaPkg = pkgs.unstablePkgs.ollama;
in in
{ {
# environment.systemPackages = [ ollama ]; environment = {
architect.vhost.${domain} = { systemPackages = [ ollamaPkg ];
variables = {
OLLAMA_ORIGINS = "*";
};
};
services.ollama = {
inherit listenAddress;
enable = true;
acceleration = "cuda";
package = ollamaPkg;
};
architect.vhost.${frontendDomain} = {
dnsInterfaces = [ "tailscale" ]; dnsInterfaces = [ "tailscale" ];
locations."/" = { locations."/" = {
host = "172.17.0.1"; host = "127.0.0.1";
port = frontendPort; port = frontendPort;
allowLan = true; allowLan = true;
allowWAN = true; allowWAN = true;
# allow = [ config.architect.networks."tailscale".net ];
extraConfig = '' extraConfig = ''
proxy_read_timeout 600s; proxy_read_timeout 600s;
''; '';
}; };
}; };
architect.vhost."ollama.giugl.io" = { architect.vhost.${backendDomain} = {
dnsInterfaces = [ "tailscale" ]; dnsInterfaces = [ "tailscale" ];
locations."/" = { locations."/" = {
host = "172.17.0.1"; host = "127.0.0.1";
port = 11434; port = 11434;
allowLan = true; allowLan = true;
allowWAN = true; allowWAN = true;
# allow = [ config.architect.networks."tailscale".net ];
extraConfig = '' extraConfig = ''
proxy_read_timeout 600s; proxy_read_timeout 600s;
''; '';
@ -46,12 +59,11 @@ in
autoStart = true; autoStart = true;
ports = [ ports = [
"172.17.0.1:${toString frontendPort}:${toString backendPort}" "127.0.0.1:${toString frontendPort}:8080"
]; ];
environment = { environment = {
PORT = "${toString backendPort}"; OLLAMA_API_BASE_URL = "https://${backendDomain}/api";
OLLAMA_API_BASE_URL = "http://172.17.0.1:11434/api";
}; };
extraOptions = [ extraOptions = [
@ -61,25 +73,6 @@ in
"/var/lib/ollama-webui:/app/backend/data" "/var/lib/ollama-webui:/app/backend/data"
]; ];
}; };
ollama = {
image = "ollama/ollama:latest";
autoStart = true;
extraOptions = [
"--pull=always"
"--gpus=all"
];
environment = {
OLLAMA_ORIGINS = "*";
};
volumes = [
"/ollama:/root/.ollama"
];
ports = [
"127.0.0.1:11434:11434"
"172.17.0.1:11434:11434"
];
};
}; };
}; };
} }