peperunas/nixos
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

architect: use docker ollama

Browse Source
This commit is contained in:
Giulio De Pasquale 2024-02-21 11:33:54 +00:00
parent 31a41642bb
commit 9b21c7d2ef
1 changed files with 55 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
hosts/architect/llm.nix
View File

@ -2,13 +2,13 @@
let
domain = "pino.giugl.io";
backendPort = 3000;
frontendPort = 3002;
llama-cpp = pkgs.unstablePkgs.llama-cpp.override { cudaSupport = true; };
ollama = pkgs.unstablePkgs.ollama.override { inherit llama-cpp; };
backendPort = 8080;
frontendPort = 3030;
# llama-cpp = pkgs.unstablePkgs.llama-cpp.override { cudaSupport = true; };
# ollama = pkgs.unstablePkgs.ollama.override { inherit llama-cpp; };
in
{
environment.systemPackages = [ ollama ];
# environment.systemPackages = [ ollama ];
architect.vhost.${domain} = {
dnsInterfaces = [ "tailscale" ];
@ -16,25 +16,69 @@ in
host = "172.17.0.1";
port = frontendPort;
allowLan = true;
allow = [ config.architect.networks."tailscale".net ];
allowWAN = true;
# allow = [ config.architect.networks."tailscale".net ];
extraConfig = ''
proxy_read_timeout 600s;
'';
};
};
architect.vhost."ollama.giugl.io" = {
dnsInterfaces = [ "tailscale" ];
locations."/" = {
host = "172.17.0.1";
port = 11434;
allowLan = true;
allowWAN = true;
# allow = [ config.architect.networks."tailscale".net ];
extraConfig = ''
proxy_read_timeout 600s;
'';
};
};
virtualisation.oci-containers = {
containers = {
big-agi = {
image = "ghcr.io/enricoros/big-agi:latest";
ollama-webui = {
image = "ghcr.io/open-webui/open-webui:main";
autoStart = true;
ports = [
"172.17.0.1:${toString frontendPort}:${toString backendPort}"
];
environmentFiles = [
"/var/lib/llm/big-agi.env"
];
environment = {
PORT = "${toString backendPort}";
OLLAMA_API_BASE_URL = "http://172.17.0.1:11434/api";
};
extraOptions = [
"--pull=always"
];
volumes = [
"/var/lib/ollama-webui:/app/backend/data"
];
};
ollama = {
image = "ollama/ollama:latest";
autoStart = true;
extraOptions = [
"--pull=always"
"--gpus=all"
];
environment = {
OLLAMA_ORIGINS = "*";
};
volumes = [
"/ollama:/root/.ollama"
];
ports = [
"127.0.0.1:11434:11434"
"172.17.0.1:11434:11434"
];
};
};
};