From 9b21c7d2efbdef6645d62a95b620643506b2f3ec Mon Sep 17 00:00:00 2001
From: Giulio De Pasquale <git@depasquale.giugl.io>
Date: Wed, 21 Feb 2024 11:33:54 +0000
Subject: [PATCH] architect: use docker ollama

---
 hosts/architect/llm.nix | 66 ++++++++++++++++++++++++++++++++++-------
 1 file changed, 55 insertions(+), 11 deletions(-)

diff --git a/hosts/architect/llm.nix b/hosts/architect/llm.nix
index 73db576..be2f582 100644
--- a/hosts/architect/llm.nix
+++ b/hosts/architect/llm.nix
@@ -2,13 +2,13 @@
 
 let
   domain = "pino.giugl.io";
-  backendPort = 3000;
-  frontendPort = 3002;
-  llama-cpp = pkgs.unstablePkgs.llama-cpp.override { cudaSupport = true; };
-  ollama = pkgs.unstablePkgs.ollama.override { inherit llama-cpp; };
+  backendPort = 8080;
+  frontendPort = 3030;
+  # llama-cpp = pkgs.unstablePkgs.llama-cpp.override { cudaSupport = true; };
+  # ollama = pkgs.unstablePkgs.ollama.override { inherit llama-cpp; };
 in
 {
-  environment.systemPackages = [ ollama ];
+  # environment.systemPackages = [ ollama ];
   architect.vhost.${domain} = {
     dnsInterfaces = [ "tailscale" ];
 
@@ -16,25 +16,69 @@ in
       host = "172.17.0.1";
       port = frontendPort;
       allowLan = true;
-      allow = [ config.architect.networks."tailscale".net ];
+      allowWAN = true;
+      # allow = [ config.architect.networks."tailscale".net ];
+      extraConfig = ''
+        proxy_read_timeout 600s;
+      '';
+    };
+  };
+
+  architect.vhost."ollama.giugl.io" = {
+    dnsInterfaces = [ "tailscale" ];
+
+    locations."/" = {
+      host = "172.17.0.1";
+      port = 11434;
+      allowLan = true;
+      allowWAN = true;
+      # allow = [ config.architect.networks."tailscale".net ];
+      extraConfig = ''
+        proxy_read_timeout 600s;
+      '';
     };
   };
 
   virtualisation.oci-containers = {
     containers = {
-      big-agi = {
-        image = "ghcr.io/enricoros/big-agi:latest";
+      ollama-webui = {
+        image = "ghcr.io/open-webui/open-webui:main";
         autoStart = true;
 
         ports = [
           "172.17.0.1:${toString frontendPort}:${toString backendPort}"
         ];
-        environmentFiles = [
-          "/var/lib/llm/big-agi.env"
-        ];
+
+        environment = {
+          PORT = "${toString backendPort}";
+          OLLAMA_API_BASE_URL = "http://172.17.0.1:11434/api";
+        };
+
         extraOptions = [
           "--pull=always"
         ];
+        volumes = [
+          "/var/lib/ollama-webui:/app/backend/data"
+        ];
+      };
+
+      ollama = {
+        image = "ollama/ollama:latest";
+        autoStart = true;
+        extraOptions = [
+          "--pull=always"
+          "--gpus=all"
+        ];
+        environment = {
+          OLLAMA_ORIGINS = "*";
+        };
+        volumes = [
+          "/ollama:/root/.ollama"
+        ];
+        ports = [
+          "127.0.0.1:11434:11434"
+          "172.17.0.1:11434:11434"
+        ];
       };
     };
   };