peperunas/nixos
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
29692c5f29
nixos/roles/home/aichat.nix

721 lines
26 KiB
Nix
Raw Blame History

{ pkgs, ... }:
let
configDir = "$HOME/.config/aichat";
config = ''
clients:
- type: ollama
api_base: https://ollama.giugl.io
models:
- name: mistral:7b-instruct-v0.3-fp16
max_input_tokens: 32000
max_output_tokens: 8192
- name: llama3:8b-instruct-fp16
max_input_tokens: 8192
max_output_tokens: 8192
- name: phi3:14b-medium-4k-instruct-q8_0
max_input_tokens: 128000
max_output_tokens: 8192
- name: pino-coder
max_input_tokens: 28000
max_output_tokens: 8192
- type: openai
api_key: null
api_base: https://api.openai.com/v1
'';
roles = ''
- name: commitmessage
prompt: |-
Your task is to generate a commit message for a given git diff. The commit message should follow the Conventional Commits specification, which includes a type, optional scope, and a brief description. The message should be concise, unambiguous, and capture the technical details of the changes made.
Commit Convention Format:
<type>(<scope>): <description>
[optional body]
[optional footer(s)]
Common types include:
- feat: A new feature
- fix: A bug fix
- docs: Documentation changes
- style: Code style/formatting changes (not affecting code logic)
- refactor: Code changes that neither fix a bug nor add a feature
- test: Adding or modifying tests
- chore: Changes to build process or auxiliary tools
- perf: Performance improvements
Here are some examples of well-formatted commit messages:
### INPUT :
diff --git a/src/utils/date-formatter.js b/src/utils/date-formatter.js
index 2345678..3456789 100644
--- a/src/utils/date-formatter.js
+++ b/src/utils/date-formatter.js
@@ -5,7 +5,7 @@ export function formatDate(date) {
const month = String(date.getMonth() + 1).padStart(2, '0');
const day = String(date.getDate()).padStart(2, '0');
- return `$${year}-$${month}-$${day}`;
+ return `$${year}/$${month}/$${day}`;
}
### OUTPUT:
fix(date-formatter): modified `formatDate()` to use '/' instead of '-' as the separator
### INPUT:
diff --git a/src/app.js b/src/app.js
index 83d2e7a..b6a1c3f 100644
--- a/src/app.js
+++ b/src/app.js
@@ -10,6 +10,10 @@ function initialize() {
setupEventListeners();
}
+// TODO: add other listeners
+// https://github.com/user/project/issue/123
+function setupEventListeners() {
+ document.getElementById('submit').addEventListener('click', handleSubmit);
+ document.getElementById('reset').addEventListener('click', handleReset);
+}
+
function handleSubmit(event) {
event.preventDefault();
const data = new FormData(event.target);
@@ -20,6 +24,10 @@ function handleSubmit(event) {
console.log('Form submitted:', data);
}
+function handleReset(event) {
+ event.preventDefault();
+ event.target.form.reset();
+ console.log('Form reset');
}
### OUTPUT:
feat(app): implement form event listeners
- Added `setupEventListeners()` to handle form interactions
- Implemented `handleReset()` for form reset functionality
- Added event listeners for submit and reset buttons
- Track TODO comment for future listener additions (https://github.com/user/project/issue/123)
### INPUT:
diff --git a/pkg/database/client.go b/pkg/database/client.go
index 003740f..6fc4861 100644
--- a/pkg/database/client.go
+++ b/pkg/database/client.go
@@ -24,9 +24,12 @@ var ErrNilDatabaseClient = errors.New("database client is nil after setup")
// InitDB initializes the database with the given application name and optional dbpath for SQLite.
func InitDB(appName string, dbpath ...string) error {
- cfg := config.New()
+ var (
+ psqlReadReplica string
+ err error
+ )
- var err error
+ cfg := config.New()
// Set up a new logger with your required configuration.
newLogger := logger.New(
@@ -38,9 +41,8 @@ func InitDB(appName string, dbpath ...string) error {
},
)
- // Load PostgreSQL configurations
- var psqlReadReplica string
psqlSource, err := cfg.Load(config.PSQL.String())
+
if err != nil {
log.Println("PSQL not set, using SQLite instead.")
} else {
### OUTPUT:
style(database/client): group together `psqlReadReplica` and `err` in function's prologue
### INPUT:
diff --git a/pkg/khttp/client.go b/pkg/khttp/client.go
index a53064c..3aff938 100644
--- a/pkg/khttp/client.go
+++ b/pkg/khttp/client.go
@@ -11,14 +11,17 @@ import (
"github.pie.apple.com/kerosene/Core/structs"
)
+// TODO: https://github.pie.apple.com/Kerosene/Core/issues/43
+// feat: Centralise and remove over use of os.Environment
const (
- // Environment variables and file names.
authFilesDirEnvVar = "WHISPER_AUTH_FILES_DIR"
keyName = "decrypted_key.pem"
certName = "cert.pem"
)
// Error for missing environment variable.
+// TODO: refactor: move errors into centralized errors.go files
+// https://github.pie.apple.com/Kerosene/Core/issues/57
var errMissingEnvironmentVariable = fmt.Errorf("%s environment variable is not set", authFilesDirEnvVar)
// AuthConfig holds authentication file paths.
@@ -31,9 +34,11 @@ type AuthConfig struct {
// NewAuthConfig creates an AuthConfig from environment variables.
func NewAuthConfig() (*AuthConfig, error) {
dir := os.Getenv(authFilesDirEnvVar)
+
if dir == "" {
return nil, errMissingEnvironmentVariable
}
+
return &AuthConfig{
Dir: dir,
CertFile: filepath.Join(dir, certName),
@@ -211,7 +216,7 @@ func setupMTLSOnlyTransport(certData string, keyData string) (*http.Transport, e
// Make scheme and Auth Type separate and load from DB.
func parseProxyURL(scheme string, routing structs.Routing) (*url.URL, error) {
- return url.Parse(fmt.Sprintf("%s://%s:%s", scheme, routing.Proxy, routing.Port))
+ return url.Parse(fmt.Sprintf("%s://%s:%d", scheme, routing.Proxy, routing.Port))
}
// loadX509KeyPair loads an X509 key pair from the specified cert and key files.
### OUTPUT:
fix/refactor(khttp/client): use correct format specifier for and add TODOs
- Parsed proxy URL using `fmt.Sprintf()` with correct format specifier for port
- Added TODOs to centralize errors and remove overuse of `os.Environment` (#43, #57)
- name: createpr
prompt: |-
You are an AI language model tasked with generating a comprehensive Pull Request (PR) description. Your goal is to create a clear and informative PR description that summarizes the changes and highlights any important details or considerations.
You are given a git diff and a list of commits - use this context to generate the PR message.
# Requirements for the PR Description:
1. **Title:** Provide a concise and descriptive title for the PR.
2. **Summary:** Summarize the overall purpose and scope of the changes.
3. **Details of Changes:** Describe the key changes made, referencing specific files or functions if necessary.
4. **Impact:** Discuss any potential impact on the system, including backward compatibility, performance implications, and any new dependencies.
### INPUT:
diff --git a/cmd/api/main.go b/cmd/api/main.go
index 8b974c8..9dfd24b 100644
--- a/cmd/api/main.go
+++ b/cmd/api/main.go
@@ -33,6 +33,8 @@ func loadAppConfig() *AppConfig {
}
func main() {
+ applyMigrations := false
+ cfg := config.New()
err := config.SetSecret([]string{
"TEMPORAL_ADDRESS",
"PSQL",
@@ -42,13 +44,23 @@ func main() {
"REDASH_API_KEY",
"INFOBLOX_SECRET",
})
+
if err != nil {
log.Printf("Could not set secrets: %v", err)
os.Exit(1)
}
- database.InitDB("Core")
+ runMigrations, _ := cfg.Load(config.RunMigration.String())
+ if runMigrations != "" {
+ applyMigrations = true
+ }
+
+ err = database.InitDB("Core", applyMigrations)
+
+ if err != nil {
+ log.Fatalf("Cannot initialize DB: %v", err)
+ }
appConfig := loadAppConfig()
diff --git a/cmd/fixtures/fixtures.go b/cmd/fixtures/fixtures.go
index 3d23a6d..ff3290e 100644
--- a/cmd/fixtures/fixtures.go
+++ b/cmd/fixtures/fixtures.go
@@ -10,13 +10,27 @@ import (
"github.com/go-faker/faker/v4"
"github.com/lib/pq"
+ "github.pie.apple.com/kerosene/Core/pkg/config"
"github.pie.apple.com/kerosene/Core/pkg/database"
"github.pie.apple.com/kerosene/Core/structs"
)
// Initializing worker, database, producer and redis when the program starts.
func init() {
- database.InitDB("DatabaseManager", "W")
+ applyMigrations := false
+ cfg := config.New()
+
+ runMigrations, _ := cfg.Load(config.RunMigration.String())
+ if runMigrations != "" {
+ applyMigrations = true
+ }
+
+ err := database.InitDB("DatabaseManager", applyMigrations, "W")
+
+ if err != nil {
+ log.Fatalf("Cannot initialize DB: %s", err)
+ }
+
rand.Seed(time.Now().UnixNano())
}
diff --git a/cmd/migrator/migrator.go b/cmd/migrator/migrator.go
index 3b9961a..278cc2b 100755
--- a/cmd/migrator/migrator.go
+++ b/cmd/migrator/migrator.go
@@ -17,13 +17,12 @@ func main() {
automigrate := flag.Bool("auto", false, "Run auto-migration")
apply := flag.String("apply", "", "Apply a specific migration file")
deleteFlag := flag.Bool("delete", false, "Delete existing tables before migration")
- configDir := flag.String("configDir", "", "Optional directory for configuration files")
flag.Parse()
utils.LogStep("Connecting to the database...")
- err := database.InitDB("Migrator")
+ err := database.InitDB("Migrator", *automigrate)
if err != nil {
utils.LogFatal("Could not initialise DB: ", err)
}
@@ -40,16 +39,13 @@ func main() {
}
switch {
- case *automigrate:
- if err := database.DBAutoMigration(database.Client, *configDir); err != nil {
- utils.LogFatal("Failed to run auto-migration", err)
- }
-
case *apply != "":
if err := database.ApplyMigration(database.Client, *apply); err != nil {
utils.LogFatal("Failed to apply migration", err)
}
-
+ case *automigrate:
+ // Just log a message as the migrations are handled by InitDB
+ utils.LogInfo("Initialized DB with auto-migrations")
default:
flag.Usage()
os.Exit(1)
diff --git a/pkg/advans/connect.go b/pkg/advans/connect.go
index cbcfdb3..f65c59c 100644
--- a/pkg/advans/connect.go
+++ b/pkg/advans/connect.go
@@ -4,6 +4,7 @@ import (
"bytes"
"encoding/json"
"fmt"
+ "log"
"net/http"
"github.pie.apple.com/kerosene/Core/pkg/config"
@@ -28,7 +29,21 @@ type Payload struct {
var cfg = config.New()
func init() {
- database.InitDB("IngestionPipeline", "W")
+ applyMigrations := false
+
+ runMigrations, _ := cfg.Load(config.RunMigration.String())
+ if runMigrations != "" {
+ applyMigrations = true
+ }
+
+ err := database.InitDB("IngestionPipeline", applyMigrations, "W")
+
+ if err != nil {
+ log.Fatalf("Cannot initialize DB: %s", err)
+ }
+
+ // TODO check errors!
+ // https://github.pie.apple.com/Kerosene/Core/issues/51
config.SetSecret([]string{"ADVANS_MTLS_PRIVATE_KEY", "ADVANS_MTLS_CERT", "ADVANS_API_END_POINT"})
}
@@ -42,6 +57,7 @@ func initMTLSClient(certFilePath, keyFilePath string) (*http.Client, error) {
func sendPayload(client *http.Client, advansEndpoint string, path string, payload Payload) error {
payloadBytes, err := json.Marshal(payload)
+
if err != nil {
return fmt.Errorf("error marshalling payload: %s", err)
}
@@ -63,88 +79,3 @@ func sendPayload(client *http.Client, advansEndpoint string, path string, payloa
return nil
}
diff --git a/pkg/config/main.go b/pkg/config/main.go
index ec14c10..e80c2c2 100644
--- a/pkg/config/main.go
+++ b/pkg/config/main.go
@@ -16,6 +16,7 @@ import (
var (
errSecretNotFound = errors.New("secret not found")
errSecretDuplicateSource = errors.New("tried to retrieve secret from multiple sources")
+ errDataConfigDirNotExists = errors.New("data config directory does not exist")
errSecretNotSet = errors.New("secret not set in any provider or as an environment variable")
errUnknownProvider = errors.New("unknown provider when retrieving secret")
errWhisperAuthDirNotSet = errors.New("whisper authentication file directory not specified")
@@ -28,8 +29,12 @@ var (
errWhisperRetrieveContent = errors.New("could not retrieve whisper secret's content")
)
-type configProvider string
-type configKey string
+const defaultDataConfigDir = "Config/"
+
+type (
+ configProvider string
+ configKey string
+)
func (cg configKey) String() string {
return string(cg)
@@ -40,17 +45,18 @@ func (cp configProvider) String() string {
}
const (
- whisperProvider configProvider = "WHISPER"
- PSQL configKey = "PSQL"
- PSQLRead configKey = "PSQL_READ"
- TemporalAddress configKey = "TEMPORAL_ADDRESS"
- KafkaBrokers configKey = "KAFKA_BROKERS"
- RedisURI configKey = "REDIS"
- KeroseneEnvironment configKey = "ENVIRONMENT"
- ElasticSearchURI configKey = "ELASTICSEARCH_URL"
- Proxy configKey = "PROXY"
- Queue configKey = "QUEUE"
- RunMigration configKey = "RUN_AUTOMIGRATION"
+ whisperProvider configProvider = "WHISPER"
+ PSQL configKey = "PSQL"
+ PSQLRead configKey = "PSQL_READ"
+ TemporalAddress configKey = "TEMPORAL_ADDRESS"
+ KafkaBrokers configKey = "KAFKA_BROKERS"
+ RedisURI configKey = "REDIS"
+ KeroseneEnvironment configKey = "ENVIRONMENT"
+ ElasticSearchURI configKey = "ELASTICSEARCH_URL"
+ Proxy configKey = "PROXY"
+ Queue configKey = "QUEUE"
+ RunMigration configKey = "RUN_AUTOMIGRATION"
+ DefaultDataConfigDir configKey = "DEFAULT_DATA_CONFIG_DIR"
)
var providers = []configProvider{
@@ -353,3 +359,23 @@ func (c *Config) Load(name string) (string, error) {
return val, nil
}
+
+func (c *Config) GetDataConfigDir() (string, error) {
+ retDir := defaultDataConfigDir
+
+ overriddenConfigDir, _ := c.Load(DefaultDataConfigDir.String())
+ if overriddenConfigDir != "" {
+ retDir = overriddenConfigDir
+ }
+
+ // check appended /
+ if len(retDir) > 0 && retDir[len(retDir)-1] != '/' {
+ retDir += "/"
+ }
+
+ if _, err := os.Stat(retDir); os.IsNotExist(err) {
+ return "", errDataConfigDirNotExists
+ }
+
+ return retDir, nil
+}
diff --git a/pkg/database/client.go b/pkg/database/client.go
index 6fc4861..444bb5a 100644
--- a/pkg/database/client.go
+++ b/pkg/database/client.go
@@ -19,17 +19,27 @@ import (
"gorm.io/plugin/dbresolver"
)
-var Client *gorm.DB
-var ErrNilDatabaseClient = errors.New("database client is nil after setup")
+var (
+ Client *gorm.DB
+ errNilDatabaseClient = errors.New("database client is nil after setup")
+)
+
+func determineApplyMigration(cfg *config.Config, automigrate bool) bool {
+ overrideAutomigrate, _ := cfg.Load(config.RunMigration.String())
+
+ // the env var takes precedence!
+ return overrideAutomigrate != "" || automigrate
+}
// InitDB initializes the database with the given application name and optional dbpath for SQLite.
-func InitDB(appName string, dbpath ...string) error {
+func InitDB(appName string, automigrate bool, dbpath ...string) error {
var (
psqlReadReplica string
err error
)
cfg := config.New()
+ applyMigration := determineApplyMigration(&cfg, automigrate)
// Set up a new logger with your required configuration.
newLogger := logger.New(
@@ -87,12 +97,12 @@ func InitDB(appName string, dbpath ...string) error {
if Client == nil {
log.Println("database client is nil after setup, aborting")
- return ErrNilDatabaseClient
+ return errNilDatabaseClient
}
// Run auto-migration if using SQLite
- if isSQLiteDB(Client) {
- if err := DBAutoMigration(Client, ""); err != nil {
+ if applyMigration && isSQLiteDB(Client) {
+ if err := DBAutoMigration(Client); err != nil {
log.Printf("failed to run SQLite auto-migration: %v", err)
return err
@@ -105,6 +115,7 @@ func InitDB(appName string, dbpath ...string) error {
// setupPostgres sets up a PostgreSQL database connection.
func setupPostgres(newLogger logger.Interface, appName, dbURI string) (*gorm.DB, error) {
dbURI = appendApplicationName(dbURI, appName)
+
return gorm.Open(postgres.Open(dbURI), &gorm.Config{
Logger: newLogger,
})
@@ -132,19 +143,19 @@ func setupPostgresWithReplicas(newLogger logger.Interface, appName, sourceURI, r
// setupSQLite sets up a SQLite database connection.
func setupSQLite(newLogger logger.Interface, dbpath ...string) (*gorm.DB, error) {
- var sqliteFile string
+ sqliteFile := "file::memory:?cache=shared"
+
if len(dbpath) > 0 {
sqliteFile = dbpath[0]
+
// Ensure the SQLite file ends with .db
if filepath.Ext(sqliteFile) != ".db" {
return nil, errors.New("SQLite file must have a .db extension")
}
- } else {
- // Default is to use an in-memory db
- sqliteFile = "file::memory:?cache=shared"
}
log.Printf("Using SQLite: %s", sqliteFile)
+
return gorm.Open(sqlite.Open(sqliteFile), &gorm.Config{
Logger: newLogger,
})
diff --git a/pkg/database/migrations.go b/pkg/database/migrations.go
index 6582b99..3210216 100644
--- a/pkg/database/migrations.go
+++ b/pkg/database/migrations.go
@@ -27,6 +27,7 @@ func ApplyMigration(dbClient *gorm.DB, filename string) error {
}
utils.LogSuccess("Migration applied successfully.")
+
return nil
}
@@ -47,22 +48,26 @@ func GeneratePassword() (string, error) {
return cfg.Load("REDASH_PASSWORD")
}
-func DBAutoMigration(dbClient *gorm.DB, configDir string) error {
+func DBAutoMigration(dbClient *gorm.DB) error {
utils.LogStep("Running auto-migration...")
dbDialect := dbClient.Dialector.Name()
+
if dbDialect == "postgres" {
if err := PSQLStart(); err != nil {
return fmt.Errorf("failed to initialize database: %v", err)
}
+
utils.LogSuccess("Database initialization completed successfully.")
utils.LogStep("Generating Redash password...")
+
password, err := GeneratePassword()
if err != nil {
return fmt.Errorf("failed to generate password: %v", err)
}
utils.LogStep("Creating Redash user...")
+
if err := CreateUserRedash(password); err != nil {
utils.LogWarning("Failed to create Redash user", err)
} else {
@@ -72,11 +77,7 @@ func DBAutoMigration(dbClient *gorm.DB, configDir string) error {
ReflectionMigration()
}
- if configDir != "" {
- initialise.GenerateInitData(dbClient, configDir)
- } else {
- initialise.GenerateInitData(dbClient)
- }
+ initialise.GenerateInitData(dbClient)
return nil
}
@@ -91,12 +92,15 @@ func RunAutoMigration() error {
dbTablesValue := reflect.ValueOf(structs.DatabaseTables)
for i := 0; i < dbTablesValue.NumField(); i++ {
field := dbTablesValue.Field(i)
+
if field.Kind() == reflect.Ptr && !field.IsNil() {
structValue := field.Elem()
+
if err := Client.AutoMigrate(structValue.Addr().Interface()); err != nil {
return fmt.Errorf("failed to auto-migrate model %v: %v", structValue.Type().Name(), err)
}
}
}
+
return nil
}
diff --git a/pkg/initialise/configInit.go b/pkg/initialise/configInit.go
index d087531..323a3d7 100644
--- a/pkg/initialise/configInit.go
+++ b/pkg/initialise/configInit.go
@@ -9,6 +9,7 @@ import (
"reflect"
"strings"
+ "github.pie.apple.com/kerosene/Core/pkg/config"
"github.pie.apple.com/kerosene/Core/pkg/utils"
"github.pie.apple.com/kerosene/Core/structs"
"gopkg.in/yaml.v2"
@@ -16,30 +17,28 @@ import (
"gorm.io/gorm"
)
-//go:embed config
-var configFiles embed.FS
+var (
+ //go:embed config
+ configFiles embed.FS
+ cfg config.Config = config.New()
+)
func GenerateInitData(database *gorm.DB, additionalConfigDir ...string) {
utils.LogHeader("INITIALISING DEFAULT DATA")
- configDir := "config"
- if len(additionalConfigDir) > 0 {
- configDir = additionalConfigDir[0]
-
- utils.LogInfo(fmt.Sprintf("Using additional config directory: %s", configDir))
+ configDir, err := cfg.GetDataConfigDir()
- if _, err := os.Stat(configDir); os.IsNotExist(err) {
- utils.LogWarning(fmt.Sprintf("Config directory does not exist: %s", configDir), err)
- }
- } else {
- utils.LogInfo("Using Default Config")
+ if err != nil {
+ utils.LogFatal("Cannot open config directory: %w", err)
}
+ utils.LogInfo("Using data config directory: " + configDir)
+
// Get the list of directories in the embedded config directory
- embeddedConfigDirs := getConfigDirs("config")
+ embeddedConfigDirs := getConfigDirs(configDir)
// Process the embedded configs
- processConfigs(database, embeddedConfigDirs, "config/")
+ processConfigs(database, embeddedConfigDirs, configDir)
// If an additional config directory is provided, process it after the embedded configs
if len(additionalConfigDir) > 0 {
@@ -47,7 +46,7 @@ func GenerateInitData(database *gorm.DB, additionalConfigDir ...string) {
utils.LogHeader("ADDITIONAL DATA")
- processConfigs(database, additionalConfigDirs, configDir+"/")
+ processConfigs(database, additionalConfigDirs, configDir)
}
utils.LogInfo(fmt.Sprintf("Initialised Default Data for environment '%s'.", os.Getenv("ENVIRONMENT")))
@@ -113,9 +112,11 @@ func insertPrioritizedStruct(db *gorm.DB, configDirs []string, configPath, struc
}
func getConfigDirs(path string) []string {
- var configDirs []string
- var entries []os.DirEntry
- var err error
+ var (
+ configDirs []string
+ entries []os.DirEntry
+ err error
+ )
// Check if the path is the default embedded directory or an external path
if path == "config" {
### OUTPUT:
feat(config): add new env vars and refactor database initialization
This PR introduces new environment variables and improves our database initialization and migration process:
- Add new environment variables:
1. `RUN_AUTOMIGRATION`: to control auto-migration
2. `DEFAULT_DATA_CONFIG_DIR`: to specify the data configuration directory
- Implement conditional auto-migration support based on `RUN_AUTOMIGRATION` or function parameters
- Refactor database client initialization for better error handling and flexibility
- Streamline the migration process in the main application and various commands
Key changes:
1. Introduce `RUN_AUTOMIGRATION` and `DEFAULT_DATA_CONFIG_DIR` environment variables
2. Add `applyMigrations` flag in database initialization
3. Implement `GetDataConfigDir()` method to handle default and overridden config directories
4. Refactor `InitDB()` to accept `automigrate` parameter
5. Update `cmd/api/main.go`, `cmd/fixtures/fixtures.go`, and `pkg/advans/connect.go` to use new initialization process
6. Simplify `cmd/migrator/migrator.go` to leverage the new auto-migration feature
7. Improve error handling and logging throughout the affected files
'';
in
{
home = {
sessionVariables = {
AICHAT_CONFIG_DIR = configDir;
};
packages = [ pkgs.unstablePkgs.aichat ];
file.".config/aichat/config.yaml".text = config;
file.".config/aichat/roles.yaml".text = roles;
};
}
Reference in New Issue View Git Blame Copy Permalink