{ pkgs, ... }: let configDir = "$HOME/.config/aichat"; config = '' clients: - type: ollama api_base: https://ollama.giugl.io models: - name: mistral:7b-instruct-v0.3-fp16 max_input_tokens: 32000 max_output_tokens: 8192 - name: llama3:8b-instruct-fp16 max_input_tokens: 8192 max_output_tokens: 8192 - name: phi3:14b-medium-4k-instruct-q8_0 max_input_tokens: 128000 max_output_tokens: 8192 - name: pino-coder max_input_tokens: 28000 max_output_tokens: 8192 - type: openai api_key: null api_base: https://api.openai.com/v1 ''; roles = '' - name: commitmessage prompt: |- Your task is to generate a commit message for a given git diff. The commit message should follow the Conventional Commits specification, which includes a type, optional scope, and a brief description. The message should be concise, unambiguous, and capture the technical details of the changes made. Commit Convention Format: (): [optional body] [optional footer(s)] Common types include: - feat: A new feature - fix: A bug fix - docs: Documentation changes - style: Code style/formatting changes (not affecting code logic) - refactor: Code changes that neither fix a bug nor add a feature - test: Adding or modifying tests - chore: Changes to build process or auxiliary tools - perf: Performance improvements Here are some examples of well-formatted commit messages: ### INPUT : diff --git a/src/utils/date-formatter.js b/src/utils/date-formatter.js index 2345678..3456789 100644 --- a/src/utils/date-formatter.js +++ b/src/utils/date-formatter.js @@ -5,7 +5,7 @@ export function formatDate(date) { const month = String(date.getMonth() + 1).padStart(2, '0'); const day = String(date.getDate()).padStart(2, '0'); - return `$${year}-$${month}-$${day}`; + return `$${year}/$${month}/$${day}`; } ### OUTPUT: fix(date-formatter): modified `formatDate()` to use '/' instead of '-' as the separator ### INPUT: diff --git a/src/app.js b/src/app.js index 83d2e7a..b6a1c3f 100644 --- a/src/app.js +++ b/src/app.js @@ -10,6 +10,10 @@ function initialize() { setupEventListeners(); } +// TODO: add other listeners +// https://github.com/user/project/issue/123 +function setupEventListeners() { + document.getElementById('submit').addEventListener('click', handleSubmit); + document.getElementById('reset').addEventListener('click', handleReset); +} + function handleSubmit(event) { event.preventDefault(); const data = new FormData(event.target); @@ -20,6 +24,10 @@ function handleSubmit(event) { console.log('Form submitted:', data); } +function handleReset(event) { + event.preventDefault(); + event.target.form.reset(); + console.log('Form reset'); } ### OUTPUT: feat(app): implement form event listeners - Added `setupEventListeners()` to handle form interactions - Implemented `handleReset()` for form reset functionality - Added event listeners for submit and reset buttons - Track TODO comment for future listener additions (https://github.com/user/project/issue/123) ### INPUT: diff --git a/pkg/database/client.go b/pkg/database/client.go index 003740f..6fc4861 100644 --- a/pkg/database/client.go +++ b/pkg/database/client.go @@ -24,9 +24,12 @@ var ErrNilDatabaseClient = errors.New("database client is nil after setup") // InitDB initializes the database with the given application name and optional dbpath for SQLite. func InitDB(appName string, dbpath ...string) error { - cfg := config.New() + var ( + psqlReadReplica string + err error + ) - var err error + cfg := config.New() // Set up a new logger with your required configuration. newLogger := logger.New( @@ -38,9 +41,8 @@ func InitDB(appName string, dbpath ...string) error { }, ) - // Load PostgreSQL configurations - var psqlReadReplica string psqlSource, err := cfg.Load(config.PSQL.String()) + if err != nil { log.Println("PSQL not set, using SQLite instead.") } else { ### OUTPUT: style(database/client): group together `psqlReadReplica` and `err` in function's prologue ### INPUT: diff --git a/pkg/khttp/client.go b/pkg/khttp/client.go index a53064c..3aff938 100644 --- a/pkg/khttp/client.go +++ b/pkg/khttp/client.go @@ -11,14 +11,17 @@ import ( "github.pie.apple.com/kerosene/Core/structs" ) +// TODO: https://github.pie.apple.com/Kerosene/Core/issues/43 +// feat: Centralise and remove over use of os.Environment const ( - // Environment variables and file names. authFilesDirEnvVar = "WHISPER_AUTH_FILES_DIR" keyName = "decrypted_key.pem" certName = "cert.pem" ) // Error for missing environment variable. +// TODO: refactor: move errors into centralized errors.go files +// https://github.pie.apple.com/Kerosene/Core/issues/57 var errMissingEnvironmentVariable = fmt.Errorf("%s environment variable is not set", authFilesDirEnvVar) // AuthConfig holds authentication file paths. @@ -31,9 +34,11 @@ type AuthConfig struct { // NewAuthConfig creates an AuthConfig from environment variables. func NewAuthConfig() (*AuthConfig, error) { dir := os.Getenv(authFilesDirEnvVar) + if dir == "" { return nil, errMissingEnvironmentVariable } + return &AuthConfig{ Dir: dir, CertFile: filepath.Join(dir, certName), @@ -211,7 +216,7 @@ func setupMTLSOnlyTransport(certData string, keyData string) (*http.Transport, e // Make scheme and Auth Type separate and load from DB. func parseProxyURL(scheme string, routing structs.Routing) (*url.URL, error) { - return url.Parse(fmt.Sprintf("%s://%s:%s", scheme, routing.Proxy, routing.Port)) + return url.Parse(fmt.Sprintf("%s://%s:%d", scheme, routing.Proxy, routing.Port)) } // loadX509KeyPair loads an X509 key pair from the specified cert and key files. ### OUTPUT: fix/refactor(khttp/client): use correct format specifier for and add TODOs - Parsed proxy URL using `fmt.Sprintf()` with correct format specifier for port - Added TODOs to centralize errors and remove overuse of `os.Environment` (#43, #57) - name: createpr prompt: |- You are an AI language model tasked with generating a comprehensive Pull Request (PR) description. Your goal is to create a clear and informative PR description that summarizes the changes and highlights any important details or considerations. You are given a git diff and a list of commits - use this context to generate the PR message. # Requirements for the PR Description: 1. **Title:** Provide a concise and descriptive title for the PR. 2. **Summary:** Summarize the overall purpose and scope of the changes. 3. **Details of Changes:** Describe the key changes made, referencing specific files or functions if necessary. 4. **Impact:** Discuss any potential impact on the system, including backward compatibility, performance implications, and any new dependencies. ### INPUT: diff --git a/cmd/api/main.go b/cmd/api/main.go index 8b974c8..9dfd24b 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -33,6 +33,8 @@ func loadAppConfig() *AppConfig { } func main() { + applyMigrations := false + cfg := config.New() err := config.SetSecret([]string{ "TEMPORAL_ADDRESS", "PSQL", @@ -42,13 +44,23 @@ func main() { "REDASH_API_KEY", "INFOBLOX_SECRET", }) + if err != nil { log.Printf("Could not set secrets: %v", err) os.Exit(1) } - database.InitDB("Core") + runMigrations, _ := cfg.Load(config.RunMigration.String()) + if runMigrations != "" { + applyMigrations = true + } + + err = database.InitDB("Core", applyMigrations) + + if err != nil { + log.Fatalf("Cannot initialize DB: %v", err) + } appConfig := loadAppConfig() diff --git a/cmd/fixtures/fixtures.go b/cmd/fixtures/fixtures.go index 3d23a6d..ff3290e 100644 --- a/cmd/fixtures/fixtures.go +++ b/cmd/fixtures/fixtures.go @@ -10,13 +10,27 @@ import ( "github.com/go-faker/faker/v4" "github.com/lib/pq" + "github.pie.apple.com/kerosene/Core/pkg/config" "github.pie.apple.com/kerosene/Core/pkg/database" "github.pie.apple.com/kerosene/Core/structs" ) // Initializing worker, database, producer and redis when the program starts. func init() { - database.InitDB("DatabaseManager", "W") + applyMigrations := false + cfg := config.New() + + runMigrations, _ := cfg.Load(config.RunMigration.String()) + if runMigrations != "" { + applyMigrations = true + } + + err := database.InitDB("DatabaseManager", applyMigrations, "W") + + if err != nil { + log.Fatalf("Cannot initialize DB: %s", err) + } + rand.Seed(time.Now().UnixNano()) } diff --git a/cmd/migrator/migrator.go b/cmd/migrator/migrator.go index 3b9961a..278cc2b 100755 --- a/cmd/migrator/migrator.go +++ b/cmd/migrator/migrator.go @@ -17,13 +17,12 @@ func main() { automigrate := flag.Bool("auto", false, "Run auto-migration") apply := flag.String("apply", "", "Apply a specific migration file") deleteFlag := flag.Bool("delete", false, "Delete existing tables before migration") - configDir := flag.String("configDir", "", "Optional directory for configuration files") flag.Parse() utils.LogStep("Connecting to the database...") - err := database.InitDB("Migrator") + err := database.InitDB("Migrator", *automigrate) if err != nil { utils.LogFatal("Could not initialise DB: ", err) } @@ -40,16 +39,13 @@ func main() { } switch { - case *automigrate: - if err := database.DBAutoMigration(database.Client, *configDir); err != nil { - utils.LogFatal("Failed to run auto-migration", err) - } - case *apply != "": if err := database.ApplyMigration(database.Client, *apply); err != nil { utils.LogFatal("Failed to apply migration", err) } - + case *automigrate: + // Just log a message as the migrations are handled by InitDB + utils.LogInfo("Initialized DB with auto-migrations") default: flag.Usage() os.Exit(1) diff --git a/pkg/advans/connect.go b/pkg/advans/connect.go index cbcfdb3..f65c59c 100644 --- a/pkg/advans/connect.go +++ b/pkg/advans/connect.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "log" "net/http" "github.pie.apple.com/kerosene/Core/pkg/config" @@ -28,7 +29,21 @@ type Payload struct { var cfg = config.New() func init() { - database.InitDB("IngestionPipeline", "W") + applyMigrations := false + + runMigrations, _ := cfg.Load(config.RunMigration.String()) + if runMigrations != "" { + applyMigrations = true + } + + err := database.InitDB("IngestionPipeline", applyMigrations, "W") + + if err != nil { + log.Fatalf("Cannot initialize DB: %s", err) + } + + // TODO check errors! + // https://github.pie.apple.com/Kerosene/Core/issues/51 config.SetSecret([]string{"ADVANS_MTLS_PRIVATE_KEY", "ADVANS_MTLS_CERT", "ADVANS_API_END_POINT"}) } @@ -42,6 +57,7 @@ func initMTLSClient(certFilePath, keyFilePath string) (*http.Client, error) { func sendPayload(client *http.Client, advansEndpoint string, path string, payload Payload) error { payloadBytes, err := json.Marshal(payload) + if err != nil { return fmt.Errorf("error marshalling payload: %s", err) } @@ -63,88 +79,3 @@ func sendPayload(client *http.Client, advansEndpoint string, path string, payloa return nil } diff --git a/pkg/config/main.go b/pkg/config/main.go index ec14c10..e80c2c2 100644 --- a/pkg/config/main.go +++ b/pkg/config/main.go @@ -16,6 +16,7 @@ import ( var ( errSecretNotFound = errors.New("secret not found") errSecretDuplicateSource = errors.New("tried to retrieve secret from multiple sources") + errDataConfigDirNotExists = errors.New("data config directory does not exist") errSecretNotSet = errors.New("secret not set in any provider or as an environment variable") errUnknownProvider = errors.New("unknown provider when retrieving secret") errWhisperAuthDirNotSet = errors.New("whisper authentication file directory not specified") @@ -28,8 +29,12 @@ var ( errWhisperRetrieveContent = errors.New("could not retrieve whisper secret's content") ) -type configProvider string -type configKey string +const defaultDataConfigDir = "Config/" + +type ( + configProvider string + configKey string +) func (cg configKey) String() string { return string(cg) @@ -40,17 +45,18 @@ func (cp configProvider) String() string { } const ( - whisperProvider configProvider = "WHISPER" - PSQL configKey = "PSQL" - PSQLRead configKey = "PSQL_READ" - TemporalAddress configKey = "TEMPORAL_ADDRESS" - KafkaBrokers configKey = "KAFKA_BROKERS" - RedisURI configKey = "REDIS" - KeroseneEnvironment configKey = "ENVIRONMENT" - ElasticSearchURI configKey = "ELASTICSEARCH_URL" - Proxy configKey = "PROXY" - Queue configKey = "QUEUE" - RunMigration configKey = "RUN_AUTOMIGRATION" + whisperProvider configProvider = "WHISPER" + PSQL configKey = "PSQL" + PSQLRead configKey = "PSQL_READ" + TemporalAddress configKey = "TEMPORAL_ADDRESS" + KafkaBrokers configKey = "KAFKA_BROKERS" + RedisURI configKey = "REDIS" + KeroseneEnvironment configKey = "ENVIRONMENT" + ElasticSearchURI configKey = "ELASTICSEARCH_URL" + Proxy configKey = "PROXY" + Queue configKey = "QUEUE" + RunMigration configKey = "RUN_AUTOMIGRATION" + DefaultDataConfigDir configKey = "DEFAULT_DATA_CONFIG_DIR" ) var providers = []configProvider{ @@ -353,3 +359,23 @@ func (c *Config) Load(name string) (string, error) { return val, nil } + +func (c *Config) GetDataConfigDir() (string, error) { + retDir := defaultDataConfigDir + + overriddenConfigDir, _ := c.Load(DefaultDataConfigDir.String()) + if overriddenConfigDir != "" { + retDir = overriddenConfigDir + } + + // check appended / + if len(retDir) > 0 && retDir[len(retDir)-1] != '/' { + retDir += "/" + } + + if _, err := os.Stat(retDir); os.IsNotExist(err) { + return "", errDataConfigDirNotExists + } + + return retDir, nil +} diff --git a/pkg/database/client.go b/pkg/database/client.go index 6fc4861..444bb5a 100644 --- a/pkg/database/client.go +++ b/pkg/database/client.go @@ -19,17 +19,27 @@ import ( "gorm.io/plugin/dbresolver" ) -var Client *gorm.DB -var ErrNilDatabaseClient = errors.New("database client is nil after setup") +var ( + Client *gorm.DB + errNilDatabaseClient = errors.New("database client is nil after setup") +) + +func determineApplyMigration(cfg *config.Config, automigrate bool) bool { + overrideAutomigrate, _ := cfg.Load(config.RunMigration.String()) + + // the env var takes precedence! + return overrideAutomigrate != "" || automigrate +} // InitDB initializes the database with the given application name and optional dbpath for SQLite. -func InitDB(appName string, dbpath ...string) error { +func InitDB(appName string, automigrate bool, dbpath ...string) error { var ( psqlReadReplica string err error ) cfg := config.New() + applyMigration := determineApplyMigration(&cfg, automigrate) // Set up a new logger with your required configuration. newLogger := logger.New( @@ -87,12 +97,12 @@ func InitDB(appName string, dbpath ...string) error { if Client == nil { log.Println("database client is nil after setup, aborting") - return ErrNilDatabaseClient + return errNilDatabaseClient } // Run auto-migration if using SQLite - if isSQLiteDB(Client) { - if err := DBAutoMigration(Client, ""); err != nil { + if applyMigration && isSQLiteDB(Client) { + if err := DBAutoMigration(Client); err != nil { log.Printf("failed to run SQLite auto-migration: %v", err) return err @@ -105,6 +115,7 @@ func InitDB(appName string, dbpath ...string) error { // setupPostgres sets up a PostgreSQL database connection. func setupPostgres(newLogger logger.Interface, appName, dbURI string) (*gorm.DB, error) { dbURI = appendApplicationName(dbURI, appName) + return gorm.Open(postgres.Open(dbURI), &gorm.Config{ Logger: newLogger, }) @@ -132,19 +143,19 @@ func setupPostgresWithReplicas(newLogger logger.Interface, appName, sourceURI, r // setupSQLite sets up a SQLite database connection. func setupSQLite(newLogger logger.Interface, dbpath ...string) (*gorm.DB, error) { - var sqliteFile string + sqliteFile := "file::memory:?cache=shared" + if len(dbpath) > 0 { sqliteFile = dbpath[0] + // Ensure the SQLite file ends with .db if filepath.Ext(sqliteFile) != ".db" { return nil, errors.New("SQLite file must have a .db extension") } - } else { - // Default is to use an in-memory db - sqliteFile = "file::memory:?cache=shared" } log.Printf("Using SQLite: %s", sqliteFile) + return gorm.Open(sqlite.Open(sqliteFile), &gorm.Config{ Logger: newLogger, }) diff --git a/pkg/database/migrations.go b/pkg/database/migrations.go index 6582b99..3210216 100644 --- a/pkg/database/migrations.go +++ b/pkg/database/migrations.go @@ -27,6 +27,7 @@ func ApplyMigration(dbClient *gorm.DB, filename string) error { } utils.LogSuccess("Migration applied successfully.") + return nil } @@ -47,22 +48,26 @@ func GeneratePassword() (string, error) { return cfg.Load("REDASH_PASSWORD") } -func DBAutoMigration(dbClient *gorm.DB, configDir string) error { +func DBAutoMigration(dbClient *gorm.DB) error { utils.LogStep("Running auto-migration...") dbDialect := dbClient.Dialector.Name() + if dbDialect == "postgres" { if err := PSQLStart(); err != nil { return fmt.Errorf("failed to initialize database: %v", err) } + utils.LogSuccess("Database initialization completed successfully.") utils.LogStep("Generating Redash password...") + password, err := GeneratePassword() if err != nil { return fmt.Errorf("failed to generate password: %v", err) } utils.LogStep("Creating Redash user...") + if err := CreateUserRedash(password); err != nil { utils.LogWarning("Failed to create Redash user", err) } else { @@ -72,11 +77,7 @@ func DBAutoMigration(dbClient *gorm.DB, configDir string) error { ReflectionMigration() } - if configDir != "" { - initialise.GenerateInitData(dbClient, configDir) - } else { - initialise.GenerateInitData(dbClient) - } + initialise.GenerateInitData(dbClient) return nil } @@ -91,12 +92,15 @@ func RunAutoMigration() error { dbTablesValue := reflect.ValueOf(structs.DatabaseTables) for i := 0; i < dbTablesValue.NumField(); i++ { field := dbTablesValue.Field(i) + if field.Kind() == reflect.Ptr && !field.IsNil() { structValue := field.Elem() + if err := Client.AutoMigrate(structValue.Addr().Interface()); err != nil { return fmt.Errorf("failed to auto-migrate model %v: %v", structValue.Type().Name(), err) } } } + return nil } diff --git a/pkg/initialise/configInit.go b/pkg/initialise/configInit.go index d087531..323a3d7 100644 --- a/pkg/initialise/configInit.go +++ b/pkg/initialise/configInit.go @@ -9,6 +9,7 @@ import ( "reflect" "strings" + "github.pie.apple.com/kerosene/Core/pkg/config" "github.pie.apple.com/kerosene/Core/pkg/utils" "github.pie.apple.com/kerosene/Core/structs" "gopkg.in/yaml.v2" @@ -16,30 +17,28 @@ import ( "gorm.io/gorm" ) -//go:embed config -var configFiles embed.FS +var ( + //go:embed config + configFiles embed.FS + cfg config.Config = config.New() +) func GenerateInitData(database *gorm.DB, additionalConfigDir ...string) { utils.LogHeader("INITIALISING DEFAULT DATA") - configDir := "config" - if len(additionalConfigDir) > 0 { - configDir = additionalConfigDir[0] - - utils.LogInfo(fmt.Sprintf("Using additional config directory: %s", configDir)) + configDir, err := cfg.GetDataConfigDir() - if _, err := os.Stat(configDir); os.IsNotExist(err) { - utils.LogWarning(fmt.Sprintf("Config directory does not exist: %s", configDir), err) - } - } else { - utils.LogInfo("Using Default Config") + if err != nil { + utils.LogFatal("Cannot open config directory: %w", err) } + utils.LogInfo("Using data config directory: " + configDir) + // Get the list of directories in the embedded config directory - embeddedConfigDirs := getConfigDirs("config") + embeddedConfigDirs := getConfigDirs(configDir) // Process the embedded configs - processConfigs(database, embeddedConfigDirs, "config/") + processConfigs(database, embeddedConfigDirs, configDir) // If an additional config directory is provided, process it after the embedded configs if len(additionalConfigDir) > 0 { @@ -47,7 +46,7 @@ func GenerateInitData(database *gorm.DB, additionalConfigDir ...string) { utils.LogHeader("ADDITIONAL DATA") - processConfigs(database, additionalConfigDirs, configDir+"/") + processConfigs(database, additionalConfigDirs, configDir) } utils.LogInfo(fmt.Sprintf("Initialised Default Data for environment '%s'.", os.Getenv("ENVIRONMENT"))) @@ -113,9 +112,11 @@ func insertPrioritizedStruct(db *gorm.DB, configDirs []string, configPath, struc } func getConfigDirs(path string) []string { - var configDirs []string - var entries []os.DirEntry - var err error + var ( + configDirs []string + entries []os.DirEntry + err error + ) // Check if the path is the default embedded directory or an external path if path == "config" { ### OUTPUT: feat(config): add new env vars and refactor database initialization This PR introduces new environment variables and improves our database initialization and migration process: - Add new environment variables: 1. `RUN_AUTOMIGRATION`: to control auto-migration 2. `DEFAULT_DATA_CONFIG_DIR`: to specify the data configuration directory - Implement conditional auto-migration support based on `RUN_AUTOMIGRATION` or function parameters - Refactor database client initialization for better error handling and flexibility - Streamline the migration process in the main application and various commands Key changes: 1. Introduce `RUN_AUTOMIGRATION` and `DEFAULT_DATA_CONFIG_DIR` environment variables 2. Add `applyMigrations` flag in database initialization 3. Implement `GetDataConfigDir()` method to handle default and overridden config directories 4. Refactor `InitDB()` to accept `automigrate` parameter 5. Update `cmd/api/main.go`, `cmd/fixtures/fixtures.go`, and `pkg/advans/connect.go` to use new initialization process 6. Simplify `cmd/migrator/migrator.go` to leverage the new auto-migration feature 7. Improve error handling and logging throughout the affected files ''; in { home = { sessionVariables = { AICHAT_CONFIG_DIR = configDir; }; packages = [ pkgs.unstablePkgs.aichat ]; file.".config/aichat/config.yaml".text = config; file.".config/aichat/roles.yaml".text = roles; }; }