From 29692c5f29ec904ef2add5eaab6fb648a1b84ba7 Mon Sep 17 00:00:00 2001 From: Giulio De Pasquale Date: Wed, 3 Jul 2024 10:13:02 +0100 Subject: [PATCH] Update roles/home/aichat.nix --- roles/home/aichat.nix | 518 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 481 insertions(+), 37 deletions(-) diff --git a/roles/home/aichat.nix b/roles/home/aichat.nix index 8af249c..87e5591 100644 --- a/roles/home/aichat.nix +++ b/roles/home/aichat.nix @@ -214,54 +214,498 @@ roles = '' ### INPUT: - ``` - diff --git a/src/app.js b/src/app.js - index 83d2e7a..b6a1c3f 100644 - --- a/src/app.js - +++ b/src/app.js - @@ -10,6 +10,10 @@ function initialize() { - setupEventListeners(); + diff --git a/cmd/api/main.go b/cmd/api/main.go + index 8b974c8..9dfd24b 100644 + --- a/cmd/api/main.go + +++ b/cmd/api/main.go + @@ -33,6 +33,8 @@ func loadAppConfig() *AppConfig { } - - +// TODO: add other listeners - +// https://github.com/user/project/issue/123 - +function setupEventListeners() { - + document.getElementById('submit').addEventListener('click', handleSubmit); - + document.getElementById('reset').addEventListener('click', handleReset); - +} + + func main() { + + applyMigrations := false + + cfg := config.New() + err := config.SetSecret([]string{ + "TEMPORAL_ADDRESS", + "PSQL", + @@ -42,13 +44,23 @@ func main() { + "REDASH_API_KEY", + "INFOBLOX_SECRET", + }) + - function handleSubmit(event) { - event.preventDefault(); - const data = new FormData(event.target); - @@ -20,6 +24,10 @@ function handleSubmit(event) { - - console.log('Form submitted:', data); + if err != nil { + log.Printf("Could not set secrets: %v", err) + + os.Exit(1) + } + + - database.InitDB("Core") + + runMigrations, _ := cfg.Load(config.RunMigration.String()) + + if runMigrations != "" { + + applyMigrations = true + + } + + + + err = database.InitDB("Core", applyMigrations) + + + + if err != nil { + + log.Fatalf("Cannot initialize DB: %v", err) + + } + + appConfig := loadAppConfig() + + diff --git a/cmd/fixtures/fixtures.go b/cmd/fixtures/fixtures.go + index 3d23a6d..ff3290e 100644 + --- a/cmd/fixtures/fixtures.go + +++ b/cmd/fixtures/fixtures.go + @@ -10,13 +10,27 @@ import ( + + "github.com/go-faker/faker/v4" + "github.com/lib/pq" + + "github.pie.apple.com/kerosene/Core/pkg/config" + "github.pie.apple.com/kerosene/Core/pkg/database" + "github.pie.apple.com/kerosene/Core/structs" + ) + + // Initializing worker, database, producer and redis when the program starts. + func init() { + - database.InitDB("DatabaseManager", "W") + + applyMigrations := false + + cfg := config.New() + + + + runMigrations, _ := cfg.Load(config.RunMigration.String()) + + if runMigrations != "" { + + applyMigrations = true + + } + + + + err := database.InitDB("DatabaseManager", applyMigrations, "W") + + + + if err != nil { + + log.Fatalf("Cannot initialize DB: %s", err) + + } + + + rand.Seed(time.Now().UnixNano()) + } + + diff --git a/cmd/migrator/migrator.go b/cmd/migrator/migrator.go + index 3b9961a..278cc2b 100755 + --- a/cmd/migrator/migrator.go + +++ b/cmd/migrator/migrator.go + @@ -17,13 +17,12 @@ func main() { + automigrate := flag.Bool("auto", false, "Run auto-migration") + apply := flag.String("apply", "", "Apply a specific migration file") + deleteFlag := flag.Bool("delete", false, "Delete existing tables before migration") + - configDir := flag.String("configDir", "", "Optional directory for configuration files") + + flag.Parse() + + utils.LogStep("Connecting to the database...") + + - err := database.InitDB("Migrator") + + err := database.InitDB("Migrator", *automigrate) + if err != nil { + utils.LogFatal("Could not initialise DB: ", err) + } + @@ -40,16 +39,13 @@ func main() { + } + + switch { + - case *automigrate: + - if err := database.DBAutoMigration(database.Client, *configDir); err != nil { + - utils.LogFatal("Failed to run auto-migration", err) + - } + - + case *apply != "": + if err := database.ApplyMigration(database.Client, *apply); err != nil { + utils.LogFatal("Failed to apply migration", err) + } + - + + case *automigrate: + + // Just log a message as the migrations are handled by InitDB + + utils.LogInfo("Initialized DB with auto-migrations") + default: + flag.Usage() + os.Exit(1) + diff --git a/pkg/advans/connect.go b/pkg/advans/connect.go + index cbcfdb3..f65c59c 100644 + --- a/pkg/advans/connect.go + +++ b/pkg/advans/connect.go + @@ -4,6 +4,7 @@ import ( + "bytes" + "encoding/json" + "fmt" + + "log" + "net/http" + + "github.pie.apple.com/kerosene/Core/pkg/config" + @@ -28,7 +29,21 @@ type Payload struct { + var cfg = config.New() + + func init() { + - database.InitDB("IngestionPipeline", "W") + + applyMigrations := false + + + + runMigrations, _ := cfg.Load(config.RunMigration.String()) + + if runMigrations != "" { + + applyMigrations = true + + } + + + + err := database.InitDB("IngestionPipeline", applyMigrations, "W") + + + + if err != nil { + + log.Fatalf("Cannot initialize DB: %s", err) + + } + + + + // TODO check errors! + + // https://github.pie.apple.com/Kerosene/Core/issues/51 + config.SetSecret([]string{"ADVANS_MTLS_PRIVATE_KEY", "ADVANS_MTLS_CERT", "ADVANS_API_END_POINT"}) + } + + @@ -42,6 +57,7 @@ func initMTLSClient(certFilePath, keyFilePath string) (*http.Client, error) { + + func sendPayload(client *http.Client, advansEndpoint string, path string, payload Payload) error { + payloadBytes, err := json.Marshal(payload) + + + if err != nil { + return fmt.Errorf("error marshalling payload: %s", err) + } + @@ -63,88 +79,3 @@ func sendPayload(client *http.Client, advansEndpoint string, path string, payloa + + return nil } - +function handleReset(event) { - + event.preventDefault(); - + event.target.form.reset(); - + console.log('Form reset'); + diff --git a/pkg/config/main.go b/pkg/config/main.go + index ec14c10..e80c2c2 100644 + --- a/pkg/config/main.go + +++ b/pkg/config/main.go + @@ -16,6 +16,7 @@ import ( + var ( + errSecretNotFound = errors.New("secret not found") + errSecretDuplicateSource = errors.New("tried to retrieve secret from multiple sources") + + errDataConfigDirNotExists = errors.New("data config directory does not exist") + errSecretNotSet = errors.New("secret not set in any provider or as an environment variable") + errUnknownProvider = errors.New("unknown provider when retrieving secret") + errWhisperAuthDirNotSet = errors.New("whisper authentication file directory not specified") + @@ -28,8 +29,12 @@ var ( + errWhisperRetrieveContent = errors.New("could not retrieve whisper secret's content") + ) + + -type configProvider string + -type configKey string + +const defaultDataConfigDir = "Config/" + + + +type ( + + configProvider string + + configKey string + +) + + func (cg configKey) String() string { + return string(cg) + @@ -40,17 +45,18 @@ func (cp configProvider) String() string { } - ``` + + const ( + - whisperProvider configProvider = "WHISPER" + - PSQL configKey = "PSQL" + - PSQLRead configKey = "PSQL_READ" + - TemporalAddress configKey = "TEMPORAL_ADDRESS" + - KafkaBrokers configKey = "KAFKA_BROKERS" + - RedisURI configKey = "REDIS" + - KeroseneEnvironment configKey = "ENVIRONMENT" + - ElasticSearchURI configKey = "ELASTICSEARCH_URL" + - Proxy configKey = "PROXY" + - Queue configKey = "QUEUE" + - RunMigration configKey = "RUN_AUTOMIGRATION" + + whisperProvider configProvider = "WHISPER" + + PSQL configKey = "PSQL" + + PSQLRead configKey = "PSQL_READ" + + TemporalAddress configKey = "TEMPORAL_ADDRESS" + + KafkaBrokers configKey = "KAFKA_BROKERS" + + RedisURI configKey = "REDIS" + + KeroseneEnvironment configKey = "ENVIRONMENT" + + ElasticSearchURI configKey = "ELASTICSEARCH_URL" + + Proxy configKey = "PROXY" + + Queue configKey = "QUEUE" + + RunMigration configKey = "RUN_AUTOMIGRATION" + + DefaultDataConfigDir configKey = "DEFAULT_DATA_CONFIG_DIR" + ) + + var providers = []configProvider{ + @@ -353,3 +359,23 @@ func (c *Config) Load(name string) (string, error) { + + return val, nil + } + + + +func (c *Config) GetDataConfigDir() (string, error) { + + retDir := defaultDataConfigDir + + + + overriddenConfigDir, _ := c.Load(DefaultDataConfigDir.String()) + + if overriddenConfigDir != "" { + + retDir = overriddenConfigDir + + } + + + + // check appended / + + if len(retDir) > 0 && retDir[len(retDir)-1] != '/' { + + retDir += "/" + + } + + + + if _, err := os.Stat(retDir); os.IsNotExist(err) { + + return "", errDataConfigDirNotExists + + } + + + + return retDir, nil + +} + diff --git a/pkg/database/client.go b/pkg/database/client.go + index 6fc4861..444bb5a 100644 + --- a/pkg/database/client.go + +++ b/pkg/database/client.go + @@ -19,17 +19,27 @@ import ( + "gorm.io/plugin/dbresolver" + ) + + -var Client *gorm.DB + -var ErrNilDatabaseClient = errors.New("database client is nil after setup") + +var ( + + Client *gorm.DB + + errNilDatabaseClient = errors.New("database client is nil after setup") + +) + + + +func determineApplyMigration(cfg *config.Config, automigrate bool) bool { + + overrideAutomigrate, _ := cfg.Load(config.RunMigration.String()) + + + + // the env var takes precedence! + + return overrideAutomigrate != "" || automigrate + +} + + // InitDB initializes the database with the given application name and optional dbpath for SQLite. + -func InitDB(appName string, dbpath ...string) error { + +func InitDB(appName string, automigrate bool, dbpath ...string) error { + var ( + psqlReadReplica string + err error + ) + + cfg := config.New() + + applyMigration := determineApplyMigration(&cfg, automigrate) + + // Set up a new logger with your required configuration. + newLogger := logger.New( + @@ -87,12 +97,12 @@ func InitDB(appName string, dbpath ...string) error { + if Client == nil { + log.Println("database client is nil after setup, aborting") + + - return ErrNilDatabaseClient + + return errNilDatabaseClient + } + + // Run auto-migration if using SQLite + - if isSQLiteDB(Client) { + - if err := DBAutoMigration(Client, ""); err != nil { + + if applyMigration && isSQLiteDB(Client) { + + if err := DBAutoMigration(Client); err != nil { + log.Printf("failed to run SQLite auto-migration: %v", err) + + return err + @@ -105,6 +115,7 @@ func InitDB(appName string, dbpath ...string) error { + // setupPostgres sets up a PostgreSQL database connection. + func setupPostgres(newLogger logger.Interface, appName, dbURI string) (*gorm.DB, error) { + dbURI = appendApplicationName(dbURI, appName) + + + return gorm.Open(postgres.Open(dbURI), &gorm.Config{ + Logger: newLogger, + }) + @@ -132,19 +143,19 @@ func setupPostgresWithReplicas(newLogger logger.Interface, appName, sourceURI, r + + // setupSQLite sets up a SQLite database connection. + func setupSQLite(newLogger logger.Interface, dbpath ...string) (*gorm.DB, error) { + - var sqliteFile string + + sqliteFile := "file::memory:?cache=shared" + + + if len(dbpath) > 0 { + sqliteFile = dbpath[0] + + + // Ensure the SQLite file ends with .db + if filepath.Ext(sqliteFile) != ".db" { + return nil, errors.New("SQLite file must have a .db extension") + } + - } else { + - // Default is to use an in-memory db + - sqliteFile = "file::memory:?cache=shared" + } + + log.Printf("Using SQLite: %s", sqliteFile) + + + return gorm.Open(sqlite.Open(sqliteFile), &gorm.Config{ + Logger: newLogger, + }) + diff --git a/pkg/database/migrations.go b/pkg/database/migrations.go + index 6582b99..3210216 100644 + --- a/pkg/database/migrations.go + +++ b/pkg/database/migrations.go + @@ -27,6 +27,7 @@ func ApplyMigration(dbClient *gorm.DB, filename string) error { + } + + utils.LogSuccess("Migration applied successfully.") + + + return nil + } + + @@ -47,22 +48,26 @@ func GeneratePassword() (string, error) { + return cfg.Load("REDASH_PASSWORD") + } + + -func DBAutoMigration(dbClient *gorm.DB, configDir string) error { + +func DBAutoMigration(dbClient *gorm.DB) error { + utils.LogStep("Running auto-migration...") + + dbDialect := dbClient.Dialector.Name() + + + if dbDialect == "postgres" { + if err := PSQLStart(); err != nil { + return fmt.Errorf("failed to initialize database: %v", err) + } + + + utils.LogSuccess("Database initialization completed successfully.") + utils.LogStep("Generating Redash password...") + + + password, err := GeneratePassword() + if err != nil { + return fmt.Errorf("failed to generate password: %v", err) + } + + utils.LogStep("Creating Redash user...") + + + if err := CreateUserRedash(password); err != nil { + utils.LogWarning("Failed to create Redash user", err) + } else { + @@ -72,11 +77,7 @@ func DBAutoMigration(dbClient *gorm.DB, configDir string) error { + ReflectionMigration() + } + + - if configDir != "" { + - initialise.GenerateInitData(dbClient, configDir) + - } else { + - initialise.GenerateInitData(dbClient) + - } + + initialise.GenerateInitData(dbClient) + + return nil + } + @@ -91,12 +92,15 @@ func RunAutoMigration() error { + dbTablesValue := reflect.ValueOf(structs.DatabaseTables) + for i := 0; i < dbTablesValue.NumField(); i++ { + field := dbTablesValue.Field(i) + + + if field.Kind() == reflect.Ptr && !field.IsNil() { + structValue := field.Elem() + + + if err := Client.AutoMigrate(structValue.Addr().Interface()); err != nil { + return fmt.Errorf("failed to auto-migrate model %v: %v", structValue.Type().Name(), err) + } + } + } + + + return nil + } + diff --git a/pkg/initialise/configInit.go b/pkg/initialise/configInit.go + index d087531..323a3d7 100644 + --- a/pkg/initialise/configInit.go + +++ b/pkg/initialise/configInit.go + @@ -9,6 +9,7 @@ import ( + "reflect" + "strings" + + + "github.pie.apple.com/kerosene/Core/pkg/config" + "github.pie.apple.com/kerosene/Core/pkg/utils" + "github.pie.apple.com/kerosene/Core/structs" + "gopkg.in/yaml.v2" + @@ -16,30 +17,28 @@ import ( + "gorm.io/gorm" + ) + + -//go:embed config + -var configFiles embed.FS + +var ( + + //go:embed config + + configFiles embed.FS + + cfg config.Config = config.New() + +) + + func GenerateInitData(database *gorm.DB, additionalConfigDir ...string) { + utils.LogHeader("INITIALISING DEFAULT DATA") + + - configDir := "config" + - if len(additionalConfigDir) > 0 { + - configDir = additionalConfigDir[0] + - + - utils.LogInfo(fmt.Sprintf("Using additional config directory: %s", configDir)) + + configDir, err := cfg.GetDataConfigDir() + + - if _, err := os.Stat(configDir); os.IsNotExist(err) { + - utils.LogWarning(fmt.Sprintf("Config directory does not exist: %s", configDir), err) + - } + - } else { + - utils.LogInfo("Using Default Config") + + if err != nil { + + utils.LogFatal("Cannot open config directory: %w", err) + } + + + utils.LogInfo("Using data config directory: " + configDir) + + + // Get the list of directories in the embedded config directory + - embeddedConfigDirs := getConfigDirs("config") + + embeddedConfigDirs := getConfigDirs(configDir) + + // Process the embedded configs + - processConfigs(database, embeddedConfigDirs, "config/") + + processConfigs(database, embeddedConfigDirs, configDir) + + // If an additional config directory is provided, process it after the embedded configs + if len(additionalConfigDir) > 0 { + @@ -47,7 +46,7 @@ func GenerateInitData(database *gorm.DB, additionalConfigDir ...string) { + + utils.LogHeader("ADDITIONAL DATA") + + - processConfigs(database, additionalConfigDirs, configDir+"/") + + processConfigs(database, additionalConfigDirs, configDir) + } + + utils.LogInfo(fmt.Sprintf("Initialised Default Data for environment '%s'.", os.Getenv("ENVIRONMENT"))) + @@ -113,9 +112,11 @@ func insertPrioritizedStruct(db *gorm.DB, configDirs []string, configPath, struc + } + + func getConfigDirs(path string) []string { + - var configDirs []string + - var entries []os.DirEntry + - var err error + + var ( + + configDirs []string + + entries []os.DirEntry + + err error + + ) + + // Check if the path is the default embedded directory or an external path + if path == "config" { ### OUTPUT: - **Title:** feat(app): add event listeners for submit and reset buttons + feat(config): add new env vars and refactor database initialization - **Summary:** - This PR adds event listeners for the submit and reset buttons in the application. It introduces a new function to set up these event listeners and implements a handler for the reset button. + This PR introduces new environment variables and improves our database initialization and migration process: - **Details of Changes:** - 1. **src/app.js:** - - Added a new function `setupEventListeners` to attach click event listeners to the submit and reset buttons. - - Implemented the `handleReset` function to reset the form and log the reset action. - - Included a TODO comment to add other listeners, referencing issue [#123](https://github.com/user/project/issue/123). + - Add new environment variables: + 1. `RUN_AUTOMIGRATION`: to control auto-migration + 2. `DEFAULT_DATA_CONFIG_DIR`: to specify the data configuration directory + - Implement conditional auto-migration support based on `RUN_AUTOMIGRATION` or function parameters + - Refactor database client initialization for better error handling and flexibility + - Streamline the migration process in the main application and various commands - **Impact:** - - **Backward Compatibility:** The changes are backward compatible. - - **Performance Implications:** Minimal performance impact due to the addition of event listeners. - - **New Dependencies:** No new dependencies introduced. + Key changes: + 1. Introduce `RUN_AUTOMIGRATION` and `DEFAULT_DATA_CONFIG_DIR` environment variables + 2. Add `applyMigrations` flag in database initialization + 3. Implement `GetDataConfigDir()` method to handle default and overridden config directories + 4. Refactor `InitDB()` to accept `automigrate` parameter + 5. Update `cmd/api/main.go`, `cmd/fixtures/fixtures.go`, and `pkg/advans/connect.go` to use new initialization process + 6. Simplify `cmd/migrator/migrator.go` to leverage the new auto-migration feature + 7. Improve error handling and logging throughout the affected files ''; in {