From 55f370c9d1be04e6cb14cb8689d1346524f52ccf Mon Sep 17 00:00:00 2001 From: Giulio De Pasquale Date: Wed, 3 Jul 2024 10:18:00 +0100 Subject: [PATCH] Update roles/home/aichat.nix --- roles/home/aichat.nix | 484 +----------------------------------------- 1 file changed, 3 insertions(+), 481 deletions(-) diff --git a/roles/home/aichat.nix b/roles/home/aichat.nix index 87e5591..be75802 100644 --- a/roles/home/aichat.nix +++ b/roles/home/aichat.nix @@ -203,487 +203,9 @@ roles = '' - name: createpr prompt: |- - You are an AI language model tasked with generating a comprehensive Pull Request (PR) description. Your goal is to create a clear and informative PR description that summarizes the changes and highlights any important details or considerations. - You are given a git diff and a list of commits - use this context to generate the PR message. - - # Requirements for the PR Description: - 1. **Title:** Provide a concise and descriptive title for the PR. - 2. **Summary:** Summarize the overall purpose and scope of the changes. - 3. **Details of Changes:** Describe the key changes made, referencing specific files or functions if necessary. - 4. **Impact:** Discuss any potential impact on the system, including backward compatibility, performance implications, and any new dependencies. - - ### INPUT: - - diff --git a/cmd/api/main.go b/cmd/api/main.go - index 8b974c8..9dfd24b 100644 - --- a/cmd/api/main.go - +++ b/cmd/api/main.go - @@ -33,6 +33,8 @@ func loadAppConfig() *AppConfig { - } - - func main() { - + applyMigrations := false - + cfg := config.New() - err := config.SetSecret([]string{ - "TEMPORAL_ADDRESS", - "PSQL", - @@ -42,13 +44,23 @@ func main() { - "REDASH_API_KEY", - "INFOBLOX_SECRET", - }) - + - if err != nil { - log.Printf("Could not set secrets: %v", err) - - os.Exit(1) - } - - - database.InitDB("Core") - + runMigrations, _ := cfg.Load(config.RunMigration.String()) - + if runMigrations != "" { - + applyMigrations = true - + } - + - + err = database.InitDB("Core", applyMigrations) - + - + if err != nil { - + log.Fatalf("Cannot initialize DB: %v", err) - + } - - appConfig := loadAppConfig() - - diff --git a/cmd/fixtures/fixtures.go b/cmd/fixtures/fixtures.go - index 3d23a6d..ff3290e 100644 - --- a/cmd/fixtures/fixtures.go - +++ b/cmd/fixtures/fixtures.go - @@ -10,13 +10,27 @@ import ( - - "github.com/go-faker/faker/v4" - "github.com/lib/pq" - + "github.pie.apple.com/kerosene/Core/pkg/config" - "github.pie.apple.com/kerosene/Core/pkg/database" - "github.pie.apple.com/kerosene/Core/structs" - ) - - // Initializing worker, database, producer and redis when the program starts. - func init() { - - database.InitDB("DatabaseManager", "W") - + applyMigrations := false - + cfg := config.New() - + - + runMigrations, _ := cfg.Load(config.RunMigration.String()) - + if runMigrations != "" { - + applyMigrations = true - + } - + - + err := database.InitDB("DatabaseManager", applyMigrations, "W") - + - + if err != nil { - + log.Fatalf("Cannot initialize DB: %s", err) - + } - + - rand.Seed(time.Now().UnixNano()) - } - - diff --git a/cmd/migrator/migrator.go b/cmd/migrator/migrator.go - index 3b9961a..278cc2b 100755 - --- a/cmd/migrator/migrator.go - +++ b/cmd/migrator/migrator.go - @@ -17,13 +17,12 @@ func main() { - automigrate := flag.Bool("auto", false, "Run auto-migration") - apply := flag.String("apply", "", "Apply a specific migration file") - deleteFlag := flag.Bool("delete", false, "Delete existing tables before migration") - - configDir := flag.String("configDir", "", "Optional directory for configuration files") - - flag.Parse() - - utils.LogStep("Connecting to the database...") - - - err := database.InitDB("Migrator") - + err := database.InitDB("Migrator", *automigrate) - if err != nil { - utils.LogFatal("Could not initialise DB: ", err) - } - @@ -40,16 +39,13 @@ func main() { - } - - switch { - - case *automigrate: - - if err := database.DBAutoMigration(database.Client, *configDir); err != nil { - - utils.LogFatal("Failed to run auto-migration", err) - - } - - - case *apply != "": - if err := database.ApplyMigration(database.Client, *apply); err != nil { - utils.LogFatal("Failed to apply migration", err) - } - - - + case *automigrate: - + // Just log a message as the migrations are handled by InitDB - + utils.LogInfo("Initialized DB with auto-migrations") - default: - flag.Usage() - os.Exit(1) - diff --git a/pkg/advans/connect.go b/pkg/advans/connect.go - index cbcfdb3..f65c59c 100644 - --- a/pkg/advans/connect.go - +++ b/pkg/advans/connect.go - @@ -4,6 +4,7 @@ import ( - "bytes" - "encoding/json" - "fmt" - + "log" - "net/http" - - "github.pie.apple.com/kerosene/Core/pkg/config" - @@ -28,7 +29,21 @@ type Payload struct { - var cfg = config.New() - - func init() { - - database.InitDB("IngestionPipeline", "W") - + applyMigrations := false - + - + runMigrations, _ := cfg.Load(config.RunMigration.String()) - + if runMigrations != "" { - + applyMigrations = true - + } - + - + err := database.InitDB("IngestionPipeline", applyMigrations, "W") - + - + if err != nil { - + log.Fatalf("Cannot initialize DB: %s", err) - + } - + - + // TODO check errors! - + // https://github.pie.apple.com/Kerosene/Core/issues/51 - config.SetSecret([]string{"ADVANS_MTLS_PRIVATE_KEY", "ADVANS_MTLS_CERT", "ADVANS_API_END_POINT"}) - } - - @@ -42,6 +57,7 @@ func initMTLSClient(certFilePath, keyFilePath string) (*http.Client, error) { - - func sendPayload(client *http.Client, advansEndpoint string, path string, payload Payload) error { - payloadBytes, err := json.Marshal(payload) - + - if err != nil { - return fmt.Errorf("error marshalling payload: %s", err) - } - @@ -63,88 +79,3 @@ func sendPayload(client *http.Client, advansEndpoint string, path string, payloa - - return nil - } - - diff --git a/pkg/config/main.go b/pkg/config/main.go - index ec14c10..e80c2c2 100644 - --- a/pkg/config/main.go - +++ b/pkg/config/main.go - @@ -16,6 +16,7 @@ import ( - var ( - errSecretNotFound = errors.New("secret not found") - errSecretDuplicateSource = errors.New("tried to retrieve secret from multiple sources") - + errDataConfigDirNotExists = errors.New("data config directory does not exist") - errSecretNotSet = errors.New("secret not set in any provider or as an environment variable") - errUnknownProvider = errors.New("unknown provider when retrieving secret") - errWhisperAuthDirNotSet = errors.New("whisper authentication file directory not specified") - @@ -28,8 +29,12 @@ var ( - errWhisperRetrieveContent = errors.New("could not retrieve whisper secret's content") - ) - - -type configProvider string - -type configKey string - +const defaultDataConfigDir = "Config/" - + - +type ( - + configProvider string - + configKey string - +) - - func (cg configKey) String() string { - return string(cg) - @@ -40,17 +45,18 @@ func (cp configProvider) String() string { - } - - const ( - - whisperProvider configProvider = "WHISPER" - - PSQL configKey = "PSQL" - - PSQLRead configKey = "PSQL_READ" - - TemporalAddress configKey = "TEMPORAL_ADDRESS" - - KafkaBrokers configKey = "KAFKA_BROKERS" - - RedisURI configKey = "REDIS" - - KeroseneEnvironment configKey = "ENVIRONMENT" - - ElasticSearchURI configKey = "ELASTICSEARCH_URL" - - Proxy configKey = "PROXY" - - Queue configKey = "QUEUE" - - RunMigration configKey = "RUN_AUTOMIGRATION" - + whisperProvider configProvider = "WHISPER" - + PSQL configKey = "PSQL" - + PSQLRead configKey = "PSQL_READ" - + TemporalAddress configKey = "TEMPORAL_ADDRESS" - + KafkaBrokers configKey = "KAFKA_BROKERS" - + RedisURI configKey = "REDIS" - + KeroseneEnvironment configKey = "ENVIRONMENT" - + ElasticSearchURI configKey = "ELASTICSEARCH_URL" - + Proxy configKey = "PROXY" - + Queue configKey = "QUEUE" - + RunMigration configKey = "RUN_AUTOMIGRATION" - + DefaultDataConfigDir configKey = "DEFAULT_DATA_CONFIG_DIR" - ) - - var providers = []configProvider{ - @@ -353,3 +359,23 @@ func (c *Config) Load(name string) (string, error) { - - return val, nil - } - + - +func (c *Config) GetDataConfigDir() (string, error) { - + retDir := defaultDataConfigDir - + - + overriddenConfigDir, _ := c.Load(DefaultDataConfigDir.String()) - + if overriddenConfigDir != "" { - + retDir = overriddenConfigDir - + } - + - + // check appended / - + if len(retDir) > 0 && retDir[len(retDir)-1] != '/' { - + retDir += "/" - + } - + - + if _, err := os.Stat(retDir); os.IsNotExist(err) { - + return "", errDataConfigDirNotExists - + } - + - + return retDir, nil - +} - diff --git a/pkg/database/client.go b/pkg/database/client.go - index 6fc4861..444bb5a 100644 - --- a/pkg/database/client.go - +++ b/pkg/database/client.go - @@ -19,17 +19,27 @@ import ( - "gorm.io/plugin/dbresolver" - ) - - -var Client *gorm.DB - -var ErrNilDatabaseClient = errors.New("database client is nil after setup") - +var ( - + Client *gorm.DB - + errNilDatabaseClient = errors.New("database client is nil after setup") - +) - + - +func determineApplyMigration(cfg *config.Config, automigrate bool) bool { - + overrideAutomigrate, _ := cfg.Load(config.RunMigration.String()) - + - + // the env var takes precedence! - + return overrideAutomigrate != "" || automigrate - +} - - // InitDB initializes the database with the given application name and optional dbpath for SQLite. - -func InitDB(appName string, dbpath ...string) error { - +func InitDB(appName string, automigrate bool, dbpath ...string) error { - var ( - psqlReadReplica string - err error - ) - - cfg := config.New() - + applyMigration := determineApplyMigration(&cfg, automigrate) - - // Set up a new logger with your required configuration. - newLogger := logger.New( - @@ -87,12 +97,12 @@ func InitDB(appName string, dbpath ...string) error { - if Client == nil { - log.Println("database client is nil after setup, aborting") - - - return ErrNilDatabaseClient - + return errNilDatabaseClient - } - - // Run auto-migration if using SQLite - - if isSQLiteDB(Client) { - - if err := DBAutoMigration(Client, ""); err != nil { - + if applyMigration && isSQLiteDB(Client) { - + if err := DBAutoMigration(Client); err != nil { - log.Printf("failed to run SQLite auto-migration: %v", err) - - return err - @@ -105,6 +115,7 @@ func InitDB(appName string, dbpath ...string) error { - // setupPostgres sets up a PostgreSQL database connection. - func setupPostgres(newLogger logger.Interface, appName, dbURI string) (*gorm.DB, error) { - dbURI = appendApplicationName(dbURI, appName) - + - return gorm.Open(postgres.Open(dbURI), &gorm.Config{ - Logger: newLogger, - }) - @@ -132,19 +143,19 @@ func setupPostgresWithReplicas(newLogger logger.Interface, appName, sourceURI, r - - // setupSQLite sets up a SQLite database connection. - func setupSQLite(newLogger logger.Interface, dbpath ...string) (*gorm.DB, error) { - - var sqliteFile string - + sqliteFile := "file::memory:?cache=shared" - + - if len(dbpath) > 0 { - sqliteFile = dbpath[0] - + - // Ensure the SQLite file ends with .db - if filepath.Ext(sqliteFile) != ".db" { - return nil, errors.New("SQLite file must have a .db extension") - } - - } else { - - // Default is to use an in-memory db - - sqliteFile = "file::memory:?cache=shared" - } - - log.Printf("Using SQLite: %s", sqliteFile) - + - return gorm.Open(sqlite.Open(sqliteFile), &gorm.Config{ - Logger: newLogger, - }) - diff --git a/pkg/database/migrations.go b/pkg/database/migrations.go - index 6582b99..3210216 100644 - --- a/pkg/database/migrations.go - +++ b/pkg/database/migrations.go - @@ -27,6 +27,7 @@ func ApplyMigration(dbClient *gorm.DB, filename string) error { - } - - utils.LogSuccess("Migration applied successfully.") - + - return nil - } - - @@ -47,22 +48,26 @@ func GeneratePassword() (string, error) { - return cfg.Load("REDASH_PASSWORD") - } - - -func DBAutoMigration(dbClient *gorm.DB, configDir string) error { - +func DBAutoMigration(dbClient *gorm.DB) error { - utils.LogStep("Running auto-migration...") - - dbDialect := dbClient.Dialector.Name() - + - if dbDialect == "postgres" { - if err := PSQLStart(); err != nil { - return fmt.Errorf("failed to initialize database: %v", err) - } - + - utils.LogSuccess("Database initialization completed successfully.") - utils.LogStep("Generating Redash password...") - + - password, err := GeneratePassword() - if err != nil { - return fmt.Errorf("failed to generate password: %v", err) - } - - utils.LogStep("Creating Redash user...") - + - if err := CreateUserRedash(password); err != nil { - utils.LogWarning("Failed to create Redash user", err) - } else { - @@ -72,11 +77,7 @@ func DBAutoMigration(dbClient *gorm.DB, configDir string) error { - ReflectionMigration() - } - - - if configDir != "" { - - initialise.GenerateInitData(dbClient, configDir) - - } else { - - initialise.GenerateInitData(dbClient) - - } - + initialise.GenerateInitData(dbClient) - - return nil - } - @@ -91,12 +92,15 @@ func RunAutoMigration() error { - dbTablesValue := reflect.ValueOf(structs.DatabaseTables) - for i := 0; i < dbTablesValue.NumField(); i++ { - field := dbTablesValue.Field(i) - + - if field.Kind() == reflect.Ptr && !field.IsNil() { - structValue := field.Elem() - + - if err := Client.AutoMigrate(structValue.Addr().Interface()); err != nil { - return fmt.Errorf("failed to auto-migrate model %v: %v", structValue.Type().Name(), err) - } - } - } - + - return nil - } - diff --git a/pkg/initialise/configInit.go b/pkg/initialise/configInit.go - index d087531..323a3d7 100644 - --- a/pkg/initialise/configInit.go - +++ b/pkg/initialise/configInit.go - @@ -9,6 +9,7 @@ import ( - "reflect" - "strings" - - + "github.pie.apple.com/kerosene/Core/pkg/config" - "github.pie.apple.com/kerosene/Core/pkg/utils" - "github.pie.apple.com/kerosene/Core/structs" - "gopkg.in/yaml.v2" - @@ -16,30 +17,28 @@ import ( - "gorm.io/gorm" - ) - - -//go:embed config - -var configFiles embed.FS - +var ( - + //go:embed config - + configFiles embed.FS - + cfg config.Config = config.New() - +) - - func GenerateInitData(database *gorm.DB, additionalConfigDir ...string) { - utils.LogHeader("INITIALISING DEFAULT DATA") - - - configDir := "config" - - if len(additionalConfigDir) > 0 { - - configDir = additionalConfigDir[0] - - - - utils.LogInfo(fmt.Sprintf("Using additional config directory: %s", configDir)) - + configDir, err := cfg.GetDataConfigDir() - - - if _, err := os.Stat(configDir); os.IsNotExist(err) { - - utils.LogWarning(fmt.Sprintf("Config directory does not exist: %s", configDir), err) - - } - - } else { - - utils.LogInfo("Using Default Config") - + if err != nil { - + utils.LogFatal("Cannot open config directory: %w", err) - } - - + utils.LogInfo("Using data config directory: " + configDir) - + - // Get the list of directories in the embedded config directory - - embeddedConfigDirs := getConfigDirs("config") - + embeddedConfigDirs := getConfigDirs(configDir) - - // Process the embedded configs - - processConfigs(database, embeddedConfigDirs, "config/") - + processConfigs(database, embeddedConfigDirs, configDir) - - // If an additional config directory is provided, process it after the embedded configs - if len(additionalConfigDir) > 0 { - @@ -47,7 +46,7 @@ func GenerateInitData(database *gorm.DB, additionalConfigDir ...string) { - - utils.LogHeader("ADDITIONAL DATA") - - - processConfigs(database, additionalConfigDirs, configDir+"/") - + processConfigs(database, additionalConfigDirs, configDir) - } - - utils.LogInfo(fmt.Sprintf("Initialised Default Data for environment '%s'.", os.Getenv("ENVIRONMENT"))) - @@ -113,9 +112,11 @@ func insertPrioritizedStruct(db *gorm.DB, configDirs []string, configPath, struc - } - - func getConfigDirs(path string) []string { - - var configDirs []string - - var entries []os.DirEntry - - var err error - + var ( - + configDirs []string - + entries []os.DirEntry - + err error - + ) - - // Check if the path is the default embedded directory or an external path - if path == "config" { + Create a concise and informative PR message for a given code diff using the Conventional Commits format. + Include all significant changes, such as new environment variables, refactored functions, and added or removed TODOs. + Mention specific issue numbers for better trackability. ### OUTPUT: