peperunas/nixos
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update roles/home/aichat.nix

Browse Source
This commit is contained in:
Giulio De Pasquale 2024-07-03 10:18:00 +01:00
parent 29692c5f29
commit 55f370c9d1
1 changed files with 3 additions and 481 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

484
roles/home/aichat.nix
View File

@ -203,487 +203,9 @@ roles = ''
- name: createpr - name: createpr
prompt: |- prompt: |-
You are an AI language model tasked with generating a comprehensive Pull Request (PR) description. Your goal is to create a clear and informative PR description that summarizes the changes and highlights any important details or considerations. Create a concise and informative PR message for a given code diff using the Conventional Commits format.
You are given a git diff and a list of commits - use this context to generate the PR message. Include all significant changes, such as new environment variables, refactored functions, and added or removed TODOs.
Mention specific issue numbers for better trackability.
# Requirements for the PR Description:
1. **Title:** Provide a concise and descriptive title for the PR.
2. **Summary:** Summarize the overall purpose and scope of the changes.
3. **Details of Changes:** Describe the key changes made, referencing specific files or functions if necessary.
4. **Impact:** Discuss any potential impact on the system, including backward compatibility, performance implications, and any new dependencies.
### INPUT:
diff --git a/cmd/api/main.go b/cmd/api/main.go
index 8b974c8..9dfd24b 100644
--- a/cmd/api/main.go
+++ b/cmd/api/main.go
@@ -33,6 +33,8 @@ func loadAppConfig() *AppConfig {
}
func main() {
+ applyMigrations := false
+ cfg := config.New()
err := config.SetSecret([]string{
"TEMPORAL_ADDRESS",
"PSQL",
@@ -42,13 +44,23 @@ func main() {
"REDASH_API_KEY",
"INFOBLOX_SECRET",
})
+
if err != nil {
log.Printf("Could not set secrets: %v", err)
os.Exit(1)
}
- database.InitDB("Core")
+ runMigrations, _ := cfg.Load(config.RunMigration.String())
+ if runMigrations != "" {
+ applyMigrations = true
+ }
+
+ err = database.InitDB("Core", applyMigrations)
+
+ if err != nil {
+ log.Fatalf("Cannot initialize DB: %v", err)
+ }
appConfig := loadAppConfig()
diff --git a/cmd/fixtures/fixtures.go b/cmd/fixtures/fixtures.go
index 3d23a6d..ff3290e 100644
--- a/cmd/fixtures/fixtures.go
+++ b/cmd/fixtures/fixtures.go
@@ -10,13 +10,27 @@ import (
"github.com/go-faker/faker/v4"
"github.com/lib/pq"
+ "github.pie.apple.com/kerosene/Core/pkg/config"
"github.pie.apple.com/kerosene/Core/pkg/database"
"github.pie.apple.com/kerosene/Core/structs"
)
// Initializing worker, database, producer and redis when the program starts.
func init() {
- database.InitDB("DatabaseManager", "W")
+ applyMigrations := false
+ cfg := config.New()
+
+ runMigrations, _ := cfg.Load(config.RunMigration.String())
+ if runMigrations != "" {
+ applyMigrations = true
+ }
+
+ err := database.InitDB("DatabaseManager", applyMigrations, "W")
+
+ if err != nil {
+ log.Fatalf("Cannot initialize DB: %s", err)
+ }
+
rand.Seed(time.Now().UnixNano())
}
diff --git a/cmd/migrator/migrator.go b/cmd/migrator/migrator.go
index 3b9961a..278cc2b 100755
--- a/cmd/migrator/migrator.go
+++ b/cmd/migrator/migrator.go
@@ -17,13 +17,12 @@ func main() {
automigrate := flag.Bool("auto", false, "Run auto-migration")
apply := flag.String("apply", "", "Apply a specific migration file")
deleteFlag := flag.Bool("delete", false, "Delete existing tables before migration")
- configDir := flag.String("configDir", "", "Optional directory for configuration files")
flag.Parse()
utils.LogStep("Connecting to the database...")
- err := database.InitDB("Migrator")
+ err := database.InitDB("Migrator", *automigrate)
if err != nil {
utils.LogFatal("Could not initialise DB: ", err)
}
@@ -40,16 +39,13 @@ func main() {
}
switch {
- case *automigrate:
- if err := database.DBAutoMigration(database.Client, *configDir); err != nil {
- utils.LogFatal("Failed to run auto-migration", err)
- }
-
case *apply != "":
if err := database.ApplyMigration(database.Client, *apply); err != nil {
utils.LogFatal("Failed to apply migration", err)
}
-
+ case *automigrate:
+ // Just log a message as the migrations are handled by InitDB
+ utils.LogInfo("Initialized DB with auto-migrations")
default:
flag.Usage()
os.Exit(1)
diff --git a/pkg/advans/connect.go b/pkg/advans/connect.go
index cbcfdb3..f65c59c 100644
--- a/pkg/advans/connect.go
+++ b/pkg/advans/connect.go
@@ -4,6 +4,7 @@ import (
"bytes"
"encoding/json"
"fmt"
+ "log"
"net/http"
"github.pie.apple.com/kerosene/Core/pkg/config"
@@ -28,7 +29,21 @@ type Payload struct {
var cfg = config.New()
func init() {
- database.InitDB("IngestionPipeline", "W")
+ applyMigrations := false
+
+ runMigrations, _ := cfg.Load(config.RunMigration.String())
+ if runMigrations != "" {
+ applyMigrations = true
+ }
+
+ err := database.InitDB("IngestionPipeline", applyMigrations, "W")
+
+ if err != nil {
+ log.Fatalf("Cannot initialize DB: %s", err)
+ }
+
+ // TODO check errors!
+ // https://github.pie.apple.com/Kerosene/Core/issues/51
config.SetSecret([]string{"ADVANS_MTLS_PRIVATE_KEY", "ADVANS_MTLS_CERT", "ADVANS_API_END_POINT"})
}
@@ -42,6 +57,7 @@ func initMTLSClient(certFilePath, keyFilePath string) (*http.Client, error) {
func sendPayload(client *http.Client, advansEndpoint string, path string, payload Payload) error {
payloadBytes, err := json.Marshal(payload)
+
if err != nil {
return fmt.Errorf("error marshalling payload: %s", err)
}
@@ -63,88 +79,3 @@ func sendPayload(client *http.Client, advansEndpoint string, path string, payloa
return nil
}
diff --git a/pkg/config/main.go b/pkg/config/main.go
index ec14c10..e80c2c2 100644
--- a/pkg/config/main.go
+++ b/pkg/config/main.go
@@ -16,6 +16,7 @@ import (
var (
errSecretNotFound = errors.New("secret not found")
errSecretDuplicateSource = errors.New("tried to retrieve secret from multiple sources")
+ errDataConfigDirNotExists = errors.New("data config directory does not exist")
errSecretNotSet = errors.New("secret not set in any provider or as an environment variable")
errUnknownProvider = errors.New("unknown provider when retrieving secret")
errWhisperAuthDirNotSet = errors.New("whisper authentication file directory not specified")
@@ -28,8 +29,12 @@ var (
errWhisperRetrieveContent = errors.New("could not retrieve whisper secret's content")
)
-type configProvider string
-type configKey string
+const defaultDataConfigDir = "Config/"
+
+type (
+ configProvider string
+ configKey string
+)
func (cg configKey) String() string {
return string(cg)
@@ -40,17 +45,18 @@ func (cp configProvider) String() string {
}
const (
- whisperProvider configProvider = "WHISPER"
- PSQL configKey = "PSQL"
- PSQLRead configKey = "PSQL_READ"
- TemporalAddress configKey = "TEMPORAL_ADDRESS"
- KafkaBrokers configKey = "KAFKA_BROKERS"
- RedisURI configKey = "REDIS"
- KeroseneEnvironment configKey = "ENVIRONMENT"
- ElasticSearchURI configKey = "ELASTICSEARCH_URL"
- Proxy configKey = "PROXY"
- Queue configKey = "QUEUE"
- RunMigration configKey = "RUN_AUTOMIGRATION"
+ whisperProvider configProvider = "WHISPER"
+ PSQL configKey = "PSQL"
+ PSQLRead configKey = "PSQL_READ"
+ TemporalAddress configKey = "TEMPORAL_ADDRESS"
+ KafkaBrokers configKey = "KAFKA_BROKERS"
+ RedisURI configKey = "REDIS"
+ KeroseneEnvironment configKey = "ENVIRONMENT"
+ ElasticSearchURI configKey = "ELASTICSEARCH_URL"
+ Proxy configKey = "PROXY"
+ Queue configKey = "QUEUE"
+ RunMigration configKey = "RUN_AUTOMIGRATION"
+ DefaultDataConfigDir configKey = "DEFAULT_DATA_CONFIG_DIR"
)
var providers = []configProvider{
@@ -353,3 +359,23 @@ func (c *Config) Load(name string) (string, error) {
return val, nil
}
+
+func (c *Config) GetDataConfigDir() (string, error) {
+ retDir := defaultDataConfigDir
+
+ overriddenConfigDir, _ := c.Load(DefaultDataConfigDir.String())
+ if overriddenConfigDir != "" {
+ retDir = overriddenConfigDir
+ }
+
+ // check appended /
+ if len(retDir) > 0 && retDir[len(retDir)-1] != '/' {
+ retDir += "/"
+ }
+
+ if _, err := os.Stat(retDir); os.IsNotExist(err) {
+ return "", errDataConfigDirNotExists
+ }
+
+ return retDir, nil
+}
diff --git a/pkg/database/client.go b/pkg/database/client.go
index 6fc4861..444bb5a 100644
--- a/pkg/database/client.go
+++ b/pkg/database/client.go
@@ -19,17 +19,27 @@ import (
"gorm.io/plugin/dbresolver"
)
-var Client *gorm.DB
-var ErrNilDatabaseClient = errors.New("database client is nil after setup")
+var (
+ Client *gorm.DB
+ errNilDatabaseClient = errors.New("database client is nil after setup")
+)
+
+func determineApplyMigration(cfg *config.Config, automigrate bool) bool {
+ overrideAutomigrate, _ := cfg.Load(config.RunMigration.String())
+
+ // the env var takes precedence!
+ return overrideAutomigrate != "" || automigrate
+}
// InitDB initializes the database with the given application name and optional dbpath for SQLite.
-func InitDB(appName string, dbpath ...string) error {
+func InitDB(appName string, automigrate bool, dbpath ...string) error {
var (
psqlReadReplica string
err error
)
cfg := config.New()
+ applyMigration := determineApplyMigration(&cfg, automigrate)
// Set up a new logger with your required configuration.
newLogger := logger.New(
@@ -87,12 +97,12 @@ func InitDB(appName string, dbpath ...string) error {
if Client == nil {
log.Println("database client is nil after setup, aborting")
- return ErrNilDatabaseClient
+ return errNilDatabaseClient
}
// Run auto-migration if using SQLite
- if isSQLiteDB(Client) {
- if err := DBAutoMigration(Client, ""); err != nil {
+ if applyMigration && isSQLiteDB(Client) {
+ if err := DBAutoMigration(Client); err != nil {
log.Printf("failed to run SQLite auto-migration: %v", err)
return err
@@ -105,6 +115,7 @@ func InitDB(appName string, dbpath ...string) error {
// setupPostgres sets up a PostgreSQL database connection.
func setupPostgres(newLogger logger.Interface, appName, dbURI string) (*gorm.DB, error) {
dbURI = appendApplicationName(dbURI, appName)
+
return gorm.Open(postgres.Open(dbURI), &gorm.Config{
Logger: newLogger,
})
@@ -132,19 +143,19 @@ func setupPostgresWithReplicas(newLogger logger.Interface, appName, sourceURI, r
// setupSQLite sets up a SQLite database connection.
func setupSQLite(newLogger logger.Interface, dbpath ...string) (*gorm.DB, error) {
- var sqliteFile string
+ sqliteFile := "file::memory:?cache=shared"
+
if len(dbpath) > 0 {
sqliteFile = dbpath[0]
+
// Ensure the SQLite file ends with .db
if filepath.Ext(sqliteFile) != ".db" {
return nil, errors.New("SQLite file must have a .db extension")
}
- } else {
- // Default is to use an in-memory db
- sqliteFile = "file::memory:?cache=shared"
}
log.Printf("Using SQLite: %s", sqliteFile)
+
return gorm.Open(sqlite.Open(sqliteFile), &gorm.Config{
Logger: newLogger,
})
diff --git a/pkg/database/migrations.go b/pkg/database/migrations.go
index 6582b99..3210216 100644
--- a/pkg/database/migrations.go
+++ b/pkg/database/migrations.go
@@ -27,6 +27,7 @@ func ApplyMigration(dbClient *gorm.DB, filename string) error {
}
utils.LogSuccess("Migration applied successfully.")
+
return nil
}
@@ -47,22 +48,26 @@ func GeneratePassword() (string, error) {
return cfg.Load("REDASH_PASSWORD")
}
-func DBAutoMigration(dbClient *gorm.DB, configDir string) error {
+func DBAutoMigration(dbClient *gorm.DB) error {
utils.LogStep("Running auto-migration...")
dbDialect := dbClient.Dialector.Name()
+
if dbDialect == "postgres" {
if err := PSQLStart(); err != nil {
return fmt.Errorf("failed to initialize database: %v", err)
}
+
utils.LogSuccess("Database initialization completed successfully.")
utils.LogStep("Generating Redash password...")
+
password, err := GeneratePassword()
if err != nil {
return fmt.Errorf("failed to generate password: %v", err)
}
utils.LogStep("Creating Redash user...")
+
if err := CreateUserRedash(password); err != nil {
utils.LogWarning("Failed to create Redash user", err)
} else {
@@ -72,11 +77,7 @@ func DBAutoMigration(dbClient *gorm.DB, configDir string) error {
ReflectionMigration()
}
- if configDir != "" {
- initialise.GenerateInitData(dbClient, configDir)
- } else {
- initialise.GenerateInitData(dbClient)
- }
+ initialise.GenerateInitData(dbClient)
return nil
}
@@ -91,12 +92,15 @@ func RunAutoMigration() error {
dbTablesValue := reflect.ValueOf(structs.DatabaseTables)
for i := 0; i < dbTablesValue.NumField(); i++ {
field := dbTablesValue.Field(i)
+
if field.Kind() == reflect.Ptr && !field.IsNil() {
structValue := field.Elem()
+
if err := Client.AutoMigrate(structValue.Addr().Interface()); err != nil {
return fmt.Errorf("failed to auto-migrate model %v: %v", structValue.Type().Name(), err)
}
}
}
+
return nil
}
diff --git a/pkg/initialise/configInit.go b/pkg/initialise/configInit.go
index d087531..323a3d7 100644
--- a/pkg/initialise/configInit.go
+++ b/pkg/initialise/configInit.go
@@ -9,6 +9,7 @@ import (
"reflect"
"strings"
+ "github.pie.apple.com/kerosene/Core/pkg/config"
"github.pie.apple.com/kerosene/Core/pkg/utils"
"github.pie.apple.com/kerosene/Core/structs"
"gopkg.in/yaml.v2"
@@ -16,30 +17,28 @@ import (
"gorm.io/gorm"
)
-//go:embed config
-var configFiles embed.FS
+var (
+ //go:embed config
+ configFiles embed.FS
+ cfg config.Config = config.New()
+)
func GenerateInitData(database *gorm.DB, additionalConfigDir ...string) {
utils.LogHeader("INITIALISING DEFAULT DATA")
- configDir := "config"
- if len(additionalConfigDir) > 0 {
- configDir = additionalConfigDir[0]
-
- utils.LogInfo(fmt.Sprintf("Using additional config directory: %s", configDir))
+ configDir, err := cfg.GetDataConfigDir()
- if _, err := os.Stat(configDir); os.IsNotExist(err) {
- utils.LogWarning(fmt.Sprintf("Config directory does not exist: %s", configDir), err)
- }
- } else {
- utils.LogInfo("Using Default Config")
+ if err != nil {
+ utils.LogFatal("Cannot open config directory: %w", err)
}
+ utils.LogInfo("Using data config directory: " + configDir)
+
// Get the list of directories in the embedded config directory
- embeddedConfigDirs := getConfigDirs("config")
+ embeddedConfigDirs := getConfigDirs(configDir)
// Process the embedded configs
- processConfigs(database, embeddedConfigDirs, "config/")
+ processConfigs(database, embeddedConfigDirs, configDir)
// If an additional config directory is provided, process it after the embedded configs
if len(additionalConfigDir) > 0 {
@@ -47,7 +46,7 @@ func GenerateInitData(database *gorm.DB, additionalConfigDir ...string) {
utils.LogHeader("ADDITIONAL DATA")
- processConfigs(database, additionalConfigDirs, configDir+"/")
+ processConfigs(database, additionalConfigDirs, configDir)
}
utils.LogInfo(fmt.Sprintf("Initialised Default Data for environment '%s'.", os.Getenv("ENVIRONMENT")))
@@ -113,9 +112,11 @@ func insertPrioritizedStruct(db *gorm.DB, configDirs []string, configPath, struc
}
func getConfigDirs(path string) []string {
- var configDirs []string
- var entries []os.DirEntry
- var err error
+ var (
+ configDirs []string
+ entries []os.DirEntry
+ err error
+ )
// Check if the path is the default embedded directory or an external path
if path == "config" {
### OUTPUT: ### OUTPUT: