peperunas/nixos
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update roles/home/aichat.nix

Browse Source
This commit is contained in:
Giulio De Pasquale 2024-07-03 10:13:02 +01:00
parent abb0002c4c
commit 29692c5f29
1 changed files with 481 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

512
roles/home/aichat.nix
View File

@ -214,54 +214,498 @@ roles = ''
### INPUT: ### INPUT:
``` diff --git a/cmd/api/main.go b/cmd/api/main.go
diff --git a/src/app.js b/src/app.js index 8b974c8..9dfd24b 100644
index 83d2e7a..b6a1c3f 100644 --- a/cmd/api/main.go
--- a/src/app.js +++ b/cmd/api/main.go
+++ b/src/app.js @@ -33,6 +33,8 @@ func loadAppConfig() *AppConfig {
@@ -10,6 +10,10 @@ function initialize() {
setupEventListeners();
} }
+// TODO: add other listeners func main() {
+// https://github.com/user/project/issue/123 + applyMigrations := false
+function setupEventListeners() { + cfg := config.New()
+ document.getElementById('submit').addEventListener('click', handleSubmit); err := config.SetSecret([]string{
+ document.getElementById('reset').addEventListener('click', handleReset); "TEMPORAL_ADDRESS",
"PSQL",
@@ -42,13 +44,23 @@ func main() {
"REDASH_API_KEY",
"INFOBLOX_SECRET",
})
+
if err != nil {
log.Printf("Could not set secrets: %v", err)
os.Exit(1)
}
- database.InitDB("Core")
+ runMigrations, _ := cfg.Load(config.RunMigration.String())
+ if runMigrations != "" {
+ applyMigrations = true
+ } + }
+ +
function handleSubmit(event) { + err = database.InitDB("Core", applyMigrations)
event.preventDefault(); +
const data = new FormData(event.target); + if err != nil {
@@ -20,6 +24,10 @@ function handleSubmit(event) { + log.Fatalf("Cannot initialize DB: %v", err)
+ }
console.log('Form submitted:', data); appConfig := loadAppConfig()
diff --git a/cmd/fixtures/fixtures.go b/cmd/fixtures/fixtures.go
index 3d23a6d..ff3290e 100644
--- a/cmd/fixtures/fixtures.go
+++ b/cmd/fixtures/fixtures.go
@@ -10,13 +10,27 @@ import (
"github.com/go-faker/faker/v4"
"github.com/lib/pq"
+ "github.pie.apple.com/kerosene/Core/pkg/config"
"github.pie.apple.com/kerosene/Core/pkg/database"
"github.pie.apple.com/kerosene/Core/structs"
)
// Initializing worker, database, producer and redis when the program starts.
func init() {
- database.InitDB("DatabaseManager", "W")
+ applyMigrations := false
+ cfg := config.New()
+
+ runMigrations, _ := cfg.Load(config.RunMigration.String())
+ if runMigrations != "" {
+ applyMigrations = true
+ }
+
+ err := database.InitDB("DatabaseManager", applyMigrations, "W")
+
+ if err != nil {
+ log.Fatalf("Cannot initialize DB: %s", err)
+ }
+
rand.Seed(time.Now().UnixNano())
} }
+function handleReset(event) { diff --git a/cmd/migrator/migrator.go b/cmd/migrator/migrator.go
+ event.preventDefault(); index 3b9961a..278cc2b 100755
+ event.target.form.reset(); --- a/cmd/migrator/migrator.go
+ console.log('Form reset'); +++ b/cmd/migrator/migrator.go
@@ -17,13 +17,12 @@ func main() {
automigrate := flag.Bool("auto", false, "Run auto-migration")
apply := flag.String("apply", "", "Apply a specific migration file")
deleteFlag := flag.Bool("delete", false, "Delete existing tables before migration")
- configDir := flag.String("configDir", "", "Optional directory for configuration files")
flag.Parse()
utils.LogStep("Connecting to the database...")
- err := database.InitDB("Migrator")
+ err := database.InitDB("Migrator", *automigrate)
if err != nil {
utils.LogFatal("Could not initialise DB: ", err)
} }
``` @@ -40,16 +39,13 @@ func main() {
}
switch {
- case *automigrate:
- if err := database.DBAutoMigration(database.Client, *configDir); err != nil {
- utils.LogFatal("Failed to run auto-migration", err)
- }
-
case *apply != "":
if err := database.ApplyMigration(database.Client, *apply); err != nil {
utils.LogFatal("Failed to apply migration", err)
}
-
+ case *automigrate:
+ // Just log a message as the migrations are handled by InitDB
+ utils.LogInfo("Initialized DB with auto-migrations")
default:
flag.Usage()
os.Exit(1)
diff --git a/pkg/advans/connect.go b/pkg/advans/connect.go
index cbcfdb3..f65c59c 100644
--- a/pkg/advans/connect.go
+++ b/pkg/advans/connect.go
@@ -4,6 +4,7 @@ import (
"bytes"
"encoding/json"
"fmt"
+ "log"
"net/http"
"github.pie.apple.com/kerosene/Core/pkg/config"
@@ -28,7 +29,21 @@ type Payload struct {
var cfg = config.New()
func init() {
- database.InitDB("IngestionPipeline", "W")
+ applyMigrations := false
+
+ runMigrations, _ := cfg.Load(config.RunMigration.String())
+ if runMigrations != "" {
+ applyMigrations = true
+ }
+
+ err := database.InitDB("IngestionPipeline", applyMigrations, "W")
+
+ if err != nil {
+ log.Fatalf("Cannot initialize DB: %s", err)
+ }
+
+ // TODO check errors!
+ // https://github.pie.apple.com/Kerosene/Core/issues/51
config.SetSecret([]string{"ADVANS_MTLS_PRIVATE_KEY", "ADVANS_MTLS_CERT", "ADVANS_API_END_POINT"})
}
@@ -42,6 +57,7 @@ func initMTLSClient(certFilePath, keyFilePath string) (*http.Client, error) {
func sendPayload(client *http.Client, advansEndpoint string, path string, payload Payload) error {
payloadBytes, err := json.Marshal(payload)
+
if err != nil {
return fmt.Errorf("error marshalling payload: %s", err)
}
@@ -63,88 +79,3 @@ func sendPayload(client *http.Client, advansEndpoint string, path string, payloa
return nil
}
diff --git a/pkg/config/main.go b/pkg/config/main.go
index ec14c10..e80c2c2 100644
--- a/pkg/config/main.go
+++ b/pkg/config/main.go
@@ -16,6 +16,7 @@ import (
var (
errSecretNotFound = errors.New("secret not found")
errSecretDuplicateSource = errors.New("tried to retrieve secret from multiple sources")
+ errDataConfigDirNotExists = errors.New("data config directory does not exist")
errSecretNotSet = errors.New("secret not set in any provider or as an environment variable")
errUnknownProvider = errors.New("unknown provider when retrieving secret")
errWhisperAuthDirNotSet = errors.New("whisper authentication file directory not specified")
@@ -28,8 +29,12 @@ var (
errWhisperRetrieveContent = errors.New("could not retrieve whisper secret's content")
)
-type configProvider string
-type configKey string
+const defaultDataConfigDir = "Config/"
+
+type (
+ configProvider string
+ configKey string
+)
func (cg configKey) String() string {
return string(cg)
@@ -40,17 +45,18 @@ func (cp configProvider) String() string {
}
const (
- whisperProvider configProvider = "WHISPER"
- PSQL configKey = "PSQL"
- PSQLRead configKey = "PSQL_READ"
- TemporalAddress configKey = "TEMPORAL_ADDRESS"
- KafkaBrokers configKey = "KAFKA_BROKERS"
- RedisURI configKey = "REDIS"
- KeroseneEnvironment configKey = "ENVIRONMENT"
- ElasticSearchURI configKey = "ELASTICSEARCH_URL"
- Proxy configKey = "PROXY"
- Queue configKey = "QUEUE"
- RunMigration configKey = "RUN_AUTOMIGRATION"
+ whisperProvider configProvider = "WHISPER"
+ PSQL configKey = "PSQL"
+ PSQLRead configKey = "PSQL_READ"
+ TemporalAddress configKey = "TEMPORAL_ADDRESS"
+ KafkaBrokers configKey = "KAFKA_BROKERS"
+ RedisURI configKey = "REDIS"
+ KeroseneEnvironment configKey = "ENVIRONMENT"
+ ElasticSearchURI configKey = "ELASTICSEARCH_URL"
+ Proxy configKey = "PROXY"
+ Queue configKey = "QUEUE"
+ RunMigration configKey = "RUN_AUTOMIGRATION"
+ DefaultDataConfigDir configKey = "DEFAULT_DATA_CONFIG_DIR"
)
var providers = []configProvider{
@@ -353,3 +359,23 @@ func (c *Config) Load(name string) (string, error) {
return val, nil
}
+
+func (c *Config) GetDataConfigDir() (string, error) {
+ retDir := defaultDataConfigDir
+
+ overriddenConfigDir, _ := c.Load(DefaultDataConfigDir.String())
+ if overriddenConfigDir != "" {
+ retDir = overriddenConfigDir
+ }
+
+ // check appended /
+ if len(retDir) > 0 && retDir[len(retDir)-1] != '/' {
+ retDir += "/"
+ }
+
+ if _, err := os.Stat(retDir); os.IsNotExist(err) {
+ return "", errDataConfigDirNotExists
+ }
+
+ return retDir, nil
+}
diff --git a/pkg/database/client.go b/pkg/database/client.go
index 6fc4861..444bb5a 100644
--- a/pkg/database/client.go
+++ b/pkg/database/client.go
@@ -19,17 +19,27 @@ import (
"gorm.io/plugin/dbresolver"
)
-var Client *gorm.DB
-var ErrNilDatabaseClient = errors.New("database client is nil after setup")
+var (
+ Client *gorm.DB
+ errNilDatabaseClient = errors.New("database client is nil after setup")
+)
+
+func determineApplyMigration(cfg *config.Config, automigrate bool) bool {
+ overrideAutomigrate, _ := cfg.Load(config.RunMigration.String())
+
+ // the env var takes precedence!
+ return overrideAutomigrate != "" || automigrate
+}
// InitDB initializes the database with the given application name and optional dbpath for SQLite.
-func InitDB(appName string, dbpath ...string) error {
+func InitDB(appName string, automigrate bool, dbpath ...string) error {
var (
psqlReadReplica string
err error
)
cfg := config.New()
+ applyMigration := determineApplyMigration(&cfg, automigrate)
// Set up a new logger with your required configuration.
newLogger := logger.New(
@@ -87,12 +97,12 @@ func InitDB(appName string, dbpath ...string) error {
if Client == nil {
log.Println("database client is nil after setup, aborting")
- return ErrNilDatabaseClient
+ return errNilDatabaseClient
}
// Run auto-migration if using SQLite
- if isSQLiteDB(Client) {
- if err := DBAutoMigration(Client, ""); err != nil {
+ if applyMigration && isSQLiteDB(Client) {
+ if err := DBAutoMigration(Client); err != nil {
log.Printf("failed to run SQLite auto-migration: %v", err)
return err
@@ -105,6 +115,7 @@ func InitDB(appName string, dbpath ...string) error {
// setupPostgres sets up a PostgreSQL database connection.
func setupPostgres(newLogger logger.Interface, appName, dbURI string) (*gorm.DB, error) {
dbURI = appendApplicationName(dbURI, appName)
+
return gorm.Open(postgres.Open(dbURI), &gorm.Config{
Logger: newLogger,
})
@@ -132,19 +143,19 @@ func setupPostgresWithReplicas(newLogger logger.Interface, appName, sourceURI, r
// setupSQLite sets up a SQLite database connection.
func setupSQLite(newLogger logger.Interface, dbpath ...string) (*gorm.DB, error) {
- var sqliteFile string
+ sqliteFile := "file::memory:?cache=shared"
+
if len(dbpath) > 0 {
sqliteFile = dbpath[0]
+
// Ensure the SQLite file ends with .db
if filepath.Ext(sqliteFile) != ".db" {
return nil, errors.New("SQLite file must have a .db extension")
}
- } else {
- // Default is to use an in-memory db
- sqliteFile = "file::memory:?cache=shared"
}
log.Printf("Using SQLite: %s", sqliteFile)
+
return gorm.Open(sqlite.Open(sqliteFile), &gorm.Config{
Logger: newLogger,
})
diff --git a/pkg/database/migrations.go b/pkg/database/migrations.go
index 6582b99..3210216 100644
--- a/pkg/database/migrations.go
+++ b/pkg/database/migrations.go
@@ -27,6 +27,7 @@ func ApplyMigration(dbClient *gorm.DB, filename string) error {
}
utils.LogSuccess("Migration applied successfully.")
+
return nil
}
@@ -47,22 +48,26 @@ func GeneratePassword() (string, error) {
return cfg.Load("REDASH_PASSWORD")
}
-func DBAutoMigration(dbClient *gorm.DB, configDir string) error {
+func DBAutoMigration(dbClient *gorm.DB) error {
utils.LogStep("Running auto-migration...")
dbDialect := dbClient.Dialector.Name()
+
if dbDialect == "postgres" {
if err := PSQLStart(); err != nil {
return fmt.Errorf("failed to initialize database: %v", err)
}
+
utils.LogSuccess("Database initialization completed successfully.")
utils.LogStep("Generating Redash password...")
+
password, err := GeneratePassword()
if err != nil {
return fmt.Errorf("failed to generate password: %v", err)
}
utils.LogStep("Creating Redash user...")
+
if err := CreateUserRedash(password); err != nil {
utils.LogWarning("Failed to create Redash user", err)
} else {
@@ -72,11 +77,7 @@ func DBAutoMigration(dbClient *gorm.DB, configDir string) error {
ReflectionMigration()
}
- if configDir != "" {
- initialise.GenerateInitData(dbClient, configDir)
- } else {
- initialise.GenerateInitData(dbClient)
- }
+ initialise.GenerateInitData(dbClient)
return nil
}
@@ -91,12 +92,15 @@ func RunAutoMigration() error {
dbTablesValue := reflect.ValueOf(structs.DatabaseTables)
for i := 0; i < dbTablesValue.NumField(); i++ {
field := dbTablesValue.Field(i)
+
if field.Kind() == reflect.Ptr && !field.IsNil() {
structValue := field.Elem()
+
if err := Client.AutoMigrate(structValue.Addr().Interface()); err != nil {
return fmt.Errorf("failed to auto-migrate model %v: %v", structValue.Type().Name(), err)
}
}
}
+
return nil
}
diff --git a/pkg/initialise/configInit.go b/pkg/initialise/configInit.go
index d087531..323a3d7 100644
--- a/pkg/initialise/configInit.go
+++ b/pkg/initialise/configInit.go
@@ -9,6 +9,7 @@ import (
"reflect"
"strings"
+ "github.pie.apple.com/kerosene/Core/pkg/config"
"github.pie.apple.com/kerosene/Core/pkg/utils"
"github.pie.apple.com/kerosene/Core/structs"
"gopkg.in/yaml.v2"
@@ -16,30 +17,28 @@ import (
"gorm.io/gorm"
)
-//go:embed config
-var configFiles embed.FS
+var (
+ //go:embed config
+ configFiles embed.FS
+ cfg config.Config = config.New()
+)
func GenerateInitData(database *gorm.DB, additionalConfigDir ...string) {
utils.LogHeader("INITIALISING DEFAULT DATA")
- configDir := "config"
- if len(additionalConfigDir) > 0 {
- configDir = additionalConfigDir[0]
-
- utils.LogInfo(fmt.Sprintf("Using additional config directory: %s", configDir))
+ configDir, err := cfg.GetDataConfigDir()
- if _, err := os.Stat(configDir); os.IsNotExist(err) {
- utils.LogWarning(fmt.Sprintf("Config directory does not exist: %s", configDir), err)
- }
- } else {
- utils.LogInfo("Using Default Config")
+ if err != nil {
+ utils.LogFatal("Cannot open config directory: %w", err)
}
+ utils.LogInfo("Using data config directory: " + configDir)
+
// Get the list of directories in the embedded config directory
- embeddedConfigDirs := getConfigDirs("config")
+ embeddedConfigDirs := getConfigDirs(configDir)
// Process the embedded configs
- processConfigs(database, embeddedConfigDirs, "config/")
+ processConfigs(database, embeddedConfigDirs, configDir)
// If an additional config directory is provided, process it after the embedded configs
if len(additionalConfigDir) > 0 {
@@ -47,7 +46,7 @@ func GenerateInitData(database *gorm.DB, additionalConfigDir ...string) {
utils.LogHeader("ADDITIONAL DATA")
- processConfigs(database, additionalConfigDirs, configDir+"/")
+ processConfigs(database, additionalConfigDirs, configDir)
}
utils.LogInfo(fmt.Sprintf("Initialised Default Data for environment '%s'.", os.Getenv("ENVIRONMENT")))
@@ -113,9 +112,11 @@ func insertPrioritizedStruct(db *gorm.DB, configDirs []string, configPath, struc
}
func getConfigDirs(path string) []string {
- var configDirs []string
- var entries []os.DirEntry
- var err error
+ var (
+ configDirs []string
+ entries []os.DirEntry
+ err error
+ )
// Check if the path is the default embedded directory or an external path
if path == "config" {
### OUTPUT: ### OUTPUT:
**Title:** feat(app): add event listeners for submit and reset buttons feat(config): add new env vars and refactor database initialization
**Summary:** This PR introduces new environment variables and improves our database initialization and migration process:
This PR adds event listeners for the submit and reset buttons in the application. It introduces a new function to set up these event listeners and implements a handler for the reset button.
**Details of Changes:** - Add new environment variables:
1. **src/app.js:** 1. `RUN_AUTOMIGRATION`: to control auto-migration
- Added a new function `setupEventListeners` to attach click event listeners to the submit and reset buttons. 2. `DEFAULT_DATA_CONFIG_DIR`: to specify the data configuration directory
- Implemented the `handleReset` function to reset the form and log the reset action. - Implement conditional auto-migration support based on `RUN_AUTOMIGRATION` or function parameters
- Included a TODO comment to add other listeners, referencing issue [#123](https://github.com/user/project/issue/123). - Refactor database client initialization for better error handling and flexibility
- Streamline the migration process in the main application and various commands
**Impact:** Key changes:
- **Backward Compatibility:** The changes are backward compatible. 1. Introduce `RUN_AUTOMIGRATION` and `DEFAULT_DATA_CONFIG_DIR` environment variables
- **Performance Implications:** Minimal performance impact due to the addition of event listeners. 2. Add `applyMigrations` flag in database initialization
- **New Dependencies:** No new dependencies introduced. 3. Implement `GetDataConfigDir()` method to handle default and overridden config directories
4. Refactor `InitDB()` to accept `automigrate` parameter
5. Update `cmd/api/main.go`, `cmd/fixtures/fixtures.go`, and `pkg/advans/connect.go` to use new initialization process
6. Simplify `cmd/migrator/migrator.go` to leverage the new auto-migration feature
7. Improve error handling and logging throughout the affected files
''; '';
in in
{ {