diff --git a/graphql/service/config/mutation_utils.go b/graphql/service/config/mutation_utils.go index 7a3c33f..fd61554 100644 --- a/graphql/service/config/mutation_utils.go +++ b/graphql/service/config/mutation_utils.go @@ -9,11 +9,13 @@ import ( "context" "errors" "fmt" + "os" "reflect" "regexp" "sort" "strings" "sync" + "time" "github.com/daeuniverse/dae-wing/common" "github.com/daeuniverse/dae-wing/dae" @@ -258,6 +260,15 @@ func Rename(ctx context.Context, _id graphql.ID, name string) (n int32, err erro var runLock sync.Mutex +const reloadWaitTimeout = 75 * time.Second + +func restartAfterReloadTimeout() { + go func() { + time.Sleep(time.Second) + os.Exit(1) + }() +} + func Run(d *gorm.DB, noLoad bool) (n int32, err error) { if ok := runLock.TryLock(); !ok { return 0, fmt.Errorf("the last request didn't complete; make a cup of tea and take a break") @@ -455,13 +466,24 @@ func Run(d *gorm.DB, noLoad bool) (n int32, err error) { /// Reload with current config. chReloadCallback := make(chan error) - dae.ChReloadConfigs <- &dae.ReloadMessage{ + reloadMsg := &dae.ReloadMessage{ Config: c, Callback: chReloadCallback, } - errReload := <-chReloadCallback - if errReload != nil { - return 0, fmt.Errorf("failed to load new config: %w; see more in log", errReload) + select { + case dae.ChReloadConfigs <- reloadMsg: + case <-time.After(reloadWaitTimeout): + restartAfterReloadTimeout() + return 0, fmt.Errorf("failed to submit reload within %s; restarting daed to recover runtime", reloadWaitTimeout) + } + select { + case errReload := <-chReloadCallback: + if errReload != nil { + return 0, fmt.Errorf("failed to load new config: %w; see more in log", errReload) + } + case <-time.After(reloadWaitTimeout): + restartAfterReloadTimeout() + return 0, fmt.Errorf("reload timed out after %s; restarting daed to recover runtime", reloadWaitTimeout) } // Save running status