103 lines
2.7 KiB
Go
103 lines
2.7 KiB
Go
package scaler
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
|
|
"git.stinnesbeck.com/go/jynx/backend/hetzner"
|
|
"git.stinnesbeck.com/go/jynx/ci/woodpecker"
|
|
"git.stinnesbeck.com/go/log"
|
|
)
|
|
|
|
func Start(w *woodpecker.Server) error {
|
|
stats, err := w.GetStats()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// get the prefix for jynx managed servers
|
|
prefix := os.Getenv("HETZNER_CLOUD_PREFIX")
|
|
|
|
// set default prefix if none was provided
|
|
if prefix == "" {
|
|
prefix = "jynx"
|
|
}
|
|
|
|
// get a list of all servers from hetzner (from jynx)
|
|
servers, err := hetzner.ListJynxServers(prefix)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
newServers := stats.PendingJobs - len(servers) + stats.RunningJobs
|
|
startingUp := len(servers) - stats.RunningJobs - stats.IdlingWorkers
|
|
// check if the result is negative, if so set starting servers as 0
|
|
// happens when servers are queued for deletion and api is polled in the
|
|
// meantime
|
|
if startingUp < 0 {
|
|
startingUp = 0
|
|
}
|
|
|
|
log.PrintInfo(stats.RunningJobs, "running jobs,", stats.PendingJobs, "pending jobs,", stats.IdlingWorkers, "idling workers,", startingUp, "servers starting up")
|
|
|
|
// check if there are servers to be removed
|
|
if stats.PendingJobs == 0 && stats.RunningJobs == 0 {
|
|
// we need to remove servers
|
|
if err := hetzner.CheckServerDeletions(servers); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// check if we need to create new servers
|
|
// if stats.PendingJobs < 1 || len(servers)-stats.RunningJobs >= stats.PendingJobs {
|
|
if stats.PendingJobs < 1 || newServers < 1 {
|
|
// if stats.PendingJobs == 0 || stats.IdlingWorkers+stats.RunningJobs >= stats.PendingJobs {
|
|
// skip rest of this run
|
|
return nil
|
|
}
|
|
|
|
if maxServersString := os.Getenv("HETZNER_SERVER_MAX"); maxServersString != "" {
|
|
maxServers, err := strconv.Atoi(maxServersString)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// check if the maximum amount of servers is reached if one is provided
|
|
if newServers+len(servers) > maxServers && maxServers != 0 {
|
|
newServers = maxServers - len(servers)
|
|
}
|
|
}
|
|
|
|
// if there are no servers to be created, exit here
|
|
if newServers < 1 {
|
|
return nil
|
|
}
|
|
|
|
// slice to hold servers to be created
|
|
var ScaleUpServers []string
|
|
|
|
// there are servers to be created
|
|
// get the suffix for the starting server
|
|
suffix, err := hetzner.GetNewSuffix(servers)
|
|
for i := 0; i < newServers; i++ {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// assemble server name and add it to the list of servers to scale up
|
|
serverName := fmt.Sprintf("%s-worker-%d", prefix, suffix)
|
|
ScaleUpServers = append(ScaleUpServers, serverName)
|
|
|
|
// add 1 to the suffix for the next server
|
|
suffix++
|
|
}
|
|
|
|
log.PrintInfo("starting", newServers, "workers:", ScaleUpServers)
|
|
|
|
// create servers on hetzner cloud
|
|
if err := hetzner.CreateServers(ScaleUpServers); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|