From 6f11eaa8916ab641bf6334909190b38d44af2de0 Mon Sep 17 00:00:00 2001 From: Nils Stinnesbeck Date: Sat, 12 Aug 2023 21:34:02 +0200 Subject: [PATCH] * added HETZNER_SERVER_IMAGE * added HETZNER_SERVER_LOCATION * added HETZNER_SERVER_MAX * added HETZNER_SERVER_MIN * added HETZNER_SERVER_TYPE * exposing SCALER_POLL_INTERVAL in Dockerfile * now logging error instead of quitting program * added support for newer woodpecker server version 1.0.x --- Dockerfile | 16 +++++++---- backend/hetzner/servers.go | 55 +++++++++++++++++++++++++++---------- ci/woodpecker/woodpecker.go | 6 ++-- main.go | 12 ++++---- scaler/scaler.go | 22 +++++++++++++-- 5 files changed, 80 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9714293..efa8025 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,9 +9,15 @@ COPY --from=builder /jynx/jynx /jynx COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ ENTRYPOINT ["/jynx"] -ENV HCLOUD_API_TOKEN="" -ENV WOODPECKER_HOSTNAME="" -ENV WOODPECKER_AGENT_SECRET="" -ENV HCLOUD_PREFIX="" -ENV WOODPECKER_PROMETHEUS_AUTH_TOKEN="" +ENV HETZNER_CLOUD_API_TOKEN="" +ENV HETZNER_CLOUD_PREFIX="" +ENV HETZNER_SERVER_IMAGE="" +ENV HETZNER_SERVER_LOCATION="" +ENV HETZNER_SERVER_MAX="" +ENV HETZNER_SERVER_MIN="" +ENV HETZNER_SERVER_TYPE="" ENV LOGLEVEL="" +ENV SCALER_POLL_INTERVAL="" +ENV WOODPECKER_AGENT_SECRET="" +ENV WOODPECKER_HOSTNAME="" +ENV WOODPECKER_PROMETHEUS_AUTH_TOKEN="" diff --git a/backend/hetzner/servers.go b/backend/hetzner/servers.go index c16552c..0acc12b 100644 --- a/backend/hetzner/servers.go +++ b/backend/hetzner/servers.go @@ -22,12 +22,7 @@ type VersionInfo struct { } func connect() *hcloud.Client { - token := os.Getenv("HCLOUD_API_TOKEN") - if token == "" { - log.PrintError("please specify HCLOUD_API_TOKEN") - os.Exit(1) - } - return hcloud.NewClient(hcloud.WithToken(token)) + return hcloud.NewClient(hcloud.WithToken(os.Getenv("HETZNER_CLOUD_API_TOKEN"))) } func GetNewSuffix(servers []*hcloud.Server) (int, error) { @@ -100,7 +95,7 @@ func CheckServerDeletions(servers []*hcloud.Server) error { pollIntervalSeconds %= 3600 } - for _, server := range servers { + for i, server := range servers { safeInterval := time.Duration(3600-pollIntervalSeconds) * time.Second optimalDeleteTime := server.Created.Add(safeInterval).Local() log.PrintDebug(server.Name, "optimal delete time:", optimalDeleteTime) @@ -108,6 +103,21 @@ func CheckServerDeletions(servers []*hcloud.Server) error { continue } + // check if HETZNER_SERVER_MIN was provided as an environment variable + if minServersString := os.Getenv("HETZNER_SERVER_MIN"); minServersString != "" { + minServers, err := strconv.Atoi(minServersString) + if err != nil { + return err + } + + // if the amount of servers is less than the minimal amount required + // leave this server running + if i < minServers { + log.PrintDebug("not deleting server", server.Name, "because HETZNER_SERVER_MIN is", minServers) + continue + } + } + // optimal delete time has passed, delete the server if err := DeleteServer(server); err != nil { return err @@ -199,16 +209,31 @@ func CreateServers(servers []string) error { return nil } + // get location and server type from environment + serverImage := os.Getenv("HETZNER_SERVER_IMAGE") + serverLocation := os.Getenv("HETZNER_SERVER_LOCATION") + serverType := os.Getenv("HETZNER_SERVER_TYPE") + + // set default values for servers if none were provided + switch "" { + case serverImage: + serverImage = "docker-ce" + case serverLocation: + serverLocation = "fsn1" + case serverType: + serverType = "cax11" + } + serverOpts := hcloud.ServerCreateOpts{ - Name: server, - StartAfterCreate: hcloud.Ptr(true), - SSHKeys: sshkeys, - Image: &hcloud.Image{Name: "docker-ce"}, - ServerType: &hcloud.ServerType{Name: "cax11"}, - Location: &hcloud.Location{Name: "fsn1"}, - UserData: userdata, - PublicNet: &hcloud.ServerCreatePublicNet{EnableIPv4: true}, + Image: &hcloud.Image{Name: serverImage}, Labels: labels, + Location: &hcloud.Location{Name: serverLocation}, + Name: server, + PublicNet: &hcloud.ServerCreatePublicNet{EnableIPv4: true}, + SSHKeys: sshkeys, + ServerType: &hcloud.ServerType{Name: serverType}, + StartAfterCreate: hcloud.Ptr(true), + UserData: userdata, } _, _, err = client.Server.Create(context.Background(), serverOpts) if err != nil { diff --git a/ci/woodpecker/woodpecker.go b/ci/woodpecker/woodpecker.go index 32215f0..79f3504 100644 --- a/ci/woodpecker/woodpecker.go +++ b/ci/woodpecker/woodpecker.go @@ -61,7 +61,7 @@ func (s Server) GetStats() (*Stats, error) { lines := strings.Split(string(b), "\n") - r := regexp.MustCompile(`^((woodpecker_)(worker_count|pending_jobs|running_jobs)) (\d+)`) + r := regexp.MustCompile(`^((woodpecker_)(worker_count|pending_jobs|pending_steps|running_jobs|running_steps)) (\d+)`) var stats Stats @@ -76,9 +76,9 @@ func (s Server) GetStats() (*Stats, error) { // assign stats to struct switch matches[3] { - case "pending_jobs": + case "pending_jobs", "pending_steps": stats.PendingJobs = parseStringToInt(matches[4]) - case "running_jobs": + case "running_jobs", "running_steps": stats.RunningJobs = parseStringToInt(matches[4]) case "worker_count": stats.IdlingWorkers = parseStringToInt(matches[4]) diff --git a/main.go b/main.go index 89f9820..670e970 100644 --- a/main.go +++ b/main.go @@ -35,13 +35,13 @@ func main() { // start scaler once if err := scaler.Start(&w); err != nil { - panic(err) + log.PrintError(err) } // start scaler every tick for range ticker { if err := scaler.Start(&w); err != nil { - panic(err) + log.PrintError(err) } } } @@ -50,18 +50,18 @@ func checkEnv() { switch "" { case os.Getenv("WOODPECKER_HOSTNAME"): log.PrintError("WOODPECKER_HOSTNAME not set") + os.Exit(1) case os.Getenv("WOODPECKER_AGENT_SECRET"): log.PrintError("WOODPECKER_AGENT_SECRET not set") os.Exit(1) - case os.Getenv("HCLOUD_PREFIX"): - log.PrintError("HCLOUD_PREFIX not set") - os.Exit(1) case os.Getenv("WOODPECKER_PROMETHEUS_AUTH_TOKEN"): log.PrintError("WOODPECKER_PROMETHEUS_AUTH_TOKEN not set") os.Exit(1) case os.Getenv("HETZNER_SSH_KEY_NAMES"): log.PrintError("HETZNER_SSH_KEY_NAMES not set") os.Exit(1) + case os.Getenv("HETZNER_CLOUD_API_TOKEN"): + log.PrintError("please specify HETZNER_CLOUD_API_TOKEN") + os.Exit(1) } - } diff --git a/scaler/scaler.go b/scaler/scaler.go index 2174832..2d5c676 100644 --- a/scaler/scaler.go +++ b/scaler/scaler.go @@ -3,6 +3,7 @@ package scaler import ( "fmt" "os" + "strconv" "git.stinnesbeck.com/go/jynx/backend/hetzner" "git.stinnesbeck.com/go/jynx/ci/woodpecker" @@ -16,7 +17,12 @@ func Start(w *woodpecker.Server) error { } // get the prefix for jynx managed servers - prefix := os.Getenv("HCLOUD_PREFIX") + prefix := os.Getenv("HETZNER_CLOUD_PREFIX") + + // set default prefix if none was provided + if prefix == "" { + prefix = "jynx" + } // get a list of all servers from hetzner (from jynx) servers, err := hetzner.ListJynxServers(prefix) @@ -51,14 +57,26 @@ func Start(w *woodpecker.Server) error { return nil } - // there are servers to be created + if maxServersString := os.Getenv("HETZNER_SERVER_MAX"); maxServersString != "" { + maxServers, err := strconv.Atoi(maxServersString) + if err != nil { + return err + } + // check if the maximum amount of servers is reached if one is provided + if newServers+len(servers) > maxServers && maxServers != 0 { + newServers = maxServers - len(servers) + } + } + // if there are no servers to be created, exit here if newServers < 1 { return nil } + // slice to hold servers to be created var ScaleUpServers []string + // there are servers to be created // get the suffix for the starting server suffix, err := hetzner.GetNewSuffix(servers) for i := 0; i < newServers; i++ {