From ca9c90cf3c18c4afc94a6ece4f6eafe10e9aa607 Mon Sep 17 00:00:00 2001 From: Nils Stinnesbeck Date: Mon, 29 May 2023 23:21:50 +0200 Subject: [PATCH] [WIP] first commit, not done yet --- .gitignore | 2 + backend/common/common.go | 9 +++ backend/hetzner/servers.go | 107 ++++++++++++++++++++++++++++++++++++ backend/hetzner/sshKeys.go | 38 +++++++++++++ ci/woodpecker/woodpecker.go | 99 +++++++++++++++++++++++++++++++++ db/db.go | 70 +++++++++++++++++++++++ go.mod | 26 +++++++++ go.sum | 49 +++++++++++++++++ main.go | 62 +++++++++++++++++++++ scaler/scaler.go | 59 ++++++++++++++++++++ 10 files changed, 521 insertions(+) create mode 100644 .gitignore create mode 100644 backend/common/common.go create mode 100644 backend/hetzner/servers.go create mode 100644 backend/hetzner/sshKeys.go create mode 100644 ci/woodpecker/woodpecker.go create mode 100644 db/db.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 scaler/scaler.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bee402f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +# databases +*.json diff --git a/backend/common/common.go b/backend/common/common.go new file mode 100644 index 0000000..5656214 --- /dev/null +++ b/backend/common/common.go @@ -0,0 +1,9 @@ +package common + +import "time" + +type Server struct { + Name string `json:"name"` + Created time.Time `json:"started_at"` + Deployed bool `json:"deployed"` +} diff --git a/backend/hetzner/servers.go b/backend/hetzner/servers.go new file mode 100644 index 0000000..5ee00b8 --- /dev/null +++ b/backend/hetzner/servers.go @@ -0,0 +1,107 @@ +package hetzner + +import ( + "context" + "fmt" + "io" + "os" + "strings" + "time" + + "git.stinnesbeck.com/go/jynx/backend/common" + "git.stinnesbeck.com/go/log" + "github.com/hetznercloud/hcloud-go/hcloud" +) + +func connect() *hcloud.Client { + token := os.Getenv("HCLOUD_API_TOKEN") + if token == "" { + log.PrintError("please specify HCLOUD_API_TOKEN") + os.Exit(1) + } + return hcloud.NewClient(hcloud.WithToken(token)) +} + +func DestroyServer(s *hcloud.Server) error { + client := connect() + serverResponse, response, err := client.Server.DeleteWithResult(context.Background(), s) + if err != nil { + return err + } + + log.PrintInfo(*serverResponse, *response) + + return nil +} + +func ListServers() ([]common.Server, error) { + client := connect() + servers, err := client.Server.All(context.Background()) + if err != nil { + return nil, err + } + + var managedServers []common.Server + + for i := range servers { + // only look at servers, that we created + if !strings.HasPrefix(servers[i].Name, os.Getenv("HCLOUD_PREFIX")) { + continue + } + + // append this server to our managed servers + managedServers = append(managedServers, common.Server{ + Name: servers[i].Name, + Created: servers[i].Created, + Deployed: true, + }) + } + + return managedServers, nil +} + +func AddServer(s common.Server) error { + client := connect() + woodpeckerHost := os.Getenv("WOODPECKER_HOSTNAME") + agentSecret := os.Getenv("WOODPECKER_AGENT_SECRET") + AgentCMD := fmt.Sprintf("docker run -d -v /var/run/docker.sock:/var/run/docker.sock -e WOODPECKER_GRPC_SECURE=false -e WOODPECKER_SERVER=%s -e WOODPECKER_AGENT_SECRET=%s -e WOODPECKER_MAX_PROCS=%d -e WOODPECKER_HOSTNAME=%s", woodpeckerHost, agentSecret, 1, s.Name) + + userdata := fmt.Sprintf("#cloud-config\nruncmd:\n\t- %s", AgentCMD) + fmt.Println(userdata) + labels := make(map[string]string) + labels["created_by"] = "jynx" + labels["created"] = fmt.Sprintf("%d", time.Now().Unix()) + + // sshkeys, err := client.SSHKey.All(context.Background()) + // if err != nil { + // return err + // } + + var sshkeys []*hcloud.SSHKey + + sshkeys, err := getSSHKeys("stefan", "nils") + if err != nil { + return nil + } + + serverOpts := hcloud.ServerCreateOpts{ + Name: s.Name, + StartAfterCreate: hcloud.Ptr(true), + SSHKeys: sshkeys, + Image: &hcloud.Image{Name: "docker-ce"}, + ServerType: &hcloud.ServerType{Name: "cax11"}, + Location: &hcloud.Location{Name: "fsn1"}, + UserData: userdata, + PublicNet: &hcloud.ServerCreatePublicNet{EnableIPv4: true}, + Labels: labels, + } + os.Exit(0) + result, response, err := client.Server.Create(context.Background(), serverOpts) + if err != nil { + return err + } + + io.ReadAll(response.Body) + log.PrintInfo(fmt.Sprintf("%+v\n%+v\n", result, response.Response.Body)) + return nil +} diff --git a/backend/hetzner/sshKeys.go b/backend/hetzner/sshKeys.go new file mode 100644 index 0000000..0f51517 --- /dev/null +++ b/backend/hetzner/sshKeys.go @@ -0,0 +1,38 @@ +package hetzner + +import ( + "context" + + "github.com/hetznercloud/hcloud-go/hcloud" +) + +func getSSHKeys(names ...string) ([]*hcloud.SSHKey, error) { + // if no names were provided return nil object + if len(names) == 0 { + return nil, nil + } + + // connect to hcloud client + client := connect() + + // get all ssh keys from hcloud + keys, err := client.SSHKey.All(context.Background()) + if err != nil { + return nil, err + } + + // variable to hold found keys + var foundKeys []*hcloud.SSHKey + + // check if name is found in keys + for i := range keys { + for j := range names { + if keys[i].Name == names[j] { + foundKeys = append(foundKeys, keys[i]) + } + } + } + + // return found keys + return foundKeys, nil +} diff --git a/ci/woodpecker/woodpecker.go b/ci/woodpecker/woodpecker.go new file mode 100644 index 0000000..4de5d5e --- /dev/null +++ b/ci/woodpecker/woodpecker.go @@ -0,0 +1,99 @@ +package woodpecker + +import ( + "fmt" + "io" + "net/http" + "os" + "regexp" + "strconv" + "strings" + + "git.stinnesbeck.com/go/log" +) + +type Server struct { + Host string `json:"host"` + Token string `json:"token"` +} + +type Stats struct { + PendingJobs uint `json:"woodpecker_pending_jobs"` + RunningJobs uint `json:"woodpecker_running_jobs"` + IdlingWorkers uint `json:"woodpecker_worker_count"` +} + +// NewServer returns a Server object for host +func NewServer(host string, token string) Server { + switch { + case strings.HasPrefix(host, "http://"), strings.HasPrefix(host, "https://"): + return Server{ + Host: host, + Token: token, + } + default: + log.PrintError("host needs to start with 'http://' or 'https://'") + os.Exit(1) + return Server{} + } +} + +func (s Server) GetStats() (*Stats, error) { + // prepare request + req, err := http.NewRequest(http.MethodGet, s.Host+"/metrics/", nil) + if err != nil { + return nil, err + } + + // add authorization header + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", s.Token)) + + c := http.DefaultClient + response, err := c.Do(req) + if err != nil { + return nil, err + } + + b, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + + lines := strings.Split(string(b), "\n") + + r := regexp.MustCompile(`^((woodpecker_)(worker_count|pending_jobs|running_jobs)) (\d+)`) + + var stats Stats + + for i := range lines { + // skip rest of loop if this line doesn't match regex + if !r.MatchString(lines[i]) { + continue + } + + // line matches regex + matches := r.FindStringSubmatch(lines[i]) + + // assign stats to struct + switch matches[3] { + case "pending_jobs": + stats.PendingJobs = parseStringToUint(matches[4]) + case "running_jobs": + stats.RunningJobs = parseStringToUint(matches[4]) + case "worker_count": + stats.IdlingWorkers = parseStringToUint(matches[4]) + } + } + + return &stats, nil +} + +// parseStringToUint parses a string and returns the resulting uint +func parseStringToUint(s string) uint { + num, err := strconv.Atoi(s) + if err != nil { + log.PrintError("can't parse string to uint,", err) + os.Exit(1) + } + return uint(num) +} diff --git a/db/db.go b/db/db.go new file mode 100644 index 0000000..fcff348 --- /dev/null +++ b/db/db.go @@ -0,0 +1,70 @@ +package db + +import ( + "encoding/json" + "io" + "os" + + "git.stinnesbeck.com/go/jynx/backend/common" + "git.stinnesbeck.com/go/jynx/backend/hetzner" +) + +func Read() ([]common.Server, error) { + // open file for reading, create it when necessary + f, err := os.OpenFile("db.json", os.O_RDONLY|os.O_CREATE, os.ModePerm) + if err != nil { + return nil, err + } + + servers, err := hetzner.ListServers() + if err != nil { + return nil, err + } + + // close file when we are done + defer f.Close() + + // read file as byte array + b, err := io.ReadAll(f) + if err != nil { + return nil, err + } + + if len(b) == 0 { + return nil, nil + } + + // parse byte array to []common.Server + var serversFromDB []common.Server + if err := json.Unmarshal(b, &serversFromDB); err != nil { + return nil, err + } + + // merge servers from db and hcloud + servers = append(servers, serversFromDB...) + + // return read in servers + return servers, nil +} + +func Save(servers []common.Server) error { + // open file with provided flags, create it when necessary + f, err := os.OpenFile("db.json", os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm) + if err != nil { + return err + } + + // close file when we are done + defer f.Close() + + // Marshal data as JSON and put into byte array + b, err := json.MarshalIndent(servers, "", " ") + if err != nil { + return err + } + + // write byte array to file + _, err = f.Write(b) + + return err +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..07c83b0 --- /dev/null +++ b/go.mod @@ -0,0 +1,26 @@ +module git.stinnesbeck.com/go/jynx + +go 1.20 + +require ( + git.stinnesbeck.com/go/log v0.0.1 + github.com/hetznercloud/hcloud-go v1.45.1 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/fatih/color v1.15.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/prometheus/client_golang v1.15.1 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a3b7950 --- /dev/null +++ b/go.sum @@ -0,0 +1,49 @@ +git.stinnesbeck.com/go/log v0.0.1 h1:aH3jTgw6NZiJXxNrxE6XiTF3ymsIg+HIGH4A6PPpIcA= +git.stinnesbeck.com/go/log v0.0.1/go.mod h1:9Op2dPZAALo1sHUSaZa/6Gs6YyU1qM/e2RNZg6RXtyE= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/hetznercloud/hcloud-go v1.45.1 h1:nl0OOklFfQT5J6AaNIOhl5Ruh3fhmGmhvZEqHbibVuk= +github.com/hetznercloud/hcloud-go v1.45.1/go.mod h1:aAUGxSfSnB8/lVXHNEDxtCT1jykaul8kqjD7f5KQXF8= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/main.go b/main.go new file mode 100644 index 0000000..86bc9f0 --- /dev/null +++ b/main.go @@ -0,0 +1,62 @@ +package main + +import ( + "os" + "strconv" + "time" + + "git.stinnesbeck.com/go/jynx/ci/woodpecker" + "git.stinnesbeck.com/go/jynx/scaler" + "git.stinnesbeck.com/go/log" +) + +func main() { + checkEnv() + + w := woodpecker.NewServer( + "https://woodpecker.fftdf.de", + os.Getenv("WOODPECKER_PROMETHEUS_AUTH_TOKEN"), + ) + + interval := time.Duration(5) + if i := os.Getenv("SCALER_POLL_INTERVAL"); i != "" { + envInterval, err := strconv.Atoi(i) + if err != nil { + log.PrintError("can't parse SCALER_POLL_INTERVAL", err) + os.Exit(1) + } + + // overwrite default interval with the one provided by env variable + interval = time.Duration(envInterval) + } + + ticker := time.Tick(interval * time.Second) + + // start scaler once + if err := scaler.Start(&w); err != nil { + panic(err) + } + + // start scaler every tick + for range ticker { + if err := scaler.Start(&w); err != nil { + panic(err) + } + } +} + +func checkEnv() { + switch "" { + case os.Getenv("WOODPECKER_HOSTNAME"): + log.PrintError("WOODPECKER_HOSTNAME not set") + case os.Getenv("WOODPECKER_AGENT_SECRET"): + log.PrintError("WOODPECKER_AGENT_SECRET not set") + os.Exit(1) + case os.Getenv("HCLOUD_PREFIX"): + log.PrintError("HCLOUD_PREFIX not set") + os.Exit(1) + case os.Getenv("WOODPECKER_PROMETHEUS_AUTH_TOKEN"): + log.PrintError("WOODPECKER_PROMETHEUS_AUTH_TOKEN not set") + os.Exit(1) + } +} diff --git a/scaler/scaler.go b/scaler/scaler.go new file mode 100644 index 0000000..1f04ca2 --- /dev/null +++ b/scaler/scaler.go @@ -0,0 +1,59 @@ +package scaler + +import ( + "fmt" + "time" + + "git.stinnesbeck.com/go/jynx/backend/common" + "git.stinnesbeck.com/go/jynx/backend/hetzner" + "git.stinnesbeck.com/go/jynx/ci/woodpecker" + "git.stinnesbeck.com/go/jynx/db" + "git.stinnesbeck.com/go/log" +) + +func Start(w *woodpecker.Server) error { + log.PrintDebug("Starting autoscaler") + stats, err := w.GetStats() + if err != nil { + return err + } + + // read info from database + servers, err := db.Read() + if err != nil { + return err + } + + var newServers int + + if stats.PendingJobs == 0 || stats.IdlingWorkers+stats.RunningJobs+uint(len(servers)) >= stats.PendingJobs { + return nil + } + + log.PrintInfo("starting", stats.PendingJobs, "workers") + + newServers += int(stats.PendingJobs) + var ScaleUpServers []common.Server + + for i := 0; i < newServers; i++ { + name := fmt.Sprintf("jynx-worker-%d", i+1) + + ScaleUpServers = append(ScaleUpServers, common.Server{ + Name: name, + Created: time.Now(), + }) + if err := hetzner.AddServer(ScaleUpServers[i]); err != nil { + return err + } + } + + // save servers to database + servers = append(servers, ScaleUpServers...) + if err := db.Save(servers); err != nil { + return err + } + + log.PrintInfo(stats.RunningJobs, "running jobs,", stats.PendingJobs, "pending jobs,", stats.IdlingWorkers, "idling workers,", len(servers), "servers starting up") + + return nil +}