diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 0000000..0e07f9a --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,13 @@ +--- +pipeline: + dockerfile: + image: woodpeckerci/plugin-docker-buildx + settings: + platforms: linux/amd64,linux/arm64/v8 + repo: git.stinnesbeck.com:52433/go/jynx + registry: https://git.stinnesbeck.com + tags: latest + auto_tag: true + username: nils + password: + from_secret: gitpw diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2d8aac9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM golang:alpine AS builder +RUN apk update && apk add --no-cache git +COPY . /jynx/ +WORKDIR /jynx/ +RUN go build . + +FROM scratch +COPY --from=builder /jynx/jynx /jynx +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +ENTRYPOINT ["/jynx"] + +ENV HCLOUD_API_TOKEN="" +ENV WOODPECKER_HOSTNAME="" +ENV WOODPECKER_AGENT_SECRET="" +ENV HCLOUD_PREFIX="" +ENV WOODPECKER_PROMETHEUS_AUTH_TOKEN="" +ENV LOGLEVEL="" diff --git a/backend/hetzner/servers.go b/backend/hetzner/servers.go index 5ee00b8..1b30b7e 100644 --- a/backend/hetzner/servers.go +++ b/backend/hetzner/servers.go @@ -3,12 +3,11 @@ package hetzner import ( "context" "fmt" - "io" "os" + "strconv" "strings" "time" - "git.stinnesbeck.com/go/jynx/backend/common" "git.stinnesbeck.com/go/log" "github.com/hetznercloud/hcloud-go/hcloud" ) @@ -22,86 +21,146 @@ func connect() *hcloud.Client { return hcloud.NewClient(hcloud.WithToken(token)) } -func DestroyServer(s *hcloud.Server) error { - client := connect() - serverResponse, response, err := client.Server.DeleteWithResult(context.Background(), s) - if err != nil { - return err +func GetNewSuffix(servers []*hcloud.Server) (int, error) { + // cop out if there are no servers yet + if len(servers) == 0 { + return 1, nil } - log.PrintInfo(*serverResponse, *response) + // variable to hold highest number + var highestNumber int - return nil + // loop through all servers + for i := range servers { + serverNameSlice := strings.Split(servers[i].Name, "-") + suffix, err := strconv.Atoi(serverNameSlice[len(serverNameSlice)-1]) + if err != nil { + return 0, err + } + + // compare our current number agains the highest yet + if suffix > highestNumber { + highestNumber = suffix + } + } + // return one number higher than highest number + return highestNumber + 1, nil } -func ListServers() ([]common.Server, error) { +// ListJynxServers lists all servers managed by Jynx hosted on Hetzner Cloud +func ListJynxServers(prefix string) ([]*hcloud.Server, error) { + // connect to the hcloud API client := connect() + + // get all servers servers, err := client.Server.All(context.Background()) if err != nil { return nil, err } - var managedServers []common.Server + // create object that holds all servers managed by Jynx + var jynxServers []*hcloud.Server - for i := range servers { - // only look at servers, that we created - if !strings.HasPrefix(servers[i].Name, os.Getenv("HCLOUD_PREFIX")) { - continue + // loop through servers and filter out managed servers + for _, server := range servers { + if strings.HasPrefix(server.Name, prefix) { + jynxServers = append(jynxServers, server) } - - // append this server to our managed servers - managedServers = append(managedServers, common.Server{ - Name: servers[i].Name, - Created: servers[i].Created, - Deployed: true, - }) } - return managedServers, nil + // return managed servers + return jynxServers, nil } -func AddServer(s common.Server) error { - client := connect() - woodpeckerHost := os.Getenv("WOODPECKER_HOSTNAME") - agentSecret := os.Getenv("WOODPECKER_AGENT_SECRET") - AgentCMD := fmt.Sprintf("docker run -d -v /var/run/docker.sock:/var/run/docker.sock -e WOODPECKER_GRPC_SECURE=false -e WOODPECKER_SERVER=%s -e WOODPECKER_AGENT_SECRET=%s -e WOODPECKER_MAX_PROCS=%d -e WOODPECKER_HOSTNAME=%s", woodpeckerHost, agentSecret, 1, s.Name) - - userdata := fmt.Sprintf("#cloud-config\nruncmd:\n\t- %s", AgentCMD) - fmt.Println(userdata) - labels := make(map[string]string) - labels["created_by"] = "jynx" - labels["created"] = fmt.Sprintf("%d", time.Now().Unix()) - - // sshkeys, err := client.SSHKey.All(context.Background()) - // if err != nil { - // return err - // } - - var sshkeys []*hcloud.SSHKey - - sshkeys, err := getSSHKeys("stefan", "nils") - if err != nil { - return nil +func CheckServerDeletions(servers []*hcloud.Server) error { + interval := os.Getenv("SCALER_POLL_INTERVAL") + if interval == "" { + interval = "5" } - serverOpts := hcloud.ServerCreateOpts{ - Name: s.Name, - StartAfterCreate: hcloud.Ptr(true), - SSHKeys: sshkeys, - Image: &hcloud.Image{Name: "docker-ce"}, - ServerType: &hcloud.ServerType{Name: "cax11"}, - Location: &hcloud.Location{Name: "fsn1"}, - UserData: userdata, - PublicNet: &hcloud.ServerCreatePublicNet{EnableIPv4: true}, - Labels: labels, - } - os.Exit(0) - result, response, err := client.Server.Create(context.Background(), serverOpts) + pollIntervalSeconds, err := strconv.Atoi(interval) if err != nil { return err } - io.ReadAll(response.Body) - log.PrintInfo(fmt.Sprintf("%+v\n%+v\n", result, response.Response.Body)) + if pollIntervalSeconds < 60 { + pollIntervalSeconds = 60 + } + + if pollIntervalSeconds > 3600 { + pollIntervalSeconds %= 3600 + } + + for _, server := range servers { + safeInterval := time.Duration(3600-pollIntervalSeconds) * time.Second + optimalDeleteTime := server.Created.Add(safeInterval).Local() + log.PrintDebug(server.Name, "optimal delete time:", optimalDeleteTime) + if time.Now().Local().Before(optimalDeleteTime) { + continue + } + + // optimal delete time has passed, delete the server + if err := DeleteServer(server); err != nil { + return err + } + } + + return nil +} + +func DeleteServer(server *hcloud.Server) error { + client := connect() + log.PrintInfo("Deleting", server.Name) + result, response, err := client.Server.DeleteWithResult(context.Background(), server) + if err != nil { + return err + } + + log.PrintWarning(result.Action.Status) + log.PrintWarning(response.StatusCode) + return nil +} +func CreateServers(servers []string) error { + // connect to hcloud API + client := connect() + + // read in some environment variables + woodpeckerHost := os.Getenv("WOODPECKER_HOSTNAME") + agentSecret := os.Getenv("WOODPECKER_AGENT_SECRET") + + // loop through each server + for _, server := range servers { + // create unique AgentCMD for cloud init + AgentCMD := fmt.Sprintf("docker run -d -v /var/run/docker.sock:/var/run/docker.sock -e WOODPECKER_GRPC_SECURE=false -e WOODPECKER_SERVER=%s -e WOODPECKER_AGENT_SECRET=%s -e WOODPECKER_MAX_PROCS=%d -e WOODPECKER_HOSTNAME=%s woodpeckerci/woodpecker-agent", woodpeckerHost, agentSecret, 1, server) + + userdata := fmt.Sprintf("#cloud-config\nruncmd:\n - %s", AgentCMD) + labels := make(map[string]string) + labels["created_by"] = "jynx" + labels["created"] = fmt.Sprintf("%d", time.Now().Unix()) + + var sshkeys []*hcloud.SSHKey + + sshkeys, err := getSSHKeys("stefan", "nils") + if err != nil { + return nil + } + + serverOpts := hcloud.ServerCreateOpts{ + Name: server, + StartAfterCreate: hcloud.Ptr(true), + SSHKeys: sshkeys, + Image: &hcloud.Image{Name: "docker-ce"}, + ServerType: &hcloud.ServerType{Name: "cax11"}, + Location: &hcloud.Location{Name: "fsn1"}, + UserData: userdata, + PublicNet: &hcloud.ServerCreatePublicNet{EnableIPv4: true}, + Labels: labels, + } + _, _, err = client.Server.Create(context.Background(), serverOpts) + if err != nil { + return err + } + } + return nil } diff --git a/ci/woodpecker/woodpecker.go b/ci/woodpecker/woodpecker.go index 4de5d5e..32215f0 100644 --- a/ci/woodpecker/woodpecker.go +++ b/ci/woodpecker/woodpecker.go @@ -18,9 +18,9 @@ type Server struct { } type Stats struct { - PendingJobs uint `json:"woodpecker_pending_jobs"` - RunningJobs uint `json:"woodpecker_running_jobs"` - IdlingWorkers uint `json:"woodpecker_worker_count"` + PendingJobs int `json:"woodpecker_pending_jobs"` + RunningJobs int `json:"woodpecker_running_jobs"` + IdlingWorkers int `json:"woodpecker_worker_count"` } // NewServer returns a Server object for host @@ -77,23 +77,23 @@ func (s Server) GetStats() (*Stats, error) { // assign stats to struct switch matches[3] { case "pending_jobs": - stats.PendingJobs = parseStringToUint(matches[4]) + stats.PendingJobs = parseStringToInt(matches[4]) case "running_jobs": - stats.RunningJobs = parseStringToUint(matches[4]) + stats.RunningJobs = parseStringToInt(matches[4]) case "worker_count": - stats.IdlingWorkers = parseStringToUint(matches[4]) + stats.IdlingWorkers = parseStringToInt(matches[4]) } } return &stats, nil } -// parseStringToUint parses a string and returns the resulting uint -func parseStringToUint(s string) uint { +// parseStringToInt parses a string and returns the resulting uint +func parseStringToInt(s string) int { num, err := strconv.Atoi(s) if err != nil { log.PrintError("can't parse string to uint,", err) os.Exit(1) } - return uint(num) + return num } diff --git a/db/db.go b/db/db.go index fcff348..910fbed 100644 --- a/db/db.go +++ b/db/db.go @@ -1,70 +1,61 @@ package db -import ( - "encoding/json" - "io" - "os" +// func Read() ([]common.Server, error) { +// // open file for reading, create it when necessary +// f, err := os.OpenFile("db.json", os.O_RDONLY|os.O_CREATE, os.ModePerm) +// if err != nil { +// return nil, err +// } - "git.stinnesbeck.com/go/jynx/backend/common" - "git.stinnesbeck.com/go/jynx/backend/hetzner" -) +// servers, err := hetzner.ListServers() +// if err != nil { +// return nil, err +// } -func Read() ([]common.Server, error) { - // open file for reading, create it when necessary - f, err := os.OpenFile("db.json", os.O_RDONLY|os.O_CREATE, os.ModePerm) - if err != nil { - return nil, err - } +// // close file when we are done +// defer f.Close() - servers, err := hetzner.ListServers() - if err != nil { - return nil, err - } +// // read file as byte array +// b, err := io.ReadAll(f) +// if err != nil { +// return nil, err +// } - // close file when we are done - defer f.Close() +// if len(b) == 0 { +// return nil, nil +// } - // read file as byte array - b, err := io.ReadAll(f) - if err != nil { - return nil, err - } +// // parse byte array to []common.Server +// // var serversFromDB []common.Server +// // if err := json.Unmarshal(b, &serversFromDB); err != nil { +// // return nil, err +// // } - if len(b) == 0 { - return nil, nil - } +// // // merge servers from db and hcloud +// // servers = append(servers, serversFromDB...) - // parse byte array to []common.Server - var serversFromDB []common.Server - if err := json.Unmarshal(b, &serversFromDB); err != nil { - return nil, err - } +// // return read in servers +// return servers, nil +// } - // merge servers from db and hcloud - servers = append(servers, serversFromDB...) +// func Save(servers []common.Server) error { +// // open file with provided flags, create it when necessary +// f, err := os.OpenFile("db.json", os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm) +// if err != nil { +// return err +// } - // return read in servers - return servers, nil -} +// // close file when we are done +// defer f.Close() -func Save(servers []common.Server) error { - // open file with provided flags, create it when necessary - f, err := os.OpenFile("db.json", os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm) - if err != nil { - return err - } +// // Marshal data as JSON and put into byte array +// b, err := json.MarshalIndent(servers, "", " ") +// if err != nil { +// return err +// } - // close file when we are done - defer f.Close() +// // write byte array to file +// _, err = f.Write(b) - // Marshal data as JSON and put into byte array - b, err := json.MarshalIndent(servers, "", " ") - if err != nil { - return err - } - - // write byte array to file - _, err = f.Write(b) - - return err -} +// return err +// } diff --git a/scaler/scaler.go b/scaler/scaler.go index 1f04ca2..3deef55 100644 --- a/scaler/scaler.go +++ b/scaler/scaler.go @@ -2,12 +2,10 @@ package scaler import ( "fmt" - "time" + "os" - "git.stinnesbeck.com/go/jynx/backend/common" "git.stinnesbeck.com/go/jynx/backend/hetzner" "git.stinnesbeck.com/go/jynx/ci/woodpecker" - "git.stinnesbeck.com/go/jynx/db" "git.stinnesbeck.com/go/log" ) @@ -18,42 +16,62 @@ func Start(w *woodpecker.Server) error { return err } - // read info from database - servers, err := db.Read() + // get the prefix for jynx managed servers + prefix := os.Getenv("HCLOUD_PREFIX") + + // get a list of all servers from hetzner (from jynx) + servers, err := hetzner.ListJynxServers(prefix) if err != nil { return err } - var newServers int + newServers := stats.PendingJobs - len(servers) + stats.RunningJobs + log.PrintDebug(stats.RunningJobs, "running jobs,", stats.PendingJobs, "pending jobs,", stats.IdlingWorkers, "idling workers,", len(servers)-stats.RunningJobs-stats.IdlingWorkers, "servers starting up") - if stats.PendingJobs == 0 || stats.IdlingWorkers+stats.RunningJobs+uint(len(servers)) >= stats.PendingJobs { - return nil - } - - log.PrintInfo("starting", stats.PendingJobs, "workers") - - newServers += int(stats.PendingJobs) - var ScaleUpServers []common.Server - - for i := 0; i < newServers; i++ { - name := fmt.Sprintf("jynx-worker-%d", i+1) - - ScaleUpServers = append(ScaleUpServers, common.Server{ - Name: name, - Created: time.Now(), - }) - if err := hetzner.AddServer(ScaleUpServers[i]); err != nil { + // check if there are servers to be removed + if stats.PendingJobs == 0 && stats.RunningJobs == 0 { + // we need to remove servers + if err := hetzner.CheckServerDeletions(servers); err != nil { return err } } - // save servers to database - servers = append(servers, ScaleUpServers...) - if err := db.Save(servers); err != nil { + // check if we need to create new servers + // if stats.PendingJobs < 1 || len(servers)-stats.RunningJobs >= stats.PendingJobs { + if stats.PendingJobs < 1 || newServers < 1 { + // if stats.PendingJobs == 0 || stats.IdlingWorkers+stats.RunningJobs >= stats.PendingJobs { + // skip rest of this run + return nil + } + + // there are servers to be created + + if newServers < 1 { + return nil + } + + var ScaleUpServers []string + + // get the suffix for the starting server + suffix, err := hetzner.GetNewSuffix(servers) + for i := 0; i < newServers; i++ { + if err != nil { + return err + } + // assemble server name and add it to the list of servers to scale up + serverName := fmt.Sprintf("%s-worker-%d", prefix, suffix) + ScaleUpServers = append(ScaleUpServers, serverName) + + // add 1 to the suffix for the next server + suffix++ + } + + log.PrintInfo("starting", newServers, "workers:", ScaleUpServers) + + // create servers on hetzner cloud + if err := hetzner.CreateServers(ScaleUpServers); err != nil { return err } - log.PrintInfo(stats.RunningJobs, "running jobs,", stats.PendingJobs, "pending jobs,", stats.IdlingWorkers, "idling workers,", len(servers), "servers starting up") - return nil }