first test build

.woodpecker.yml

@@ -0,0 +1,13 @@
+---
+pipeline:
+  dockerfile:
+    image: woodpeckerci/plugin-docker-buildx
+    settings:
+      platforms: linux/amd64,linux/arm64/v8
+      repo: git.stinnesbeck.com:52433/go/jynx
+      registry: https://git.stinnesbeck.com
+      tags: latest
+      auto_tag: true
+      username: nils
+      password:
+        from_secret: gitpw

Dockerfile

@@ -0,0 +1,17 @@
+FROM golang:alpine AS builder
+RUN apk update && apk add --no-cache git
+COPY . /jynx/
+WORKDIR /jynx/
+RUN go build .
+
+FROM scratch
+COPY --from=builder /jynx/jynx /jynx
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+ENTRYPOINT ["/jynx"]
+
+ENV HCLOUD_API_TOKEN=""
+ENV WOODPECKER_HOSTNAME=""
+ENV WOODPECKER_AGENT_SECRET=""
+ENV HCLOUD_PREFIX=""
+ENV WOODPECKER_PROMETHEUS_AUTH_TOKEN=""
+ENV LOGLEVEL=""

backend/hetzner/servers.go

@@ -3,12 +3,11 @@ package hetzner
 import (
 	"context"
 	"fmt"
-	"io"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
-	"git.stinnesbeck.com/go/jynx/backend/common"
 	"git.stinnesbeck.com/go/log"
 	"github.com/hetznercloud/hcloud-go/hcloud"
 )
@@ -22,61 +21,123 @@ func connect() *hcloud.Client {
 	return hcloud.NewClient(hcloud.WithToken(token))
 }
 
-func DestroyServer(s *hcloud.Server) error {
+func GetNewSuffix(servers []*hcloud.Server) (int, error) {
-	client := connect()
+	// cop out if there are no servers yet
-	serverResponse, response, err := client.Server.DeleteWithResult(context.Background(), s)
+	if len(servers) == 0 {
+		return 1, nil
+	}
+
+	// variable to hold highest number
+	var highestNumber int
+
+	// loop through all servers
+	for i := range servers {
+		serverNameSlice := strings.Split(servers[i].Name, "-")
+		suffix, err := strconv.Atoi(serverNameSlice[len(serverNameSlice)-1])
 		if err != nil {
-		return err
+			return 0, err
 		}
 
-	log.PrintInfo(*serverResponse, *response)
+		// compare our current number agains the highest yet
-
+		if suffix > highestNumber {
-	return nil
+			highestNumber = suffix
+		}
+	}
+	// return one number higher than highest number
+	return highestNumber + 1, nil
 }
 
-func ListServers() ([]common.Server, error) {
+// ListJynxServers lists all servers managed by Jynx hosted on Hetzner Cloud
+func ListJynxServers(prefix string) ([]*hcloud.Server, error) {
+	// connect to the hcloud API
 	client := connect()
+
+	// get all servers
 	servers, err := client.Server.All(context.Background())
 	if err != nil {
 		return nil, err
 	}
 
-	var managedServers []common.Server
+	// create object that holds all servers managed by Jynx
+	var jynxServers []*hcloud.Server
 
-	for i := range servers {
+	// loop through servers and filter out managed servers
-		// only look at servers, that we created
+	for _, server := range servers {
-		if !strings.HasPrefix(servers[i].Name, os.Getenv("HCLOUD_PREFIX")) {
+		if strings.HasPrefix(server.Name, prefix) {
+			jynxServers = append(jynxServers, server)
+		}
+	}
+
+	// return managed servers
+	return jynxServers, nil
+}
+
+func CheckServerDeletions(servers []*hcloud.Server) error {
+	interval := os.Getenv("SCALER_POLL_INTERVAL")
+	if interval == "" {
+		interval = "5"
+	}
+
+	pollIntervalSeconds, err := strconv.Atoi(interval)
+	if err != nil {
+		return err
+	}
+
+	if pollIntervalSeconds < 60 {
+		pollIntervalSeconds = 60
+	}
+
+	if pollIntervalSeconds > 3600 {
+		pollIntervalSeconds %= 3600
+	}
+
+	for _, server := range servers {
+		safeInterval := time.Duration(3600-pollIntervalSeconds) * time.Second
+		optimalDeleteTime := server.Created.Add(safeInterval).Local()
+		log.PrintDebug(server.Name, "optimal delete time:", optimalDeleteTime)
+		if time.Now().Local().Before(optimalDeleteTime) {
 			continue
 		}
 
-		// append this server to our managed servers
+		// optimal delete time has passed, delete the server
-		managedServers = append(managedServers, common.Server{
+		if err := DeleteServer(server); err != nil {
-			Name:     servers[i].Name,
+			return err
-			Created:  servers[i].Created,
+		}
-			Deployed: true,
-		})
 	}
 
-	return managedServers, nil
+	return nil
 }
 
-func AddServer(s common.Server) error {
+func DeleteServer(server *hcloud.Server) error {
 	client := connect()
+	log.PrintInfo("Deleting", server.Name)
+	result, response, err := client.Server.DeleteWithResult(context.Background(), server)
+	if err != nil {
+		return err
+	}
+
+	log.PrintWarning(result.Action.Status)
+	log.PrintWarning(response.StatusCode)
+	return nil
+}
+func CreateServers(servers []string) error {
+	// connect to hcloud API
+	client := connect()
+
+	// read in some environment variables
 	woodpeckerHost := os.Getenv("WOODPECKER_HOSTNAME")
 	agentSecret := os.Getenv("WOODPECKER_AGENT_SECRET")
-	AgentCMD := fmt.Sprintf("docker run -d -v /var/run/docker.sock:/var/run/docker.sock -e WOODPECKER_GRPC_SECURE=false -e WOODPECKER_SERVER=%s -e WOODPECKER_AGENT_SECRET=%s -e WOODPECKER_MAX_PROCS=%d -e WOODPECKER_HOSTNAME=%s", woodpeckerHost, agentSecret, 1, s.Name)
 
-	userdata := fmt.Sprintf("#cloud-config\nruncmd:\n\t- %s", AgentCMD)
+	// loop through each server
-	fmt.Println(userdata)
+	for _, server := range servers {
+		// create unique AgentCMD for cloud init
+		AgentCMD := fmt.Sprintf("docker run -d -v /var/run/docker.sock:/var/run/docker.sock -e WOODPECKER_GRPC_SECURE=false -e WOODPECKER_SERVER=%s -e WOODPECKER_AGENT_SECRET=%s -e WOODPECKER_MAX_PROCS=%d -e WOODPECKER_HOSTNAME=%s woodpeckerci/woodpecker-agent", woodpeckerHost, agentSecret, 1, server)
+
+		userdata := fmt.Sprintf("#cloud-config\nruncmd:\n  - %s", AgentCMD)
 		labels := make(map[string]string)
 		labels["created_by"] = "jynx"
 		labels["created"] = fmt.Sprintf("%d", time.Now().Unix())
 
-	// sshkeys, err := client.SSHKey.All(context.Background())
-	// if err != nil {
-	// 	return err
-	// }
-
 		var sshkeys []*hcloud.SSHKey
 
 		sshkeys, err := getSSHKeys("stefan", "nils")
@@ -85,7 +146,7 @@ func AddServer(s common.Server) error {
 		}
 
 		serverOpts := hcloud.ServerCreateOpts{
-		Name:             s.Name,
+			Name:             server,
 			StartAfterCreate: hcloud.Ptr(true),
 			SSHKeys:          sshkeys,
 			Image:            &hcloud.Image{Name: "docker-ce"},
@@ -95,13 +156,11 @@ func AddServer(s common.Server) error {
 			PublicNet:        &hcloud.ServerCreatePublicNet{EnableIPv4: true},
 			Labels:           labels,
 		}
-	os.Exit(0)
+		_, _, err = client.Server.Create(context.Background(), serverOpts)
-	result, response, err := client.Server.Create(context.Background(), serverOpts)
 		if err != nil {
 			return err
 		}
+	}
 
-	io.ReadAll(response.Body)
-	log.PrintInfo(fmt.Sprintf("%+v\n%+v\n", result, response.Response.Body))
 	return nil
 }

ci/woodpecker/woodpecker.go

@@ -18,9 +18,9 @@ type Server struct {
 }
 
 type Stats struct {
-	PendingJobs   uint `json:"woodpecker_pending_jobs"`
+	PendingJobs   int `json:"woodpecker_pending_jobs"`
-	RunningJobs   uint `json:"woodpecker_running_jobs"`
+	RunningJobs   int `json:"woodpecker_running_jobs"`
-	IdlingWorkers uint `json:"woodpecker_worker_count"`
+	IdlingWorkers int `json:"woodpecker_worker_count"`
 }
 
 // NewServer returns a Server object for host
@@ -77,23 +77,23 @@ func (s Server) GetStats() (*Stats, error) {
 		// assign stats to struct
 		switch matches[3] {
 		case "pending_jobs":
-			stats.PendingJobs = parseStringToUint(matches[4])
+			stats.PendingJobs = parseStringToInt(matches[4])
 		case "running_jobs":
-			stats.RunningJobs = parseStringToUint(matches[4])
+			stats.RunningJobs = parseStringToInt(matches[4])
 		case "worker_count":
-			stats.IdlingWorkers = parseStringToUint(matches[4])
+			stats.IdlingWorkers = parseStringToInt(matches[4])
 		}
 	}
 
 	return &stats, nil
 }
 
-// parseStringToUint parses a string and returns the resulting uint
+// parseStringToInt parses a string and returns the resulting uint
-func parseStringToUint(s string) uint {
+func parseStringToInt(s string) int {
 	num, err := strconv.Atoi(s)
 	if err != nil {
 		log.PrintError("can't parse string to uint,", err)
 		os.Exit(1)
 	}
-	return uint(num)
+	return num
 }

db/db.go

@@ -1,70 +1,61 @@
 package db
 
-import (
+// func Read() ([]common.Server, error) {
-	"encoding/json"
+// 	// open file for reading, create it when necessary
-	"io"
+// 	f, err := os.OpenFile("db.json", os.O_RDONLY|os.O_CREATE, os.ModePerm)
-	"os"
+// 	if err != nil {
+// 		return nil, err
+// 	}
 
-	"git.stinnesbeck.com/go/jynx/backend/common"
+// 	servers, err := hetzner.ListServers()
-	"git.stinnesbeck.com/go/jynx/backend/hetzner"
+// 	if err != nil {
-)
+// 		return nil, err
+// 	}
 
-func Read() ([]common.Server, error) {
+// 	// close file when we are done
-	// open file for reading, create it when necessary
+// 	defer f.Close()
-	f, err := os.OpenFile("db.json", os.O_RDONLY|os.O_CREATE, os.ModePerm)
-	if err != nil {
-		return nil, err
-	}
 
-	servers, err := hetzner.ListServers()
+// 	// read file as byte array
-	if err != nil {
+// 	b, err := io.ReadAll(f)
-		return nil, err
+// 	if err != nil {
-	}
+// 		return nil, err
+// 	}
 
-	// close file when we are done
+// 	if len(b) == 0 {
-	defer f.Close()
+// 		return nil, nil
+// 	}
 
-	// read file as byte array
+// 	// parse byte array to []common.Server
-	b, err := io.ReadAll(f)
+// 	// var serversFromDB []common.Server
-	if err != nil {
+// 	// if err := json.Unmarshal(b, &serversFromDB); err != nil {
-		return nil, err
+// 	// 	return nil, err
-	}
+// 	// }
 
-	if len(b) == 0 {
+// 	// // merge servers from db and hcloud
-		return nil, nil
+// 	// servers = append(servers, serversFromDB...)
-	}
 
-	// parse byte array to []common.Server
+// 	// return read in servers
-	var serversFromDB []common.Server
+// 	return servers, nil
-	if err := json.Unmarshal(b, &serversFromDB); err != nil {
+// }
-		return nil, err
-	}
 
-	// merge servers from db and hcloud
+// func Save(servers []common.Server) error {
-	servers = append(servers, serversFromDB...)
+// 	// open file with provided flags, create it when necessary
+// 	f, err := os.OpenFile("db.json", os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm)
+// 	if err != nil {
+// 		return err
+// 	}
 
-	// return read in servers
+// 	// close file when we are done
-	return servers, nil
+// 	defer f.Close()
-}
 
-func Save(servers []common.Server) error {
+// 	// Marshal data as JSON and put into byte array
-	// open file with provided flags, create it when necessary
+// 	b, err := json.MarshalIndent(servers, "", "  ")
-	f, err := os.OpenFile("db.json", os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm)
+// 	if err != nil {
-	if err != nil {
+// 		return err
-		return err
+// 	}
-	}
 
-	// close file when we are done
+// 	// write byte array to file
-	defer f.Close()
+// 	_, err = f.Write(b)
 
-	// Marshal data as JSON and put into byte array
+// 	return err
-	b, err := json.MarshalIndent(servers, "", "  ")
+// }
-	if err != nil {
-		return err
-	}
-
-	// write byte array to file
-	_, err = f.Write(b)
-
-	return err
-}

scaler/scaler.go

@@ -2,12 +2,10 @@ package scaler
 
 import (
 	"fmt"
-	"time"
+	"os"
 
-	"git.stinnesbeck.com/go/jynx/backend/common"
 	"git.stinnesbeck.com/go/jynx/backend/hetzner"
 	"git.stinnesbeck.com/go/jynx/ci/woodpecker"
-	"git.stinnesbeck.com/go/jynx/db"
 	"git.stinnesbeck.com/go/log"
 )
 
@@ -18,42 +16,62 @@ func Start(w *woodpecker.Server) error {
 		return err
 	}
 
-	// read info from database
+	// get the prefix for jynx managed servers
-	servers, err := db.Read()
+	prefix := os.Getenv("HCLOUD_PREFIX")
+
+	// get a list of all servers from hetzner (from jynx)
+	servers, err := hetzner.ListJynxServers(prefix)
 	if err != nil {
 		return err
 	}
 
-	var newServers int
+	newServers := stats.PendingJobs - len(servers) + stats.RunningJobs
+	log.PrintDebug(stats.RunningJobs, "running jobs,", stats.PendingJobs, "pending jobs,", stats.IdlingWorkers, "idling workers,", len(servers)-stats.RunningJobs-stats.IdlingWorkers, "servers starting up")
 
-	if stats.PendingJobs == 0 || stats.IdlingWorkers+stats.RunningJobs+uint(len(servers)) >= stats.PendingJobs {
+	// check if there are servers to be removed
+	if stats.PendingJobs == 0 && stats.RunningJobs == 0 {
+		// we need to remove servers
+		if err := hetzner.CheckServerDeletions(servers); err != nil {
+			return err
+		}
+	}
+
+	// check if we need to create new servers
+	// if stats.PendingJobs < 1 || len(servers)-stats.RunningJobs >= stats.PendingJobs {
+	if stats.PendingJobs < 1 || newServers < 1 {
+		// if stats.PendingJobs == 0 || stats.IdlingWorkers+stats.RunningJobs >= stats.PendingJobs {
+		// skip rest of this run
 		return nil
 	}
 
-	log.PrintInfo("starting", stats.PendingJobs, "workers")
+	// there are servers to be created
 
-	newServers += int(stats.PendingJobs)
+	if newServers < 1 {
-	var ScaleUpServers []common.Server
+		return nil
+	}
 
+	var ScaleUpServers []string
+
+	// get the suffix for the starting server
+	suffix, err := hetzner.GetNewSuffix(servers)
 	for i := 0; i < newServers; i++ {
-		name := fmt.Sprintf("jynx-worker-%d", i+1)
+		if err != nil {
-
-		ScaleUpServers = append(ScaleUpServers, common.Server{
-			Name:    name,
-			Created: time.Now(),
-		})
-		if err := hetzner.AddServer(ScaleUpServers[i]); err != nil {
 			return err
 		}
+		// assemble server name and add it to the list of servers to scale up
+		serverName := fmt.Sprintf("%s-worker-%d", prefix, suffix)
+		ScaleUpServers = append(ScaleUpServers, serverName)
+
+		// add 1 to the suffix for the next server
+		suffix++
 	}
 
-	// save servers to database
+	log.PrintInfo("starting", newServers, "workers:", ScaleUpServers)
-	servers = append(servers, ScaleUpServers...)
+
-	if err := db.Save(servers); err != nil {
+	// create servers on hetzner cloud
+	if err := hetzner.CreateServers(ScaleUpServers); err != nil {
 		return err
 	}
 
-	log.PrintInfo(stats.RunningJobs, "running jobs,", stats.PendingJobs, "pending jobs,", stats.IdlingWorkers, "idling workers,", len(servers), "servers starting up")
-
 	return nil
 }