go/jynx
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

first test build
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
Details

Browse Source
This commit is contained in:
Nils Stinnesbeck 2023-07-15 21:51:31 +02:00
parent ca9c90cf3c
commit c7da3f2147
Signed by: nils
GPG Key ID: C52E07422A136076
6 changed files with 251 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.woodpecker.yml Normal file
View File

@ -0,0 +1,13 @@
---
pipeline:
dockerfile:
image: woodpeckerci/plugin-docker-buildx
settings:
platforms: linux/amd64,linux/arm64/v8
repo: git.stinnesbeck.com:52433/go/jynx
registry: https://git.stinnesbeck.com
tags: latest
auto_tag: true
username: nils
password:
from_secret: gitpw

17
Dockerfile Normal file
View File

@ -0,0 +1,17 @@
FROM golang:alpine AS builder
RUN apk update && apk add --no-cache git
COPY . /jynx/
WORKDIR /jynx/
RUN go build .
FROM scratch
COPY --from=builder /jynx/jynx /jynx
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
ENTRYPOINT ["/jynx"]
ENV HCLOUD_API_TOKEN=""
ENV WOODPECKER_HOSTNAME=""
ENV WOODPECKER_AGENT_SECRET=""
ENV HCLOUD_PREFIX=""
ENV WOODPECKER_PROMETHEUS_AUTH_TOKEN=""
ENV LOGLEVEL=""

129
backend/hetzner/servers.go
View File

@ -3,12 +3,11 @@ package hetzner
import (
"context"
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
"git.stinnesbeck.com/go/jynx/backend/common"
"git.stinnesbeck.com/go/log"
"github.com/hetznercloud/hcloud-go/hcloud"
)
@ -22,61 +21,123 @@ func connect() *hcloud.Client {
return hcloud.NewClient(hcloud.WithToken(token))
}
func DestroyServer(s *hcloud.Server) error {
client := connect()
serverResponse, response, err := client.Server.DeleteWithResult(context.Background(), s)
func GetNewSuffix(servers []*hcloud.Server) (int, error) {
// cop out if there are no servers yet
if len(servers) == 0 {
return 1, nil
}
// variable to hold highest number
var highestNumber int
// loop through all servers
for i := range servers {
serverNameSlice := strings.Split(servers[i].Name, "-")
suffix, err := strconv.Atoi(serverNameSlice[len(serverNameSlice)-1])
if err != nil {
return err
return 0, err
}
log.PrintInfo(*serverResponse, *response)
return nil
// compare our current number agains the highest yet
if suffix > highestNumber {
highestNumber = suffix
}
}
// return one number higher than highest number
return highestNumber + 1, nil
}
func ListServers() ([]common.Server, error) {
// ListJynxServers lists all servers managed by Jynx hosted on Hetzner Cloud
func ListJynxServers(prefix string) ([]*hcloud.Server, error) {
// connect to the hcloud API
client := connect()
// get all servers
servers, err := client.Server.All(context.Background())
if err != nil {
return nil, err
}
var managedServers []common.Server
// create object that holds all servers managed by Jynx
var jynxServers []*hcloud.Server
for i := range servers {
// only look at servers, that we created
if !strings.HasPrefix(servers[i].Name, os.Getenv("HCLOUD_PREFIX")) {
// loop through servers and filter out managed servers
for _, server := range servers {
if strings.HasPrefix(server.Name, prefix) {
jynxServers = append(jynxServers, server)
}
}
// return managed servers
return jynxServers, nil
}
func CheckServerDeletions(servers []*hcloud.Server) error {
interval := os.Getenv("SCALER_POLL_INTERVAL")
if interval == "" {
interval = "5"
}
pollIntervalSeconds, err := strconv.Atoi(interval)
if err != nil {
return err
}
if pollIntervalSeconds < 60 {
pollIntervalSeconds = 60
}
if pollIntervalSeconds > 3600 {
pollIntervalSeconds %= 3600
}
for _, server := range servers {
safeInterval := time.Duration(3600-pollIntervalSeconds) * time.Second
optimalDeleteTime := server.Created.Add(safeInterval).Local()
log.PrintDebug(server.Name, "optimal delete time:", optimalDeleteTime)
if time.Now().Local().Before(optimalDeleteTime) {
continue
}
// append this server to our managed servers
managedServers = append(managedServers, common.Server{
Name: servers[i].Name,
Created: servers[i].Created,
Deployed: true,
})
// optimal delete time has passed, delete the server
if err := DeleteServer(server); err != nil {
return err
}
}
return managedServers, nil
return nil
}
func AddServer(s common.Server) error {
func DeleteServer(server *hcloud.Server) error {
client := connect()
log.PrintInfo("Deleting", server.Name)
result, response, err := client.Server.DeleteWithResult(context.Background(), server)
if err != nil {
return err
}
log.PrintWarning(result.Action.Status)
log.PrintWarning(response.StatusCode)
return nil
}
func CreateServers(servers []string) error {
// connect to hcloud API
client := connect()
// read in some environment variables
woodpeckerHost := os.Getenv("WOODPECKER_HOSTNAME")
agentSecret := os.Getenv("WOODPECKER_AGENT_SECRET")
AgentCMD := fmt.Sprintf("docker run -d -v /var/run/docker.sock:/var/run/docker.sock -e WOODPECKER_GRPC_SECURE=false -e WOODPECKER_SERVER=%s -e WOODPECKER_AGENT_SECRET=%s -e WOODPECKER_MAX_PROCS=%d -e WOODPECKER_HOSTNAME=%s", woodpeckerHost, agentSecret, 1, s.Name)
userdata := fmt.Sprintf("#cloud-config\nruncmd:\n\t- %s", AgentCMD)
fmt.Println(userdata)
// loop through each server
for _, server := range servers {
// create unique AgentCMD for cloud init
AgentCMD := fmt.Sprintf("docker run -d -v /var/run/docker.sock:/var/run/docker.sock -e WOODPECKER_GRPC_SECURE=false -e WOODPECKER_SERVER=%s -e WOODPECKER_AGENT_SECRET=%s -e WOODPECKER_MAX_PROCS=%d -e WOODPECKER_HOSTNAME=%s woodpeckerci/woodpecker-agent", woodpeckerHost, agentSecret, 1, server)
userdata := fmt.Sprintf("#cloud-config\nruncmd:\n - %s", AgentCMD)
labels := make(map[string]string)
labels["created_by"] = "jynx"
labels["created"] = fmt.Sprintf("%d", time.Now().Unix())
// sshkeys, err := client.SSHKey.All(context.Background())
// if err != nil {
// return err
// }
var sshkeys []*hcloud.SSHKey
sshkeys, err := getSSHKeys("stefan", "nils")
@ -85,7 +146,7 @@ func AddServer(s common.Server) error {
}
serverOpts := hcloud.ServerCreateOpts{
Name: s.Name,
Name: server,
StartAfterCreate: hcloud.Ptr(true),
SSHKeys: sshkeys,
Image: &hcloud.Image{Name: "docker-ce"},
@ -95,13 +156,11 @@ func AddServer(s common.Server) error {
PublicNet: &hcloud.ServerCreatePublicNet{EnableIPv4: true},
Labels: labels,
}
os.Exit(0)
result, response, err := client.Server.Create(context.Background(), serverOpts)
_, _, err = client.Server.Create(context.Background(), serverOpts)
if err != nil {
return err
}
}
io.ReadAll(response.Body)
log.PrintInfo(fmt.Sprintf("%+v\n%+v\n", result, response.Response.Body))
return nil
}

18
ci/woodpecker/woodpecker.go
View File

@ -18,9 +18,9 @@ type Server struct {
}
type Stats struct {
PendingJobs uint `json:"woodpecker_pending_jobs"`
RunningJobs uint `json:"woodpecker_running_jobs"`
IdlingWorkers uint `json:"woodpecker_worker_count"`
PendingJobs int `json:"woodpecker_pending_jobs"`
RunningJobs int `json:"woodpecker_running_jobs"`
IdlingWorkers int `json:"woodpecker_worker_count"`
}
// NewServer returns a Server object for host
@ -77,23 +77,23 @@ func (s Server) GetStats() (*Stats, error) {
// assign stats to struct
switch matches[3] {
case "pending_jobs":
stats.PendingJobs = parseStringToUint(matches[4])
stats.PendingJobs = parseStringToInt(matches[4])
case "running_jobs":
stats.RunningJobs = parseStringToUint(matches[4])
stats.RunningJobs = parseStringToInt(matches[4])
case "worker_count":
stats.IdlingWorkers = parseStringToUint(matches[4])
stats.IdlingWorkers = parseStringToInt(matches[4])
}
}
return &stats, nil
}
// parseStringToUint parses a string and returns the resulting uint
func parseStringToUint(s string) uint {
// parseStringToInt parses a string and returns the resulting uint
func parseStringToInt(s string) int {
num, err := strconv.Atoi(s)
if err != nil {
log.PrintError("can't parse string to uint,", err)
os.Exit(1)
}
return uint(num)
return num
}

103
db/db.go
View File

@ -1,70 +1,61 @@
package db
import (
"encoding/json"
"io"
"os"
// func Read() ([]common.Server, error) {
// // open file for reading, create it when necessary
// f, err := os.OpenFile("db.json", os.O_RDONLY|os.O_CREATE, os.ModePerm)
// if err != nil {
// return nil, err
// }
"git.stinnesbeck.com/go/jynx/backend/common"
"git.stinnesbeck.com/go/jynx/backend/hetzner"
)
// servers, err := hetzner.ListServers()
// if err != nil {
// return nil, err
// }
func Read() ([]common.Server, error) {
// open file for reading, create it when necessary
f, err := os.OpenFile("db.json", os.O_RDONLY|os.O_CREATE, os.ModePerm)
if err != nil {
return nil, err
}
// // close file when we are done
// defer f.Close()
servers, err := hetzner.ListServers()
if err != nil {
return nil, err
}
// // read file as byte array
// b, err := io.ReadAll(f)
// if err != nil {
// return nil, err
// }
// close file when we are done
defer f.Close()
// if len(b) == 0 {
// return nil, nil
// }
// read file as byte array
b, err := io.ReadAll(f)
if err != nil {
return nil, err
}
// // parse byte array to []common.Server
// // var serversFromDB []common.Server
// // if err := json.Unmarshal(b, &serversFromDB); err != nil {
// // return nil, err
// // }
if len(b) == 0 {
return nil, nil
}
// // // merge servers from db and hcloud
// // servers = append(servers, serversFromDB...)
// parse byte array to []common.Server
var serversFromDB []common.Server
if err := json.Unmarshal(b, &serversFromDB); err != nil {
return nil, err
}
// // return read in servers
// return servers, nil
// }
// merge servers from db and hcloud
servers = append(servers, serversFromDB...)
// func Save(servers []common.Server) error {
// // open file with provided flags, create it when necessary
// f, err := os.OpenFile("db.json", os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm)
// if err != nil {
// return err
// }
// return read in servers
return servers, nil
}
// // close file when we are done
// defer f.Close()
func Save(servers []common.Server) error {
// open file with provided flags, create it when necessary
f, err := os.OpenFile("db.json", os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm)
if err != nil {
return err
}
// // Marshal data as JSON and put into byte array
// b, err := json.MarshalIndent(servers, "", " ")
// if err != nil {
// return err
// }
// close file when we are done
defer f.Close()
// // write byte array to file
// _, err = f.Write(b)
// Marshal data as JSON and put into byte array
b, err := json.MarshalIndent(servers, "", " ")
if err != nil {
return err
}
// write byte array to file
_, err = f.Write(b)
return err
}
// return err
// }

62
scaler/scaler.go
View File

@ -2,12 +2,10 @@ package scaler
import (
"fmt"
"time"
"os"
"git.stinnesbeck.com/go/jynx/backend/common"
"git.stinnesbeck.com/go/jynx/backend/hetzner"
"git.stinnesbeck.com/go/jynx/ci/woodpecker"
"git.stinnesbeck.com/go/jynx/db"
"git.stinnesbeck.com/go/log"
)
@ -18,42 +16,62 @@ func Start(w *woodpecker.Server) error {
return err
}
// read info from database
servers, err := db.Read()
// get the prefix for jynx managed servers
prefix := os.Getenv("HCLOUD_PREFIX")
// get a list of all servers from hetzner (from jynx)
servers, err := hetzner.ListJynxServers(prefix)
if err != nil {
return err
}
var newServers int
newServers := stats.PendingJobs - len(servers) + stats.RunningJobs
log.PrintDebug(stats.RunningJobs, "running jobs,", stats.PendingJobs, "pending jobs,", stats.IdlingWorkers, "idling workers,", len(servers)-stats.RunningJobs-stats.IdlingWorkers, "servers starting up")
if stats.PendingJobs == 0 || stats.IdlingWorkers+stats.RunningJobs+uint(len(servers)) >= stats.PendingJobs {
// check if there are servers to be removed
if stats.PendingJobs == 0 && stats.RunningJobs == 0 {
// we need to remove servers
if err := hetzner.CheckServerDeletions(servers); err != nil {
return err
}
}
// check if we need to create new servers
// if stats.PendingJobs < 1 || len(servers)-stats.RunningJobs >= stats.PendingJobs {
if stats.PendingJobs < 1 || newServers < 1 {
// if stats.PendingJobs == 0 || stats.IdlingWorkers+stats.RunningJobs >= stats.PendingJobs {
// skip rest of this run
return nil
}
log.PrintInfo("starting", stats.PendingJobs, "workers")
// there are servers to be created
newServers += int(stats.PendingJobs)
var ScaleUpServers []common.Server
if newServers < 1 {
return nil
}
var ScaleUpServers []string
// get the suffix for the starting server
suffix, err := hetzner.GetNewSuffix(servers)
for i := 0; i < newServers; i++ {
name := fmt.Sprintf("jynx-worker-%d", i+1)
ScaleUpServers = append(ScaleUpServers, common.Server{
Name: name,
Created: time.Now(),
})
if err := hetzner.AddServer(ScaleUpServers[i]); err != nil {
if err != nil {
return err
}
// assemble server name and add it to the list of servers to scale up
serverName := fmt.Sprintf("%s-worker-%d", prefix, suffix)
ScaleUpServers = append(ScaleUpServers, serverName)
// add 1 to the suffix for the next server
suffix++
}
// save servers to database
servers = append(servers, ScaleUpServers...)
if err := db.Save(servers); err != nil {
log.PrintInfo("starting", newServers, "workers:", ScaleUpServers)
// create servers on hetzner cloud
if err := hetzner.CreateServers(ScaleUpServers); err != nil {
return err
}
log.PrintInfo(stats.RunningJobs, "running jobs,", stats.PendingJobs, "pending jobs,", stats.IdlingWorkers, "idling workers,", len(servers), "servers starting up")
return nil
}