refactor: complete rewrite #17

Merged
Aire-One merged 5 commits from refactor/complete-rewrite into main 2024-09-23 23:55:44 +02:00
28 changed files with 616 additions and 208 deletions

View File

@ -1,20 +1,26 @@
{
"ignorePaths": [
".golangci.yaml", // Configuration file with a lot of Go specific words
"go.mod", // This file is managed by Go
"go.sum", // This file is managed by Go
"LICENSE" // This file was automatically generated by Gitea
],
"words": [
"aireone",
"buildx",
"distroless",
"Gitea",
"gocron",
"golangci",
"hadolint",
"healthcheckers",
"labtime",
"nonroot",
"promhttp",
"prommonitors",
"proxied",
"sigchan",
"woodpeckerci"
"woodpeckerci",
"yamlconfig"
]
}

9
.dockerignore Normal file
View File

@ -0,0 +1,9 @@
.vscode
.woodpecker
build
configs
.*
renovate.json

View File

@ -8,8 +8,14 @@ indent_style = space
indent_size = 4
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = false
trim_trailing_whitespace = true
insert_final_newline = true
[*.{json,yaml}]
indent_size = 2
[*.go]
indent_style = tab
[Makefile]
indent_style = tab

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/build

View File

@ -1,7 +1,50 @@
---
run:
timeout: 5m
# this settings is for the CI pipeline because it fails with the default value
# we should have used the --timeout flag in the CI pipeline
# but to make the linter work locally, I need to have a config file
# so I pushed this config file until we have something to do here
linters:
enable:
- asasalint
- bodyclose
- contextcheck
- copyloopvar
- depguard
- durationcheck
- errcheck
- errname
- errorlint
- gocritic
- godot
- gofumpt
- goimports
- gosec
- gosimple
- govet
- ineffassign
- misspell
- nilerr
- nilnil
- noctx
- nolintlint
- prealloc
- predeclared
- promlinter
- reassign
- revive
- staticcheck
- tenv
- testableexamples
- thelper
- tparallel
- typecheck
- unconvert
- unused
- unparam
- usestdlibvars
- wastedassign
linters-settings:
depguard:
rules:
main:
list-mode: lax
allow: [aireone.xyz/labtime] # It's mendatory to have at least one package in allow or deny list
errcheck:
check-type-assertions: true
check-blank: true

16
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,16 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Launch Package",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/cmd/labtime/main.go",
"args": ["-config", "../../configs/example-config.yaml"]
}
]
}

15
.vscode/settings.json vendored
View File

@ -1,11 +1,8 @@
{
"[go]": {
"editor.codeActionsOnSave": {
// "source.organizeImports": "always"
},
"editor.formatOnSave": true
},
"go.formatTool": "goimports",
"go.lintTool": "golangci-lint"
// "go.lintFlags": ["--fast"] // We only have the default linters and it seems they are not marked as fast, but it's fast enough for me
"go.lintTool": "golangci-lint",
"go.lintFlags": ["--fast"],
"go.useLanguageServer": true,
"gopls": {
"formatting.gofumpt": true
}
}

9
.woodpecker/test.yaml Normal file
View File

@ -0,0 +1,9 @@
---
when:
event: pull_request
steps:
golang:
image: golang:1.23.1
commands:
- go test -v ./...

View File

@ -4,9 +4,11 @@ FROM golang:1.23.1 AS builder
WORKDIR /app
COPY go.mod go.sum main.go ./
COPY go.mod go.sum ./
RUN go mod download
RUN go build
COPY . .
RUN go build -o labtime cmd/labtime/main.go
FROM gcr.io/distroless/base-debian12
@ -14,12 +16,10 @@ WORKDIR /
COPY --from=builder /app/labtime /labtime
# This is the port currently hardcoded in the application
EXPOSE 2112
# For now the config file path/name are hardcoded in the application
VOLUME ["/config"]
USER nonroot:nonroot
ENTRYPOINT ["/labtime"]
ENTRYPOINT ["/labtime"]

24
Makefile Normal file
View File

@ -0,0 +1,24 @@
BINARY_NAME=labtime
.PHONY: all
all: lint test build
.PHONY: clean
clean:
go clean -i ./...
rm -rf build
.PHONY: lint
lint:
@echo "Running golangci-lint"
golangci-lint run
.PHONY: test
test:
@echo "Running tests"
go test -v ./...
.PHONY: build
build:
@echo "Building binary"
go build -o build/$(BINARY_NAME) cmd/labtime/main.go

26
cmd/labtime/main.go Normal file
View File

@ -0,0 +1,26 @@
package main
import (
"log"
"os"
"aireone.xyz/labtime/internal/apps/labtime"
)
const (
loggerPrefix = "main:"
)
func main() {
logger := log.New(os.Stdout, loggerPrefix, log.LstdFlags|log.Lshortfile)
cfg := labtime.LoadFlag()
app, err := labtime.NewApp(cfg, logger)
if err != nil {
logger.Fatalf("Error creating app: %v", err)
}
if err := app.Start(); err != nil {
logger.Fatalf("Error starting app: %v", err)
}
}

View File

@ -4,4 +4,4 @@ targets:
url: https://aireone.xyz
interval: 2
- name: Gitea
url: https://gitea.aireone.xyz
url: https://gitea.aireone.xyz

View File

@ -0,0 +1 @@
This is a docker compose stack example to run Prometheus with Grafana to play with the Labtime metrics.

View File

@ -5,12 +5,12 @@ services:
ports:
- 9090:9090
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
- prometheus-data/prometheus.yaml:/etc/prometheus/prometheus.yaml
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--config.file=/etc/prometheus/prometheus.yaml'
extra_hosts:
- "host.docker.internal:host-gateway"
grafana:
image: grafana/grafana
ports:
- 3000:3000
- 3000:3000

1
go.mod
View File

@ -15,6 +15,7 @@ require (
github.com/jonboulle/clockwork v0.4.0 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect

2
go.sum
View File

@ -18,6 +18,8 @@ github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg=

View File

@ -0,0 +1,107 @@
package labtime
import (
"context"
"log"
"net/http"
"os"
"time"
"aireone.xyz/labtime/internal/monitors"
"aireone.xyz/labtime/internal/scheduler"
"aireone.xyz/labtime/internal/yamlconfig"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
type App struct {
scheduler *scheduler.Scheduler
prometheusHTTPServer *http.Server
logger *log.Logger
}
func NewApp(flag *FlagConfig, logger *log.Logger) (*App, error) {
config, err := loadYamlConfig(flag.configFile)
if err != nil {
return nil, errors.Wrap(err, "error loading yaml config")
}
scheduler, err := createScheduler(config, logger)
if err != nil {
return nil, errors.Wrap(err, "error creating scheduler")
}
return &App{
logger: logger,
scheduler: scheduler,
prometheusHTTPServer: &http.Server{
Addr: ":2112",
ReadTimeout: 5 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 15 * time.Second,
},
}, nil
}
func loadYamlConfig(configFile string) (*yamlconfig.YamlConfig, error) {
file, err := os.Open(configFile)
if err != nil {
return nil, errors.Wrap(err, "error opening config file")
}
defer file.Close()
return yamlconfig.NewYamlConfig(file)
}
func createScheduler(config *yamlconfig.YamlConfig, logger *log.Logger) (*scheduler.Scheduler, error) {
scheduler, err := scheduler.NewScheduler(logger)
if err != nil {
return nil, errors.Wrap(err, "error creating scheduler")
}
httpMonitor := prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "labtime_response_time_duration",
Help: "The ping time.",
}, []string{"target_name"})
prometheus.MustRegister(httpMonitor)
for _, t := range config.Targets {
if err := scheduler.AddJob(&monitors.HTTPMonitor{
Label: t.Name,
URL: t.URL,
Logger: logger,
ResponseTimeMonitor: httpMonitor,
}, t.Interval); err != nil {
return nil, errors.Wrap(err, "error adding job")
}
}
return scheduler, nil
}
func (a *App) Start() error {
a.scheduler.Start()
// Serve Prometheus metrics
http.Handle("/metrics", promhttp.Handler())
if err := a.prometheusHTTPServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
return errors.Wrap(err, "error starting prometheus http server")
}
return nil
}
func (a *App) Shutdown() error {
if err := a.scheduler.Shutdown(); err != nil {
return errors.Wrap(err, "error shutting down scheduler")
}
if err := a.prometheusHTTPServer.Shutdown(context.TODO()); err != nil {
return errors.Wrap(err, "error shutting down prometheus http server")
}
return nil
}

View File

@ -0,0 +1,31 @@
package labtime
import (
"log"
"os"
"testing"
)
func TestNewApp(t *testing.T) {
flag := &FlagConfig{
configFile: "../../../configs/example-config.yaml", // we shouldn't rely on the actual file
}
logger := log.New(os.Stdout, "", 0) // silent?
app, err := NewApp(flag, logger)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if app.logger != logger {
t.Errorf("expected logger to be set")
}
if app.scheduler == nil {
t.Errorf("expected scheduler to be initialized")
}
if app.prometheusHTTPServer == nil {
t.Errorf("expected prometheusHTTPServer to be initialized")
}
}

View File

@ -0,0 +1,18 @@
package labtime
import "flag"
const (
defaultConfigFile = "config.yaml"
)
type FlagConfig struct {
configFile string
}
func LoadFlag() *FlagConfig {
cfg := FlagConfig{}
flag.StringVar(&cfg.configFile, "config", defaultConfigFile, "Path to the configuration file")
flag.Parse()
return &cfg
}

View File

@ -0,0 +1,45 @@
package labtime
import (
"flag"
"os"
"testing"
)
func TestLoadFlag(t *testing.T) {
tests := []struct {
name string
args []string
expectedFile string
}{
{
name: "No flag provided",
args: []string{},
expectedFile: defaultConfigFile,
},
{
name: "Flag provided",
args: []string{"-config", "custom.yaml"},
expectedFile: "custom.yaml",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Save and restore the original os.Args
oldArgs := os.Args
defer func() { os.Args = oldArgs }()
// Make sure we are using a clean flag set
flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
os.Args = append([]string{"cmd"}, tt.args...)
cfg := LoadFlag()
if cfg.configFile != tt.expectedFile {
t.Errorf("got %s, want %s", cfg.configFile, tt.expectedFile)
}
})
}
}

View File

@ -0,0 +1,33 @@
package http
import (
"log"
"net/http"
)
type RoundTripperMiddleware struct {
Proxied http.RoundTripper
OnBefore func(req *http.Request)
OnAfter func(res *http.Response)
}
func (m RoundTripperMiddleware) RoundTrip(req *http.Request) (res *http.Response, err error) {
m.OnBefore(req)
res, err = m.Proxied.RoundTrip(req)
m.OnAfter(res)
return res, err
}
func NewLoggerMiddleware(logger *log.Logger, proxied http.RoundTripper) *RoundTripperMiddleware {
return &RoundTripperMiddleware{
Proxied: proxied,
OnBefore: func(req *http.Request) {
logger.Printf("Request: %s %s", req.Method, req.URL.String())
},
OnAfter: func(res *http.Response) {
logger.Printf("Response: %d", res.StatusCode)
},
}
}

View File

@ -0,0 +1,88 @@
package monitors
import (
"context"
"fmt"
"log"
"net/http"
"time"
aireoneHttp "aireone.xyz/labtime/internal/http"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
)
var ErrInvalidStatusCode = errors.New("expected status code 200")
type HTTPMonitor struct {
Label string
URL string
Logger *log.Logger
ResponseTimeMonitor *prometheus.GaugeVec
}
func (h *HTTPMonitor) ID() string {
return h.Label
}
func (h *HTTPMonitor) Run() error {
d, err := h.httpHealthCheck()
if err != nil {
return errors.Wrap(err, "error running http health check")
}
h.pushToPrometheus(d)
return nil
}
func newHTTPDurationMiddleware(duration *time.Duration, proxied http.RoundTripper) *aireoneHttp.RoundTripperMiddleware {
var t time.Time
return &aireoneHttp.RoundTripperMiddleware{
Proxied: proxied,
OnBefore: func(_ *http.Request) {
t = time.Now()
},
OnAfter: func(_ *http.Response) {
*duration = time.Since(t)
},
}
}
type HTTPHealthCheckerData struct {
Duration time.Duration
}
func (h *HTTPMonitor) httpHealthCheck() (*HTTPHealthCheckerData, error) {
r, err := http.NewRequest(http.MethodHead, h.URL, http.NoBody)
if err != nil {
return nil, errors.Wrap(err, "error creating http request")
}
req := r.WithContext(context.TODO())
var duration time.Duration
client := &http.Client{
Transport: aireoneHttp.NewLoggerMiddleware(h.Logger, newHTTPDurationMiddleware(&duration, http.DefaultTransport)),
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, errors.Wrap(ErrInvalidStatusCode, fmt.Sprintf("got status code %d", resp.StatusCode))
}
return &HTTPHealthCheckerData{
Duration: duration,
}, nil
}
func (h *HTTPMonitor) pushToPrometheus(d *HTTPHealthCheckerData) {
h.Logger.Printf("Push metrics to Prometheus: Response time for %s: %v", h.Label, d.Duration.Seconds())
h.ResponseTimeMonitor.With(prometheus.Labels{"target_name": h.Label}).Set(d.Duration.Seconds())
}

View File

@ -0,0 +1,16 @@
package monitors
import "testing"
func TestHTTPMonitor_ID(t *testing.T) {
monitor := &HTTPMonitor{
Label: "example",
}
expectedID := "example"
actualID := monitor.ID()
if actualID != expectedID {
t.Errorf("expected ID to be %s, but got %s", expectedID, actualID)
}
}

View File

@ -0,0 +1,6 @@
package monitors
type Monitor interface {
ID() string
Run() error
}

View File

@ -0,0 +1,60 @@
package scheduler
import (
"log"
"time"
"aireone.xyz/labtime/internal/monitors"
"github.com/go-co-op/gocron/v2"
"github.com/pkg/errors"
)
type Scheduler struct {
scheduler gocron.Scheduler
logger *log.Logger
}
func NewScheduler(logger *log.Logger) (*Scheduler, error) {
s, err := gocron.NewScheduler()
if err != nil {
return nil, errors.Wrap(err, "error creating scheduler")
}
return &Scheduler{
scheduler: s,
logger: logger,
}, nil
}
func (s *Scheduler) AddJob(m monitors.Monitor, interval int) error {
j, err := s.scheduler.NewJob(
gocron.DurationJob(time.Duration(interval)*time.Second),
gocron.NewTask(func() {
s.logger.Printf("Running job for HTTP monitor %s\n", m.ID())
if err := m.Run(); err != nil {
s.logger.Printf("Error running job for HTTP monitor %s: %s\n", m.ID(), err)
}
s.logger.Printf("Job finished for HTTP monitor %s\n", m.ID())
}),
)
if err != nil {
return errors.Wrap(err, "error creating job")
}
s.logger.Printf("Job started for HTTP monitor %s with ID: %s\n", m.ID(), j.ID().String())
return nil
}
func (s *Scheduler) Start() {
s.scheduler.Start()
}
func (s *Scheduler) Shutdown() error {
if err := s.scheduler.Shutdown(); err != nil {
return errors.Wrap(err, "error shutting down scheduler")
}
return nil
}

View File

@ -0,0 +1,45 @@
package yamlconfig
import (
"io"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
)
var ErrYAMLDecode = errors.New("error decoding YAML file")
type YamlConfig struct {
// List of targets to ping.
Targets []struct {
// Name of the target. Used to identify the target from Prometheus.
Name string `yaml:"name"`
// URL of the target. The target should be accessible from the machine running the exporter.
// The URL should contain the protocol (http:// or https://) and the port if it's not the default one.
URL string `yaml:"url"`
// Interval to ping the target. Default is 5 seconds.
Interval int `yaml:"interval,omitempty"`
} `yaml:"targets"`
}
func NewYamlConfig(r io.Reader) (*YamlConfig, error) {
d := yaml.NewDecoder(r)
config := &YamlConfig{}
if err := d.Decode(&config); err != nil {
return nil, errors.Wrap(ErrYAMLDecode, err.Error())
}
applyDefault(config)
return config, nil
}
func applyDefault(config *YamlConfig) {
// Set default interval to 5 seconds if not provided
for i := range config.Targets {
if config.Targets[i].Interval == 0 {
config.Targets[i].Interval = 5
}
}
}

182
main.go
View File

@ -1,182 +0,0 @@
package main
import (
"crypto/tls"
"fmt"
"log"
"net/http"
"net/http/httptrace"
"os"
"os/signal"
"time"
"github.com/go-co-op/gocron/v2"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"gopkg.in/yaml.v3"
)
type Config struct {
// List of targets to ping
Targets []struct {
// Name of the target. Used to identify the target from Prometheus.
Name string `yaml:"name"`
// URL of the target. The target should be accessible from the machine running the exporter.
// The URL should contain the protocol (http:// or https://) and the port if it's not the default one.
Url string `yaml:"url"`
// Interval to ping the target. Default is 5 seconds
Interval int `yaml:"interval,omitempty"`
} `yaml:"targets"`
}
func NewConfig(configPath string) (*Config, error) {
// Create config structure
config := &Config{}
// Open config file
file, err := os.Open(configPath)
if err != nil {
return nil, err
}
defer file.Close()
// Init new YAML decode
d := yaml.NewDecoder(file)
// Start YAML decoding from file
if err := d.Decode(&config); err != nil {
return nil, err
}
return config, nil
}
func Ping(url string) (int, time.Duration, error) {
req, err := http.NewRequest(http.MethodHead, url, nil)
if err != nil {
return 0, 0, err
}
var start, connect, dns, tlsHandshake time.Time
trace := &httptrace.ClientTrace{
DNSStart: func(dsi httptrace.DNSStartInfo) { dns = time.Now() },
DNSDone: func(ddi httptrace.DNSDoneInfo) {
fmt.Printf("DNS Done: %v\n", time.Since(dns))
},
TLSHandshakeStart: func() { tlsHandshake = time.Now() },
TLSHandshakeDone: func(cs tls.ConnectionState, err error) {
fmt.Printf("TLS Handshake: %v\n", time.Since(tlsHandshake))
},
ConnectStart: func(network, addr string) { connect = time.Now() },
ConnectDone: func(network, addr string, err error) {
fmt.Printf("Connect time: %v\n", time.Since(connect))
},
GotFirstResponseByte: func() {
fmt.Printf("Time from start to first byte: %v\n", time.Since(start))
},
}
req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
start = time.Now()
resp, err := http.DefaultTransport.RoundTrip(req)
if err != nil {
return 0, 0, err
}
defer resp.Body.Close()
return resp.StatusCode, time.Since(start), nil
}
func main() {
// Load config
// TODO: Use a flag to specify the config file
config, err := NewConfig("config/config.yaml")
if err != nil {
log.Fatalf("Error loading config: %s", err)
}
// Print config
fmt.Printf("%+v\n", config)
// create a scheduler
s, err := gocron.NewScheduler()
if err != nil {
log.Fatalf("Error creating the scheduler: %s",err)
}
// Prometheus metrics
responseTimeMonitor := prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "labtime_response_time_duration",
Help: "The ping time.",
}, []string{"target_name"})
prometheus.MustRegister(responseTimeMonitor)
// Intercept the signal to stop the program
go func() {
sigchan := make(chan os.Signal, 1)
signal.Notify(sigchan, os.Interrupt)
<-sigchan
log.Println("Program killed !")
// do last actions and wait for all write operations to end
// when you're done, shut it down
err = s.Shutdown()
if err != nil {
log.Fatalf("Error shutting down scheduler: %s", err)
}
os.Exit(0)
}()
// add a job to the scheduler
for _, target := range config.Targets {
j, err := s.NewJob(
gocron.DurationJob(
func(interval int) time.Duration {
if interval == 0 {
const defaultInterval = 5
return time.Duration(defaultInterval) * time.Second
}
return time.Duration(interval) * time.Second
}(target.Interval),
),
gocron.NewTask(
func(url string) {
status, elapsedTime, err := Ping(url)
if err != nil {
fmt.Println(err)
return
}
fmt.Printf("%s - Status: %d in %v\n", target.Name, status, elapsedTime.Seconds())
// push to Prometheus
responseTimeMonitor.With(prometheus.Labels{"target_name": target.Name}).Set(elapsedTime.Seconds())
},
target.Url,
),
)
if err != nil {
log.Fatalf("Error creating job: %s", err)
}
// each job has a unique id
fmt.Printf("Job %s started with ID: %s\n", target.Name, j.ID().String())
}
// start the scheduler
s.Start()
// Serve Prometheus metrics
http.Handle("/metrics", promhttp.Handler())
if err := http.ListenAndServe(":2112", nil); err != nil {
log.Fatalf("Error starting server: %s", err)
}
}