Add display of identifying labels + reworked the structure

src/metrics-alerting/alert/alert.go

@@ -4,54 +4,25 @@ import (
 	"fmt"
 
 	"metrics-alerting/config"
-	"metrics-alerting/warp10"
-)
-
-func ProcessNumber(
-	client warp10.Warp10Client,
-	script config.Script,
-	ms config.MailSettings,
-) error {
-	value, err := client.ReadNumber(script.Script)
-	if err != nil {
-		return err
-	}
-
-	if value < script.Threshold {
-		// Nothing to alert about
-		return nil
-	}
 
-	return alert(script, value, ms)
-}
-
-func ProcessBool(
-	client warp10.Warp10Client,
-	script config.Script,
-	ms config.MailSettings,
-) error {
-	value, err := client.ReadBool(script.Script)
-	if err != nil {
-		return err
-	}
-
-	if value {
-		return nil
-	}
+	"gopkg.in/gomail.v2"
+)
 
-	return alert(script, value, ms)
+type Alerter struct {
+	Dialer *gomail.Dialer
+	Sender string
 }
 
-func alert(
+func (a *Alerter) Alert(
 	script config.Script,
 	result interface{},
-	ms config.MailSettings,
+	labels map[string]string,
 ) error {
 	switch script.Action {
 	case "http":
-		return alertHttp(script, result)
+		return a.alertHttp(script, result, labels)
 	case "email":
-		return alertEmail(script, result, ms)
+		return a.alertEmail(script, result, labels)
 	default:
 		return fmt.Errorf("invalid action type: %s", script.Action)
 	}

src/metrics-alerting/alert/email.go

@@ -1,55 +1,79 @@
 package alert
 
 import (
+	"encoding/json"
 	"fmt"
+	"strings"
 
 	"metrics-alerting/config"
 
 	"gopkg.in/gomail.v2"
 )
 
-func alertEmail(
+func (a *Alerter) alertEmail(
 	script config.Script,
 	result interface{},
-	ms config.MailSettings,
+	labels map[string]string,
 ) error {
-	formatNumber := `
-Script %s just exceeded its threshold of %f and now returns %f
-
-Script:
-
-%s
-	`
-
-	formatBool := `
-Test for script %s and returned false instead of true
-
-Script:
-
-%s
-	`
+	formatNumber := "Script %s just exceeded its threshold of %.2f and now returns %f"
+	formatBool := "Test for script %s and returned false instead of true"
 
 	var body, subject string
 	switch script.Type {
-	case "number":
-		subject = fmt.Sprintf("Threshold exceeded for script %s", script.Key)
+	case "number", "series":
+		subject = fmt.Sprintf(
+			"Threshold exceeded for script %s %s", script.Key,
+			getIdentifyingLabels(script, labels),
+		)
 		body = fmt.Sprintf(
 			formatNumber, script.Key, script.Threshold, result.(float64),
-			script.Script,
 		)
 	case "bool":
-		subject = fmt.Sprintf("Test for script %s failed", script.Key)
-		body = fmt.Sprintf(formatBool, script.Key, script.Script)
+		subject = fmt.Sprintf(
+			"Test for script %s failed %s", script.Key,
+			getIdentifyingLabels(script, labels),
+		)
+		body = fmt.Sprintf(formatBool, script.Key)
 	}
 
+	if labels != nil {
+		jsonLabels, err := json.Marshal(labels)
+		if err != nil {
+			return err
+		}
+		body = fmt.Sprintf("%s\n\nLabels: %+v", body, string(jsonLabels))
+	}
+
+	body = fmt.Sprintf("%s\n\nScript:\n%s", body, script.Script)
+
 	m := gomail.NewMessage()
-	m.SetHeader("From", ms.Sender)
-	m.SetHeader("To", ms.Recipient)
+	m.SetHeader("From", a.Sender)
+	m.SetHeader("To", script.Target)
 	m.SetHeader("Subject", subject)
 	m.SetBody("text/plain", body)
 
-	d := gomail.NewDialer(
-		ms.SMTP.Host, ms.SMTP.Port, ms.SMTP.Username, ms.SMTP.Password,
-	)
-	return d.DialAndSend(m)
+	return a.Dialer.DialAndSend(m)
+}
+
+func getIdentifyingLabels(
+	script config.Script,
+	labels map[string]string,
+) string {
+	if len(script.IdentifyingLabels) == 0 {
+		return ""
+	}
+
+	identifyingLabels := make(map[string]string)
+	for _, label := range script.IdentifyingLabels {
+		identifyingLabels[label] = labels[label]
+	}
+
+	labelsAsStrs := []string{}
+	var labelAsStr string
+	for key, value := range identifyingLabels {
+		labelAsStr = key + ": " + value
+		labelsAsStrs = append(labelsAsStrs, labelAsStr)
+	}
+
+	return "(" + strings.Join(labelsAsStrs, ", ") + ")"
 }

src/metrics-alerting/alert/http.go

@@ -11,11 +11,16 @@ import (
 )
 
 type alertBody struct {
-	Key   string `json:"scriptKey"`
-	Value string `json:"value"`
+	Key    string            `json:"scriptKey"`
+	Value  string            `json:"value"`
+	Labels map[string]string `json:"labels"`
 }
 
-func alertHttp(script config.Script, result interface{}) error {
+func (a *Alerter) alertHttp(
+	script config.Script,
+	result interface{},
+	labels map[string]string,
+) error {
 	var value string
 	switch script.Type {
 	case "number":
@@ -25,8 +30,9 @@ func alertHttp(script config.Script, result interface{}) error {
 	}
 
 	alert := alertBody{
-		Key:   script.Key,
-		Value: value,
+		Key:    script.Key,
+		Value:  value,
+		Labels: labels,
 	}
 
 	body, err := json.Marshal(alert)

src/metrics-alerting/config/config.go

@@ -9,8 +9,6 @@ import (
 type MailSettings struct {
 	// Sender of the alert emails
 	Sender string `yaml:"sender"`
-	// Recipient of the alert emails
-	Recipient string `yaml:"recipient"`
 	// Settings to connect to the mail server
 	SMTP SMTPSettings `yaml:"smtp"`
 }
@@ -40,6 +38,9 @@ type Script struct {
 	Action string `yaml:"action"`
 	// The action's target
 	Target string `yaml:"target"`
+	// The labels that will be mentioned in the email subject, only required if
+	// the action is "email"
+	IdentifyingLabels []string `yaml:"identifying_labels,omitempty"`
 }
 
 type Config struct {

src/metrics-alerting/main.go

@@ -6,9 +6,11 @@ import (
 
 	"metrics-alerting/alert"
 	"metrics-alerting/config"
+	"metrics-alerting/process"
 	"metrics-alerting/warp10"
 
 	"github.com/sirupsen/logrus"
+	"gopkg.in/gomail.v2"
 )
 
 var (
@@ -23,15 +25,26 @@ func main() {
 		ExecEndpoint: cfg.Warp10Exec,
 		ReadToken:    cfg.ReadToken,
 	}
+	dialer := gomail.NewDialer(
+		cfg.Mail.SMTP.Host, cfg.Mail.SMTP.Port, cfg.Mail.SMTP.Username,
+		cfg.Mail.SMTP.Password,
+	)
+	alerter := alert.Alerter{
+		Dialer: dialer,
+		Sender: cfg.Mail.Sender,
+	}
 
 	for _, script := range cfg.Scripts {
 		var err error
 		switch script.Type {
 		case "number":
-			err = alert.ProcessNumber(client, script, cfg.Mail)
+			err = process.ProcessNumber(client, script, alerter)
 			break
 		case "bool":
-			err = alert.ProcessBool(client, script, cfg.Mail)
+			err = process.ProcessBool(client, script, alerter)
+			break
+		case "series":
+			err = process.ProcessSeries(client, script, alerter)
 			break
 		default:
 			err = fmt.Errorf("invalid return type: %s", script.Type)

src/metrics-alerting/process/process.go

@@ -0,0 +1,97 @@
+package process
+
+import (
+	"time"
+
+	"metrics-alerting/alert"
+	"metrics-alerting/config"
+	"metrics-alerting/warp10"
+)
+
+func ProcessNumber(
+	client warp10.Warp10Client,
+	script config.Script,
+	alerter alert.Alerter,
+) error {
+	value, err := client.ReadNumber(script.Script)
+	if err != nil {
+		return err
+	}
+
+	return processFloat(value, script, alerter, nil)
+}
+
+func ProcessBool(
+	client warp10.Warp10Client,
+	script config.Script,
+	alerter alert.Alerter,
+) error {
+	value, err := client.ReadBool(script.Script)
+	if err != nil {
+		return err
+	}
+
+	if value {
+		return nil
+	}
+
+	return alerter.Alert(script, value, nil)
+}
+
+func ProcessSeries(
+	client warp10.Warp10Client,
+	script config.Script,
+	alerter alert.Alerter,
+) error {
+	series, err := client.ReadSeriesOfNumbers(script.Script)
+
+	for _, serie := range series {
+		if !isRecentEnough(serie.Datapoints[0]) {
+			// If the serie hasn't been active in the last 10min, don't consider
+			// it
+			// TODO: If the serie was active at the previous run, send an alert
+			continue
+		}
+
+		// Remove useless ".app" label
+		_, ok := serie.Labels[".app"]
+		if ok {
+			delete(serie.Labels, ".app")
+		}
+		// TODO: Currently we only process the most recent point.
+		// If the datapoint is above the threshold, we should crawl back in time
+		// to find when the situation began so we can add info about time in the
+		// alert.
+		if err = processFloat(
+			serie.Datapoints[0][1], script, alerter, serie.Labels,
+		); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func processFloat(
+	value float64,
+	script config.Script,
+	alerter alert.Alerter,
+	labels map[string]string,
+) error {
+	if value < script.Threshold {
+		// Nothing to alert about
+		return nil
+	}
+
+	return alerter.Alert(script, value, labels)
+}
+
+func isRecentEnough(datapoint []float64) bool {
+	// Allowed offset between a point and now is 10min
+	allowedOffset := int64(600000000)
+
+	now := time.Now().UnixNano() / 1000 // Current timestamp (seconds)
+
+	return now-int64(datapoint[0]) <= allowedOffset
+	return false
+}

src/metrics-alerting/warp10/serie.go

@@ -0,0 +1,14 @@
+package warp10
+
+type FloatTimeSerie struct {
+	// Class name of the serie
+	Class string `json:"c"`
+	// Labels of the serie
+	Labels map[string]string `json:"l"`
+	// Attributes of the serie
+	Attributes map[string]string `json:"a"`
+	// Datapoints: each element of the slice is a point, represented as a slice
+	// with the timestamp as its first element and the value of the point as its
+	// second element
+	Datapoints [][]float64 `json:"v"`
+}

src/metrics-alerting/warp10/warp10.go

@@ -42,6 +42,16 @@ func (w *Warp10Client) ReadNumber(script string) (f float64, err error) {
 	return
 }
 
+func (w *Warp10Client) ReadSeriesOfNumbers(script string) (f []FloatTimeSerie, err error) {
+	resp, err := w.sendRequest(script)
+	if err != nil {
+		return
+	}
+
+	err = json.NewDecoder(resp.Body).Decode(&f)
+	return
+}
+
 func (w *Warp10Client) appendToken(script string) string {
 	return fmt.Sprintf("'%s' 'token' STORE\n%s", w.ReadToken, script)
 }