alert on error by default, add configuration to control behaviour (#6)

Author: r2k1

README.md

@@ -19,6 +19,7 @@ Example:
 
 ```toml
 reminder_interval = "3h"
+alert_on_error = false
 
 [alerts]
     [alerts.test-alert-1]
@@ -39,6 +40,7 @@ reminder_interval = "3h"
     dbs = ["my-mysql-db"]
     destinations = ["slacks.my-slack"]
     interval = "1h30m"
+    alert_on_error = true
 
 
 [dbs]
@@ -60,16 +62,17 @@ reminder_interval = "3h"
 
 
 Configuration is defined in [TOML](https://github.com/toml-lang/toml) format.
-- `reminder_interval` - Default interval for all alerts after which alert will be triggered again. Set to 0 if you don't need reminders. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". Examples: "300s", "1.5h" or "2h45m". Default value is 0.
-- `alerts.{alert-name}.reminder_interval` - Same as above, but for individual alert.
+- `reminder_interval` - Optional. Default interval for all alerts after which alert will be triggered again. Set to 0 if you don't need reminders. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". Examples: "300s", "1.5h" or "2h45m". Default value is 0.
+- `alert_on_error` - Option. Defines behavior during error encounters (connection errors, sql syntax errors, timeouts etc ). If set to true any error will trigger an alert. If set to false then error will be logged and ignored. Default value is true.
+- `alerts.{alert-name}.reminder_interval` - Optional. Same as above, but for individual alert.
 - `alerts.{alert-name}.query` - Required. SQL query to execute at regular interval.
 - `alerts.{alert-name}.message` - Optional. Message to pass with alert.
 - `alerts.{alert-name}.dbs` - Required. List of database references. Provided query will be executed against each database and trigger an individual alert for each database. All databases must be defined in `dbs` section of the configuration. Example: `["my-postgres-db", "my-mysql-db"]`.
-- `alerts.{alert-name}.destinations` - List of destination references to report the alert. Each destination should be defined in related section. Example: `["slacks.channel-1-webhook", "slacks.channel-2-webhook"]`
-- `alerts.{alert-name}.interval` - An interval between consecutive query execution. For simplicity it does not take in account time required to execute the query. For example if query execution time is 5s and interval is 10s then interval between two consecutive queries will be 15s. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". Examples: "300s", "1.5h" or "2h45m"
-- `dbs.{db-name}.driver` - SQL driver. Supported drivers: mysql, postgres.
-- `dbs.{db-name}.connection` - Connection string for the database. Documentation for databases: [postgres](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING), [mysql](https://github.com/go-sql-driver/mysql#dsn-data-source-name).
-- `slacks.{destination-name}.webhook_url` - Webhook URL for slack integration. More information how to set it up in [official slack documentation](https://api.slack.com/messaging/webhooks).
+- `alerts.{alert-name}.destinations` - Required. List of destination references to report the alert. Each destination should be defined in related section. Example: `["slacks.channel-1-webhook", "slacks.channel-2-webhook"]`
+- `alerts.{alert-name}.interval` - Required. An interval between consecutive query execution. For simplicity it does not take in account time required to execute the query. For example if query execution time is 5s and interval is 10s then interval between two consecutive queries will be 15s. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". Examples: "300s", "1.5h" or "2h45m"
+- `dbs.{db-name}.driver` - Required. SQL driver. Supported drivers: mysql, postgres.
+- `dbs.{db-name}.connection` - Required. Connection string for the database. Documentation for databases: [postgres](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING), [mysql](https://github.com/go-sql-driver/mysql#dsn-data-source-name).
+- `slacks.{destination-name}.webhook_url` - Required. Webhook URL for slack integration. More information how to set it up in [official slack documentation](https://api.slack.com/messaging/webhooks).
 
 ### Environment substitution.
 

app/alert/alert.go

@@ -19,6 +19,8 @@ type Alert struct {
 	Interval         time.Duration
 	ReminderInterval time.Duration
 	AlertingAt       time.Time
+	LastAlertSentAt  time.Time
+	AlertOnError     bool
 }
 
 type Destination interface {
@@ -84,46 +86,51 @@ func (a *Alert) Check() {
 	msg, err := a.ExecQuery()
 	if err != nil {
 		LogError(fmt.Errorf("couldn't check conditions for %s: %s", a.Name, err))
+		if a.AlertOnError {
+			a.SendAlert(err.Error())
+		}
 		return
 	}
 	if msg == "" {
-		log.Printf("INFO: %s is OK", a.Name)
-		if !a.AlertingAt.IsZero() {
-			log.Printf("INFO: %s is resolved", a.Name)
-			a.Resolve(fmt.Sprintf("Resolved after %s", humanizeDuration(time.Now().Sub(a.AlertingAt))))
-			a.AlertingAt = time.Time{}
-		}
+		a.Resolve()
 		return
 	}
-	log.Printf("INFO: %s is alerting", a.Name)
-	if a.AlertingAt.IsZero() {
-		a.AlertingAt = time.Now()
-		a.SendAlert(msg)
-	} else if a.ReminderInterval > 0 && time.Now().Sub(a.AlertingAt) > a.ReminderInterval {
-		a.SendAlert(msg)
-	}
+	a.SendAlert(msg)
 
 }
 
 func (a *Alert) SendAlert(msg string) {
-	if !a.AlertingAt.IsZero() {
+	log.Printf("INFO: %s is alerting", a.Name)
+	if a.AlertingAt.IsZero() {
 		a.AlertingAt = time.Now()
 	}
-	for i := range a.Destinations {
-		err := a.Destinations[i].SendAlert(a, msg)
-		if err != nil {
-			LogError(fmt.Errorf("couldn't send alert %s to %s", a.Name, a.Destinations[i].Name()))
+	if a.LastAlertSentAt.IsZero() || (a.ReminderInterval > 0 && time.Since(a.LastAlertSentAt) > a.ReminderInterval) {
+		for i := range a.Destinations {
+			err := a.Destinations[i].SendAlert(a, msg)
+			if err != nil {
+				LogError(fmt.Errorf("couldn't send alert %s to %s", a.Name, a.Destinations[i].Name()))
+			}
 		}
+		a.LastAlertSentAt = time.Now()
 	}
 }
 
-func (a *Alert) Resolve(msg string) {
+func (a *Alert) Resolve() {
+	log.Printf("INFO: %s is OK", a.Name)
+	if a.AlertingAt.IsZero() {
+		return
+	}
+	alertDuration := humanizeDuration(time.Now().Sub(a.AlertingAt))
+	resolveMessage := fmt.Sprintf("Resolved after %s", alertDuration)
 	for i := range a.Destinations {
-		err := a.Destinations[i].ResolveAlert(a, msg)
+		err := a.Destinations[i].ResolveAlert(a, resolveMessage)
 		if err != nil {
 			LogError(fmt.Errorf("error during resolver alert %s to %s", a.Name, a.Destinations))
 		}
 	}
+	a.AlertingAt = time.Time{}
+	a.LastAlertSentAt = time.Time{}
+	log.Printf("INFO: %s is resolved after %s", a.Name, alertDuration)
 }
 
 func LogError(err error) {

app/alert/alert_test.go

@@ -3,6 +3,7 @@ package alert
 import (
 	"strings"
 	"testing"
+	"time"
 
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/lib/pq"
@@ -64,7 +65,7 @@ func TestAlert_ExecQuery_Mysql(t *testing.T) {
 func TestAlert_Check(t *testing.T) {
 	d := new(MockDestination)
 	d.On("SendAlert", mock.Anything, mock.Anything).Return(nil)
-	mysql := sqltest.GetTestMysql()
+	mysql := sqltest.GetTestPostgres()
 	a := Alert{
 		Name:         "my_alert",
 		Source:       NewSource(mysql, "test"),
@@ -75,10 +76,55 @@ func TestAlert_Check(t *testing.T) {
 			SELECT 2 as col1, 'string2' as col2`,
 	}
 	a.Check()
+	assert.NotEqual(t, a.LastAlertSentAt, time.Time{})
+	assert.NotEqual(t, a.AlertingAt, time.Time{})
+	d.AssertNumberOfCalls(t, "SendAlert", 1)
+	a.Check()
+	a.Check()
+	d.AssertNumberOfCalls(t, "SendAlert", 1)
+}
+
+func TestAlert_Check_Error(t *testing.T) {
+	d := new(MockDestination)
+	d.On("SendAlert", mock.Anything, mock.Anything).Return(nil)
+	mysql := sqltest.GetTestPostgres()
+	a := Alert{
+		Name:         "my_alert",
+		Source:       NewSource(mysql, "test"),
+		Destinations: []Destination{d},
+		AlertOnError: false,
+		Query: `
+			SELECT 1 as col1, 'string' as col2 FROM`, // invalid query
+	}
+	a.Check()
+	d.AssertNumberOfCalls(t, "SendAlert", 0)
+	a.AlertOnError = true
+	a.Check()
+	assert.NotEqual(t, a.LastAlertSentAt, time.Time{})
+	assert.NotEqual(t, a.AlertingAt, time.Time{})
 	d.AssertNumberOfCalls(t, "SendAlert", 1)
+}
+
+
+func TestAlert_Reminder(t *testing.T) {
+	d := new(MockDestination)
+	d.On("SendAlert", mock.Anything, mock.Anything).Return(nil)
+	mysql := sqltest.GetTestPostgres()
+	a := Alert{
+		Name:         "my_alert",
+		Source:       NewSource(mysql, "test"),
+		Destinations: []Destination{d},
+		ReminderInterval: time.Millisecond,
+		Query: `
+			SELECT 1 as col1, 'string' as col2`,
+	}
 	a.Check()
+	d.AssertNumberOfCalls(t, "SendAlert", 1)
 	a.Check()
 	d.AssertNumberOfCalls(t, "SendAlert", 1)
+	time.Sleep(time.Millisecond)
+	a.Check()
+	d.AssertNumberOfCalls(t, "SendAlert", 2)
 }
 
 func assertTables(t *testing.T, actual string, expected string) {

app/config/config.go

@@ -31,6 +31,7 @@ type tomlAlert struct {
 	Interval         tomlDuration `toml:"interval"`
 	ReminderInterval tomlDuration `toml:"reminder_interval"`
 	Message          string       `toml:"message"`
+	AlertOnError     *bool        `toml:"alert_on_error"` // can hold 3 possible values: true/false/not-specified
 }
 
 type tomlDuration struct {
@@ -62,6 +63,7 @@ type tomlConfig struct {
 	PagerDuty        map[string]tomlPagerDuty `toml:"pager_duties"`
 	Message          string                   `toml:"message"`
 	ReminderInterval tomlDuration             `toml:"reminder_interval"`
+	AlertOnError     *bool                     `toml:"alert_on_error"` // can hold 3 possible values: true/false/not-specified
 }
 
 func loadTOMLConfig(path string) (tomlConfig, error) {
@@ -102,6 +104,16 @@ func prepareAlerts(config tomlConfig) ([]alert.Alert, error) {
 	}
 
 	for alertName, alertConfig := range config.Alerts {
+		var alertOnError bool
+		if alertConfig.AlertOnError == nil {
+			if config.AlertOnError == nil {
+				alertOnError = true
+			} else {
+				alertOnError = *config.AlertOnError
+			}
+		} else {
+			alertOnError = *alertConfig.AlertOnError
+		}
 		reminderInterval := alertConfig.ReminderInterval.Duration
 		if reminderInterval == 0 {
 			reminderInterval = config.ReminderInterval.Duration
@@ -130,6 +142,7 @@ func prepareAlerts(config tomlConfig) ([]alert.Alert, error) {
 				Interval:         alertConfig.Interval.Duration,
 				ReminderInterval: reminderInterval,
 				Destinations:     alertDestinations,
+				AlertOnError:     alertOnError,
 			})
 		}
 	}

app/config/config_test.go

@@ -22,7 +22,10 @@ func TestMain(m *testing.M) {
 func Test_loadTOMLConfig(t *testing.T) {
 	config, err := loadTOMLConfig("example.toml")
 	require.NoError(t, err)
+	falseB := false
+	trueB := true
 	expected := tomlConfig{
+		AlertOnError: &trueB,
 		ReminderInterval: tomlDuration{Duration: 3 * time.Hour},
 		Alerts: map[string]tomlAlert{
 			"test-alert-1": {
@@ -40,6 +43,7 @@ func Test_loadTOMLConfig(t *testing.T) {
 				Destinations:     []string{"slacks.my-slack"},
 				Interval:         tomlDuration{Duration: time.Minute * 90},
 				ReminderInterval: tomlDuration{Duration: 0},
+				AlertOnError:     &falseB,
 			},
 		},
 		DB: map[string]tomlDB{

app/config/example.toml

@@ -1,4 +1,5 @@
 reminder_interval = "3h"
+alert_on_error = true
 
 [alerts]
     [alerts.test-alert-1]
@@ -19,6 +20,7 @@ reminder_interval = "3h"
     dbs = ["my-mysql-db"]
     destinations = ["slacks.my-slack"]
     interval = "1h30m"
+    alert_on_error = false
 
 
 [dbs]