diff --git a/config/config.go b/config/config.go index 487ec83e4a..8bab8e0277 100644 --- a/config/config.go +++ b/config/config.go @@ -460,6 +460,17 @@ func (c *Config) UnmarshalYAML(unmarshal func(any) error) error { } sc.APIURL = (*amcommoncfg.SecretURL)(sc.AppURL) } + // update_message and post_updates_to_thread require the bot-token API. + // The endpoint can only be verified for URLs known at load time; + // api_url_file is read at notification time and is accepted as-is. + if len(sc.APIURLFile) == 0 && (sc.APIURL == nil || sc.APIURL.String() != "https://slack.com/api/chat.postMessage") { + if sc.UpdateMessage { + return errors.New("update_message can only be used with bot tokens. api_url must be set to https://slack.com/api/chat.postMessage") + } + if sc.PostUpdatesToThread { + return errors.New("post_updates_to_thread can only be used with bot tokens. api_url must be set to https://slack.com/api/chat.postMessage") + } + } } for _, poc := range rcv.PushoverConfigs { if poc == nil { diff --git a/config/config_test.go b/config/config_test.go index c70fdaf3d2..4e29a58c00 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1321,6 +1321,41 @@ func TestSlackUpdateMessageWebhookURL(t *testing.T) { } } +func TestSlackPostUpdatesToThreadWebhookURL(t *testing.T) { + _, err := LoadFile("testdata/conf.slack-post-updates-to-thread-and-webhook.yml") + if err == nil { + t.Fatalf("Expected an error parsing %s: %s", "testdata/conf.slack-post-updates-to-thread-and-webhook", err) + } + if err.Error() != "post_updates_to_thread can only be used with bot tokens. api_url must be set to https://slack.com/api/chat.postMessage" { + t.Errorf("Expected: %s\nGot: %s", "post_updates_to_thread can only be used with bot tokens. api_url must be set to https://slack.com/api/chat.postMessage", err.Error()) + } +} + +func TestSlackUpdateMessageWithAppToken(t *testing.T) { + // The app token flow resolves api_url to the Slack bot API during global + // config resolution, so update_message must be accepted with it. + _, err := LoadFile("testdata/conf.slack-update-message-and-app-token.yml") + if err != nil { + t.Fatalf("Error parsing %s: %s", "testdata/conf.slack-update-message-and-app-token.yml", err) + } +} + +func TestSlackPostUpdatesToThreadWithAppToken(t *testing.T) { + _, err := LoadFile("testdata/conf.slack-post-updates-to-thread-and-app-token.yml") + if err != nil { + t.Fatalf("Error parsing %s: %s", "testdata/conf.slack-post-updates-to-thread-and-app-token.yml", err) + } +} + +func TestSlackUpdateMessageWithAPIURLFile(t *testing.T) { + // api_url_file is read at notification time, so its content cannot be + // verified at load time and the configuration must be accepted. + _, err := LoadFile("testdata/conf.slack-update-message-and-api-url-file.yml") + if err != nil { + t.Fatalf("Error parsing %s: %s", "testdata/conf.slack-update-message-and-api-url-file.yml", err) + } +} + func TestSlackGlobalAppToken(t *testing.T) { conf, err := LoadFile("testdata/conf.slack-default-app-token.yml") if err != nil { diff --git a/config/notifiers.go b/config/notifiers.go index 8ae478eca7..3d6ce5bf5c 100644 --- a/config/notifiers.go +++ b/config/notifiers.go @@ -336,6 +336,13 @@ type SlackConfig struct { // Requires bot token with chat:write scope. Webhook URLs do not support updates. UpdateMessage bool `yaml:"update_message" json:"update_message,omitempty"` + + // PostUpdatesToThread enables posting subsequent notifications for an alert group + // as replies in the thread of the initial message. When combined with UpdateMessage, + // the initial message is updated in place and a reply is also posted to its thread. + // Requires bot token with chat:write scope. Webhook URLs do not support threads. + + PostUpdatesToThread bool `yaml:"post_updates_to_thread" json:"post_updates_to_thread,omitempty"` // Timeout is the maximum time allowed to invoke the slack. Setting this to 0 // does not impose a timeout. Timeout time.Duration `yaml:"timeout" json:"timeout"` @@ -351,6 +358,10 @@ func (c *SlackConfig) UnmarshalYAML(unmarshal func(any) error) error { return c.Validate() } +// Validate checks that the Slack configuration endpoints and credentials are +// mutually consistent. The endpoint requirements of update_message and +// post_updates_to_thread are checked during global config resolution, once +// api_url has been resolved from the global section or an app token. func (c *SlackConfig) Validate() error { if c.APIURL != nil && len(c.APIURLFile) > 0 { return errors.New("at most one of api_url & api_url_file must be configured") @@ -362,10 +373,6 @@ func (c *SlackConfig) Validate() error { return errors.New("at most one of api_url/api_url_file & app_token/app_token_file must be configured") } - if c.UpdateMessage && c.APIURL.String() != "https://slack.com/api/chat.postMessage" { - return errors.New("update_message can only be used with bot tokens. api_url must be set to https://slack.com/api/chat.postMessage") - } - return nil } diff --git a/config/testdata/conf.slack-post-updates-to-thread-and-app-token.yml b/config/testdata/conf.slack-post-updates-to-thread-and-app-token.yml new file mode 100644 index 0000000000..236551c77b --- /dev/null +++ b/config/testdata/conf.slack-post-updates-to-thread-and-app-token.yml @@ -0,0 +1,12 @@ +route: + receiver: 'slack-notifications' + group_by: [alertname] +receivers: + - name: 'slack-notifications' + slack_configs: + # bot token flow without explicit api_url + - channel: '#alerts1' + text: 'test' + send_resolved: true + app_token: 'xoxb-some-token' + post_updates_to_thread: true diff --git a/config/testdata/conf.slack-post-updates-to-thread-and-webhook.yml b/config/testdata/conf.slack-post-updates-to-thread-and-webhook.yml new file mode 100644 index 0000000000..b39e538ed9 --- /dev/null +++ b/config/testdata/conf.slack-post-updates-to-thread-and-webhook.yml @@ -0,0 +1,13 @@ +route: + receiver: 'slack-notifications' + group_by: [alertname] +receivers: + - name: 'slack-notifications' + slack_configs: + # use global + - channel: '#alerts1' + text: 'test' + send_resolved: true + # trying to use webhook urls with post_updates_to_thread + api_url: 'https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX' + post_updates_to_thread: true diff --git a/config/testdata/conf.slack-update-message-and-api-url-file.yml b/config/testdata/conf.slack-update-message-and-api-url-file.yml new file mode 100644 index 0000000000..293317514f --- /dev/null +++ b/config/testdata/conf.slack-update-message-and-api-url-file.yml @@ -0,0 +1,13 @@ +route: + receiver: 'slack-notifications' + group_by: [alertname] +receivers: + - name: 'slack-notifications' + slack_configs: + # api_url_file is read at notification time; accepted at load time + - channel: '#alerts1' + text: 'test' + send_resolved: true + api_url_file: '/etc/slack/api_url' + update_message: true + post_updates_to_thread: true diff --git a/config/testdata/conf.slack-update-message-and-app-token.yml b/config/testdata/conf.slack-update-message-and-app-token.yml new file mode 100644 index 0000000000..d8d69c2fa6 --- /dev/null +++ b/config/testdata/conf.slack-update-message-and-app-token.yml @@ -0,0 +1,12 @@ +route: + receiver: 'slack-notifications' + group_by: [alertname] +receivers: + - name: 'slack-notifications' + slack_configs: + # bot token flow without explicit api_url + - channel: '#alerts1' + text: 'test' + send_resolved: true + app_token: 'xoxb-some-token' + update_message: true diff --git a/docs/configuration.md b/docs/configuration.md index 80c1a2bd1a..cf315f5374 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1747,6 +1747,12 @@ fields: # Enables updating existing Slack messages instead of creating new ones on alert state change. # Webhook URLs do not support updates. [ update_message: | default = false ] + +# Posts subsequent notifications for an alert group as replies in the thread of the +# initial message instead of new channel messages. When combined with update_message, +# the initial message is updated in place and a reply is also posted to its thread. +# Webhook URLs do not support threads. +[ post_updates_to_thread: | default = false ] ``` #### `` (Slack) diff --git a/notify/slack/slack.go b/notify/slack/slack.go index 64964db4ea..1ae9ca87e1 100644 --- a/notify/slack/slack.go +++ b/notify/slack/slack.go @@ -171,27 +171,67 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) Attachments: []attachment{*att}, } - // If a notification for this alert group has already been sent and `update_message` config is set - // edit API endpoint and payload to update notification instead of sending a new one. + // If a notification for this alert group has already been sent, `update_message` + // edits the initial message instead of sending a new one and `post_updates_to_thread` + // posts the notification as a reply in the initial message's thread. var store *nflog.Store + var threadTs, channelId string - if n.conf.UpdateMessage { + if n.conf.UpdateMessage || n.conf.PostUpdatesToThread { var ok bool store, ok = notify.NflogStore(ctx) if !ok { - logger.Warn("cannot create NflogStore, updatable messages will be disabled.") + logger.Warn("cannot create NflogStore, updatable and threaded messages will be disabled.") } else { - threadTs, _ := store.GetStr("threadTs") - channelId, _ := store.GetStr("channelId") - logger.Debug("attempt recovering threadTs and channelId to update an existing message", "threadTs", threadTs, "channelId", channelId) - if threadTs != "" && channelId != "" { - u = "https://slack.com/api/chat.update" - req.Timestamp = threadTs - req.Channel = channelId - logger.Debug("updating previously sent message", "threadTs", threadTs, "channelId", channelId) - } + threadTs, _ = store.GetStr("threadTs") + channelId, _ = store.GetStr("channelId") + logger.Debug("attempt recovering threadTs and channelId of the initial message", "threadTs", threadTs, "channelId", channelId) + } + } + + postURL := u + initialMessageSent := threadTs != "" && channelId != "" + if initialMessageSent { + switch { + case n.conf.UpdateMessage: + u = "https://slack.com/api/chat.update" + req.Timestamp = threadTs + req.Channel = channelId + logger.Debug("updating previously sent message", "threadTs", threadTs, "channelId", channelId) + case n.conf.PostUpdatesToThread: + req.ThreadTimestamp = threadTs + req.Channel = channelId + logger.Debug("posting to thread of previously sent message", "threadTs", threadTs, "channelId", channelId) } } + + // The thread reply must not overwrite the initial message's timestamp in the + // nflog store, so no store is passed when the request targets a thread. + responseStore := store + if initialMessageSent { + responseStore = nil + } + retry, err := n.postRequest(ctx, u, req, responseStore) + if err != nil { + return retry, err + } + + // When update_message and post_updates_to_thread are combined, the initial + // message was just updated in place; additionally post a reply to its thread. + if initialMessageSent && n.conf.UpdateMessage && n.conf.PostUpdatesToThread { + threadReq := *req + threadReq.Timestamp = "" + threadReq.ThreadTimestamp = threadTs + logger.Debug("posting update to thread of previously sent message", "threadTs", threadTs, "channelId", channelId) + return n.postRequest(ctx, postURL, &threadReq, nil) + } + + return retry, nil +} + +// postRequest encodes and sends a single request to the Slack API, classifies +// errors as retriable or not, and hands the response to slackResponseHandler. +func (n *Notifier) postRequest(ctx context.Context, u string, req *request, store *nflog.Store) (bool, error) { var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(req); err != nil { return false, err diff --git a/notify/slack/slack_test.go b/notify/slack/slack_test.go index ba0e793907..5cb34c87a8 100644 --- a/notify/slack/slack_test.go +++ b/notify/slack/slack_test.go @@ -34,6 +34,7 @@ import ( amcommoncfg "github.com/prometheus/alertmanager/config/common" "github.com/prometheus/alertmanager/config" + "github.com/prometheus/alertmanager/nflog" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/notify/test" "github.com/prometheus/alertmanager/template" @@ -466,3 +467,124 @@ func TestNotifier_Notify_RetryAfterContextCancelled(t *testing.T) { require.Error(t, err) require.Less(t, elapsed, 2*time.Second, "should not have waited the full Retry-After duration") } + +func TestSlackPostUpdatesToThread(t *testing.T) { + type capturedRequest struct { + url string + body map[string]any + } + + newTestNotifier := func(t *testing.T, conf *config.SlackConfig, captured *[]capturedRequest, respTs string) *Notifier { + t.Helper() + u, _ := url.Parse("https://slack.com/api/chat.postMessage") + conf.APIURL = &amcommoncfg.SecretURL{URL: u} + conf.Channel = "#test-channel" + conf.HTTPConfig = &commoncfg.HTTPClientConfig{} + + tmpl, err := template.FromGlobs([]string{}) + require.NoError(t, err) + tmpl.ExternalURL = u + + notifier, err := New(conf, tmpl, slog.New(slog.DiscardHandler)) + require.NoError(t, err) + + notifier.postJSONFunc = func(ctx context.Context, client *http.Client, reqURL string, body io.Reader) (*http.Response, error) { + var decoded map[string]any + require.NoError(t, json.NewDecoder(body).Decode(&decoded)) + *captured = append(*captured, capturedRequest{url: reqURL, body: decoded}) + resp := &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"ok": true, "channel": "C123", "ts": "` + respTs + `"}`)), + } + return resp, nil + } + return notifier + } + + newCtx := func(store *nflog.Store) context.Context { + ctx := notify.WithGroupKey(context.Background(), "test-group-key") + return notify.WithNflogStore(ctx, store) + } + + t.Run("first notification posts to channel and stores thread ts", func(t *testing.T) { + var captured []capturedRequest + notifier := newTestNotifier(t, &config.SlackConfig{UpdateMessage: true, PostUpdatesToThread: true}, &captured, "111.222") + store := nflog.NewStore(nil) + + _, err := notifier.Notify(newCtx(store)) + require.NoError(t, err) + + require.Len(t, captured, 1) + require.Equal(t, "https://slack.com/api/chat.postMessage", captured[0].url) + require.NotContains(t, captured[0].body, "ts") + require.NotContains(t, captured[0].body, "thread_ts") + + threadTs, _ := store.GetStr("threadTs") + channelId, _ := store.GetStr("channelId") + require.Equal(t, "111.222", threadTs) + require.Equal(t, "C123", channelId) + }) + + t.Run("subsequent notification updates message and posts thread reply", func(t *testing.T) { + var captured []capturedRequest + notifier := newTestNotifier(t, &config.SlackConfig{UpdateMessage: true, PostUpdatesToThread: true}, &captured, "999.999") + store := nflog.NewStore(nil) + store.SetStr("threadTs", "111.222") + store.SetStr("channelId", "C123") + + _, err := notifier.Notify(newCtx(store)) + require.NoError(t, err) + + require.Len(t, captured, 2) + require.Equal(t, "https://slack.com/api/chat.update", captured[0].url) + require.Equal(t, "111.222", captured[0].body["ts"]) + require.Equal(t, "C123", captured[0].body["channel"]) + require.NotContains(t, captured[0].body, "thread_ts") + + require.Equal(t, "https://slack.com/api/chat.postMessage", captured[1].url) + require.Equal(t, "111.222", captured[1].body["thread_ts"]) + require.Equal(t, "C123", captured[1].body["channel"]) + require.NotContains(t, captured[1].body, "ts") + + // The stored root message ts must not be overwritten by the responses. + threadTs, _ := store.GetStr("threadTs") + require.Equal(t, "111.222", threadTs) + }) + + t.Run("subsequent notification posts only to thread without update_message", func(t *testing.T) { + var captured []capturedRequest + notifier := newTestNotifier(t, &config.SlackConfig{PostUpdatesToThread: true}, &captured, "999.999") + store := nflog.NewStore(nil) + store.SetStr("threadTs", "111.222") + store.SetStr("channelId", "C123") + + _, err := notifier.Notify(newCtx(store)) + require.NoError(t, err) + + require.Len(t, captured, 1) + require.Equal(t, "https://slack.com/api/chat.postMessage", captured[0].url) + require.Equal(t, "111.222", captured[0].body["thread_ts"]) + require.Equal(t, "C123", captured[0].body["channel"]) + require.NotContains(t, captured[0].body, "ts") + + threadTs, _ := store.GetStr("threadTs") + require.Equal(t, "111.222", threadTs) + }) + + t.Run("update_message alone does not post thread reply", func(t *testing.T) { + var captured []capturedRequest + notifier := newTestNotifier(t, &config.SlackConfig{UpdateMessage: true}, &captured, "999.999") + store := nflog.NewStore(nil) + store.SetStr("threadTs", "111.222") + store.SetStr("channelId", "C123") + + _, err := notifier.Notify(newCtx(store)) + require.NoError(t, err) + + require.Len(t, captured, 1) + require.Equal(t, "https://slack.com/api/chat.update", captured[0].url) + require.Equal(t, "111.222", captured[0].body["ts"]) + require.NotContains(t, captured[0].body, "thread_ts") + }) +} diff --git a/notify/slack/types.go b/notify/slack/types.go index a85734e1ad..4bf2146e7e 100644 --- a/notify/slack/types.go +++ b/notify/slack/types.go @@ -37,14 +37,15 @@ type Notifier struct { // request is the request for sending a Slack notification. type request struct { - Channel string `json:"channel,omitempty"` - Timestamp string `json:"ts,omitempty"` - Username string `json:"username,omitempty"` - IconEmoji string `json:"icon_emoji,omitempty"` - IconURL string `json:"icon_url,omitempty"` - LinkNames bool `json:"link_names,omitempty"` - Text string `json:"text,omitempty"` - Attachments []attachment `json:"attachments"` + Channel string `json:"channel,omitempty"` + Timestamp string `json:"ts,omitempty"` + ThreadTimestamp string `json:"thread_ts,omitempty"` + Username string `json:"username,omitempty"` + IconEmoji string `json:"icon_emoji,omitempty"` + IconURL string `json:"icon_url,omitempty"` + LinkNames bool `json:"link_names,omitempty"` + Text string `json:"text,omitempty"` + Attachments []attachment `json:"attachments"` } // attachment is used to display a richly formatted message block.