From ec7c9f876b88c13c86a47cb7d5e29a65dfab723b Mon Sep 17 00:00:00 2001 From: Cody Kaczynski Date: Fri, 18 Sep 2026 04:45:58 +0000 Subject: [PATCH 1/5] notify/slack: add thread_replies for follow-up notifications Follow-up notifications for an alert group can be posted as Slack thread replies on the group's first message. The parent timestamp is read from the nflog store already used by update_message. When both thread_replies and update_message are set, a state-change notification edits the parent and adds a reply. A repeat-interval notification only edits the parent, so the thread is not filled with copies of a message that is already current. thread_replies alone still posts a reply on repeats, otherwise the repeat would be silent. Incoming webhooks cannot do this: they do not return a message timestamp. The option requires the chat.postMessage bot API. Signed-off-by: Cody Kaczynski --- config/config.go | 3 + config/config_test.go | 32 +++ config/notifiers.go | 28 ++- ....slack-thread-replies-and-api-url-file.yml | 12 ++ ...onf.slack-thread-replies-and-app-token.yml | 11 ++ .../conf.slack-thread-replies-and-webhook.yml | 11 ++ ...onf.slack-update-message-and-app-token.yml | 12 ++ docs/configuration.md | 14 ++ notify/slack/slack.go | 98 +++++++--- notify/slack/slack_test.go | 185 ++++++++++++++++++ notify/slack/types.go | 17 +- 11 files changed, 389 insertions(+), 34 deletions(-) create mode 100644 config/testdata/conf.slack-thread-replies-and-api-url-file.yml create mode 100644 config/testdata/conf.slack-thread-replies-and-app-token.yml create mode 100644 config/testdata/conf.slack-thread-replies-and-webhook.yml create mode 100644 config/testdata/conf.slack-update-message-and-app-token.yml diff --git a/config/config.go b/config/config.go index 487ec83e4a..b507baccfa 100644 --- a/config/config.go +++ b/config/config.go @@ -460,6 +460,9 @@ func (c *Config) UnmarshalYAML(unmarshal func(any) error) error { } sc.APIURL = (*amcommoncfg.SecretURL)(sc.AppURL) } + if err := sc.validateMessageAPIURL(); err != nil { + return err + } } for _, poc := range rcv.PushoverConfigs { if poc == nil { diff --git a/config/config_test.go b/config/config_test.go index c70fdaf3d2..687fadcb3e 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1321,6 +1321,38 @@ func TestSlackUpdateMessageWebhookURL(t *testing.T) { } } +func TestSlackThreadRepliesWebhookURL(t *testing.T) { + _, err := LoadFile("testdata/conf.slack-thread-replies-and-webhook.yml") + if err == nil { + t.Fatalf("Expected an error parsing testdata/conf.slack-thread-replies-and-webhook.yml") + } + want := "thread_replies can only be used with bot tokens. api_url must be set to https://slack.com/api/chat.postMessage" + if err.Error() != want { + t.Errorf("Expected: %s\nGot: %s", want, err.Error()) + } +} + +func TestSlackUpdateMessageWithAppToken(t *testing.T) { + _, err := LoadFile("testdata/conf.slack-update-message-and-app-token.yml") + if err != nil { + t.Fatalf("Error parsing testdata/conf.slack-update-message-and-app-token.yml: %s", err) + } +} + +func TestSlackThreadRepliesWithAppToken(t *testing.T) { + _, err := LoadFile("testdata/conf.slack-thread-replies-and-app-token.yml") + if err != nil { + t.Fatalf("Error parsing testdata/conf.slack-thread-replies-and-app-token.yml: %s", err) + } +} + +func TestSlackThreadRepliesWithAPIURLFile(t *testing.T) { + _, err := LoadFile("testdata/conf.slack-thread-replies-and-api-url-file.yml") + if err != nil { + t.Fatalf("Error parsing testdata/conf.slack-thread-replies-and-api-url-file.yml: %s", err) + } +} + func TestSlackGlobalAppToken(t *testing.T) { conf, err := LoadFile("testdata/conf.slack-default-app-token.yml") if err != nil { diff --git a/config/notifiers.go b/config/notifiers.go index 8ae478eca7..daf941ce82 100644 --- a/config/notifiers.go +++ b/config/notifiers.go @@ -334,8 +334,11 @@ type SlackConfig struct { // UpdateMessage enables updating existing Slack messages instead of creating new ones. // Requires bot token with chat:write scope. Webhook URLs do not support updates. - UpdateMessage bool `yaml:"update_message" json:"update_message,omitempty"` + // ThreadReplies posts follow-up notifications for an alert group as replies + // in the Slack thread of the group's first message. Requires bot token with + // chat:write scope. Incoming webhooks cannot start a thread. + ThreadReplies bool `yaml:"thread_replies" json:"thread_replies,omitempty"` // Timeout is the maximum time allowed to invoke the slack. Setting this to 0 // does not impose a timeout. Timeout time.Duration `yaml:"timeout" json:"timeout"` @@ -351,6 +354,10 @@ func (c *SlackConfig) UnmarshalYAML(unmarshal func(any) error) error { return c.Validate() } +// Validate checks that Slack credential fields are mutually exclusive. The +// chat.postMessage requirement of update_message and thread_replies is checked +// during global config resolution, after api_url is filled in from the global +// section or an app token. func (c *SlackConfig) Validate() error { if c.APIURL != nil && len(c.APIURLFile) > 0 { return errors.New("at most one of api_url & api_url_file must be configured") @@ -362,11 +369,24 @@ func (c *SlackConfig) Validate() error { return errors.New("at most one of api_url/api_url_file & app_token/app_token_file must be configured") } - if c.UpdateMessage && c.APIURL.String() != "https://slack.com/api/chat.postMessage" { + return nil +} + +func (c *SlackConfig) validateMessageAPIURL() error { + if !c.UpdateMessage && !c.ThreadReplies { + return nil + } + // File-backed URLs are read when the notification is sent. + if len(c.APIURLFile) > 0 { + return nil + } + if c.APIURL != nil && c.APIURL.String() == "https://slack.com/api/chat.postMessage" { + return nil + } + if c.UpdateMessage { return errors.New("update_message can only be used with bot tokens. api_url must be set to https://slack.com/api/chat.postMessage") } - - return nil + return errors.New("thread_replies can only be used with bot tokens. api_url must be set to https://slack.com/api/chat.postMessage") } // WechatConfig configures notifications via Wechat. diff --git a/config/testdata/conf.slack-thread-replies-and-api-url-file.yml b/config/testdata/conf.slack-thread-replies-and-api-url-file.yml new file mode 100644 index 0000000000..7e7e960f3d --- /dev/null +++ b/config/testdata/conf.slack-thread-replies-and-api-url-file.yml @@ -0,0 +1,12 @@ +route: + receiver: 'slack-notifications' + group_by: [alertname] +receivers: + - name: 'slack-notifications' + slack_configs: + - channel: '#alerts1' + text: 'test' + send_resolved: true + api_url_file: '/etc/slack/api_url' + update_message: true + thread_replies: true diff --git a/config/testdata/conf.slack-thread-replies-and-app-token.yml b/config/testdata/conf.slack-thread-replies-and-app-token.yml new file mode 100644 index 0000000000..f00dc5d982 --- /dev/null +++ b/config/testdata/conf.slack-thread-replies-and-app-token.yml @@ -0,0 +1,11 @@ +route: + receiver: 'slack-notifications' + group_by: [alertname] +receivers: + - name: 'slack-notifications' + slack_configs: + - channel: '#alerts1' + text: 'test' + send_resolved: true + app_token: 'xoxb-some-token' + thread_replies: true diff --git a/config/testdata/conf.slack-thread-replies-and-webhook.yml b/config/testdata/conf.slack-thread-replies-and-webhook.yml new file mode 100644 index 0000000000..99b0bb1614 --- /dev/null +++ b/config/testdata/conf.slack-thread-replies-and-webhook.yml @@ -0,0 +1,11 @@ +route: + receiver: 'slack-notifications' + group_by: [alertname] +receivers: + - name: 'slack-notifications' + slack_configs: + - channel: '#alerts1' + text: 'test' + send_resolved: true + api_url: 'https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX' + thread_replies: true diff --git a/config/testdata/conf.slack-update-message-and-app-token.yml b/config/testdata/conf.slack-update-message-and-app-token.yml new file mode 100644 index 0000000000..d8d69c2fa6 --- /dev/null +++ b/config/testdata/conf.slack-update-message-and-app-token.yml @@ -0,0 +1,12 @@ +route: + receiver: 'slack-notifications' + group_by: [alertname] +receivers: + - name: 'slack-notifications' + slack_configs: + # bot token flow without explicit api_url + - channel: '#alerts1' + text: 'test' + send_resolved: true + app_token: 'xoxb-some-token' + update_message: true diff --git a/docs/configuration.md b/docs/configuration.md index ead7ee0482..4a621cef9d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1759,6 +1759,20 @@ fields: # Enables updating existing Slack messages instead of creating new ones on alert state change. # Webhook URLs do not support updates. [ update_message: | default = false ] + +# Post follow-up notifications for the same alert group as replies in the +# Slack thread of the group's first message. A later firing after the group +# has fully resolved starts a new thread. +# +# Together with update_message, state changes edit the first message and +# also add a reply. Repeat-interval notifications then only edit the first +# message, so the thread is not filled with copies of a message that is +# already current. thread_replies without update_message still posts a +# reply on repeats; otherwise the repeat would not appear in Slack at all. +# +# Requires a bot token (api_url https://slack.com/api/chat.postMessage). +# Incoming webhooks do not return a message timestamp and cannot be used. +[ thread_replies: | default = false ] ``` #### `` (Slack) diff --git a/notify/slack/slack.go b/notify/slack/slack.go index adfe53acdd..ba7916a231 100644 --- a/notify/slack/slack.go +++ b/notify/slack/slack.go @@ -170,27 +170,82 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) Attachments: []attachment{*att}, } - // If a notification for this alert group has already been sent and `update_message` config is set - // edit API endpoint and payload to update notification instead of sending a new one. - var store *nflog.Store - - if n.conf.UpdateMessage { - var ok bool - store, ok = notify.NflogStore(ctx) - if !ok { - logger.Warn("cannot create NflogStore, updatable messages will be disabled.") - } else { - threadTs, _ := store.GetStr("threadTs") - channelId, _ := store.GetStr("channelId") - logger.Debug("attempt recovering threadTs and channelId to update an existing message", "threadTs", threadTs, "channelId", channelId) - if threadTs != "" && channelId != "" { - u = "https://slack.com/api/chat.update" - req.Timestamp = threadTs - req.Channel = channelId - logger.Debug("updating previously sent message", "threadTs", threadTs, "channelId", channelId) - } - } + store := n.nflogStore(ctx, logger) + parentTS, parentChannel := storedParent(store) + haveParent := parentTS != "" && parentChannel != "" + + edit := n.conf.UpdateMessage && haveParent + reply := n.conf.ThreadReplies && haveParent && !skipThreadReply(ctx, edit) + + // Follow-ups must not replace the parent timestamp with a reply's ts. + var record *nflog.Store + if !edit && !reply { + record = store + } + + firstURL := u + if edit { + u = "https://slack.com/api/chat.update" + req.Timestamp = parentTS + req.Channel = parentChannel + logger.Debug("editing existing Slack message", "ts", parentTS, "channel", parentChannel) + } else if reply { + req.ThreadTimestamp = parentTS + req.Channel = parentChannel + logger.Debug("replying in existing Slack thread", "ts", parentTS, "channel", parentChannel) + } + + retry, err := n.post(ctx, u, req, record) + if err != nil { + return retry, err + } + if !edit || !reply { + return retry, nil + } + + followUp := *req + followUp.Timestamp = "" + followUp.ThreadTimestamp = parentTS + logger.Debug("adding thread reply after editing parent", "ts", parentTS, "channel", parentChannel) + return n.post(ctx, firstURL, &followUp, nil) +} + +func (n *Notifier) nflogStore(ctx context.Context, logger *slog.Logger) *nflog.Store { + if !n.conf.UpdateMessage && !n.conf.ThreadReplies { + return nil + } + store, ok := notify.NflogStore(ctx) + if !ok { + logger.Warn("nflog store missing; Slack message editing and thread replies disabled") + return nil + } + return store +} + +func storedParent(store *nflog.Store) (ts, channel string) { + if store == nil { + return "", "" } + ts, _ = store.GetStr("threadTs") + channel, _ = store.GetStr("channelId") + if ts == "" || channel == "" { + return "", "" + } + return ts, channel +} + +// skipThreadReply reports whether a thread reply would only duplicate a parent +// that update_message is already rewriting. Repeats with thread_replies alone +// still post a reply, otherwise Slack would receive nothing. +func skipThreadReply(ctx context.Context, editingParent bool) bool { + if !editingParent { + return false + } + reason, ok := notify.NotificationReason(ctx) + return ok && reason == notify.ReasonRepeatIntervalElapsed +} + +func (n *Notifier) post(ctx context.Context, u string, req *request, store *nflog.Store) (bool, error) { var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(req); err != nil { return false, err @@ -248,11 +303,10 @@ func (n *Notifier) slackResponseHandler(resp *http.Response, store *nflog.Store) if !data.OK { return false, fmt.Errorf("error response from Slack: %s", data.Error) } - // If store, TS and Channel are set, store the threadTS and channelId if store != nil && data.Timestamp != "" && data.Channel != "" { store.SetStr("threadTs", data.Timestamp) store.SetStr("channelId", data.Channel) - n.logger.Debug("stored threadTs and channelId", "threadTs", data.Timestamp, "channelId", data.Channel) + n.logger.Debug("stored Slack message identity", "ts", data.Timestamp, "channel", data.Channel) } return false, nil } diff --git a/notify/slack/slack_test.go b/notify/slack/slack_test.go index 076d1d7ca7..4189bcfa88 100644 --- a/notify/slack/slack_test.go +++ b/notify/slack/slack_test.go @@ -34,6 +34,7 @@ import ( amcommoncfg "github.com/prometheus/alertmanager/config/common" "github.com/prometheus/alertmanager/config" + "github.com/prometheus/alertmanager/nflog" "github.com/prometheus/alertmanager/notify" "github.com/prometheus/alertmanager/notify/test" "github.com/prometheus/alertmanager/template" @@ -446,3 +447,187 @@ func TestNotifier_Notify_RetryAfterContextCancelled(t *testing.T) { require.Error(t, err) require.Less(t, elapsed, 2*time.Second, "should not have waited the full Retry-After duration") } + +func TestSkipThreadReply(t *testing.T) { + tests := []struct { + name string + edit bool + reason notify.NotifyReason + omit bool + want bool + }{ + {name: "repeat while editing parent", edit: true, reason: notify.ReasonRepeatIntervalElapsed, want: true}, + {name: "resolved while editing parent", edit: true, reason: notify.ReasonAllAlertsResolved}, + {name: "repeat without parent edit", reason: notify.ReasonRepeatIntervalElapsed}, + {name: "no reason in context while editing", edit: true, omit: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + if !tt.omit { + ctx = notify.WithNotificationReason(ctx, tt.reason) + } + require.Equal(t, tt.want, skipThreadReply(ctx, tt.edit)) + }) + } +} + +type slackCall struct { + endpoint string + payload map[string]any +} + +func notifierRecordingCalls(t *testing.T, conf *config.SlackConfig, calls *[]slackCall, respTS string) *Notifier { + t.Helper() + u, err := url.Parse("https://slack.com/api/chat.postMessage") + require.NoError(t, err) + conf.APIURL = &amcommoncfg.SecretURL{URL: u} + conf.Channel = "#test-channel" + conf.HTTPConfig = &commoncfg.HTTPClientConfig{} + + notifier, err := New(conf, test.CreateTmpl(t), promslog.NewNopLogger()) + require.NoError(t, err) + + notifier.postJSONFunc = func(ctx context.Context, client *http.Client, endpoint string, body io.Reader) (*http.Response, error) { + var payload map[string]any + require.NoError(t, json.NewDecoder(body).Decode(&payload)) + *calls = append(*calls, slackCall{endpoint: endpoint, payload: payload}) + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"ok": true, "channel": "C123", "ts": "` + respTS + `"}`)), + }, nil + } + return notifier +} + +func notifyCtx(store *nflog.Store, reason notify.NotifyReason) context.Context { + ctx := notify.WithGroupKey(context.Background(), "test-group-key") + ctx = notify.WithNflogStore(ctx, store) + return notify.WithNotificationReason(ctx, reason) +} + +func TestSlackThreadReplies(t *testing.T) { + parentStore := func() *nflog.Store { + store := nflog.NewStore(nil) + store.SetStr("threadTs", "111.222") + store.SetStr("channelId", "C123") + return store + } + + t.Run("first message is posted to the channel and remembered", func(t *testing.T) { + var calls []slackCall + n := notifierRecordingCalls(t, &config.SlackConfig{UpdateMessage: true, ThreadReplies: true}, &calls, "111.222") + store := nflog.NewStore(nil) + + _, err := n.Notify(notifyCtx(store, notify.ReasonFirstNotification)) + require.NoError(t, err) + + require.Len(t, calls, 1) + require.Equal(t, "https://slack.com/api/chat.postMessage", calls[0].endpoint) + require.NotContains(t, calls[0].payload, "ts") + require.NotContains(t, calls[0].payload, "thread_ts") + ts, ok := store.GetStr("threadTs") + require.True(t, ok) + require.Equal(t, "111.222", ts) + channel, ok := store.GetStr("channelId") + require.True(t, ok) + require.Equal(t, "C123", channel) + }) + + t.Run("resolve edits the parent and adds a reply", func(t *testing.T) { + var calls []slackCall + n := notifierRecordingCalls(t, &config.SlackConfig{UpdateMessage: true, ThreadReplies: true}, &calls, "999.999") + store := parentStore() + + _, err := n.Notify(notifyCtx(store, notify.ReasonAllAlertsResolved)) + require.NoError(t, err) + + require.Len(t, calls, 2) + require.Equal(t, "https://slack.com/api/chat.update", calls[0].endpoint) + require.Equal(t, "111.222", calls[0].payload["ts"]) + require.Equal(t, "C123", calls[0].payload["channel"]) + require.NotContains(t, calls[0].payload, "thread_ts") + + require.Equal(t, "https://slack.com/api/chat.postMessage", calls[1].endpoint) + require.Equal(t, "111.222", calls[1].payload["thread_ts"]) + require.Equal(t, "C123", calls[1].payload["channel"]) + require.NotContains(t, calls[1].payload, "ts") + + ts, _ := store.GetStr("threadTs") + require.Equal(t, "111.222", ts) + }) + + t.Run("repeat only edits the parent when both options are set", func(t *testing.T) { + var calls []slackCall + n := notifierRecordingCalls(t, &config.SlackConfig{UpdateMessage: true, ThreadReplies: true}, &calls, "999.999") + store := parentStore() + + _, err := n.Notify(notifyCtx(store, notify.ReasonRepeatIntervalElapsed)) + require.NoError(t, err) + + require.Len(t, calls, 1) + require.Equal(t, "https://slack.com/api/chat.update", calls[0].endpoint) + require.Equal(t, "111.222", calls[0].payload["ts"]) + require.NotContains(t, calls[0].payload, "thread_ts") + }) + + t.Run("follow-up without update_message is a thread reply", func(t *testing.T) { + var calls []slackCall + n := notifierRecordingCalls(t, &config.SlackConfig{ThreadReplies: true}, &calls, "999.999") + store := parentStore() + + _, err := n.Notify(notifyCtx(store, notify.ReasonNewAlertsInGroup)) + require.NoError(t, err) + + require.Len(t, calls, 1) + require.Equal(t, "https://slack.com/api/chat.postMessage", calls[0].endpoint) + require.Equal(t, "111.222", calls[0].payload["thread_ts"]) + require.NotContains(t, calls[0].payload, "ts") + ts, _ := store.GetStr("threadTs") + require.Equal(t, "111.222", ts) + }) + + t.Run("repeat without update_message still replies", func(t *testing.T) { + var calls []slackCall + n := notifierRecordingCalls(t, &config.SlackConfig{ThreadReplies: true}, &calls, "999.999") + store := parentStore() + + _, err := n.Notify(notifyCtx(store, notify.ReasonRepeatIntervalElapsed)) + require.NoError(t, err) + + require.Len(t, calls, 1) + require.Equal(t, "111.222", calls[0].payload["thread_ts"]) + }) + + t.Run("update_message does not open a thread", func(t *testing.T) { + var calls []slackCall + n := notifierRecordingCalls(t, &config.SlackConfig{UpdateMessage: true}, &calls, "999.999") + store := parentStore() + + _, err := n.Notify(notifyCtx(store, notify.ReasonNewAlertsInGroup)) + require.NoError(t, err) + + require.Len(t, calls, 1) + require.Equal(t, "https://slack.com/api/chat.update", calls[0].endpoint) + require.NotContains(t, calls[0].payload, "thread_ts") + }) + + t.Run("incomplete stored identity starts a new message", func(t *testing.T) { + var calls []slackCall + n := notifierRecordingCalls(t, &config.SlackConfig{ThreadReplies: true}, &calls, "333.444") + store := nflog.NewStore(nil) + store.SetStr("threadTs", "111.222") + + _, err := n.Notify(notifyCtx(store, notify.ReasonNewAlertsInGroup)) + require.NoError(t, err) + + require.Len(t, calls, 1) + require.NotContains(t, calls[0].payload, "thread_ts") + require.NotContains(t, calls[0].payload, "ts") + ts, _ := store.GetStr("threadTs") + require.Equal(t, "333.444", ts) + channel, _ := store.GetStr("channelId") + require.Equal(t, "C123", channel) + }) +} diff --git a/notify/slack/types.go b/notify/slack/types.go index a85734e1ad..4bf2146e7e 100644 --- a/notify/slack/types.go +++ b/notify/slack/types.go @@ -37,14 +37,15 @@ type Notifier struct { // request is the request for sending a Slack notification. type request struct { - Channel string `json:"channel,omitempty"` - Timestamp string `json:"ts,omitempty"` - Username string `json:"username,omitempty"` - IconEmoji string `json:"icon_emoji,omitempty"` - IconURL string `json:"icon_url,omitempty"` - LinkNames bool `json:"link_names,omitempty"` - Text string `json:"text,omitempty"` - Attachments []attachment `json:"attachments"` + Channel string `json:"channel,omitempty"` + Timestamp string `json:"ts,omitempty"` + ThreadTimestamp string `json:"thread_ts,omitempty"` + Username string `json:"username,omitempty"` + IconEmoji string `json:"icon_emoji,omitempty"` + IconURL string `json:"icon_url,omitempty"` + LinkNames bool `json:"link_names,omitempty"` + Text string `json:"text,omitempty"` + Attachments []attachment `json:"attachments"` } // attachment is used to display a richly formatted message block. From 54700e6d9be2607b14bae48e3181b6409abecf74 Mon Sep 17 00:00:00 2001 From: Cody Kaczynski Date: Fri, 18 Sep 2026 04:58:17 +0000 Subject: [PATCH 2/5] config: validate slack api_url_file for thread_replies and update_message When those options are enabled, read api_url_file at load time and require the resolved URL to be https://slack.com/api/chat.postMessage. Configs that do not use them still skip the file read. Signed-off-by: Cody Kaczynski --- config/config_test.go | 44 +++++++++++++++++-- config/notifiers.go | 14 ++++-- ....slack-thread-replies-and-api-url-file.yml | 12 ----- docs/configuration.md | 5 ++- 4 files changed, 55 insertions(+), 20 deletions(-) delete mode 100644 config/testdata/conf.slack-thread-replies-and-api-url-file.yml diff --git a/config/config_test.go b/config/config_test.go index 687fadcb3e..43867738c5 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1347,9 +1347,47 @@ func TestSlackThreadRepliesWithAppToken(t *testing.T) { } func TestSlackThreadRepliesWithAPIURLFile(t *testing.T) { - _, err := LoadFile("testdata/conf.slack-thread-replies-and-api-url-file.yml") - if err != nil { - t.Fatalf("Error parsing testdata/conf.slack-thread-replies-and-api-url-file.yml: %s", err) + urlFile := t.TempDir() + "/api_url" + if err := os.WriteFile(urlFile, []byte("https://slack.com/api/chat.postMessage\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg := fmt.Sprintf(` +route: + receiver: slack +receivers: + - name: slack + slack_configs: + - channel: '#alerts' + api_url_file: %q + thread_replies: true +`, urlFile) + if _, err := Load(cfg); err != nil { + t.Fatalf("Load() error = %v, want nil", err) + } +} + +func TestSlackThreadRepliesAPIURLFileWebhook(t *testing.T) { + urlFile := t.TempDir() + "/api_url" + if err := os.WriteFile(urlFile, []byte("https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg := fmt.Sprintf(` +route: + receiver: slack +receivers: + - name: slack + slack_configs: + - channel: '#alerts' + api_url_file: %q + thread_replies: true +`, urlFile) + _, err := Load(cfg) + if err == nil { + t.Fatal("Load() error = nil, want webhook rejected") + } + want := "thread_replies can only be used with bot tokens. api_url must be set to https://slack.com/api/chat.postMessage" + if err.Error() != want { + t.Errorf("Expected: %s\nGot: %s", want, err.Error()) } } diff --git a/config/notifiers.go b/config/notifiers.go index daf941ce82..327aa6f410 100644 --- a/config/notifiers.go +++ b/config/notifiers.go @@ -17,8 +17,10 @@ import ( "errors" "fmt" "net/textproto" + "os" "regexp" "slices" + "strings" "time" commoncfg "github.com/prometheus/common/config" @@ -376,11 +378,17 @@ func (c *SlackConfig) validateMessageAPIURL() error { if !c.UpdateMessage && !c.ThreadReplies { return nil } - // File-backed URLs are read when the notification is sent. + apiURL := "" if len(c.APIURLFile) > 0 { - return nil + content, err := os.ReadFile(c.APIURLFile) + if err != nil { + return fmt.Errorf("reading api_url_file: %w", err) + } + apiURL = strings.TrimSpace(string(content)) + } else if c.APIURL != nil { + apiURL = c.APIURL.String() } - if c.APIURL != nil && c.APIURL.String() == "https://slack.com/api/chat.postMessage" { + if apiURL == "https://slack.com/api/chat.postMessage" { return nil } if c.UpdateMessage { diff --git a/config/testdata/conf.slack-thread-replies-and-api-url-file.yml b/config/testdata/conf.slack-thread-replies-and-api-url-file.yml deleted file mode 100644 index 7e7e960f3d..0000000000 --- a/config/testdata/conf.slack-thread-replies-and-api-url-file.yml +++ /dev/null @@ -1,12 +0,0 @@ -route: - receiver: 'slack-notifications' - group_by: [alertname] -receivers: - - name: 'slack-notifications' - slack_configs: - - channel: '#alerts1' - text: 'test' - send_resolved: true - api_url_file: '/etc/slack/api_url' - update_message: true - thread_replies: true diff --git a/docs/configuration.md b/docs/configuration.md index 4a621cef9d..5122370af9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1770,8 +1770,9 @@ fields: # already current. thread_replies without update_message still posts a # reply on repeats; otherwise the repeat would not appear in Slack at all. # -# Requires a bot token (api_url https://slack.com/api/chat.postMessage). -# Incoming webhooks do not return a message timestamp and cannot be used. +# Requires a bot token. api_url or the contents of api_url_file must be +# https://slack.com/api/chat.postMessage. Incoming webhooks do not return a +# message timestamp and cannot be used. [ thread_replies: | default = false ] ``` From ed4ac0e2c5af8e44b3a8d67be620330654b7564f Mon Sep 17 00:00:00 2001 From: Cody Kaczynski Date: Fri, 18 Sep 2026 05:07:14 +0000 Subject: [PATCH 3/5] notify/slack: reject non-bot api_url_file at send time Configuration load validates api_url_file, but Notify rereads the file on every notification. If the file later changes to a webhook URL, thread replies and first posts would go there without a reload. When update_message or thread_replies is set, require the resolved endpoint to be https://slack.com/api/chat.postMessage before sending. Signed-off-by: Cody Kaczynski --- notify/slack/slack.go | 20 ++++++++++++++++++++ notify/slack/slack_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/notify/slack/slack.go b/notify/slack/slack.go index ba7916a231..63779eb806 100644 --- a/notify/slack/slack.go +++ b/notify/slack/slack.go @@ -17,6 +17,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "log/slog" @@ -37,6 +38,8 @@ import ( // https://api.slack.com/reference/messaging/attachments#legacy_fields - 1024, no units given, assuming runes or characters. const maxTitleLenRunes = 1024 +const slackChatPostMessageURL = "https://slack.com/api/chat.postMessage" + // New returns a new Slack notification handler. func New(c *config.SlackConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) { client, err := notify.NewClientWithTracing(*c.HTTPConfig, "slack", httpOpts...) @@ -153,6 +156,9 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) } u = strings.TrimSpace(string(content)) } + if err := requireBotAPIURL(n.conf, u); err != nil { + return false, err + } if n.conf.Timeout > 0 { postCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, fmt.Errorf("configured slack timeout reached (%s)", n.conf.Timeout)) @@ -210,6 +216,20 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) return n.post(ctx, firstURL, &followUp, nil) } +// requireBotAPIURL rejects non-bot Slack endpoints when message updates or thread replies are enabled. +func requireBotAPIURL(conf *config.SlackConfig, u string) error { + if !conf.UpdateMessage && !conf.ThreadReplies { + return nil + } + if u == slackChatPostMessageURL { + return nil + } + if conf.UpdateMessage { + return errors.New("update_message can only be used with bot tokens. api_url must be set to https://slack.com/api/chat.postMessage") + } + return errors.New("thread_replies can only be used with bot tokens. api_url must be set to https://slack.com/api/chat.postMessage") +} + func (n *Notifier) nflogStore(ctx context.Context, logger *slog.Logger) *nflog.Store { if !n.conf.UpdateMessage && !n.conf.ThreadReplies { return nil diff --git a/notify/slack/slack_test.go b/notify/slack/slack_test.go index 4189bcfa88..7a46f5b329 100644 --- a/notify/slack/slack_test.go +++ b/notify/slack/slack_test.go @@ -631,3 +631,32 @@ func TestSlackThreadReplies(t *testing.T) { require.Equal(t, "C123", channel) }) } + +func TestNotifyRejectsChangedAPIURLFile(t *testing.T) { + urlFile := t.TempDir() + "/api_url" + require.NoError(t, os.WriteFile(urlFile, []byte("https://hooks.slack.com/services/T/B/X\n"), 0o600)) + + conf := &config.SlackConfig{ + APIURLFile: urlFile, + ThreadReplies: true, + Channel: "#test-channel", + HTTPConfig: &commoncfg.HTTPClientConfig{}, + } + n, err := New(conf, test.CreateTmpl(t), promslog.NewNopLogger()) + require.NoError(t, err) + + called := false + n.postJSONFunc = func(ctx context.Context, client *http.Client, endpoint string, body io.Reader) (*http.Response, error) { + called = true + return nil, nil + } + + store := nflog.NewStore(nil) + store.SetStr("threadTs", "111.222") + store.SetStr("channelId", "C123") + + retry, err := n.Notify(notifyCtx(store, notify.ReasonNewAlertsInGroup)) + require.False(t, retry) + require.EqualError(t, err, "thread_replies can only be used with bot tokens. api_url must be set to https://slack.com/api/chat.postMessage") + require.False(t, called) +} From 9151a76e7342096025301499a89fc628006d4084 Mon Sep 17 00:00:00 2001 From: Cody Kaczynski Date: Fri, 18 Sep 2026 05:13:57 +0000 Subject: [PATCH 4/5] docs: state that Slack thread_replies requires a Slack app Incoming webhooks cannot edit messages or start threads because they do not return a message timestamp. Spell that out in the slack_config docs and on the update_message and thread_replies fields. Signed-off-by: Cody Kaczynski --- config/notifiers.go | 9 ++++++--- docs/configuration.md | 15 ++++++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/config/notifiers.go b/config/notifiers.go index 327aa6f410..66cdfe5446 100644 --- a/config/notifiers.go +++ b/config/notifiers.go @@ -335,11 +335,14 @@ type SlackConfig struct { Actions []*SlackAction `yaml:"actions,omitempty" json:"actions,omitempty"` // UpdateMessage enables updating existing Slack messages instead of creating new ones. - // Requires bot token with chat:write scope. Webhook URLs do not support updates. + // Incoming webhooks cannot be used. Requires a Slack app with a bot token + // (chat:write) and api_url https://slack.com/api/chat.postMessage. UpdateMessage bool `yaml:"update_message" json:"update_message,omitempty"` // ThreadReplies posts follow-up notifications for an alert group as replies - // in the Slack thread of the group's first message. Requires bot token with - // chat:write scope. Incoming webhooks cannot start a thread. + // in the Slack thread of the group's first message. Incoming webhooks + // (hooks.slack.com) cannot be used; they do not return a message timestamp. + // Requires a Slack app with a bot token (chat:write) and + // api_url https://slack.com/api/chat.postMessage. ThreadReplies bool `yaml:"thread_replies" json:"thread_replies,omitempty"` // Timeout is the maximum time allowed to invoke the slack. Setting this to 0 // does not impose a timeout. diff --git a/docs/configuration.md b/docs/configuration.md index 5122370af9..2c5b5d4d86 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1699,6 +1699,8 @@ If using an incoming webhook then `api_url` must be set to the URL of the incomi If using Bot tokens then `api_url` must be set to [`https://slack.com/api/chat.postMessage`](https://api.slack.com/methods/chat.postMessage), the bot token must be set as the authorization credentials in `http_config`, and `channel` must contain either the name of the channel or Channel ID to send notifications to. If using the name of the channel the # is optional. +`update_message` and `thread_replies` do **not** work with incoming webhooks (`https://hooks.slack.com/services/...`). Incoming webhooks only return `ok` and never a message timestamp, so Alertmanager cannot edit a message or reply in its thread. Both options require a [Slack app](https://api.slack.com/authentication/basics) with a bot token (`chat:write`) and `api_url: https://slack.com/api/chat.postMessage`. Invite the bot into the channel. + The notification contains an [attachment](https://docs.slack.dev/legacy/legacy-messaging/legacy-secondary-message-attachments/). ```yaml @@ -1757,22 +1759,25 @@ fields: [ timeout: | default = 0s ] # Enables updating existing Slack messages instead of creating new ones on alert state change. -# Webhook URLs do not support updates. +# Incoming webhooks (https://hooks.slack.com/services/...) cannot be used. +# Requires a Slack app with a bot token and api_url https://slack.com/api/chat.postMessage. [ update_message: | default = false ] # Post follow-up notifications for the same alert group as replies in the # Slack thread of the group's first message. A later firing after the group # has fully resolved starts a new thread. # +# Incoming webhooks (https://hooks.slack.com/services/...) cannot be used. +# Slack does not return a message timestamp from a webhook, so there is +# nothing to thread onto. Requires a Slack app with a bot token (chat:write) +# and api_url or api_url_file equal to https://slack.com/api/chat.postMessage. +# Invite the bot into the destination channel. +# # Together with update_message, state changes edit the first message and # also add a reply. Repeat-interval notifications then only edit the first # message, so the thread is not filled with copies of a message that is # already current. thread_replies without update_message still posts a # reply on repeats; otherwise the repeat would not appear in Slack at all. -# -# Requires a bot token. api_url or the contents of api_url_file must be -# https://slack.com/api/chat.postMessage. Incoming webhooks do not return a -# message timestamp and cannot be used. [ thread_replies: | default = false ] ``` From cfdda02f1ec2df84b3fc23b9aaccc334f7e35505 Mon Sep 17 00:00:00 2001 From: Cody Kaczynski Date: Fri, 18 Sep 2026 05:21:45 +0000 Subject: [PATCH 5/5] config: prefer resolved slack api_url over api_url_file validateMessageAPIURL checked api_url_file first, while Notify uses api_url when it is set. App-token receivers set api_url from the Slack app URL and can still inherit a global api_url_file, so validation must follow the same order as send time. Signed-off-by: Cody Kaczynski --- config/config_test.go | 22 ++++++++++++++++++++++ config/notifiers.go | 6 +++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 43867738c5..2714c74456 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1346,6 +1346,28 @@ func TestSlackThreadRepliesWithAppToken(t *testing.T) { } } +func TestSlackThreadRepliesAppTokenIgnoresGlobalAPIURLFile(t *testing.T) { + urlFile := t.TempDir() + "/api_url" + if err := os.WriteFile(urlFile, []byte("https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg := fmt.Sprintf(` +global: + slack_api_url_file: %q +route: + receiver: slack +receivers: + - name: slack + slack_configs: + - channel: '#alerts' + app_token: 'xoxb-some-token' + thread_replies: true +`, urlFile) + if _, err := Load(cfg); err != nil { + t.Fatalf("Load() error = %v, want nil", err) + } +} + func TestSlackThreadRepliesWithAPIURLFile(t *testing.T) { urlFile := t.TempDir() + "/api_url" if err := os.WriteFile(urlFile, []byte("https://slack.com/api/chat.postMessage\n"), 0o600); err != nil { diff --git a/config/notifiers.go b/config/notifiers.go index 66cdfe5446..787114bcd8 100644 --- a/config/notifiers.go +++ b/config/notifiers.go @@ -382,14 +382,14 @@ func (c *SlackConfig) validateMessageAPIURL() error { return nil } apiURL := "" - if len(c.APIURLFile) > 0 { + if c.APIURL != nil { + apiURL = c.APIURL.String() + } else if len(c.APIURLFile) > 0 { content, err := os.ReadFile(c.APIURLFile) if err != nil { return fmt.Errorf("reading api_url_file: %w", err) } apiURL = strings.TrimSpace(string(content)) - } else if c.APIURL != nil { - apiURL = c.APIURL.String() } if apiURL == "https://slack.com/api/chat.postMessage" { return nil