[pull] master from mattermost:master

docs/main/administration-guide/configure/compliance-configuration-settings.mdx

@@ -412,6 +412,36 @@ The SMTP server port that will receive your Global Relay EML file when a [custom
 </tbody>
 </table>
 
+### Custom header name
+
+An optional custom header added to each exported EML file when a [custom customer account type](#global-relay-customer-account) is configured. Both the custom header name and value must be set, or both left blank to omit. A save is rejected if only one of the two is set, or if the name reuses a header the export already writes (such as `From`, `To`, `Subject`, or `X-GlobalRelay-MsgType`).
+
+<table>
+<colgroup>
+<col style={{width: '100%'}} />
+</colgroup>
+<tbody>
+<tr>
+<td>This feature's <code>config.json</code> setting is <code>".MessageExportSettings.GlobalRelaySettings.CustomHeaderName": ""</code> with string input.</td>
+</tr>
+</tbody>
+</table>
+
+### Custom header value
+
+The value sent with the custom header when a [custom customer account type](#global-relay-customer-account) is configured. Both the custom header name and value must be set, or both left blank to omit. A save is rejected if only one of the two is set, or if the name reuses a header the export already writes.
+
+<table>
+<colgroup>
+<col style={{width: '100%'}} />
+</colgroup>
+<tbody>
+<tr>
+<td>This feature's <code>config.json</code> setting is <code>".MessageExportSettings.GlobalRelaySettings.CustomHeaderValue": ""</code> with string input.</td>
+</tr>
+</tbody>
+</table>
+
 ### Message export batch size
 
 This setting isn't available in the System Console and can only be set in `config.json`.

e2e-tests/cypress/tests/support/api/cloud_default_config.json

@@ -383,7 +383,9 @@
             "EmailAddress": "",
             "SMTPServerTimeout": 1800,
             "CustomSMTPServerName": "",
-            "CustomSMTPPort": "25"
+            "CustomSMTPPort": "25",
+            "CustomHeaderName": "",
+            "CustomHeaderValue": ""
         }
     },
     "PluginSettings": {

e2e-tests/cypress/tests/support/api/on_prem_default_config.json

@@ -522,7 +522,9 @@
             "EmailAddress": "",
             "SMTPServerTimeout": 1800,
             "CustomSMTPServerName": "",
-            "CustomSMTPPort": "25"
+            "CustomSMTPPort": "25",
+            "CustomHeaderName": "",
+            "CustomHeaderValue": ""
         }
     },
     "JobSettings": {

e2e-tests/playwright/lib/src/server/default_config.ts

@@ -719,6 +719,8 @@
             SMTPServerTimeout: 1800,
             CustomSMTPServerName: '',
             CustomSMTPPort: '25',
+            CustomHeaderName: '',
+            CustomHeaderValue: '',
         },
     },
     JobSettings: {
@@ -813,7 +815,7 @@
         SessionAttributes: false,
         DiscoverableChannels: false,
         MobileEphemeralMode: false,
         PropertyFieldRank: false,
         TeamMembershipAccessControl: true,
         MmBlocksEnabled: true,
         EnableConcurrentReact: true,

server/channels/app/file.go

@@ -808,7 +808,6 @@ func (a *App) UploadFileX(rctx request.CTX, channelID, name string, input io.Rea
 	}
 
 	rctx = rctx.WithLogFields(
-		mlog.String("file_name", name),
 		mlog.String("channel_id", channelID),
 		mlog.String("user_id", t.UserId),
 	)
@@ -822,6 +821,10 @@ func (a *App) UploadFileX(rctx request.CTX, channelID, name string, input io.Rea
 
 	t.init(a)
 
+	rctx = rctx.WithLogFields(
+		mlog.String("file_info_id", t.fileinfo.Id),
+	)
+
 	var aerr *model.AppError
 	if !t.Raw && t.fileinfo.IsImage() {
 		aerr = t.preprocessImage()
@@ -879,10 +882,10 @@ func (a *App) UploadFileX(rctx request.CTX, channelID, name string, input io.Rea
 		if !a.Srv().GoExtraction(func() {
 			err := a.ExtractContentFromFileInfo(rctx, &infoCopy)
 			if err != nil {
-				rctx.Logger().Error("Failed to extract file content", mlog.Err(err), mlog.String("file_info_id", infoCopy.Id))
+				rctx.Logger().Error("Failed to extract file content", mlog.Err(err))
 			}
 		}) {
-			rctx.Logger().Warn("Content extraction queue is full, skipping inline extraction; this file's content will not be searchable until the scheduled content extraction catch-up job runs or an admin runs a content extraction job (e.g. mmctl extract)", mlog.String("file_info_id", infoCopy.Id))
+			rctx.Logger().Warn("Content extraction queue is full, skipping inline extraction; this file's content will not be searchable until the scheduled content extraction catch-up job runs or an admin runs a content extraction job (e.g. mmctl extract)")
 		}
 	}
 
@@ -1713,6 +1716,9 @@ func (a *App) ExtractContentFromFileInfo(rctx request.CTX, fileInfo *model.FileI
 		return nil
 	}
 
+	logger := rctx.Logger().With(mlog.String("file_info_id", fileInfo.Id))
+	logger.Debug("Extracting content from file", mlog.String("extension", fileInfo.Extension))
+
 	file, aerr := a.FileReader(fileInfo.Path)
 	if aerr != nil {
 		return errors.Wrap(aerr, "failed to open file for extract file content")
@@ -1721,7 +1727,7 @@ func (a *App) ExtractContentFromFileInfo(rctx request.CTX, fileInfo *model.FileI
 	// ReaderCloser: with a timeout configured, extraction may continue on a
 	// detached goroutine after Extract returns, so closing the file here would
 	// race with that goroutine still reading it.
-	text, err := docextractor.Extract(rctx.Logger(), fileInfo.Name, file, docextractor.ExtractSettings{
+	text, err := docextractor.Extract(logger, fileInfo.Name, file, docextractor.ExtractSettings{
 		Ctx:              rctx.Context(),
 		ArchiveRecursion: *a.Config().FileSettings.ArchiveRecursion,
 		MaxFileSize:      *a.Config().FileSettings.MaxFileSize,
@@ -1740,7 +1746,7 @@ func (a *App) ExtractContentFromFileInfo(rctx request.CTX, fileInfo *model.FileI
 		}
 		reloadFileInfo, storeErr := a.Srv().Store().FileInfo().Get(fileInfo.Id)
 		if storeErr != nil {
-			rctx.Logger().Warn("Failed to invalidate the fileInfo cache.", mlog.Err(storeErr), mlog.String("file_info_id", fileInfo.Id))
+			logger.Warn("Failed to invalidate the fileInfo cache.", mlog.Err(storeErr))
 		} else {
 			a.Srv().Store().FileInfo().InvalidateFileInfosForPostCache(reloadFileInfo.PostId, false)
 		}

server/channels/app/migrations_test.go

@@ -545,7 +545,7 @@ func TestDoSetupSessionAttributesProperties(t *testing.T) {
 		}
 		require.Len(t, trimmed, len(persisted)-1)
 		field.Attrs[model.PropertyFieldAttributeOptions] = trimmed
-		_, _, _, err := th.Server.propertyService.UpdatePropertyFields(nil, group.ID, []*model.PropertyField{field})
+		_, _, _, err := th.Server.propertyService.UpdatePropertyFields(properties.SystemCallerContext(th.Context), group.ID, []*model.PropertyField{field})
 		require.NoError(t, err)
 
 		before := sessionAttributeOptionIDsByName(t, sessionAttributeFieldByName(t, th, group.ID, model.SessionAttributesPropertyFieldUserAgentPlatform))

server/channels/jobs/extract_content/worker.go

@@ -74,8 +74,6 @@ func runCatchupExtraction(logger mlog.LoggerIFace, job *model.Job, jobServer *jo
 				continue
 			}
 
-			logger.Debug("Extracting file", mlog.String("filename", fileInfo.Name), mlog.String("filepath", fileInfo.Path))
-
 			err = app.ExtractContentFromFileInfo(request.EmptyContext(logger), fileInfo)
 			if err != nil {
 				logger.Warn("Failed to extract file content", mlog.Err(err), mlog.String("file_info_id", fileInfo.Id))
@@ -135,8 +133,6 @@ func runRangeExtraction(logger mlog.LoggerIFace, job *model.Job, jobServer *jobs
 		}
 		for _, fileInfo := range fileInfos {
 			if !ignoredFiles[fileInfo.Extension] {
-				logger.Debug("Extracting file", mlog.String("filename", fileInfo.Name), mlog.String("filepath", fileInfo.Path))
-
 				err = app.ExtractContentFromFileInfo(request.EmptyContext(logger), fileInfo)
 				if err != nil {
 					logger.Warn("Failed to extract file content", mlog.Err(err), mlog.String("file_info_id", fileInfo.Id))

server/i18n/en.json

@@ -12282,6 +12282,22 @@
     "id": "model.config.is_valid.message_export.global_relay.config_missing.app_error",
     "translation": "Message export job ExportFormat is set to 'globalrelay', but GlobalRelaySettings are missing."
   },
+  {
+    "id": "model.config.is_valid.message_export.global_relay.custom_header_incomplete.app_error",
+    "translation": "Message export GlobalRelaySettings.CustomHeaderName and CustomHeaderValue must both be set, or both left empty."
+  },
+  {
+    "id": "model.config.is_valid.message_export.global_relay.custom_header_name.app_error",
+    "translation": "Message export GlobalRelaySettings.CustomHeaderName must be a valid header name with no spaces, colons, separators, or line breaks."
+  },
+  {
+    "id": "model.config.is_valid.message_export.global_relay.custom_header_reserved.app_error",
+    "translation": "Message export GlobalRelaySettings.CustomHeaderName must not reuse a header that the Global Relay export already writes."
+  },
+  {
+    "id": "model.config.is_valid.message_export.global_relay.custom_header_value.app_error",
+    "translation": "Message export GlobalRelaySettings.CustomHeaderValue must not contain control characters or line breaks."
+  },
   {
     "id": "model.config.is_valid.message_export.global_relay.customer_type.app_error",
     "translation": "Message export GlobalRelaySettings.CustomerType must be set to one of either 'A9', 'A10' or 'CUSTOM'."

server/platform/services/docextractor/archive.go

@@ -100,14 +100,14 @@ func (ae *archiveExtractor) Extract(ctx context.Context, name string, r io.ReadS
 
 			data, err := io.ReadAll(reader)
 			if err != nil {
-				return fmt.Errorf("error reading archive entry %s: %w", path, err)
+				return fmt.Errorf("error reading archive entry: %w", err)
 			}
 
 			subtext, extractErr := ae.SubExtractor.Extract(ctx, filename, bytes.NewReader(data), maxFileSize)
 			if extractErr == nil {
 				text.WriteString(subtext + " ")
 			} else if errors.Is(extractErr, context.Canceled) || errors.Is(extractErr, context.DeadlineExceeded) {
-				return fmt.Errorf("error extracting %q: %w", filename, extractErr)
+				return fmt.Errorf("error extracting archive entry: %w", extractErr)
 			}
 		}
 		return nil

server/platform/services/docextractor/archive_test.go

@@ -4,8 +4,11 @@
 package docextractor
 
 import (
+	"archive/zip"
 	"bytes"
 	"context"
+	"io"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -51,3 +54,64 @@ func TestArchiveExtractorSkips7zip(t *testing.T) {
 		assert.Error(t, err) // fails to extract as any valid archive format
 	})
 }
+
+// contextErrorExtractor fails every extraction with the given context error.
+type contextErrorExtractor struct {
+	err error
+}
+
+func (ce *contextErrorExtractor) Name() string {
+	return "contextErrorExtractor"
+}
+
+func (ce *contextErrorExtractor) Match(filename string) bool {
+	return true
+}
+
+func (ce *contextErrorExtractor) Extract(_ context.Context, _ string, _ io.ReadSeeker, _ int64) (string, error) {
+	return "", ce.err
+}
+
+func TestArchiveExtractorErrorOmitsEntryName(t *testing.T) {
+	// The entry name is transformed before reaching the nested extraction error
+	// (separators become spaces), so assert on the stem token as well: it
+	// survives that transformation and would appear in either leaky error.
+	const entryName = "confidential-customer-list.txt"
+	const entryStem = "confidential"
+
+	var archive bytes.Buffer
+	zw := zip.NewWriter(&archive)
+	entry, err := zw.Create(entryName)
+	require.NoError(t, err)
+	_, err = entry.Write([]byte(strings.Repeat("a", 1024)))
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+
+	requireNoEntryName := func(t *testing.T, err error) {
+		t.Helper()
+		require.Error(t, err)
+		assert.NotContains(t, err.Error(), entryName)
+		assert.NotContains(t, err.Error(), entryStem)
+	}
+
+	t.Run("entry read failure", func(t *testing.T) {
+		ae := &archiveExtractor{SubExtractor: &plainExtractor{}}
+
+		// A maxFileSize below the entry size fails the entry read.
+		_, err := ae.Extract(context.Background(), "archive.zip", bytes.NewReader(archive.Bytes()), 8)
+		requireNoEntryName(t, err)
+	})
+
+	for name, contextErr := range map[string]error{
+		"cancelled":         context.Canceled,
+		"deadline exceeded": context.DeadlineExceeded,
+	} {
+		t.Run("nested extraction "+name, func(t *testing.T) {
+			ae := &archiveExtractor{SubExtractor: &contextErrorExtractor{err: contextErr}}
+
+			_, err := ae.Extract(context.Background(), "archive.zip", bytes.NewReader(archive.Bytes()), 0)
+			requireNoEntryName(t, err)
+			assert.ErrorIs(t, err, contextErr)
+		})
+	}
+}

server/platform/services/docextractor/combine.go

@@ -44,7 +44,8 @@ func (ce *combineExtractor) Extract(ctx context.Context, filename string, r io.R
 				if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
 					return "", err
 				}
-				ce.logger.Warn("Unable to extract file content", mlog.String("file_name", filename), mlog.String("extractor", extractor.Name()), mlog.Err(err))
+
+				ce.logger.Warn("Unable to extract file content", mlog.String("extractor", extractor.Name()), mlog.Err(err))
 				continue
 			}
 			return text, nil

server/public/model/config.go

@@ -23,6 +23,7 @@ import (
 	"github.com/Masterminds/semver/v3"
 	"github.com/mattermost/ldap"
 	"github.com/pkg/errors"
+	"golang.org/x/net/http/httpguts"
 
 	"github.com/mattermost/mattermost/server/public/shared/mlog"
 	"github.com/mattermost/mattermost/server/public/utils"
@@ -286,6 +287,11 @@ const (
 	GlobalrelayCustomerTypeA10    = "A10"
 	GlobalrelayCustomerTypeCustom = "CUSTOM"
 
+	GlobalRelayMsgTypeHeader     = "X-GlobalRelay-MsgType"
+	GlobalRelayChannelNameHeader = "X-Mattermost-ChannelName"
+	GlobalRelayChannelIDHeader   = "X-Mattermost-ChannelID"
+	GlobalRelayChannelTypeHeader = "X-Mattermost-ChannelType"
+
 	ImageProxyTypeLocal           = "local"
 	ImageProxyTypeLegacyAtmosCamo = "atmos/camo"
 
@@ -335,6 +341,31 @@ func GetDefaultAppCustomURLSchemes() []string {
 	return []string{"mmauth://", "mmauthbeta://"}
 }
 
+// globalRelayReservedHeaders are the headers the Global Relay EML export writes itself.
+// A custom header may not reuse any of them: delivery re-parses From out of the generated
+// EML to set the SMTP envelope sender, Global Relay routes on X-GlobalRelay-MsgType, and a
+// duplicate Content-Type or Mime-Version makes the message unparseable.
+var globalRelayReservedHeaders = map[string]struct{}{
+	"from":                      {},
+	"to":                        {},
+	"subject":                   {},
+	"content-transfer-encoding": {},
+	"auto-submitted":            {},
+	"precedence":                {},
+	strings.ToLower(GlobalRelayMsgTypeHeader):     {},
+	strings.ToLower(GlobalRelayChannelNameHeader): {},
+	strings.ToLower(GlobalRelayChannelIDHeader):   {},
+	strings.ToLower(GlobalRelayChannelTypeHeader): {},
+	"date":         {},
+	"mime-version": {},
+	"content-type": {},
+}
+
+func IsGlobalRelayReservedHeader(name string) bool {
+	_, ok := globalRelayReservedHeaders[strings.ToLower(name)]
+	return ok
+}
+
 var ServerTLSSupportedCiphers = map[string]uint16{
 	"TLS_RSA_WITH_RC4_128_SHA":                tls.TLS_RSA_WITH_RC4_128_SHA,
 	"TLS_RSA_WITH_3DES_EDE_CBC_SHA":           tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,
@@ -3830,6 +3861,8 @@ type GlobalRelayMessageExportSettings struct {
 	SMTPServerTimeout    *int    `access:"compliance_compliance_export"`
 	CustomSMTPServerName *string `access:"compliance_compliance_export"`
 	CustomSMTPPort       *string `access:"compliance_compliance_export"`
+	CustomHeaderName     *string `access:"compliance_compliance_export"` // optional custom header name added to each exported EML
+	CustomHeaderValue    *string `access:"compliance_compliance_export"` // value sent with the custom header
 }
 
 func (s *GlobalRelayMessageExportSettings) SetDefaults() {
@@ -3854,6 +3887,38 @@ func (s *GlobalRelayMessageExportSettings) SetDefaults() {
 	if s.CustomSMTPPort == nil {
 		s.CustomSMTPPort = new("25")
 	}
+	if s.CustomHeaderName == nil {
+		s.CustomHeaderName = new("")
+	}
+	if s.CustomHeaderValue == nil {
+		s.CustomHeaderValue = new("")
+	}
+}
+
+// The custom header name is written verbatim into the exported EML, so it must be a
+// valid header field name; the value must be free of control characters that could
+// inject additional headers.
+func (s *GlobalRelayMessageExportSettings) isValidCustomHeader() *AppError {
+	name := SafeDereference(s.CustomHeaderName)
+	value := strings.TrimSpace(SafeDereference(s.CustomHeaderValue))
+
+	// No custom header configured.
+	if name == "" && value == "" {
+		return nil
+	}
+	if name == "" || value == "" {
+		return NewAppError("Config.IsValid", "model.config.is_valid.message_export.global_relay.custom_header_incomplete.app_error", nil, "", http.StatusBadRequest)
+	}
+	if !httpguts.ValidHeaderFieldName(name) {
+		return NewAppError("Config.IsValid", "model.config.is_valid.message_export.global_relay.custom_header_name.app_error", nil, "", http.StatusBadRequest)
+	}
+	if IsGlobalRelayReservedHeader(name) {
+		return NewAppError("Config.IsValid", "model.config.is_valid.message_export.global_relay.custom_header_reserved.app_error", nil, "", http.StatusBadRequest)
+	}
+	if !httpguts.ValidHeaderFieldValue(value) {
+		return NewAppError("Config.IsValid", "model.config.is_valid.message_export.global_relay.custom_header_value.app_error", nil, "", http.StatusBadRequest)
+	}
+	return nil
 }
 
 type MessageExportSettings struct {
@@ -5251,6 +5316,14 @@ func (s *MessageExportSettings) isValid() *AppError {
 				return NewAppError("Config.IsValid", "model.config.is_valid.message_export.global_relay.smtp_password.app_error", nil, "", http.StatusBadRequest)
 			}
 		}
+
+		if (*s.ExportFormat == ComplianceExportTypeGlobalrelay || *s.ExportFormat == ComplianceExportTypeGlobalrelayZip) &&
+			s.GlobalRelaySettings != nil &&
+			SafeDereference(s.GlobalRelaySettings.CustomerType) == GlobalrelayCustomerTypeCustom {
+			if appErr := s.GlobalRelaySettings.isValidCustomHeader(); appErr != nil {
+				return appErr
+			}
+		}
 	}
 	return nil
 }