diff --git a/code/backend/internal/ml/client.go b/code/backend/internal/ml/client.go index b3f8f3f..55fc042 100644 --- a/code/backend/internal/ml/client.go +++ b/code/backend/internal/ml/client.go @@ -102,6 +102,7 @@ func GenerateDummyParser(rawLog string, features fingerprint.FingerprintFeatures pattern5 := `^(?P\S+ \S+) (?P[A-Za-z]+) (?P\S+) (?P\S+) (?P.*)$` if re, err := regexp.Compile(pattern5); err == nil && re.MatchString(trimmed) { return &parser.Parser{ + Name: "Generic Structured Log Parser", Pattern: pattern5, Mapping: map[string]string{ "timestamp": "timestamp", @@ -127,6 +128,7 @@ func GenerateDummyParser(rawLog string, features fingerprint.FingerprintFeatures pattern4 := `^(?P\S+) \[(?P[A-Za-z]+)\] \[(?P\S+)\] (?P.*)$` if re, err := regexp.Compile(pattern4); err == nil && re.MatchString(trimmed) { return &parser.Parser{ + Name: "Bracketed Service Log Parser", Pattern: pattern4, Mapping: map[string]string{ "timestamp": "timestamp", @@ -150,6 +152,7 @@ func GenerateDummyParser(rawLog string, features fingerprint.FingerprintFeatures pattern3 := `^(?P\S+ \S+) (?P[A-Za-z]+) (?P.*)$` if re, err := regexp.Compile(pattern3); err == nil && re.MatchString(trimmed) { return &parser.Parser{ + Name: "Timestamp Severity Log Parser", Pattern: pattern3, Mapping: map[string]string{ "timestamp": "timestamp", diff --git a/code/backend/internal/parser/engine.go b/code/backend/internal/parser/engine.go index 5cc6cb8..8672efc 100644 --- a/code/backend/internal/parser/engine.go +++ b/code/backend/internal/parser/engine.go @@ -162,6 +162,20 @@ func applyTransformation(value string, t Transformation) (string, error) { case "trim", "trim_space": return strings.TrimSpace(value), nil + case "integer", "int": + number, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + if err != nil { + return "", fmt.Errorf("invalid integer value %q: %w", value, err) + } + return strconv.FormatInt(number, 10), nil + + case "float", "decimal", "number": + number, err := strconv.ParseFloat(strings.TrimSpace(value), 64) + if err != nil { + return "", fmt.Errorf("invalid float value %q: %w", value, err) + } + return strconv.FormatFloat(number, 'f', -1, 64), nil + case "datetime", "date", "timestamp", "time": return parseAndFormatDateTime(value, t.Format) diff --git a/code/backend/internal/parser/engine_test.go b/code/backend/internal/parser/engine_test.go index 68b34f0..e89a0e0 100644 --- a/code/backend/internal/parser/engine_test.go +++ b/code/backend/internal/parser/engine_test.go @@ -203,6 +203,20 @@ func TestParseLog_Transformations(t *testing.T) { expected: "padded message", expectError: false, }, + { + name: "integer", + input: "001204", + trans: Transformation{Type: "integer"}, + expected: "1204", + expectError: false, + }, + { + name: "float", + input: "249.990", + trans: Transformation{Type: "float"}, + expected: "249.99", + expectError: false, + }, { name: "datetime with Go layout", input: "2026-09-05 18:30:21", diff --git a/code/backend/internal/parser/model.go b/code/backend/internal/parser/model.go index 49b0040..1f384be 100644 --- a/code/backend/internal/parser/model.go +++ b/code/backend/internal/parser/model.go @@ -16,6 +16,7 @@ type NormalizedLog struct { } type Parser struct { + Name string `json:"name,omitempty" bson:"name,omitempty"` Pattern string `json:"pattern" bson:"pattern"` Mapping map[string]string `json:"mapping" bson:"mapping"` Transformations map[string]Transformation `json:"transformations,omitempty" bson:"transformations,omitempty"` diff --git a/code/backend/internal/parsergen/parser_generator.go b/code/backend/internal/parsergen/parser_generator.go index 6a95445..054aac0 100644 --- a/code/backend/internal/parsergen/parser_generator.go +++ b/code/backend/internal/parsergen/parser_generator.go @@ -141,6 +141,7 @@ func (pg *ParserGenerator) ProcessMessage(ctx context.Context, body []byte) erro } doc := database.ParserDoc{ + Name: newParser.Name, Hash: msg.Hash, Pattern: newParser.Pattern, Mapping: newParser.Mapping, diff --git a/code/frontend/gologgo-client/components/log-dashboard.jsx b/code/frontend/gologgo-client/components/log-dashboard.jsx index 1856339..3d0ab34 100644 --- a/code/frontend/gologgo-client/components/log-dashboard.jsx +++ b/code/frontend/gologgo-client/components/log-dashboard.jsx @@ -55,22 +55,30 @@ export function LogDashboard() { const handleClearLogs = () => setRawLogs('') const handleProcessLogs = async () => { - if (!rawLogs.trim()) return; + const logLines = rawLogs + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + + if (logLines.length === 0) return setIsProcessing(true); try { - const res = await fetch('/api/logs', { + const responses = await Promise.all(logLines.map((line) => fetch('/api/logs', { method: 'POST', headers: { 'Content-Type': 'text/plain' }, - body: rawLogs - }); - if (res.ok) { + body: line + }))) + + const failedResponse = responses.find((response) => !response.ok) + if (!failedResponse) { setRawLogs(''); - toast.success('Logs successfully sent for processing!'); + toast.success(`${logLines.length} log${logLines.length === 1 ? '' : 's'} sent for processing!`); } else { - const errorData = await res.json().catch(() => ({})); - toast.error(`Failed to process logs: ${errorData.error || 'Unknown error'}`); + const errorData = await failedResponse.json().catch(() => ({})) + const successfulCount = responses.length - responses.filter((response) => !response.ok).length + toast.error(`Processed ${successfulCount}/${logLines.length} logs. ${errorData.error || 'Some logs failed.'}`) } } catch (err) { console.error(err); diff --git a/code/frontend/gologgo-client/components/logs-explorer.jsx b/code/frontend/gologgo-client/components/logs-explorer.jsx index 24ff46c..12057c3 100644 --- a/code/frontend/gologgo-client/components/logs-explorer.jsx +++ b/code/frontend/gologgo-client/components/logs-explorer.jsx @@ -86,6 +86,8 @@ export function LogsExplorer() { // Filter / pagination state const [search, setSearch] = useState('') const [statusFilter, setStatusFilter] = useState('all') + const [parserFilter, setParserFilter] = useState('all') + const [parserOptions, setParserOptions] = useState([{ value: 'all', label: 'All Parsers' }]) const [currentPage, setCurrentPage] = useState(1) const [pageSize, setPageSize] = useState(10) @@ -138,6 +140,24 @@ export function LogsExplorer() { // Initial load useEffect(() => { fetchLogs() }, []) // eslint-disable-line react-hooks/exhaustive-deps + useEffect(() => { + fetch('/api/parsers', { cache: 'no-store' }) + .then((response) => response.ok ? response.json() : null) + .then((data) => { + const parsers = data?.parsers ?? data ?? [] + setParserOptions([ + { value: 'all', label: 'All Parsers' }, + ...parsers + .map((parser) => ({ + value: parser.hash || parser.name, + label: parser.name || parser.hash, + })) + .filter((parser) => parser.value), + ]) + }) + .catch(() => setParserOptions([{ value: 'all', label: 'All Parsers' }])) + }, []) + // Re-fetch on filter/page change (skip on mount, handled above) useEffect(() => { fetchLogs({ page: currentPage, limit: pageSize, status: statusFilter }) @@ -150,21 +170,24 @@ export function LogsExplorer() { const clearFilters = () => { setSearch('') setStatusFilter('all') + setParserFilter('all') setCurrentPage(1) } // Client-side search on the current page results (search within payload text) const filtered = useMemo(() => { - if (!search.trim()) return requests + if (!search.trim() && parserFilter === 'all') return requests const q = search.toLowerCase() return requests.filter((r) => { const payloadMatch = r.payload?.toLowerCase().includes(q) const idMatch = r.id?.toLowerCase().includes(q) const statusMatch = r.status?.toLowerCase().includes(q) const logsMatch = r.logs?.some?.((l) => JSON.stringify(l.normalizedLog).toLowerCase().includes(q)) - return payloadMatch || idMatch || statusMatch || logsMatch + const parserMatch = parserFilter === 'all' || r.logs?.some?.((l) => l.hash === parserFilter) + const textMatch = !search.trim() || payloadMatch || idMatch || statusMatch || logsMatch + return textMatch && parserMatch }) - }, [requests, search]) + }, [requests, search, parserFilter]) // ── Format helpers ───────────────────────────────────────────────────────── @@ -299,6 +322,12 @@ export function LogsExplorer() { onChange={(v) => { setStatusFilter(v); setCurrentPage(1) }} options={statusOptions} /> + { setParserFilter(v); setCurrentPage(1) }} + options={parserOptions} + /> )} diff --git a/code/ml/app/models/rule_synthesizer.py b/code/ml/app/models/rule_synthesizer.py index 1520d92..a4e3214 100644 --- a/code/ml/app/models/rule_synthesizer.py +++ b/code/ml/app/models/rule_synthesizer.py @@ -24,7 +24,8 @@ "transformations": { "timestamp": { "type": "datetime", "format": "2006-01-02 15:04:05" }, "severity": { "type": "uppercase" } - } + }, + "name": "Nginx Access Log Parser" } CRITICAL RULES: @@ -35,8 +36,9 @@ - If a field is a timestamp/date/time, set `type` to "datetime" and provide a Go layout format string in `format` (e.g. "2006-01-02 15:04:05", "Jan _2 15:04:05", RFC3339). - If a field is severity or level, set `type` to "uppercase". - If a field is numeric, set `type` to "integer" or "float". -5. Do NOT hardcode dynamic values inside the regex; capture them generically. -6. Return ONLY raw JSON. No markdown backticks (no ```json). +5. "name" MUST be concise, human-readable, and describe the log source or format. Do not use generic names like "Parser" or hash values. +6. Do NOT hardcode dynamic values inside the regex; capture them generically. +7. Return ONLY raw JSON. No markdown backticks (no ```json). """ class RuleSynthesizer: diff --git a/code/ml/app/schemas.py b/code/ml/app/schemas.py index 7d78c5d..7b6020d 100644 --- a/code/ml/app/schemas.py +++ b/code/ml/app/schemas.py @@ -21,6 +21,9 @@ class Transformation(BaseModel): class RuleGeneratorOutput(BaseModel): model_config = ConfigDict(extra="forbid") + name: str = Field( + description="Concise human-readable name for the parser, describing the log source or format" + ) pattern: str = Field( description="Named capture group regex matching the whole log string, e.g. ^(?\\S+ \\S+) (?\\w+) ...$" )