From d21e3a28d7f5d0a22ed96c6923fa36e367409378 Mon Sep 17 00:00:00 2001 From: Umu Date: Mon, 9 Nov 2026 01:53:37 -0800 Subject: [PATCH 01/15] test: add synthetic CSV contract fixture valid.csv --- projects/10-soc-triage/fixtures/splunk_csv_v1/valid.csv | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 projects/10-soc-triage/fixtures/splunk_csv_v1/valid.csv diff --git a/projects/10-soc-triage/fixtures/splunk_csv_v1/valid.csv b/projects/10-soc-triage/fixtures/splunk_csv_v1/valid.csv new file mode 100644 index 0000000..75035ba --- /dev/null +++ b/projects/10-soc-triage/fixtures/splunk_csv_v1/valid.csv @@ -0,0 +1,9 @@ +_time,username,host,source_ip,event_type,event_uid +2026-11-08T12:00:00+02:00,lab_burst,SYNTHETIC-EXPORT-01,192.0.2.77,login_failure,burst-01 +2026-11-08T10:01:00Z,lab_burst,SYNTHETIC-EXPORT-01,192.0.2.77,login_failure,burst-02 +2026-11-08T10:02:00Z,lab_burst,SYNTHETIC-EXPORT-01,192.0.2.77,login_failure,burst-03 +2026-11-08T10:03:00Z,lab_burst,SYNTHETIC-EXPORT-01,192.0.2.77,login_failure,burst-04 +2026-11-08T10:04:00Z,lab_burst,SYNTHETIC-EXPORT-01,192.0.2.77,login_failure,burst-05 +2026-11-08T10:10:00Z,lab_burst,SYNTHETIC-EXPORT-01,192.0.2.77,login_success,burst-06 +2026-11-08T11:00:00Z,lab_benign,SYNTHETIC-EXPORT-01,198.51.100.23,login_failure,benign-01 +2026-11-08T11:01:00Z,lab_benign,SYNTHETIC-EXPORT-01,198.51.100.23,login_success,benign-02 From 579a68ebdd07c84b129c909dc849984ba29740f4 Mon Sep 17 00:00:00 2001 From: Umu Date: Mon, 9 Nov 2026 01:53:38 -0800 Subject: [PATCH 02/15] test: add synthetic CSV contract fixture malformed.csv --- .../10-soc-triage/fixtures/splunk_csv_v1/malformed.csv | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 projects/10-soc-triage/fixtures/splunk_csv_v1/malformed.csv diff --git a/projects/10-soc-triage/fixtures/splunk_csv_v1/malformed.csv b/projects/10-soc-triage/fixtures/splunk_csv_v1/malformed.csv new file mode 100644 index 0000000..726c999 --- /dev/null +++ b/projects/10-soc-triage/fixtures/splunk_csv_v1/malformed.csv @@ -0,0 +1,7 @@ +_time,username,host,source_ip,event_type,event_uid +not-a-time,lab_burst,SYNTHETIC-EXPORT-01,192.0.2.77,login_failure,burst-01 +2026-11-08T12:01:00,lab_bad,SYNTHETIC-EXPORT-02,192.0.2.78,login_failure,bad-timezone +2026-11-08T12:02:00Z,lab_bad,SYNTHETIC-EXPORT-02,999.0.0.1,login_failure,bad-ip +2026-11-08T12:03:00Z,lab_bad,,192.0.2.78,login_failure,bad-host +2026-11-08T12:04:00Z,lab_bad,SYNTHETIC-EXPORT-02,192.0.2.78,success,bad-kind +2026-11-08T12:05:00Z,lab_bad,SYNTHETIC-EXPORT-02,192.0.2.78,login_failure,bad-columns,unexpected From 10a90b6785aeb6a33c074102ce31e880b2355c5e Mon Sep 17 00:00:00 2001 From: Umu Date: Mon, 9 Nov 2026 01:53:57 -0800 Subject: [PATCH 03/15] docs: define offline CSV contract and retain diagnostic checkpoint --- .../fixtures/splunk_csv_v1/README.md | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 projects/10-soc-triage/fixtures/splunk_csv_v1/README.md diff --git a/projects/10-soc-triage/fixtures/splunk_csv_v1/README.md b/projects/10-soc-triage/fixtures/splunk_csv_v1/README.md new file mode 100644 index 0000000..de53e97 --- /dev/null +++ b/projects/10-soc-triage/fixtures/splunk_csv_v1/README.md @@ -0,0 +1,93 @@ +# Offline authentication CSV contract — v1 + +Status: **synthetic contract and fixtures prepared; importer change pending**. +These files were authored for an offline test, not exported from Splunk. They +do not establish live connectivity or native Splunk export compatibility. +The production importer does not yet recognize `_time`. + +## Input format + +Use UTF-8 CSV with one authentication event per record and this header: + +```csv +_time,username,host,source_ip,event_type,event_uid +``` + +| Column | Existing normalized field | Contract | +| --- | --- | --- | +| `_time` | `timestamp` | ISO 8601 with `T` and an explicit timezone (`Z` or numeric offset); normalized to UTC. No epoch values or inferred local timezone. | +| `username` | `username` | Required, nonempty user name. | +| `host` | `host` | Required host; existing normalization case-folds it. | +| `source_ip` | `source_ip` | Required valid IP address; existing canonicalization applies. | +| `event_type` | `event_type` | Exactly `login_failure` or `login_success`. The producer must provide these values; no vendor-field inference. | +| `event_uid` | `event_uid` | Optional stable identity within a host across supplied files. Blank or absent IDs remain distinct; do not invent IDs to deduplicate events. | + +This is an explicitly shaped event export, not an aggregate alert-count input. +Unrecognized named fields are preserved but not interpreted; recognized aliases +retain existing validation and conflict checks. Existing restrictions remain: +at most eight files, 5 MiB and 5,000 records per file, and unique nonempty +headers. Incorrect field counts are rejected; extra values remain in the source +record. Do not silently repair malformed records. + +## Evidence and rejection behavior + +Reuse the existing reader, normalizer, investigation logic and report generator. +Retain each input file's SHA-256, parsed raw fields, record hash, source reference +and rejection reason. Parsed records are not a byte-for-byte representation of +CSV quoting: retain the original CSV file alongside its hash. + +Preserve existing alias-conflict behavior: stripped strings are compared before +timestamp parsing. If `_time` and another timestamp alias disagree, reject the +record even if the two strings represent the same instant. Keep host-aware +grouping, same-time ordering rules and invalid/conflicting-identity quarantine. + +## Fixture pair and previously observed diagnostic + +The completed diagnostic used the unchanged source at commit +`0a3049fdf864c55867ee198b038821404769837b`, then temporarily appended `_time` to +`ALIASES['timestamp']` in memory and restored the tuple afterward. It did not edit +the importer. It exited 0; the results below summarize that earlier diagnostic, +which was **not rerun during this save-only slice**. No generated case JSON or +HTML report was retained. These are diagnostic observations, not a shipped +integration test or genuine-export validation. + +| Input and mode | Input records | Accepted | Rejected | Review candidates | +| --- | ---: | ---: | ---: | ---: | +| Both fixtures, unchanged importer | 14 | 0 | 14 | 0 | +| `valid.csv`, temporary alias | 8 | 8 | 0 | 1 | +| `malformed.csv`, temporary alias | 6 | 0 | 6 | 0 | +| Both fixtures, temporary alias | 14 | 7 | 7 | 0 | + +- `valid.csv`: five failures followed by success at the inclusive 600-second + boundary; a separate one-failure/success negative control. The first timestamp + normalizes from `12:00:00+02:00` to `10:00:00Z`. Candidate references cover rows + 1–6; original `_time` remains in the parsed raw record. +- `malformed.csv`: invalid timestamp, absent timezone, invalid IP, absent host, + unsupported event type and an extra CSV field. The extra value is retained. +- Combined: the invalid first record in `malformed.csv` intentionally shares + host and UID with the first valid record. Both copies are quarantined, leaving + four burst failures and no candidate. All 14 parsed source records are retained. + +The diagnostic also checked source references in the in-memory HTML report. + +| Artifact | SHA-256 | +| --- | --- | +| `valid.csv` | `b76966b8cdef9dfc0e14e26f18a16384815d8797c2973dfc22df70c391265514` | +| `malformed.csv` | `ea92256f387b8cb2aaeca4a18a7ab0b81b3fd100a13fb2d42934d738b8ed01ce` | +| `triage.py` baseline | `f74800fa490303e2a62d061b53ca2d0ffdc6ebbbcc5b83800495a84ae1787fd7` | +| `process_triage.py` baseline | `cd8784969935356a724d55a2a5f1e7885f6338fe5b1d5d4137e0578398412064` | + +## Smallest next implementation slice + +1. Append `_time` to the existing timestamp alias tuple in `triage.py`. No + separate converter, new detection rule or intermediate normalized file. +2. Add focused tests for the fixture outcomes above, conflicting timestamp + aliases, rejected naive/epoch timestamps and preserved blank-UID behavior. +3. Retain a small CLI case/report with hashes tied to the changed source before + publishing an implemented outcome. Do not rerun the whole lab or full suite. + +The alias table is also used by existing JSON-array input; this change would not +add support for native Splunk JSON envelopes or streaming exports. Keep the new +integration labeled **offline synthetic validation** until an authorized genuine +event export is processed. Live Splunk, PC access, Commander, credentials, +notifications and containment are outside this slice. From 19c7d8626a3cdedb4253211705c33a5e19dff76f Mon Sep 17 00:00:00 2001 From: Umu Date: Mon, 9 Nov 2026 02:07:24 -0800 Subject: [PATCH 04/15] feat: update triage.py for authentication export --- projects/10-soc-triage/triage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/10-soc-triage/triage.py b/projects/10-soc-triage/triage.py index 4505a16..36c5c1d 100644 --- a/projects/10-soc-triage/triage.py +++ b/projects/10-soc-triage/triage.py @@ -1,4 +1,4 @@ -"""Offline CSV/JSON triage. No network, command execution or response actions. +"""CSV/JSON security-export triage. No network, command execution or response actions. Run from a clone of this repository; process patterns reuse process_triage.py. Exit 0: complete scoped processing; 2: partial, rejected rows retained; 1: fatal. @@ -19,7 +19,7 @@ HERE = Path(__file__).resolve().parent PROCESS_SOURCE = HERE.parent / '02-windows-process-triage/process_triage.py' ALIASES = { - 'timestamp': ('timestamp', 'time', 'TimeCreated'), + 'timestamp': ('timestamp', 'time', 'TimeCreated', '_time'), 'username': ('username', 'user', 'User'), 'host': ('host', 'computer', 'Computer'), 'source_ip': ('source_ip', 'src_ip', 'IpAddress'), @@ -208,7 +208,7 @@ def analyze(paths): rejected = sum(r['status'] == 'rejected' for r in records) findings = review(events) return {'schema': 'portfolio.soc-triage.v1', 'status': 'PARTIAL' if rejected else 'COMPLETE', - 'scope': 'Offline exported-log processing; review candidates, not confirmed incidents. No endpoint collection, network calls or response actions.', + 'scope': 'Exported-log processing; review candidates, not confirmed incidents. No endpoint collection, network calls or response actions.', 'source_sha256': {'triage.py': digest(Path(__file__).read_bytes()), 'process_triage.py': digest(PROCESS_SOURCE.read_bytes())}, 'summary': {'input_rows': len(records), 'accepted_events': len(events), 'duplicate_rows': sum(r['status'] == 'duplicate' for r in records), From 07a3cd9abfd5dcf164e6f0d28a00aaeaa7d0d6c4 Mon Sep 17 00:00:00 2001 From: Umu Date: Mon, 9 Nov 2026 02:07:26 -0800 Subject: [PATCH 05/15] feat: add test_splunk_csv.py for authentication export --- projects/10-soc-triage/test_splunk_csv.py | 127 ++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 projects/10-soc-triage/test_splunk_csv.py diff --git a/projects/10-soc-triage/test_splunk_csv.py b/projects/10-soc-triage/test_splunk_csv.py new file mode 100644 index 0000000..53a2437 --- /dev/null +++ b/projects/10-soc-triage/test_splunk_csv.py @@ -0,0 +1,127 @@ +"""Focused authentication-export contract checks; no live service or mocks.""" +import csv +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +FIXTURES = HERE / 'fixtures/splunk_csv_v1' + + +class ExportContractTests(unittest.TestCase): + def setUp(self): + spec = importlib.util.spec_from_file_location('triage', HERE / 'triage.py') + self.tool = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.tool) + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.folder = Path(self.temp.name) + + def row(self, **changes): + return dict(_time='2026-11-08T10:00:00Z', username='lab_user', + host='SYNTHETIC-01', source_ip='192.0.2.1', + event_type='login_failure', event_uid='', **changes) + + def analyze_csv(self, rows): + path = self.folder / 'events.csv' + with path.open('w', newline='', encoding='utf-8') as stream: + writer = csv.DictWriter(stream, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + return self.tool.analyze([path]) + + def test_valid_export_normalizes_time_and_preserves_candidate_sources(self): + report = self.tool.analyze([FIXTURES / 'valid.csv']) + self.assertEqual(report['summary'], dict(input_rows=8, accepted_events=8, + duplicate_rows=0, rejected_rows=0, review_candidates=1)) + self.assertEqual(report['status'], 'COMPLETE') + self.assertEqual(report['findings'][0]['source_refs'], + [f'S01:R{i:06d}' for i in range(1, 7)]) + self.assertEqual(report['timeline'][0]['timestamp'], '2026-11-08T10:00:00Z') + self.assertEqual(report['source_records'][0]['raw']['_time'], + '2026-11-08T12:00:00+02:00') + self.assertEqual(report['inputs'][0]['sha256'], + 'b76966b8cdef9dfc0e14e26f18a16384815d8797c2973dfc22df70c391265514') + self.assertEqual(len(report['source_records']), 8) + + def test_malformed_export_keeps_reasons_and_extra_values(self): + report = self.tool.analyze([FIXTURES / 'malformed.csv']) + self.assertEqual(report['summary'], dict(input_rows=6, accepted_events=0, + duplicate_rows=0, rejected_rows=6, review_candidates=0)) + self.assertIn('explicit timezone', report['source_records'][1]['reason']) + self.assertIn('Unsupported event_type', report['source_records'][4]['reason']) + self.assertEqual(report['source_records'][5]['raw']['extra_values'], ['unexpected']) + self.assertEqual(report['source_records'][5]['id'], 'S01:R000006') + + def test_invalid_identified_copy_quarantines_valid_match(self): + report = self.tool.analyze([FIXTURES / 'valid.csv', FIXTURES / 'malformed.csv']) + self.assertEqual(report['summary'], dict(input_rows=14, accepted_events=7, + duplicate_rows=0, rejected_rows=7, review_candidates=0)) + self.assertEqual(report['status'], 'PARTIAL') + self.assertEqual(report['source_records'][0]['status'], 'rejected') + self.assertEqual(report['source_records'][8]['status'], 'rejected') + self.assertEqual(len(report['source_records']), 14) + + def test_timestamp_aliases_compare_strings_before_parsing(self): + row = self.row(timestamp='2026-11-08T10:00:00Z') + self.assertEqual(self.analyze_csv([row])['summary']['accepted_events'], 1) + for other in ('2026-11-08T11:00:00Z', '2026-11-08T12:00:00+02:00'): + with self.subTest(other=other): + row['timestamp'] = other + report = self.analyze_csv([row]) + self.assertEqual(report['summary']['rejected_rows'], 1) + self.assertIn('Conflicting aliases: timestamp', report['source_records'][0]['reason']) + + def test_requires_timezone_and_rejects_epoch_text(self): + row = self.row() + row['_time'] = '2026-11-08T10:00:00' + with self.assertRaisesRegex(ValueError, 'explicit timezone'): + self.tool.normalize(row) + row['_time'] = '1794132000' + with self.assertRaisesRegex(ValueError, 'Invalid isoformat'): + self.tool.normalize(row) + + def test_fractional_timestamp_is_preserved(self): + row = self.row() + row['_time'] = '2026-11-08T12:00:00.123456+02:00' + report = self.analyze_csv([row]) + self.assertEqual(report['summary']['accepted_events'], 1) + self.assertEqual(report['timeline'][0]['timestamp'], '2026-11-08T10:00:00.123456Z') + + def test_blank_or_absent_uid_does_not_coalesce_records(self): + for absent in (False, True): + with self.subTest(absent=absent): + row = self.row() + if absent: + del row['event_uid'] + report = self.analyze_csv([row, row.copy()]) + self.assertEqual(report['summary']['accepted_events'], 2) + self.assertEqual(report['summary']['duplicate_rows'], 0) + self.assertEqual(len(report['source_records']), 2) + + def test_unknown_fields_are_retained_and_html_escaped(self): + report = self.analyze_csv([self.row(note='')]) + self.assertEqual(report['summary']['accepted_events'], 1) + self.assertEqual(report['source_records'][0]['raw']['note'], '') + html = self.tool.render_html(report) + self.assertNotIn('