Add tsk_notes for comments, AI answers and summaries

bindings/java/src/org/sleuthkit/datamodel/BlackboardAttribute.java

@@ -612,6 +612,15 @@ public static final class Type implements Serializable {
 		public static final Type TSK_CORRELATION_TYPE = new Type(157, "TSK_CORRELATION_TYPE", bundle.getString("BlackboardAttribute.tskCorrelationType.text"), TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING);
 		public static final Type TSK_CORRELATION_VALUE = new Type(158, "TSK_CORRELATION_VALUE", bundle.getString("BlackboardAttribute.tskCorrelationValue.text"), TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING);
 		public static final Type TSK_OTHER_CASES = new Type(159, "TSK_OTHER_CASES", bundle.getString("BlackboardAttribute.tskOtherCases.text"), TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING);
+		/*
+		 * The note that holds the reasoning behind an analysis result, so that a
+		 * result is self-describing and anything loading it finds the detail without
+		 * having to know the notes feature exists. The value is the note's
+		 * original_note_id and never a revision id, so the attribute is written once
+		 * when the result is created and stays correct when the note is revised. See
+		 * NoteManager.getCurrentRevision().
+		 */
+		public static final Type TSK_NOTE_ID = new Type(160, "TSK_NOTE_ID", bundle.getString("BlackboardAttribute.tskNoteId.text"), TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.LONG);
 
 		// NOTE: When adding a new standard BlackboardAttribute.Type, add the instance and then add to the STANDARD_TYPES list.
 		/**
@@ -767,7 +776,8 @@ public static final class Type implements Serializable {
 				TSK_IS_ADMIN,
 				TSK_CORRELATION_TYPE,
 				TSK_CORRELATION_VALUE,
-				TSK_OTHER_CASES
+				TSK_OTHER_CASES,
+				TSK_NOTE_ID
 		));
 
 		private static final long serialVersionUID = 1L;
@@ -1544,7 +1554,10 @@ public enum ATTRIBUTE_TYPE {
 				TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING),
 		TSK_OTHER_CASES(159, "TSK_OTHER_CASES",
 				bundle.getString("BlackboardAttribute.tskOtherCases.text"),
-				TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING),;
+				TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING),
+		TSK_NOTE_ID(160, "TSK_NOTE_ID",
+				bundle.getString("BlackboardAttribute.tskNoteId.text"),
+				TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.LONG),;
 
 		private final int typeID;
 		private final String typeName;

bindings/java/src/org/sleuthkit/datamodel/Bundle.properties

@@ -224,6 +224,7 @@ BlackboardAttribute.tskIsAdmin.text=Is Administrator
 BlackboardAttribute.tskCorrelationType.text=Correlation Type
 BlackboardAttribute.tskCorrelationValue.text=Correlation Value
 BlackboardAttribute.tskOtherCases.text=Other Cases
+BlackboardAttribute.tskNoteId.text=Note
 AbstractFile.readLocal.exception.msg4.text=Error reading local file\: {0}
 AbstractFile.readLocal.exception.msg1.text=Error reading local file, local path is not set
 AbstractFile.readLocal.exception.msg2.text=Error reading local file, it does not exist at local path\: {0}
@@ -459,5 +460,6 @@ TskData.ObjectType.Report.name=Report
 TskData.ObjectType.Pool.name=Pool
 TskData.ObjectType.OsAccount.name=OS Account
 TskData.ObjectType.HostAddress.name=Host Address
+TskData.ObjectType.Case.name=Case
 TskData.ObjectType.Unsupported.name=Unsupported
 

bindings/java/src/org/sleuthkit/datamodel/Bundle.properties-MERGED

@@ -224,6 +224,7 @@ BlackboardAttribute.tskIsAdmin.text=Is Administrator
 BlackboardAttribute.tskCorrelationType.text=Correlation Type
 BlackboardAttribute.tskCorrelationValue.text=Correlation Value
 BlackboardAttribute.tskOtherCases.text=Other Cases
+BlackboardAttribute.tskNoteId.text=Note
 AbstractFile.readLocal.exception.msg4.text=Error reading local file\: {0}
 AbstractFile.readLocal.exception.msg1.text=Error reading local file, local path is not set
 AbstractFile.readLocal.exception.msg2.text=Error reading local file, it does not exist at local path\: {0}
@@ -459,5 +460,6 @@ TskData.ObjectType.Report.name=Report
 TskData.ObjectType.Pool.name=Pool
 TskData.ObjectType.OsAccount.name=OS Account
 TskData.ObjectType.HostAddress.name=Host Address
+TskData.ObjectType.Case.name=Case
 TskData.ObjectType.Unsupported.name=Unsupported
 

bindings/java/src/org/sleuthkit/datamodel/CaseDatabaseFactory.java

@@ -182,6 +182,7 @@ private void addTables(Connection conn) throws TskCoreException {
 			createFileTables(stmt);
 			createArtifactTables(stmt);
 			createAnalysisResultsTables(stmt);
+			createNoteTables(stmt);
 			createTagTables(stmt);
 			createIngestTables(stmt);
 			createEventTables(stmt);
@@ -341,6 +342,52 @@ private void createAnalysisResultsTables(Statement stmt) throws SQLException  {
 				+ ")");	
 
 	}
+	// Must be called after createTskObjects() and createAnalysisResultsTables().
+	private void createNoteTables(Statement stmt) throws SQLException {
+
+		// Note types are open. The Sleuth Kit seeds the built-in ones on every case
+		// open and consumers add their own by name at runtime, so adding a type does
+		// not need a schema change. The type carries no behaviour: whether a machine
+		// wrote a note is answered by tsk_notes.author_kind on the row, never here,
+		// because a type such as chat has rows from both people and models.
+		stmt.execute("CREATE TABLE tsk_note_types (note_type_id " + dbQueryHelper.getPrimaryKey() + " PRIMARY KEY, "
+				+ "type_name TEXT NOT NULL UNIQUE, "	// COMMENT, AI_ENRICHMENT, REMEDIATION, AI_SUMMARY, ...
+				+ "display_name TEXT, "
+				+ "description TEXT)");
+
+		// References tsk_objects, tsk_note_types, tsk_analysis_results
+		// Text about an object in the case that has no score and can change after it is
+		// written: comments, AI enrichment, remediation advice, summaries. Analysis
+		// results must not change, which is why this is a separate table rather than
+		// more columns on one - a finding that can be quietly rewritten is not evidence.
+		// The table is append-only: an edit inserts a new row carrying the same
+		// original_note_id and clears is_current on the row it replaces.
+		stmt.execute("CREATE TABLE tsk_notes (note_id " + dbQueryHelper.getPrimaryKey() + " PRIMARY KEY, "
+				+ "obj_id " + dbQueryHelper.getBigIntType() + " NOT NULL, "	// file, artifact, data source, or the case object
+				+ "data_source_obj_id " + dbQueryHelper.getBigIntType() + ", "	// derived from obj_id; null for a case level note
+				+ "note_type_id " + dbQueryHelper.getBigIntType() + " NOT NULL, "
+				+ "body TEXT NOT NULL, "	// the prose a person reads
+				+ "details TEXT, "	// structured payload as JSON; The Sleuth Kit never parses it
+				+ "author_kind INTEGER NOT NULL, "	// USER/AI/MODULE; the only answer to "did a machine write this"
+				+ "author_id TEXT NOT NULL, "	// stable id of the writer: a user id, a model id, or a module name
+				+ "author_display TEXT NOT NULL, "	// what the UI renders
+				+ "config_id TEXT, "	// prompt or module configuration version; null for people
+				+ "created_time " + dbQueryHelper.getBigIntType() + " NOT NULL, "	// epoch MILLIS, since comment ordering needs sub-second resolution
+				+ "parent_note_id " + dbQueryHelper.getBigIntType() + ", "	// note this one replies to; null on a thread root
+				+ "root_note_id " + dbQueryHelper.getBigIntType() + ", "	// root of the thread; own note_id on a root, set just after the insert
+				+ "original_note_id " + dbQueryHelper.getBigIntType() + ", "	// stable id across edits; own note_id on a first version, set just after the insert
+				+ "is_current INTEGER NOT NULL DEFAULT 1, "	// boolean, the live revision of this note
+				+ "is_deleted INTEGER NOT NULL DEFAULT 0, "	// boolean, retracted but kept so replies stay reachable
+				+ "analysis_result_id " + dbQueryHelper.getBigIntType() + ", "	// the scored finding this note explains
+				+ "FOREIGN KEY(obj_id) REFERENCES tsk_objects(obj_id) ON DELETE CASCADE, "
+				+ "FOREIGN KEY(data_source_obj_id) REFERENCES tsk_objects(obj_id) ON DELETE CASCADE, "
+				+ "FOREIGN KEY(note_type_id) REFERENCES tsk_note_types(note_type_id), "
+				+ "FOREIGN KEY(parent_note_id) REFERENCES tsk_notes(note_id) ON DELETE CASCADE, "
+				+ "FOREIGN KEY(root_note_id) REFERENCES tsk_notes(note_id), "
+				+ "FOREIGN KEY(original_note_id) REFERENCES tsk_notes(note_id), "
+				+ "FOREIGN KEY(analysis_result_id) REFERENCES tsk_analysis_results(artifact_obj_id) ON DELETE SET NULL)");
+	}
+
 	private void createTagTables(Statement stmt) throws SQLException {
 		stmt.execute("CREATE TABLE tsk_tag_sets (tag_set_id " + dbQueryHelper.getPrimaryKey() + " PRIMARY KEY, name TEXT UNIQUE)");
 		stmt.execute("CREATE TABLE tag_names (tag_name_id " + dbQueryHelper.getPrimaryKey() + " PRIMARY KEY, display_name TEXT UNIQUE, "
@@ -419,7 +466,22 @@ private void addIndexes(Connection conn) throws TskCoreException {
 
 			stmt.execute("CREATE INDEX tsk_os_account_realms_realm_name_idx  ON tsk_os_account_realms(realm_name)");
 			stmt.execute("CREATE INDEX tsk_os_account_realms_realm_addr_idx  ON tsk_os_account_realms(realm_addr)");
-
+
+			// note indexes
+			stmt.execute("CREATE INDEX tsk_notes_obj_id_created_index ON tsk_notes(obj_id, created_time)");
+			stmt.execute("CREATE INDEX tsk_notes_datasrc_type_index ON tsk_notes(data_source_obj_id, note_type_id)");
+			stmt.execute("CREATE INDEX tsk_notes_root_index ON tsk_notes(root_note_id)");
+			stmt.execute("CREATE INDEX tsk_notes_original_index ON tsk_notes(original_note_id, is_current)");
+
+			// Makes two current revisions of one note impossible rather than merely unlikely.
+			// The revision flip - clear the old current row, set the new one - is a
+			// check-then-act with no lock behind it on PostgreSQL, so enforcing it here
+			// means the second writer gets a constraint violation it can retry rather than
+			// every call site having to remember. Partial on is_current so the index holds
+			// one entry per note rather than one per revision; SQLite has supported partial
+			// indexes since 3.8.0, so this one is not PostgreSQL-only.
+			stmt.execute("CREATE UNIQUE INDEX tsk_notes_current_revision_index ON tsk_notes(original_note_id) WHERE is_current = 1");
+
 		} catch (SQLException ex) {
 			throw new TskCoreException("Error initializing db_info tables", ex);
 		}
@@ -819,6 +881,9 @@ void performPostTableInitialization(Connection conn) throws TskCoreException {
 				// is carried as a payload column so extension filters are satisfied without a heap fetch. Not partial,
 				// so it covers files of any size and the same index serves name lookups regardless of file size.
 				stmt.execute("CREATE INDEX tsk_files_datasrc_name_size_index ON tsk_files(data_source_obj_id, name, size, extension)");
+
+				// Most notes are not tied to a finding, so this one is partial.
+				stmt.execute("CREATE INDEX tsk_notes_ar_partial_index ON tsk_notes(analysis_result_id) WHERE analysis_result_id IS NOT NULL");
 			} catch (SQLException ex) {
 				throw new TskCoreException("Error performing PostgreSQL post table initialization", ex);
 			}
@@ -906,6 +971,10 @@ void performPostTableInitialization(Connection conn) throws TskCoreException {
 				// is carried as a payload column so extension filters are satisfied without a heap fetch. Not partial,
 				// so it covers files of any size and the same index serves name lookups regardless of file size.
 				stmt.execute("CREATE INDEX tsk_files_datasrc_name_size_index ON tsk_files(data_source_obj_id, name, size, extension)");
+
+				// The PostgreSQL variant of this index is partial on analysis_result_id IS NOT
+				// NULL. SQLite is kept full here for consistency with the indexes above.
+				stmt.execute("CREATE INDEX tsk_notes_ar_index ON tsk_notes(analysis_result_id)");
 			} catch (SQLException ex) {
 				throw new TskCoreException("Error performing SQLite post table initialization", ex);
 			}