Related docs: Graph Schema · Toolchain · Architecture
CodeGraph is intended to be language-agnostic. This document specifies a proposed parser interface, but no parser package or Graph Builder implementation is shipped in the public repository. A new implementation must therefore validate both this interface and its interchange contract rather than assuming an importable framework already exists.
The proof-of-concept targeted a proprietary 4GL language. Its parser is not published. The Python code below is an illustrative, non-runnable skeleton and has not been validated against the PoC pipeline.
Portability contract gap: the origin provenance enum cannot represent AST detection. This guide uses
source="ast"as aPROPOSED-EXPERIMENT. A real implementation must version and test that interchange change; it must not emit the value while claiming conformance to the unchanged origin contract.
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class GraphNode:
"""A node to be merged into the knowledge graph."""
label: str # Neo4j node label (e.g., "SourceFile", "Table")
qualified_name: str # Unique identifier: "module:name"
properties: dict # Additional node properties
@dataclass
class GraphEdge:
"""A relationship edge with provenance properties."""
type: str # Relationship type (e.g., "CALLS", "READS")
from_node: str # qualified_name of source node
to_node: str # qualified_name of target node
# Provenance — required on every edge
source: str # baseline: "xref" | "regex" | "schema" | "derived"
source_file: str # Path to the file that provided the evidence
source_line: int # Line number in that file (0 if not applicable)
confidence: str # "compiler" | "regex" | "schema" | "derived"
@dataclass
class ParseResult:
"""Output of parsing a single file."""
nodes: list[GraphNode] = field(default_factory=list)
edges: list[GraphEdge] = field(default_factory=list)
errors: list[str] = field(default_factory=list) # Non-fatal parse errors
skipped: bool = False # True if file should not be indexed
class LanguageParser(ABC):
"""
Interface for language-specific source file parsers.
Implement this interface to add CodeGraph support for a new language.
Each parser is responsible for one job: given a source file, return
the graph nodes and edges that represent its structure and relationships.
All edges MUST carry provenance properties.
"""
@abstractmethod
def parse_file(self, file_path: str, module: str) -> ParseResult:
"""
Parse a single source file.
Args:
file_path: Absolute path to the source file.
module: The module name this file belongs to (used for qualified names).
Returns:
ParseResult containing nodes, edges, and any non-fatal errors.
Return skipped=True for files that should not be indexed
(e.g., generated files, test fixtures).
"""
...
@abstractmethod
def get_supported_extensions(self) -> list[str]:
"""
Return the file extensions this parser handles.
Example: [".py", ".pyi"]
"""
...
@abstractmethod
def get_confidence_tier(self, detection_method: str) -> str:
"""
Map a detection method to its confidence tier.
Args:
detection_method: How the relationship was detected
(e.g., "ast", "import", "regex", "type_inference")
Returns:
One of: "compiler", "regex", "schema", "derived"
"""
...
def get_auxiliary_extensions(self) -> list[str]:
"""
Return extensions for auxiliary files this parser can use as
high-confidence evidence sources (e.g., compiler output, type stubs).
Override if your language produces cross-reference or type files.
Default: empty list.
"""
return []In a future implementation, add your parser to the proposed path
tools/phase1/parser_registry.py:
from parsers.python_parser import PythonParser
from parsers.typescript_parser import TypeScriptParser
PARSER_REGISTRY = {
# Your new parser:
".py": PythonParser(),
".pyi": PythonParser(),
# TypeScript/JavaScript
".ts": TypeScriptParser(),
".tsx": TypeScriptParser(),
".js": TypeScriptParser(),
".jsx": TypeScriptParser(),
}The Graph Builder automatically uses the correct parser for each file based on its extension.
import ast
import os
from pathlib import Path
from typing import Optional
from ..interface import GraphNode, GraphEdge, ParseResult, LanguageParser
class PythonParser(LanguageParser):
"""
Python AST-based parser for CodeGraph.
Detection methods and their confidence tiers:
- "ast_reference_resolved": AST reference resolved against known symbols — high confidence
- "ast_reference_unresolved": Syntax observed, target not resolved — medium confidence
- "ast_attribute": Attribute access (module.function), target unresolved — medium confidence
- "regex_string": String-based reference — low confidence, mark as regex tier
"""
def get_supported_extensions(self) -> list[str]:
return [".py", ".pyi"]
def get_confidence_tier(self, detection_method: str) -> str:
tiers = {
"ast_reference_resolved": "compiler",
"ast_reference_unresolved": "regex",
"ast_attribute": "regex",
"regex_string": "regex",
}
return tiers.get(detection_method, "derived")
def parse_file(self, file_path: str, module: str) -> ParseResult:
result = ParseResult()
try:
with open(file_path, "r", encoding="utf-8", errors="replace") as f:
source = f.read()
except (OSError, PermissionError) as e:
result.errors.append(f"Cannot read {file_path}: {e}")
result.skipped = True
return result
# Skip generated files, test fixtures, etc.
if self._should_skip(file_path, source):
result.skipped = True
return result
# Build the SourceFile node for this file
file_name = Path(file_path).stem
qualified_name = f"{module}:{file_name}"
result.nodes.append(GraphNode(
label="SourceFile",
qualified_name=qualified_name,
properties={
"name": file_name,
"module": module,
"file_type": ".py",
"path": file_path,
"line_count": source.count("\n") + 1,
}
))
# Parse the AST
try:
tree = ast.parse(source, filename=file_path)
except SyntaxError as e:
result.errors.append(f"Syntax error in {file_path}: {e}")
return result
# Walk the AST to detect imports (INCLUDES edges) and calls (CALLS edges)
for node in ast.walk(tree):
if isinstance(node, (ast.Import, ast.ImportFrom)):
edges = self._handle_import(node, qualified_name, file_path, module)
result.edges.extend(edges)
elif isinstance(node, ast.Call):
edge = self._handle_call(node, qualified_name, file_path, source)
if edge:
result.edges.append(edge)
return result
def _handle_import(
self,
node: ast.Import | ast.ImportFrom,
from_qualified: str,
file_path: str,
module: str,
) -> list[GraphEdge]:
"""Convert import statements to INCLUDES edges."""
edges = []
line = node.lineno
if isinstance(node, ast.Import):
for alias in node.names:
edges.append(GraphEdge(
type="INCLUDES",
from_node=from_qualified,
to_node=f"{module}:{alias.name}",
source="ast",
source_file=file_path,
source_line=line,
# The import name is syntactically present, but this skeleton
# does not resolve it against an analysed module set.
confidence=self.get_confidence_tier("ast_reference_unresolved"),
))
elif isinstance(node, ast.ImportFrom) and node.module:
edges.append(GraphEdge(
type="INCLUDES",
from_node=from_qualified,
to_node=f"{module}:{node.module}",
source="ast",
source_file=file_path,
source_line=line,
confidence=self.get_confidence_tier("ast_reference_unresolved"),
))
return edges
def _handle_call(
self,
node: ast.Call,
from_qualified: str,
file_path: str,
source: str,
) -> Optional[GraphEdge]:
"""Convert direct function calls to CALLS edges where the target can be resolved."""
if not isinstance(node.func, ast.Name):
return None # Skip method calls and attribute calls for now
callee_name = node.func.id
line = node.lineno
return GraphEdge(
type="CALLS",
from_node=from_qualified,
to_node=f"unknown:{callee_name}", # Resolved during graph merge
source="ast",
source_file=file_path,
source_line=line,
# ast.Name proves a call expression exists, not which runtime symbol
# it binds to. Promote only after graph/build-context resolution.
confidence=self.get_confidence_tier("ast_reference_unresolved"),
)
def _should_skip(self, file_path: str, source: str) -> bool:
"""Return True for files that should not be indexed."""
skip_patterns = [
"/test_", "/_test", "/tests/", "/fixtures/",
"# AUTO-GENERATED", "# This file was generated",
]
path_lower = file_path.lower()
return any(p in path_lower or p in source[:200] for p in skip_patterns)These are possible evidence sources, not a promise that an entire parser operates at one tier. Assign confidence per relationship after target resolution. Dynamic dispatch, dependency injection, reflection, generated code, and missing build context can leave a mixed-tier graph.
| Language | Potential evidence | Tier condition |
|---|---|---|
| Java / Kotlin | compiler symbols, bytecode, build metadata | compiler only for resolved symbols/relationships |
| TypeScript | compiler API and declaration/type information | compiler only for resolved symbols/relationships |
| Python | AST plus import/module resolution | compiler for resolved targets; syntax-only facts remain lower tier |
| Go | compiler/package metadata | compiler only for resolved packages and symbols |
| C / C++ | compiler database, symbols, object metadata | compiler only for resolved relationships represented by that evidence |
| Ruby / PHP | AST and runtime/framework metadata where available | mixed; unresolved dynamic relationships remain lower tier |
| COBOL / 4GLs | compiler cross-reference where available, otherwise source patterns | compiler for genuine xref; otherwise typically regex |
After contributing an implementation and test harness, create tests using the public
repository convention tests/phase1/test_your_language_parser.py. The proposed
suite should expect:
def test_parser_returns_source_file_node():
"""Every parsed file must produce at least one SourceFile node."""
...
def test_all_edges_have_provenance():
"""Every edge must have all four provenance properties set."""
...
def test_idempotent():
"""Parsing the same file twice produces the same result."""
...
def test_syntax_error_does_not_raise():
"""Parser must handle malformed files gracefully (errors, not exceptions)."""
...
def test_generated_files_are_skipped():
"""Files matching skip patterns must return skipped=True."""
...These commands describe the proposed repository convention after an implementation and tests have been contributed; they do not run in the current specification-only repository:
pytest tests/phase1/test_your_language_parser.py -v
pytest tests/ -v # Full suiteWhen your parser is ready:
- Ensure all tests pass
- Include a sample
ParseResultJSON intests/phase1/fixtures/your_language/showing what a real file produces - Update the language evidence-status table in
README.md - Open a PR with the label
parserand describe: which detection methods you implemented, what you chose not to implement and why, and any language-specific edge cases to be aware of
See CONTRIBUTING.md for the full PR process.