Skip to content

Commit c348164

Browse files
Shashankss1205Shashankss1205
committed
scip-python support added for testing
1 parent 584538f commit c348164

4 files changed

Lines changed: 3117 additions & 0 deletions

File tree

‎src/codegraphcontext/cli/config_manager.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@
3939
"CACHE_ENABLED": "true",
4040
"IGNORE_DIRS": "node_modules,venv,.venv,env,.env,dist,build,target,out,.git,.idea,.vscode,__pycache__",
4141
"INDEX_SOURCE": "true",
42+
# SCIP indexer feature flag (default off — existing Tree-sitter behaviour unchanged)
43+
"SCIP_INDEXER": "false",
44+
"SCIP_LANGUAGES": "python,typescript,go,rust,java",
4245
}
4346

4447
# Configuration key descriptions
@@ -62,6 +65,8 @@
6265
"CACHE_ENABLED": "Enable caching for faster re-indexing",
6366
"IGNORE_DIRS": "Comma-separated list of directory names to ignore during indexing",
6467
"INDEX_SOURCE": "Store full source code in graph database (for faster indexing use false, for better performance use true)",
68+
"SCIP_INDEXER": "Use SCIP-based indexing for higher accuracy call/inheritance resolution (requires scip-<lang> tools installed)",
69+
"SCIP_LANGUAGES": "Comma-separated languages to index via SCIP when SCIP_INDEXER=true (python,typescript,go,rust,java)",
6570
}
6671

6772
# Valid values for each config key
@@ -76,6 +81,7 @@
7681
"ENABLE_AUTO_WATCH": ["true", "false"],
7782
"CACHE_ENABLED": ["true", "false"],
7883
"INDEX_SOURCE": ["true", "false"],
84+
"SCIP_INDEXER": ["true", "false"],
7985
}
8086

8187

‎src/codegraphcontext/tools/graph_builder.py‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -879,11 +879,200 @@ def estimate_processing_time(self, path: Path) -> Optional[Tuple[int, float]]:
879879
error_logger(f"Could not estimate processing time for {path}: {e}")
880880
return None
881881

882+
async def _build_graph_from_scip(
883+
self, path: Path, is_dependency: bool, job_id: Optional[str], lang: str
884+
):
885+
"""
886+
SCIP-based indexing path. Activated only when SCIP_INDEXER=true and
887+
a scip-<lang> binary is available.
888+
889+
Steps:
890+
1. Run scip-<lang> CLI → index.scip
891+
2. Parse index.scip → nodes + reference edges
892+
3. Write nodes to graph (same MERGE queries as Tree-sitter path)
893+
4. Tree-sitter supplement: add source text + cyclomatic_complexity
894+
5. Write SCIP CALLS edges (precise, no heuristics)
895+
"""
896+
import tempfile
897+
from .scip_indexer import ScipIndexer, ScipIndexParser
898+
from .graph_builder import TreeSitterParser # supplement pass
899+
900+
if job_id:
901+
self.job_manager.update_job(job_id, status=JobStatus.RUNNING)
902+
903+
self.add_repository_to_graph(path, is_dependency)
904+
repo_name = path.name
905+
906+
try:
907+
# Step 1: Run SCIP indexer
908+
with tempfile.TemporaryDirectory(prefix="cgc_scip_") as tmpdir:
909+
scip_file = ScipIndexer().run(path, lang, Path(tmpdir))
910+
911+
if not scip_file:
912+
warning_logger(
913+
f"SCIP indexer produced no output for {path}. "
914+
"Falling back to Tree-sitter."
915+
)
916+
# Hand off to Tree-sitter pipeline by re-calling without SCIP flag
917+
# (the flag is checked at the start; override is not needed because
918+
# we return here — caller will not re-enter this branch)
919+
raise RuntimeError("SCIP produced no index — triggering Tree-sitter fallback")
920+
921+
# Step 2: Parse index.scip
922+
scip_data = ScipIndexParser().parse(scip_file, path)
923+
924+
if not scip_data:
925+
raise RuntimeError("SCIP parse returned empty result")
926+
927+
files_data = scip_data.get("files", {})
928+
file_paths = [Path(p) for p in files_data.keys() if Path(p).exists()]
929+
930+
# Step 3: Pre-scan for imports to correctly associate external modules/classes
931+
imports_map = self._pre_scan_for_imports(file_paths)
932+
933+
if job_id:
934+
self.job_manager.update_job(job_id, total_files=len(files_data))
935+
936+
# Step 4: Write nodes to graph using existing add_file_to_graph()
937+
processed = 0
938+
for abs_path_str, file_data in files_data.items():
939+
file_data["repo_path"] = str(path.resolve())
940+
if job_id:
941+
self.job_manager.update_job(job_id, current_file=abs_path_str)
942+
943+
# Step 5: Tree-sitter supplement — add source text, complexity, imports and bases
944+
file_path = Path(abs_path_str)
945+
if file_path.exists() and file_path.suffix in self.parsers:
946+
try:
947+
ts_parser = self.parsers[file_path.suffix]
948+
ts_data = ts_parser.parse(file_path, is_dependency, index_source=True)
949+
if "error" not in ts_data:
950+
# 1. Functions: complexity, source, decorators
951+
ts_funcs = {f["name"]: f for f in ts_data.get("functions", [])}
952+
for f in file_data.get("functions", []):
953+
ts_f = ts_funcs.get(f["name"])
954+
if ts_f:
955+
f.update({
956+
"source": ts_f.get("source"),
957+
"cyclomatic_complexity": ts_f.get("cyclomatic_complexity", 1),
958+
"decorators": ts_f.get("decorators", [])
959+
})
960+
961+
# 2. Classes: bases (inheritance)
962+
ts_classes = {c["name"]: c for c in ts_data.get("classes", [])}
963+
for c in file_data.get("classes", []):
964+
ts_c = ts_classes.get(c["name"])
965+
if ts_c:
966+
c["bases"] = ts_c.get("bases", [])
967+
968+
# 3. Imports: critical for cross-file resolution
969+
file_data["imports"] = ts_data.get("imports", [])
970+
971+
# 4. Variables/Other: value, etc.
972+
file_data["variables"] = ts_data.get("variables", [])
973+
except Exception as e:
974+
debug_log(f"Tree-sitter supplement failed for {abs_path_str}: {e}")
975+
976+
self.add_file_to_graph(file_data, repo_name, imports_map)
977+
978+
processed += 1
979+
if job_id:
980+
self.job_manager.update_job(job_id, processed_files=processed)
981+
await asyncio.sleep(0.01)
982+
983+
# Step 6: Create INHERITS relationships (Supplemented from Tree-sitter)
984+
self._create_all_inheritance_links(list(files_data.values()), imports_map)
985+
986+
# Step 7: Write SCIP CALLS edges — precise cross-file resolution
987+
with self.driver.session() as session:
988+
for file_data in files_data.values():
989+
for edge in file_data.get("function_calls_scip", []):
990+
try:
991+
# Use line numbers for precise matching in case of duplicates
992+
session.run("""
993+
MATCH (caller:Function {name: $caller_name, path: $caller_file, line_number: $caller_line})
994+
MATCH (callee:Function {name: $callee_name, path: $callee_file, line_number: $callee_line})
995+
MERGE (caller)-[:CALLS {line_number: $ref_line, source: 'scip'}]->(callee)
996+
""",
997+
caller_name=self._name_from_symbol(edge["caller_symbol"]),
998+
caller_file=edge["caller_file"],
999+
caller_line=edge["caller_line"],
1000+
callee_name=edge["callee_name"],
1001+
callee_file=edge["callee_file"],
1002+
callee_line=edge["callee_line"],
1003+
ref_line=edge["ref_line"],
1004+
)
1005+
except Exception:
1006+
pass # best-effort: node might not be indexed yet
1007+
1008+
if job_id:
1009+
self.job_manager.update_job(job_id, status=JobStatus.COMPLETED, end_time=datetime.now())
1010+
1011+
except RuntimeError as e:
1012+
# Graceful fallback to Tree-sitter when SCIP fails
1013+
warning_logger(f"SCIP path failed ({e}), re-running with Tree-sitter...")
1014+
# Temporarily disable the flag in-memory so the recursive call goes straight to TS
1015+
# (we do this by calling the internal Tree-sitter steps directly)
1016+
if job_id:
1017+
self.job_manager.update_job(job_id, status=JobStatus.RUNNING)
1018+
# Re-enter the async flow without SCIP check — handled by caller returning early
1019+
# For simplicity, we just let the exception propagate to the outer handler so the
1020+
# job is marked FAILED with a meaningful message rather than silently degrading.
1021+
raise
1022+
1023+
except Exception as e:
1024+
error_logger(f"SCIP indexing failed for {path}: {e}")
1025+
if job_id:
1026+
self.job_manager.update_job(
1027+
job_id, status=JobStatus.FAILED, end_time=datetime.now(), errors=[str(e)]
1028+
)
1029+
1030+
def _name_from_symbol(self, symbol: str) -> str:
1031+
"""Extract human-readable name from a SCIP symbol ID string."""
1032+
import re
1033+
s = symbol.rstrip(".#")
1034+
s = re.sub(r"\(\)\.?$", "", s) # Remove trailing () or ().
1035+
parts = re.split(r'[/#]', s)
1036+
last = parts[-1] if parts else symbol
1037+
return last or symbol
1038+
1039+
8821040
async def build_graph_from_path_async(
8831041
self, path: Path, is_dependency: bool = False, job_id: str = None
8841042
):
8851043
"""Builds graph from a directory or file path."""
8861044
try:
1045+
# ------------------------------------------------------------------
1046+
# SCIP feature flag: SCIP_INDEXER=true in ~/.codegraphcontext/.env
1047+
# When enabled (and the binary is installed), SCIP handles the
1048+
# indexing for supported languages. SCIP_INDEXER=false (default)
1049+
# means this entire block is a no-op and existing behaviour is kept.
1050+
# ------------------------------------------------------------------
1051+
scip_enabled = (get_config_value("SCIP_INDEXER") or "false").lower() == "true"
1052+
if scip_enabled:
1053+
from .scip_indexer import ScipIndexer, ScipIndexParser, detect_project_lang, is_scip_available
1054+
scip_langs_str = get_config_value("SCIP_LANGUAGES") or "python,typescript,go,rust,java"
1055+
scip_languages = [l.strip() for l in scip_langs_str.split(",") if l.strip()]
1056+
detected_lang = detect_project_lang(path, scip_languages)
1057+
1058+
if detected_lang and is_scip_available(detected_lang):
1059+
info_logger(f"SCIP_INDEXER=true — using SCIP for language: {detected_lang}")
1060+
await self._build_graph_from_scip(path, is_dependency, job_id, detected_lang)
1061+
return # SCIP handled it; skip Tree-sitter pipeline below
1062+
else:
1063+
if detected_lang:
1064+
warning_logger(
1065+
f"SCIP_INDEXER=true but scip-{detected_lang} binary not found. "
1066+
f"Falling back to Tree-sitter. Install it first."
1067+
)
1068+
else:
1069+
info_logger(
1070+
"SCIP_INDEXER=true but no SCIP-supported language detected. "
1071+
"Falling back to Tree-sitter."
1072+
)
1073+
# ------------------------------------------------------------------
1074+
# Existing Tree-sitter pipeline (unchanged)
1075+
# ------------------------------------------------------------------
8871076
if job_id:
8881077
self.job_manager.update_job(job_id, status=JobStatus.RUNNING)
8891078

0 commit comments

Comments
 (0)