Skip to content

Commit 1a419c4

Browse files
Shashankss1205Shashankss1205
committed
1 parent 80bcccc commit 1a419c4

3 files changed

Lines changed: 290 additions & 12 deletions

File tree

‎src/codegraphcontext/core/bundle_registry.py‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
import requests
2+
from pathlib import Path
3+
from typing import Optional, List, Dict, Any, Tuple
4+
import logging
5+
6+
logger = logging.getLogger(__name__)
7+
8+
GITHUB_ORG = "CodeGraphContext"
9+
GITHUB_REPO = "CodeGraphContext"
10+
REGISTRY_API_URL = f"https://api.github.com/repos/{GITHUB_ORG}/{GITHUB_REPO}/releases"
11+
MANIFEST_URL = f"https://github.com/{GITHUB_ORG}/{GITHUB_REPO}/releases/download/on-demand-bundles/manifest.json"
12+
13+
class BundleRegistry:
14+
"""
15+
Core logic for interacting with the CodeGraphContext bundle registry.
16+
Handles fetching metadata, searching, and downloading bundles without CLI dependencies.
17+
"""
18+
19+
@staticmethod
20+
def fetch_available_bundles() -> List[Dict[str, Any]]:
21+
"""
22+
Fetch all available bundles from GitHub Releases and the on-demand manifest.
23+
Returns a list of bundle dictionaries with metadata.
24+
Preserves all versions - no deduplication.
25+
"""
26+
all_bundles = []
27+
28+
# 1. Fetch on-demand bundles from manifest
29+
try:
30+
response = requests.get(MANIFEST_URL, timeout=10)
31+
if response.status_code == 200:
32+
manifest = response.json()
33+
if manifest.get('bundles'):
34+
for bundle in manifest['bundles']:
35+
bundle['source'] = 'on-demand'
36+
# Ensure bundle has a full_name field (with version info)
37+
if 'bundle_name' in bundle:
38+
# Extract full name without .cgc extension
39+
bundle['full_name'] = bundle['bundle_name'].replace('.cgc', '')
40+
all_bundles.append(bundle)
41+
except Exception as e:
42+
logger.warning(f"Could not fetch on-demand bundles from manifest: {e}")
43+
44+
# 2. Fetch weekly pre-indexed bundles
45+
try:
46+
response = requests.get(REGISTRY_API_URL, timeout=10)
47+
if response.status_code == 200:
48+
releases = response.json()
49+
50+
# Find weekly releases (bundles-YYYYMMDD pattern)
51+
weekly_releases = [r for r in releases if r['tag_name'].startswith('bundles-') and r['tag_name'] != 'bundles-latest']
52+
53+
if weekly_releases:
54+
# Get the most recent weekly release
55+
latest_weekly = weekly_releases[0]
56+
57+
for asset in latest_weekly.get('assets', []):
58+
if asset['name'].endswith('.cgc'):
59+
# Full bundle name without extension
60+
full_name = asset['name'].replace('.cgc', '')
61+
62+
# Parse bundle name
63+
name_parts = full_name.split('-')
64+
bundle = {
65+
'name': name_parts[0], # Base package name
66+
'full_name': full_name, # Complete name with version
67+
'repo': f"{name_parts[0]}/{name_parts[0]}", # Simplified
68+
'bundle_name': asset['name'],
69+
'version': name_parts[1] if len(name_parts) > 1 else 'latest',
70+
'commit': name_parts[2] if len(name_parts) > 2 else 'unknown',
71+
'size_bytes': asset.get('size', 0),
72+
'size': f"{asset['size'] / 1024 / 1024:.1f}MB",
73+
'download_url': asset['browser_download_url'],
74+
'generated_at': asset['updated_at'],
75+
'source': 'weekly'
76+
}
77+
all_bundles.append(bundle)
78+
except Exception as e:
79+
logger.warning(f"Could not fetch weekly bundles from GitHub API: {e}")
80+
81+
# Normalize all bundles to have required fields
82+
for bundle in all_bundles:
83+
# Ensure 'name' field exists (base package name)
84+
if 'name' not in bundle:
85+
repo = bundle.get('repo', '')
86+
if '/' in repo:
87+
bundle['name'] = repo.split('/')[-1]
88+
else:
89+
# Extract from full_name or bundle_name
90+
full_name = bundle.get('full_name', bundle.get('bundle_name', 'unknown'))
91+
bundle['name'] = full_name.split('-')[0]
92+
93+
# Ensure 'full_name' exists
94+
if 'full_name' not in bundle:
95+
bundle['full_name'] = bundle.get('bundle_name', bundle.get('name', 'unknown')).replace('.cgc', '')
96+
97+
return all_bundles
98+
99+
@staticmethod
100+
def find_bundle_download_info(name: str) -> Tuple[Optional[str], Optional[Dict[str, Any]], str]:
101+
"""
102+
Find a download URL and metadata for a bundle by name.
103+
104+
Strategies:
105+
1. Exact match on full_name (e.g., 'flask-main-abc123')
106+
2. Match on base name (e.g., 'flask') - returns most recent version
107+
108+
Returns:
109+
(download_url, bundle_metadata, error_message)
110+
"""
111+
bundles = BundleRegistry.fetch_available_bundles()
112+
113+
if not bundles:
114+
return None, None, "Could not fetch bundle registry."
115+
116+
name_lower = name.lower()
117+
118+
# Strategy 1: Exact match on full_name
119+
for b in bundles:
120+
if b.get('full_name', '').lower() == name_lower:
121+
url = b.get('download_url')
122+
if url:
123+
return url, b, ""
124+
return None, b, f"No download URL found for bundle '{name}'"
125+
126+
# Strategy 2: Match base package name (most recent)
127+
matching_bundles = []
128+
for b in bundles:
129+
if b.get('name', '').lower() == name_lower:
130+
matching_bundles.append(b)
131+
132+
if matching_bundles:
133+
# Sort by timestamp (newest first)
134+
matching_bundles.sort(key=lambda x: x.get('generated_at', ''), reverse=True)
135+
bundle = matching_bundles[0]
136+
url = bundle.get('download_url')
137+
if url:
138+
return url, bundle, ""
139+
return None, bundle, f"No download URL found for bundle '{name}'"
140+
141+
return None, None, f"Bundle '{name}' not found in registry."
142+
143+
@staticmethod
144+
def download_file(url: str, output_path: Path, progress_callback=None) -> bool:
145+
"""
146+
Download a file from a URL to a local path.
147+
148+
Args:
149+
url: The URL to download from
150+
output_path: Local path to save the file
151+
progress_callback: Optional callable(chunk_size) to report progress
152+
153+
Returns:
154+
True if successful, raises exception otherwise
155+
"""
156+
try:
157+
response = requests.get(url, stream=True, timeout=30)
158+
response.raise_for_status()
159+
160+
with open(output_path, 'wb') as f:
161+
for chunk in response.iter_content(chunk_size=8192):
162+
if chunk:
163+
f.write(chunk)
164+
if progress_callback:
165+
progress_callback(len(chunk))
166+
return True
167+
except Exception as e:
168+
# Clean up partial file
169+
if output_path.exists():
170+
output_path.unlink()
171+
raise e

‎src/codegraphcontext/core/cgc_bundle.py‎

Lines changed: 70 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -179,10 +179,21 @@ def import_from_bundle(
179179
info_logger(f"Loading bundle: {metadata.get('repo', 'unknown')}")
180180
info_logger(f"Bundle version: {metadata.get('cgc_version', 'unknown')}")
181181

182-
# Step 4: Clear existing data if requested
182+
# Step 4: Handle existing data
183+
repo_name = metadata.get('repo', 'unknown')
184+
repo_path = metadata.get('repo_path')
185+
183186
if clear_existing:
184-
info_logger("Clearing existing graph data...")
187+
# User explicitly wants to clear - remove everything
188+
info_logger("Clearing all existing graph data...")
185189
self._clear_graph()
190+
else:
191+
# Check if this repository already exists (only when NOT clearing)
192+
existing_repo = self._check_existing_repository(repo_name, repo_path)
193+
194+
if existing_repo:
195+
return False, f"Repository '{repo_name}' already exists in the database. Use clear_existing=True to replace it."
196+
186197

187198
# Step 5: Create schema
188199
info_logger("Creating schema...")
@@ -584,6 +595,63 @@ def _validate_bundle(self, bundle_dir: Path) -> Tuple[bool, str]:
584595

585596
return True, "Valid bundle"
586597

598+
def _check_existing_repository(self, repo_name: str, repo_path: Optional[str]) -> bool:
599+
"""Check if a repository already exists in the database."""
600+
with self.db_manager.get_driver().session() as session:
601+
# Try to find by name first
602+
result = session.run(
603+
"MATCH (r:Repository {name: $name}) RETURN r LIMIT 1",
604+
name=repo_name
605+
)
606+
if result.single():
607+
return True
608+
609+
# If repo_path is provided, also check by path
610+
if repo_path:
611+
result = session.run(
612+
"MATCH (r:Repository {path: $path}) RETURN r LIMIT 1",
613+
path=repo_path
614+
)
615+
if result.single():
616+
return True
617+
618+
return False
619+
620+
def _delete_repository(self, repo_identifier: str):
621+
"""Delete a specific repository and all its related nodes from the graph."""
622+
with self.db_manager.get_driver().session() as session:
623+
# First, try to find the repository by name or path
624+
result = session.run("""
625+
MATCH (r:Repository)
626+
WHERE r.name = $identifier OR r.path = $identifier
627+
RETURN r.path as path
628+
LIMIT 1
629+
""", identifier=repo_identifier)
630+
631+
record = result.single()
632+
if not record:
633+
warning_logger(f"Repository '{repo_identifier}' not found for deletion")
634+
return
635+
636+
repo_path = record['path']
637+
638+
# Delete all nodes that belong to this repository
639+
# Files, Functions, Classes, Modules all have paths that start with repo_path
640+
session.run("""
641+
MATCH (n)
642+
WHERE n.path STARTS WITH $repo_path
643+
DETACH DELETE n
644+
""", repo_path=repo_path)
645+
646+
# Delete the repository node itself
647+
session.run("""
648+
MATCH (r:Repository)
649+
WHERE r.path = $repo_path
650+
DELETE r
651+
""", repo_path=repo_path)
652+
653+
info_logger(f"Deleted repository: {repo_identifier}")
654+
587655
def _clear_graph(self):
588656
"""Clear all nodes and relationships from the graph."""
589657
with self.db_manager.get_driver().session() as session:

‎src/codegraphcontext/tools/handlers/management_handlers.py‎

Lines changed: 49 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,8 @@ def list_jobs(job_manager: JobManager) -> Dict[str, Any]:
116116
def load_bundle(code_finder: CodeFinder, **args) -> Dict[str, Any]:
117117
"""Tool to load a .cgc bundle into the database."""
118118
from pathlib import Path
119-
from ...cli.registry_commands import load_bundle_command
119+
from ...core.bundle_registry import BundleRegistry
120+
from ...core.cgc_bundle import CGCBundle
120121

121122
bundle_name = args.get("bundle_name")
122123
clear_existing = args.get("clear_existing", False)
@@ -127,39 +128,77 @@ def load_bundle(code_finder: CodeFinder, **args) -> Dict[str, Any]:
127128
try:
128129
debug_log(f"Loading bundle: {bundle_name}")
129130

130-
# Use the existing load_bundle_command from CLI
131-
# This handles both local files and auto-download from registry
132-
success, message, stats = load_bundle_command(
133-
bundle_name=bundle_name,
131+
# Check if bundle exists locally
132+
bundle_path = Path(bundle_name)
133+
134+
# If it doesn't exist as-is, try with .cgc extension
135+
if not bundle_path.exists() and not str(bundle_name).endswith('.cgc'):
136+
bundle_path = Path(f"{bundle_name}.cgc")
137+
138+
if not bundle_path.exists():
139+
# Try to download from registry
140+
debug_log(f"Bundle {bundle_name} not found locally, checking registry...")
141+
download_url, bundle_meta, error = BundleRegistry.find_bundle_download_info(bundle_name)
142+
143+
if not download_url:
144+
return {"error": f"Bundle not found locally or in registry: {bundle_name}. {error}"}
145+
146+
# Determine output filename from metadata
147+
filename = bundle_meta.get('bundle_name', f"{bundle_name}.cgc")
148+
# Save to current working directory
149+
target_path = Path.cwd() / filename
150+
151+
debug_log(f"Downloading bundle to {target_path}...")
152+
try:
153+
BundleRegistry.download_file(download_url, target_path)
154+
bundle_path = target_path
155+
debug_log(f"Successfully downloaded to {bundle_path}")
156+
except Exception as e:
157+
return {"error": f"Failed to download bundle: {str(e)}"}
158+
159+
# Verify the downloaded file exists
160+
if not bundle_path.exists():
161+
return {"error": f"Download completed but file not found at {bundle_path}"}
162+
163+
# Load the bundle using CGCBundle core class
164+
bundle = CGCBundle(code_finder.db_manager)
165+
success, message = bundle.import_from_bundle(
166+
bundle_path=bundle_path,
134167
clear_existing=clear_existing
135168
)
136169

137170
if success:
171+
stats = {}
172+
# Parse simple stats from message if possible, or just return success
173+
if "Nodes:" in message:
174+
# Best effort parsing, not critical
175+
pass
176+
138177
return {
139178
"success": True,
140179
"message": message,
141180
"stats": stats
142181
}
143182
else:
144-
return {"error": message}
145-
183+
return {"error": message}
184+
146185
except Exception as e:
147186
debug_log(f"Error loading bundle: {str(e)}")
148187
return {"error": f"Failed to load bundle: {str(e)}"}
149188

150189

151190
def search_registry_bundles(code_finder: CodeFinder, **args) -> Dict[str, Any]:
152191
"""Tool to search for bundles in the registry."""
153-
from ...cli.registry_commands import fetch_available_bundles
192+
from ...core.bundle_registry import BundleRegistry
154193

155194
query = args.get("query", "").lower()
156195
unique_only = args.get("unique_only", False)
157196

158197
try:
159198
debug_log(f"Searching registry for: {query}")
160199

161-
# Fetch all bundles from registry
162-
bundles = fetch_available_bundles()
200+
# Fetch directly from core registry
201+
bundles = BundleRegistry.fetch_available_bundles()
163202

164203
if not bundles:
165204
return {

0 commit comments

Comments
 (0)