Repository navigation
Expand file tree
/
Copy pathchain.py
More file actions
97 lines (75 loc) · 3.66 KB
/
Copy pathchain.py
File metadata and controls
97 lines (75 loc) · 3.66 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
import os
import time
import json
import shutil
import logging
import config
from core.repo_loader import clone_repo, get_all_files
from core.preprocess import filter_files, keep_important_files, extract_readme
from core.analyzer import Analyzer
from core.map_step import MapStep
from core.aggregator import Aggregator
from core.classifier import Classifier
logger = logging.getLogger("repomind")
class RepoExplainerChain:
def __init__(self):
"""
Initializes the individual components of the explainer pipeline.
"""
self.analyzer = Analyzer()
self.mapper = MapStep()
self.aggregator = Aggregator()
self.classifier = Classifier()
def process_repository(self, repo_url: str) -> str:
"""
Executes the full end-to-end map-reduce pipeline on a GitHub URL.
Each run clones into an isolated temp directory — no folder conflicts.
Returns the final overview Markdown.
"""
start_time = time.time()
repo_name = repo_url.rstrip('/').split('/')[-1]
if repo_name.endswith('.git'):
repo_name = repo_name[:-4]
# 1. Clone into a unique temp directory
logger.info(f"Pipeline started for: {repo_name}")
path = clone_repo(repo_url)
try:
files = get_all_files(path)
logger.info(f"[{repo_name}] Total files discovered: {len(files)}")
# 2. Extract Top-Level Context
readme_content = extract_readme(path)
logger.info(f"[{repo_name}] README extracted: {'Yes' if readme_content else 'No'}")
# 3. Classify the Repository
repo_class = self.classifier.classify(repo_name, files, readme_content)
logger.info(f"[{repo_name}] Classification: {repo_class}")
# 4. Filter Noise
files = filter_files(files)
files = keep_important_files(files)
logger.info(f"[{repo_name}] Files after filtering: {len(files)}")
if not files:
return "No relevant source files found to analyze."
# 5. Analyze Importance
important_files = self.analyzer.extract_important_files(files)
important_files = important_files[: config.MAX_IMPORTANT_FILES]
logger.info(f"[{repo_name}] Important files identified: {len(important_files)}")
if not important_files:
return "Could not identify distinct core files to explain."
# 6. Map Step (File-level context explanations)
explanations = self.mapper.generate_explanations(important_files, repo_name, repo_class)
logger.info(f"[{repo_name}] Files explained: {len(explanations)}")
# 7. Aggregator (Final holistic reduction)
final_report = self.aggregator.generate_overview(explanations, repo_name, repo_class, readme_content)
# 8. Save outputs per user request
os.makedirs("outputs/json", exist_ok=True)
os.makedirs("outputs/markdown", exist_ok=True)
with open("outputs/json/summaries.json", "w", encoding="utf-8") as f:
json.dump(explanations, f, indent=4)
with open("outputs/markdown/final_output.txt", "w", encoding="utf-8") as f:
f.write(final_report)
elapsed = round(time.time() - start_time, 2)
logger.info(f"[{repo_name}] Pipeline completed in {elapsed}s")
return final_report
finally:
# Always clean up the temp directory after processing
shutil.rmtree(path, ignore_errors=True)
logger.info(f"[{repo_name}] Temp directory cleaned up")