From 4e5f1176a11ae5d34db7921852243f35146e8857 Mon Sep 17 00:00:00 2001 From: Lingzhi Yang Date: Mon, 23 Feb 2026 16:26:34 -0500 Subject: [PATCH] Add github-research skill (v1.1.0) 6-phase pipeline for discovering, evaluating, and deeply analyzing GitHub repositories related to a research topic. Reads deep-research output, discovers repos from multiple sources (GitHub search, code search, Papers With Code), scores and ranks them, clones and analyzes code, then produces cross-repo comparison and integration blueprints. 13 Python scripts (stdlib-only), 1 SKILL.md, 1 phase-guide reference. --- .gitignore | 1 + CHANGELOG.md | 14 + README.md | 12 +- skills/github-research/SKILL.md | 414 +++++++++++++ .../github-research/references/phase-guide.md | 215 +++++++ .../scripts/analyze_repo_structure.py | 243 ++++++++ skills/github-research/scripts/clone_repo.py | 109 ++++ .../github-research/scripts/compare_repos.py | 467 +++++++++++++++ .../scripts/compile_github_report.py | 524 +++++++++++++++++ .../scripts/extract_dependencies.py | 543 ++++++++++++++++++ .../scripts/extract_research_refs.py | 311 ++++++++++ .../scripts/find_implementations.py | 279 +++++++++ skills/github-research/scripts/repo_db.py | 470 +++++++++++++++ .../github-research/scripts/repo_metadata.py | 203 +++++++ .../scripts/repo_readme_fetch.py | 119 ++++ .../github-research/scripts/search_github.py | 265 +++++++++ .../scripts/search_github_code.py | 251 ++++++++ .../scripts/search_paperswithcode.py | 413 +++++++++++++ 18 files changed, 4851 insertions(+), 2 deletions(-) create mode 100644 skills/github-research/SKILL.md create mode 100644 skills/github-research/references/phase-guide.md create mode 100755 skills/github-research/scripts/analyze_repo_structure.py create mode 100755 skills/github-research/scripts/clone_repo.py create mode 100755 skills/github-research/scripts/compare_repos.py create mode 100755 skills/github-research/scripts/compile_github_report.py create mode 100755 skills/github-research/scripts/extract_dependencies.py create mode 100755 skills/github-research/scripts/extract_research_refs.py create mode 100755 skills/github-research/scripts/find_implementations.py create mode 100755 skills/github-research/scripts/repo_db.py create mode 100755 skills/github-research/scripts/repo_metadata.py create mode 100755 skills/github-research/scripts/repo_readme_fetch.py create mode 100755 skills/github-research/scripts/search_github.py create mode 100755 skills/github-research/scripts/search_github_code.py create mode 100755 skills/github-research/scripts/search_paperswithcode.py diff --git a/.gitignore b/.gitignore index 3bbe7b6..28efc82 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ __pycache__/ *.pyc *.pyo +github/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d209b5..502239b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ All notable changes to this project will be documented in this file. +## [1.1.0] - 2026-02-23 + +### Added + +- **github-research skill**: 6-phase pipeline for discovering, evaluating, and deeply analyzing GitHub repositories related to a research topic + - Phase 1: Intake — extract refs, URLs, keywords from deep-research output + - Phase 2: Discovery — multi-source GitHub search (repos, code, Papers With Code) + - Phase 3: Filtering — score, rank, and select top repos + - Phase 4: Deep Dive — clone and deeply analyze code structure, dependencies, implementations + - Phase 5: Analysis — cross-repo comparison matrix, technique mapping, gap analysis + - Phase 6: Blueprint — integration plan, reusable components catalog +- **13 Python scripts** for github-research: `extract_research_refs.py`, `search_github.py`, `search_github_code.py`, `search_paperswithcode.py`, `repo_db.py`, `repo_metadata.py`, `repo_readme_fetch.py`, `clone_repo.py`, `analyze_repo_structure.py`, `extract_dependencies.py`, `find_implementations.py`, `compare_repos.py`, `compile_github_report.py` +- **Phase guide reference** (`references/phase-guide.md`) with search strategy matrix and per-repo analysis templates + ## [1.0.0] - 2026-02-20 ### Added diff --git a/README.md b/README.md index 020fa86..576a754 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # agent-research-skills -30 skills for [Claude Code](https://claude.ai/code) covering the full academic research paper lifecycle — from literature search to slide generation. +31 skills for [Claude Code](https://claude.ai/code) covering the full academic research paper lifecycle — from literature search to slide generation — plus GitHub repository analysis for research topics. Extracted from 17 GitHub repos studying LLM-agent-driven research automation. See [SKILLS_DESIGN.md](/Users/lingzhi/Code/research-engine/SKILLS_DESIGN.md) for the original design specifications. @@ -40,12 +40,13 @@ This installs slash commands, checks Python dependencies, and verifies script sy 2. **Output directory**: Deep research outputs go to `~/deep-research-output/` by default. -## Available Skills (30) +## Available Skills (31) ### Phase 0: Research Discovery & Planning | Skill | Description | Scripts | |-------|-------------|---------| +| **[github-research](skills/github-research/)** | 6-phase GitHub repo discovery, analysis, and integration planning for research topics | 13 scripts: search, clone, analyze structure/deps/impls, compare, compile report | | **[deep-research](skills/deep-research/)** | 6-phase systematic literature survey (frontier → survey → deep dive → code → synthesis → report) | 7 scripts: search APIs, PDF extraction, paper DB, BibTeX, report compilation | | **[literature-search](skills/literature-search/)** | Multi-source academic search (Semantic Scholar, arXiv, OpenAlex, CrossRef) with ranking | 4 scripts: `search_crossref.py`, `download_arxiv_source.py`, `search_openalex.py` + shared | | **[literature-review](skills/literature-review/)** | Multi-perspective dialogue simulation with expert personas for grounded literature review | Shares search scripts | @@ -118,6 +119,7 @@ This installs slash commands, checks Python dependencies, and verifies script sy ### Natural language (skills activate automatically) ``` +"Analyze GitHub repos for multi-agent coordination research" "Do a literature review on protein folding with LLMs" "Write the Methods section of my paper" "Generate a comparison table from results.json" @@ -130,6 +132,12 @@ This installs slash commands, checks Python dependencies, and verifies script sy ### Direct script usage ```bash +# Search GitHub repos for a research topic +python ~/.claude/skills/github-research/scripts/search_github.py --query "multi-agent LLM coordination" --max-results 50 --output repos.jsonl + +# Analyze a cloned repo's structure +python ~/.claude/skills/github-research/scripts/analyze_repo_structure.py --repo-dir ./my-repo --output analysis.json + # Search literature python ~/.claude/skills/literature-search/scripts/search_crossref.py --query "attention mechanism" --rows 10 diff --git a/skills/github-research/SKILL.md b/skills/github-research/SKILL.md new file mode 100644 index 0000000..9f9e82d --- /dev/null +++ b/skills/github-research/SKILL.md @@ -0,0 +1,414 @@ +--- +name: github-research +description: Explore and analyze GitHub repositories related to a research topic. Reads deep-research output, discovers repos from multiple sources, deeply analyzes code, and produces integration blueprints. +argument-hint: [deep-research-output-dir] +--- + +# GitHub Research Skill + +## Trigger + +Activate this skill when the user wants to: +- "Find repos for [topic]", "GitHub research on [topic]" +- "Analyze open-source code for [topic]" +- "Find implementations of [paper/technique]" +- "Which repos implement [algorithm]?" +- Uses `/github-research ` slash command + +## Overview + +This skill systematically discovers, evaluates, and deeply analyzes GitHub repositories related to a research topic. It reads **deep-research** output (paper database, phase reports, code references) and produces an actionable integration blueprint for reusing open-source code. + +**Installation**: `~/.claude/skills/github-research/` — scripts, references, and this skill definition. +**Output**: `./github-research-output/{slug}/` relative to the current working directory. +**Input**: A deep-research output directory (containing `paper_db.jsonl`, phase reports, `code_repos.md`, etc.) + +## 6-Phase Pipeline + +``` +Phase 1: Intake → Extract refs, URLs, keywords from deep-research output +Phase 2: Discovery → Multi-source broad GitHub search (50-200 repos) +Phase 3: Filtering → Score & rank → select top 15-30 repos +Phase 4: Deep Dive → Clone & deeply analyze top 8-15 repos (code reading) +Phase 5: Analysis → Per-repo reports + cross-repo comparison +Phase 6: Blueprint → Integration/reuse plan for research topic +``` + +## Output Directory Structure + +``` +github-research-output/{slug}/ +├── repo_db.jsonl # Master repo database +├── phase1_intake/ +│ ├── extracted_refs.jsonl # URLs, keywords, paper-repo links +│ └── intake_summary.md +├── phase2_discovery/ +│ ├── search_results/ # Raw JSONL from each search +│ └── discovery_log.md +├── phase3_filtering/ +│ ├── ranked_repos.jsonl # Scored & ranked subset +│ └── filtering_report.md +├── phase4_deep_dive/ +│ ├── repos/ # Cloned repos (shallow) +│ ├── analyses/ # Per-repo analysis .md files +│ └── deep_dive_summary.md +├── phase5_analysis/ +│ ├── comparison_matrix.md # Cross-repo comparison +│ ├── technique_map.md # Paper concept → code mapping +│ └── analysis_report.md +└── phase6_blueprint/ + ├── integration_plan.md # How to combine repos + ├── reuse_catalog.md # Reusable components catalog + ├── final_report.md # Complete compiled report + └── blueprint_summary.md +``` + +## Scripts Reference + +All scripts are Python 3, stdlib-only, located in `~/.claude/skills/github-research/scripts/`. + +| Script | Purpose | Key Flags | +|--------|---------|-----------| +| `extract_research_refs.py` | Parse deep-research output for GitHub URLs, paper refs, keywords | `--research-dir`, `--output` | +| `search_github.py` | Search GitHub repos via `gh api` | `--query`, `--language`, `--min-stars`, `--sort`, `--max-results`, `--topic`, `--output` | +| `search_github_code.py` | Search GitHub code for implementations | `--query`, `--language`, `--filename`, `--max-results`, `--output` | +| `search_paperswithcode.py` | Search Papers With Code for paper→repo mappings | `--paper-title`, `--arxiv-id`, `--query`, `--output` | +| `repo_db.py` | JSONL repo database management | subcommands: `merge`, `filter`, `score`, `search`, `tag`, `stats`, `export`, `rank` | +| `repo_metadata.py` | Fetch detailed metadata via `gh api` | `--repos`, `--input`, `--output`, `--delay` | +| `clone_repo.py` | Shallow-clone repos for analysis | `--repo`, `--output-dir`, `--depth`, `--branch` | +| `analyze_repo_structure.py` | Map file tree, key files, LOC stats | `--repo-dir`, `--output` | +| `extract_dependencies.py` | Extract and parse dependency files | `--repo-dir`, `--output` | +| `find_implementations.py` | Search cloned repo for specific code patterns | `--repo-dir`, `--patterns`, `--output` | +| `repo_readme_fetch.py` | Fetch README without cloning | `--repos`, `--input`, `--output`, `--max-chars` | +| `compare_repos.py` | Generate comparison matrix across repos | `--input`, `--output` | +| `compile_github_report.py` | Assemble final report from all phases | `--topic-dir` | + +--- + +## Phase 1: Intake + +**Goal**: Extract all relevant references, URLs, and keywords from the deep-research output. + +### Steps + +1. **Create output directory structure**: + ```bash + SLUG=$(echo "$TOPIC" | tr '[:upper:]' '[:lower:]' | tr ' ' '-' | tr -cd 'a-z0-9-') + mkdir -p github-research-output/$SLUG/{phase1_intake,phase2_discovery/search_results,phase3_filtering,phase4_deep_dive/{repos,analyses},phase5_analysis,phase6_blueprint} + ``` + +2. **Extract references from deep-research output**: + ```bash + python ~/.claude/skills/github-research/scripts/extract_research_refs.py \ + --research-dir \ + --output github-research-output/$SLUG/phase1_intake/extracted_refs.jsonl + ``` + +3. **Review extracted refs**: Read the generated JSONL. Note: + - GitHub URLs found directly in reports + - Paper titles and arxiv IDs (for Papers With Code lookup) + - Research keywords and themes (for GitHub search queries) + +4. **Write intake summary**: Create `phase1_intake/intake_summary.md` with: + - Number of direct GitHub URLs found + - Number of papers with potential code links + - Key research themes extracted + - Planned search queries for Phase 2 + +### Checkpoint +- `extracted_refs.jsonl` exists with entries +- `intake_summary.md` written +- Search strategy documented + +--- + +## Phase 2: Discovery + +**Goal**: Cast a wide net to find 50-200 candidate repos from multiple sources. + +### Steps + +1. **Search by direct URLs**: Any GitHub URLs from Phase 1 → fetch metadata: + ```bash + python ~/.claude/skills/github-research/scripts/repo_metadata.py \ + --repos owner1/name1 owner2/name2 ... \ + --output github-research-output/$SLUG/phase2_discovery/search_results/direct_urls.jsonl + ``` + +2. **Search Papers With Code**: For each paper with an arxiv ID: + ```bash + python ~/.claude/skills/github-research/scripts/search_paperswithcode.py \ + --arxiv-id 2401.12345 \ + --output github-research-output/$SLUG/phase2_discovery/search_results/pwc_2401.12345.jsonl + ``` + +3. **Search GitHub by keywords** (3-8 queries based on research themes): + ```bash + python ~/.claude/skills/github-research/scripts/search_github.py \ + --query "multi-agent LLM coordination" \ + --min-stars 10 --sort stars --max-results 50 \ + --output github-research-output/$SLUG/phase2_discovery/search_results/gh_query1.jsonl + ``` + +4. **Search GitHub code** (for specific implementations): + ```bash + python ~/.claude/skills/github-research/scripts/search_github_code.py \ + --query "class MultiAgentOrchestrator" \ + --language python --max-results 30 \ + --output github-research-output/$SLUG/phase2_discovery/search_results/code_query1.jsonl + ``` + +5. **Fetch READMEs** for repos that lack descriptions: + ```bash + python ~/.claude/skills/github-research/scripts/repo_readme_fetch.py \ + --input \ + --output github-research-output/$SLUG/phase2_discovery/search_results/readmes.jsonl + ``` + +6. **Merge all results** into master database: + ```bash + python ~/.claude/skills/github-research/scripts/repo_db.py merge \ + --inputs github-research-output/$SLUG/phase2_discovery/search_results/*.jsonl \ + --output github-research-output/$SLUG/repo_db.jsonl + ``` + +7. **Write discovery log**: Create `phase2_discovery/discovery_log.md` with search queries used, results per source, total unique repos found. + +### Rate Limits +- GitHub search API: 30 requests/minute (authenticated) +- Papers With Code API: No strict limit but be respectful (1 req/sec) +- Add `--delay 1.0` to batch operations when needed + +### Checkpoint +- `repo_db.jsonl` populated with 50-200 repos +- `discovery_log.md` with search details + +--- + +## Phase 3: Filtering + +**Goal**: Score and rank repos, select top 15-30 for deeper analysis. + +### Steps + +1. **Enrich metadata** for all repos: + ```bash + python ~/.claude/skills/github-research/scripts/repo_metadata.py \ + --input github-research-output/$SLUG/repo_db.jsonl \ + --output github-research-output/$SLUG/repo_db.jsonl \ + --delay 0.5 + ``` + +2. **Score repos** (quality + activity scores): + ```bash + python ~/.claude/skills/github-research/scripts/repo_db.py score \ + --input github-research-output/$SLUG/repo_db.jsonl \ + --output github-research-output/$SLUG/repo_db.jsonl + ``` + +3. **LLM relevance scoring**: Read through the top ~50 repos (by quality_score) and assign `relevance_score` (0.0-1.0) based on: + - Direct relevance to research topic + - Implementation completeness + - Code quality signals (from README, description) + - Update the relevance scores: + ```bash + python ~/.claude/skills/github-research/scripts/repo_db.py tag \ + --input github-research-output/$SLUG/repo_db.jsonl \ + --ids owner/name --tags "relevance:0.85" + ``` + +4. **Compute composite scores and rank**: + ```bash + python ~/.claude/skills/github-research/scripts/repo_db.py score \ + --input github-research-output/$SLUG/repo_db.jsonl \ + --output github-research-output/$SLUG/repo_db.jsonl + python ~/.claude/skills/github-research/scripts/repo_db.py rank \ + --input github-research-output/$SLUG/repo_db.jsonl \ + --output github-research-output/$SLUG/phase3_filtering/ranked_repos.jsonl \ + --by composite_score + ``` + +5. **Select top repos**: Filter to top 15-30: + ```bash + python ~/.claude/skills/github-research/scripts/repo_db.py filter \ + --input github-research-output/$SLUG/phase3_filtering/ranked_repos.jsonl \ + --output github-research-output/$SLUG/phase3_filtering/ranked_repos.jsonl \ + --max-repos 30 --not-archived + ``` + +6. **Write filtering report**: Create `phase3_filtering/filtering_report.md`: + - Stats before/after filtering + - Score distributions + - Top 30 repos with scores and rationale + +### Scoring Formula +``` +activity_score = sigmoid((days_since_push < 90) * 0.4 + has_recent_commits * 0.3 + open_issues_ratio * 0.3) +quality_score = normalize(log(stars+1) * 0.3 + log(forks+1) * 0.2 + has_license * 0.15 + has_readme * 0.15 + not_archived * 0.2) +composite_score = relevance * 0.4 + quality * 0.35 + activity * 0.25 +``` + +### Checkpoint +- `ranked_repos.jsonl` with 15-30 repos +- `filtering_report.md` with scoring details + +--- + +## Phase 4: Deep Dive + +**Goal**: Clone and deeply analyze the top 8-15 repos. + +### Steps + +1. **Select repos for deep dive**: Take top 8-15 from ranked list. + +2. **Clone each repo** (shallow): + ```bash + python ~/.claude/skills/github-research/scripts/clone_repo.py \ + --repo owner/name \ + --output-dir github-research-output/$SLUG/phase4_deep_dive/repos/ + ``` + +3. **Analyze structure** for each cloned repo: + ```bash + python ~/.claude/skills/github-research/scripts/analyze_repo_structure.py \ + --repo-dir github-research-output/$SLUG/phase4_deep_dive/repos/name/ \ + --output github-research-output/$SLUG/phase4_deep_dive/analyses/name_structure.json + ``` + +4. **Extract dependencies**: + ```bash + python ~/.claude/skills/github-research/scripts/extract_dependencies.py \ + --repo-dir github-research-output/$SLUG/phase4_deep_dive/repos/name/ \ + --output github-research-output/$SLUG/phase4_deep_dive/analyses/name_deps.json + ``` + +5. **Find implementations**: Search for key algorithms/concepts from research: + ```bash + python ~/.claude/skills/github-research/scripts/find_implementations.py \ + --repo-dir github-research-output/$SLUG/phase4_deep_dive/repos/name/ \ + --patterns "class Transformer" "def forward" "attention" \ + --output github-research-output/$SLUG/phase4_deep_dive/analyses/name_impls.jsonl + ``` + +6. **Deep code reading**: For each repo, READ the key source files identified by structure analysis. Write a per-repo analysis in `phase4_deep_dive/analyses/{name}_analysis.md`: + - Architecture overview + - Key algorithms implemented + - Code quality assessment + - API / interface design + - Dependencies and requirements + - Strengths and limitations + - Reusability assessment (how easy to extract components) + +7. **Write deep dive summary**: `phase4_deep_dive/deep_dive_summary.md` + +### IMPORTANT: Actually Read Code +Do NOT just summarize READMEs. You must: +- Read the main source files (entry points, core modules) +- Understand the actual implementation approach +- Identify specific functions/classes that implement research concepts +- Note code patterns, design decisions, and trade-offs + +### Checkpoint +- Repos cloned in `repos/` +- Per-repo analysis files in `analyses/` +- `deep_dive_summary.md` written + +--- + +## Phase 5: Analysis + +**Goal**: Cross-repo comparison and technique-to-code mapping. + +### Steps + +1. **Generate comparison matrix**: + ```bash + python ~/.claude/skills/github-research/scripts/compare_repos.py \ + --input github-research-output/$SLUG/phase4_deep_dive/analyses/ \ + --output github-research-output/$SLUG/phase5_analysis/comparison.json + ``` + +2. **Write comparison matrix**: Create `phase5_analysis/comparison_matrix.md`: + - Table comparing repos across dimensions (language, LOC, stars, framework, license, tests) + - Dependency overlap analysis + - Strengths/weaknesses per repo + +3. **Write technique map**: Create `phase5_analysis/technique_map.md`: + - Map each paper concept / research technique → specific repo + file + function + - Identify gaps (techniques with no implementation found) + - Note alternative implementations of the same concept + +4. **Write analysis report**: `phase5_analysis/analysis_report.md`: + - Executive summary of findings + - Key insights from code analysis + - Recommendations for which repos to use for which purposes + +### Checkpoint +- `comparison_matrix.md` with repo comparison table +- `technique_map.md` mapping concepts to code +- `analysis_report.md` with findings + +--- + +## Phase 6: Blueprint + +**Goal**: Produce an actionable integration and reuse plan. + +### Steps + +1. **Write integration plan**: `phase6_blueprint/integration_plan.md`: + - Recommended architecture for combining repos + - Step-by-step integration approach + - Dependency resolution strategy + - Potential conflicts and how to resolve them + +2. **Write reuse catalog**: `phase6_blueprint/reuse_catalog.md`: + - For each reusable component: source repo, file path, function/class, what it does, how to extract it + - License compatibility matrix + - Effort estimates (easy/medium/hard to integrate) + +3. **Compile final report**: + ```bash + python ~/.claude/skills/github-research/scripts/compile_github_report.py \ + --topic-dir github-research-output/$SLUG/ + ``` + +4. **Write blueprint summary**: `phase6_blueprint/blueprint_summary.md`: + - One-page executive summary + - Top 5 repos and why + - Recommended next steps + +### Checkpoint +- `integration_plan.md` complete +- `reuse_catalog.md` with component catalog +- `final_report.md` compiled +- `blueprint_summary.md` as executive summary + +--- + +## Quality Conventions + +1. **Repos are ranked by composite score**: `relevance × 0.4 + quality × 0.35 + activity × 0.25` +2. **Deep dive requires reading actual code**, not just READMEs +3. **Integration blueprint must map paper concepts → specific code files/functions** +4. **Incremental saves**: Each phase writes to disk immediately +5. **Checkpoint recovery**: Can resume from any phase by checking what outputs exist +6. **All scripts are stdlib-only Python** — no pip installs needed +7. **`gh` CLI is required** for GitHub API access (must be authenticated) +8. **Deduplication** by `repo_id` (owner/name) across all searches +9. **Rate limit awareness**: Respect GitHub search API limits (30 req/min) + +## Error Handling + +- If `gh` is not installed: warn user and provide installation instructions +- If a repo is archived/deleted: skip gracefully, note in log +- If clone fails: skip, note in log, continue with remaining repos +- If Papers With Code API is down: skip, rely on GitHub search only +- Always write partial progress to disk so work is not lost + +## References + +- See `references/phase-guide.md` for detailed phase execution guidance +- Deep-research skill: `~/.claude/skills/deep-research/SKILL.md` +- Paper database pattern: `~/.claude/skills/deep-research/scripts/paper_db.py` diff --git a/skills/github-research/references/phase-guide.md b/skills/github-research/references/phase-guide.md new file mode 100644 index 0000000..ab41f33 --- /dev/null +++ b/skills/github-research/references/phase-guide.md @@ -0,0 +1,215 @@ +# GitHub Research — Phase Guide + +Detailed methodology reference for the github-research skill. + +## Phase 1: Intake — Detailed Guide + +### Purpose +Extract structured information from deep-research output to seed GitHub discovery. + +### Input Requirements +- Deep-research output directory containing: + - `paper_db.jsonl` (required) + - `phase4_code/code_repos.md` (optional but valuable) + - `phase5_synthesis/synthesis.md` (optional) + - `phase6_report/report.md` (optional) + +### Keyword Extraction Strategy +- **Primary keywords**: From paper titles — extract 2-3 word technical phrases +- **Secondary keywords**: From paper tags in paper_db.jsonl +- **Tertiary keywords**: Method names, algorithm names, architecture names from synthesis +- **Author-based**: Search for prolific authors' GitHub profiles + +### Expected Output +- 5-20 GitHub URLs directly from papers +- 10-30 search keywords of varying specificity +- Clear mapping: which papers mention which repos + +### Edge Cases +- No code_repos.md: rely entirely on paper_db.jsonl keywords +- No paper_db.jsonl: ask user for manual topic keywords +- Non-English papers: extract English technical terms only + +--- + +## Phase 2: Discovery — Detailed Guide + +### Search Strategy Matrix +| Strategy | Query Pattern | Sort | When to Use | +|----------|--------------|------|-------------| +| Broad topic | "multi-agent LLM framework" | stars | Always — establishes landscape | +| Paper title | "{exact paper title}" | best-match | For each key paper | +| Method name | "{algorithm name} implementation" | stars | For specific techniques | +| Author search | "{author name}" + topic | updated | For prolific researchers | +| Code pattern | "class {ClassName}" | - | For specific implementations | +| Language-specific | topic + language:python | stars | When language matters | +| Awesome list | "awesome-{topic}" | stars | To find curated lists | + +### Rate Limiting +- GitHub search API: 30 requests/minute (unauthenticated), 10 requests/minute (code search) +- Papers With Code API: ~60 requests/minute +- Always set GITHUB_TOKEN for higher limits (5000 req/hr) + +### Deduplication +- Primary key: `repo_id` (owner/name, case-insensitive) +- When merging duplicates: keep record with more populated fields; merge paper_ids lists + +### Target Numbers +- Aim for 50-200 unique repos before filtering +- Use at least 5 different search queries +- Check Papers With Code for all papers with arxiv_ids + +--- + +## Phase 3: Filtering — Detailed Guide + +### Scoring Deep Dive + +**Activity Score** (0-1): +- Days since last push: <30d -> 0.9-1.0, 30-90d -> 0.6-0.8, 90-365d -> 0.3-0.5, >365d -> 0.0-0.2 +- Frequency weight: pushed_at recency matters most + +**Quality Score** (0-1): +- Stars (log-scaled, 30% weight): log(stars+1) normalized across set +- Forks (log-scaled, 20%): log(forks+1) normalized +- Has license (15%): any recognized license = 1.0 +- Not archived (20%): archived repos get 0 +- Has README (15%): non-empty readme_excerpt = 1.0 + +**Relevance Score** (0-1, manually assigned): +- 0.9-1.0: Direct implementation of a paper in the literature review +- 0.7-0.89: Closely related technique or framework +- 0.5-0.69: Related but tangential (e.g., general ML framework used by papers) +- 0.3-0.49: Loosely related (e.g., same domain, different approach) +- 0.0-0.29: Unlikely useful + +**Composite**: relevance x 0.4 + quality x 0.35 + activity x 0.25 + +### Selection Criteria +- Always include: repos directly linked to papers +- Prefer: repos with tests, documentation, active maintenance +- Diversity: ensure mix of approaches, not just top-starred +- Minimum: 15 repos; Maximum: 30 repos + +--- + +## Phase 4: Deep Dive — Detailed Guide + +### What "Deep Dive" Means +This is NOT a README scan. You must: +1. Clone the repo (shallow) +2. Read the directory structure +3. Open and read key source files (model definitions, training loops, core algorithms) +4. Trace the execution flow from entry point to core logic +5. Evaluate code quality, documentation, test coverage + +### Per-Repo Analysis Template +```markdown +# {owner/name} — Deep Dive Analysis + +## Overview +- **Purpose**: {one-sentence} +- **Stars**: {N} | **Language**: {lang} | **License**: {license} +- **Last active**: {date} | **Composite score**: {score} + +## Architecture +- Entry point: `{file}` -> calls `{function}` -> uses `{module}` +- Core modules: {list key files with purposes} +- Data flow: {how data moves through the system} + +## Key Algorithms +- **{Algorithm 1}**: Implemented in `{file}:{lines}`, function `{name}` + - Matches paper: {yes/no/partially} — {details} +- **{Algorithm 2}**: ... + +## Code Quality +- Documentation: {poor/fair/good/excellent} +- Test coverage: {none/minimal/moderate/comprehensive} +- Code style: {consistent/inconsistent}, {patterns used} +- Error handling: {minimal/adequate/thorough} + +## Dependencies +- Core: {list key deps} +- ML framework: {pytorch/tensorflow/jax/none} +- Hardware: {CPU only / GPU required / TPU supported} + +## Reusability Assessment +- Ease of extraction: {easy/moderate/difficult} +- Tight couplings: {what's hard to separate} +- API surface: {clean/messy} +- Recommended components: {what to reuse} + +## Limitations +- {limitation 1} +- {limitation 2} +``` + +### Prioritization +- Read model/algorithm files first (highest value) +- Then training/evaluation scripts +- Then configuration and entry points +- Skip: CI config, linting config, CHANGELOG + +--- + +## Phase 5: Analysis — Detailed Guide + +### Comparison Matrix Dimensions +Build a table comparing all deep-dived repos across: +1. Primary language & ML framework +2. Lines of code & file count +3. Code quality rating (1-5) +4. Paper fidelity (how closely it matches the paper) +5. Reusability rating (1-5) +6. Activity status (active/maintained/stale/archived) +7. Dependency count & overlap +8. License restrictiveness (permissive/copyleft/unknown) +9. Hardware requirements (CPU/GPU/TPU) + +### Technique Map Construction +For each significant concept in the deep-research papers: +1. Identify the concept (e.g., "multi-head attention", "reward shaping") +2. Find implementations in analyzed repos +3. Note the specific file, class/function, and line numbers +4. Rate fidelity: faithful / modified / inspired-by / missing +5. Note any improvements or deviations from the paper + +### Gap Analysis +Identify: +- Paper concepts with NO implementation found +- Implementations that deviate significantly from papers +- Missing components needed for a complete system +- Opportunities for novel contributions + +--- + +## Phase 6: Blueprint — Detailed Guide + +### Integration Plan Structure +1. **Objective**: What system are we building? +2. **Recommended Stack**: Best combination of repos +3. **Architecture Diagram** (text-based): How repos fit together +4. **Step-by-step Plan**: + - Step 1: Start with {repo} as base + - Step 2: Extract {component} from {repo} + - Step 3: Adapt {module} to work with base + - ... +5. **Risk Assessment**: License conflicts, version incompatibilities, maintenance risk +6. **Estimated Complexity**: Per-step effort estimate (trivial/moderate/significant) + +### Reuse Catalog Format +For each extractable component: +- Source repo and file path +- What it does +- Dependencies required +- How to extract (copy files, install deps, adapt imports) +- API surface (key classes/functions, input/output types) +- Caveats and gotchas + +### Quality Checklist +Before finalizing: +- [ ] Every paper concept has a code mapping (or explicit "not found") +- [ ] License compatibility checked for recommended stack +- [ ] Dependency conflicts identified between repos +- [ ] At least one "quick start" integration path identified +- [ ] Gaps clearly documented with suggested approaches diff --git a/skills/github-research/scripts/analyze_repo_structure.py b/skills/github-research/scripts/analyze_repo_structure.py new file mode 100755 index 0000000..634b08d --- /dev/null +++ b/skills/github-research/scripts/analyze_repo_structure.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Map repo file tree, identify key files, and compute LOC stats. + +Usage: + python analyze_repo_structure.py --repo-dir /tmp/repos/myrepo --output analysis.json [--repo-id owner/name] + +Writes analysis JSON to the specified output file. +""" + +import argparse +import json +import os +import re +import sys + +# Directories to skip during traversal +SKIP_DIRS = { + ".git", "node_modules", "__pycache__", ".venv", "venv", + ".tox", ".eggs", "dist", "build", +} + +# File extension to language mapping +EXT_TO_LANG = { + ".py": "Python", + ".js": "JavaScript", + ".ts": "TypeScript", + ".rs": "Rust", + ".go": "Go", + ".java": "Java", + ".cpp": "C++", + ".cc": "C++", + ".cxx": "C++", + ".c": "C", + ".rb": "Ruby", + ".sh": "Shell", + ".yaml": "YAML", + ".yml": "YAML", + ".json": "JSON", + ".md": "Markdown", +} + +# Known config file names +CONFIG_FILES = { + "pyproject.toml", "setup.py", "setup.cfg", "package.json", + "Cargo.toml", "go.mod", "Makefile", "Dockerfile", + "docker-compose.yml", "docker-compose.yaml", + ".env.example", "tox.ini", +} + +# Known entry-point file names +ENTRY_POINT_NAMES = {"main.py", "app.py", "run.py", "train.py", "setup.py", "Makefile", "Dockerfile"} + + +def is_egg_info_dir(name: str) -> bool: + return name.endswith(".egg-info") + + +def count_loc(filepath: str) -> int: + """Count non-empty lines in a text file.""" + try: + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + return sum(1 for line in f if line.strip()) + except (OSError, UnicodeDecodeError): + return 0 + + +def has_shebang(filepath: str) -> bool: + """Check if file starts with a shebang line.""" + try: + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + first_line = f.readline() + return first_line.startswith("#!") + except (OSError, UnicodeDecodeError): + return False + + +def has_main_guard(filepath: str) -> bool: + """Check if a Python file has `if __name__` guard.""" + try: + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + for line in f: + if re.search(r'if\s+__name__\s*==\s*["\']__main__["\']', line): + return True + except (OSError, UnicodeDecodeError): + pass + return False + + +def should_skip_dir(name: str) -> bool: + return name in SKIP_DIRS or is_egg_info_dir(name) + + +def build_tree(repo_dir: str, max_lines: int = 200) -> str: + """Generate an indented file tree string, truncated to max_lines.""" + lines = [] + base = os.path.basename(os.path.abspath(repo_dir)) + + for dirpath, dirnames, filenames in os.walk(repo_dir): + # Filter out skipped dirs in-place + dirnames[:] = sorted([d for d in dirnames if not should_skip_dir(d)]) + filenames = sorted(filenames) + + rel = os.path.relpath(dirpath, repo_dir) + depth = 0 if rel == "." else rel.count(os.sep) + 1 + indent = " " * depth + + if rel == ".": + lines.append(f"{base}/") + else: + lines.append(f"{indent}{os.path.basename(dirpath)}/") + + if len(lines) >= max_lines: + break + + file_indent = " " * (depth + 1) + for fname in filenames: + lines.append(f"{file_indent}{fname}") + if len(lines) >= max_lines: + break + if len(lines) >= max_lines: + break + + if len(lines) >= max_lines: + lines.append(" ... (truncated)") + return "\n".join(lines) + + +def analyze(repo_dir: str, repo_id: str) -> dict: + """Perform full repo structure analysis.""" + total_files = 0 + total_loc = 0 + languages: dict[str, int] = {} + key_files: list[str] = [] + entry_points: list[str] = [] + config_files: list[str] = [] + test_files: list[str] = [] + doc_files: list[str] = [] + file_loc: list[tuple[str, int]] = [] + + print(f"[analyze] Scanning {repo_dir} ...", file=sys.stderr) + + for dirpath, dirnames, filenames in os.walk(repo_dir): + dirnames[:] = [d for d in dirnames if not should_skip_dir(d)] + + rel_dir = os.path.relpath(dirpath, repo_dir) + + for fname in filenames: + filepath = os.path.join(dirpath, fname) + rel_path = os.path.join(rel_dir, fname) if rel_dir != "." else fname + # Normalize path separators + rel_path = rel_path.replace(os.sep, "/") + + total_files += 1 + + _, ext = os.path.splitext(fname) + ext_lower = ext.lower() + + # LOC counting for known languages + lang = EXT_TO_LANG.get(ext_lower) + loc = 0 + if lang: + loc = count_loc(filepath) + languages[lang] = languages.get(lang, 0) + loc + total_loc += loc + file_loc.append((rel_path, loc)) + + # Entry points detection + if fname in ENTRY_POINT_NAMES: + entry_points.append(rel_path) + elif ext_lower == ".py" and has_main_guard(filepath): + entry_points.append(rel_path) + elif has_shebang(filepath): + entry_points.append(rel_path) + + # Config files + if fname in CONFIG_FILES: + config_files.append(rel_path) + + # Test files + if (fname.startswith("test_") or fname.endswith(("_test.py", "_test.js", "_test.ts", "_test.go")) + or "/tests/" in f"/{rel_path}/" or "/test/" in f"/{rel_path}/" + or "/spec/" in f"/{rel_path}/"): + test_files.append(rel_path) + + # Doc files + if ext_lower == ".md" or "/docs/" in f"/{rel_path}/" or "/doc/" in f"/{rel_path}/": + doc_files.append(rel_path) + + # Sort files by LOC descending, take top 10 + file_loc.sort(key=lambda x: x[1], reverse=True) + file_size_top10 = file_loc[:10] + + # Key files = top LOC files + entry points + key_set = set() + for path, _ in file_size_top10: + key_set.add(path) + for path in entry_points: + key_set.add(path) + key_files = sorted(key_set) + + # Sort languages by LOC descending + languages = dict(sorted(languages.items(), key=lambda x: x[1], reverse=True)) + + tree = build_tree(repo_dir) + + print(f"[analyze] Files: {total_files}, LOC: {total_loc}, Languages: {len(languages)}", file=sys.stderr) + + return { + "repo_id": repo_id, + "total_files": total_files, + "total_loc": total_loc, + "languages": languages, + "key_files": key_files, + "entry_points": sorted(set(entry_points)), + "config_files": sorted(set(config_files)), + "test_files": sorted(set(test_files)), + "doc_files": sorted(set(doc_files)), + "tree": tree, + "file_size_top10": file_size_top10, + } + + +def main(): + parser = argparse.ArgumentParser(description="Analyze repository structure, files, and LOC stats.") + parser.add_argument("--repo-dir", required=True, help="Path to the cloned repository") + parser.add_argument("--output", required=True, help="Output JSON file path") + parser.add_argument("--repo-id", default="unknown/unknown", help="Repository identifier (owner/name)") + args = parser.parse_args() + + if not os.path.isdir(args.repo_dir): + print(f"Error: {args.repo_dir} is not a directory", file=sys.stderr) + sys.exit(1) + + result = analyze(args.repo_dir, args.repo_id) + + with open(args.output, "w", encoding="utf-8") as f: + json.dump(result, f, indent=2, ensure_ascii=False) + + print(f"[analyze] Output written to {args.output}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/skills/github-research/scripts/clone_repo.py b/skills/github-research/scripts/clone_repo.py new file mode 100755 index 0000000..2018f8e --- /dev/null +++ b/skills/github-research/scripts/clone_repo.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Shallow-clone GitHub repos for local code analysis. + +Usage: + python clone_repo.py --repo owner/name --output-dir /tmp/repos [--depth 1] [--branch main] + +Outputs JSON to stdout with clone result; stats to stderr. +""" + +import argparse +import json +import os +import subprocess +import sys + + +def get_dir_size(path: str) -> int: + """Walk directory and sum all file sizes in bytes.""" + total = 0 + for dirpath, _dirnames, filenames in os.walk(path): + for f in filenames: + fp = os.path.join(dirpath, f) + try: + total += os.path.getsize(fp) + except OSError: + pass + return total + + +def clone_repo(repo: str, output_dir: str, depth: int, branch: str | None) -> dict: + """Clone a GitHub repo and return result metadata.""" + parts = repo.split("/") + if len(parts) != 2: + return { + "repo_id": repo, + "local_path": None, + "clone_success": False, + "error": f"Invalid repo format '{repo}', expected 'owner/name'", + } + + owner, name = parts + url = f"https://github.com/{owner}/{name}.git" + target = os.path.join(output_dir, name) + + os.makedirs(output_dir, exist_ok=True) + + # Build clone command + cmd = ["git", "clone", f"--depth={depth}", "--single-branch"] + if branch: + cmd += ["--branch", branch] + cmd += [url, target] + + print(f"[clone] Running: {' '.join(cmd)}", file=sys.stderr) + + result = subprocess.run(cmd, capture_output=True, text=True) + + # If branch-specific clone fails, retry without --branch + if result.returncode != 0 and branch: + print(f"[clone] Branch '{branch}' failed, retrying with default branch...", file=sys.stderr) + cmd_retry = ["git", "clone", f"--depth={depth}", "--single-branch", url, target] + result = subprocess.run(cmd_retry, capture_output=True, text=True) + + # Validate clone + if result.returncode != 0: + return { + "repo_id": repo, + "local_path": target, + "clone_success": False, + "error": result.stderr.strip(), + } + + if not os.path.isdir(target) or not os.listdir(target): + return { + "repo_id": repo, + "local_path": target, + "clone_success": False, + "error": "Clone directory is empty or missing", + } + + # Compute clone size + size_bytes = get_dir_size(target) + size_mb = size_bytes / (1024 * 1024) + file_count = sum(len(files) for _, _, files in os.walk(target)) + + print(f"[clone] Success: {target}", file=sys.stderr) + print(f"[clone] Size: {size_mb:.1f} MB, Files: {file_count}", file=sys.stderr) + + return { + "repo_id": repo, + "local_path": os.path.abspath(target), + "clone_success": True, + "error": None, + } + + +def main(): + parser = argparse.ArgumentParser(description="Shallow-clone a GitHub repo for local analysis.") + parser.add_argument("--repo", required=True, help="Repository in owner/name format") + parser.add_argument("--output-dir", required=True, help="Parent directory for the clone") + parser.add_argument("--depth", type=int, default=1, help="Clone depth (default: 1)") + parser.add_argument("--branch", default=None, help="Branch to clone (default: repo default branch)") + args = parser.parse_args() + + result = clone_repo(args.repo, args.output_dir, args.depth, args.branch) + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/skills/github-research/scripts/compare_repos.py b/skills/github-research/scripts/compare_repos.py new file mode 100755 index 0000000..68a1863 --- /dev/null +++ b/skills/github-research/scripts/compare_repos.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python3 +"""Generate comparison matrix across analyzed repositories. + +Reads analysis JSON files and dependency files to produce a structured comparison. + +Usage: + python compare_repos.py --input phase4_deep_dive/analyses/ --output comparison.json +""" + +import argparse +import json +import os +import sys +from itertools import combinations +from pathlib import Path + + +# --------------------------------------------------------------------------- +# I/O helpers +# --------------------------------------------------------------------------- + +def load_json(path: str) -> dict | None: + """Load a JSON file, returning None on failure.""" + try: + with open(path, encoding="utf-8") as f: + return json.load(f) + except (OSError, json.JSONDecodeError) as exc: + print(f" Warning: cannot load {path}: {exc}", file=sys.stderr) + return None + + +def find_analysis_files(analyses_dir: str) -> dict[str, dict[str, dict | None]]: + """Discover *_structure.json and *_deps.json files, grouped by repo name. + + Returns {repo_name: {"structure": , "deps": }}. + """ + repos: dict[str, dict[str, dict | None]] = {} + + for fname in sorted(os.listdir(analyses_dir)): + fpath = os.path.join(analyses_dir, fname) + if not os.path.isfile(fpath): + continue + + if fname.endswith("_structure.json"): + name = fname.removesuffix("_structure.json") + repos.setdefault(name, {"structure": None, "deps": None}) + repos[name]["structure"] = load_json(fpath) + elif fname.endswith("_deps.json"): + name = fname.removesuffix("_deps.json") + repos.setdefault(name, {"structure": None, "deps": None}) + repos[name]["deps"] = load_json(fpath) + + return repos + + +# --------------------------------------------------------------------------- +# Dimension extractors +# --------------------------------------------------------------------------- + +def _get(data: dict | None, *keys: str, default: object = None) -> object: + """Try multiple keys on a dict, returning the first truthy value.""" + if data is None: + return default + for key in keys: + val = data.get(key) + if val is not None and val != "" and val != [] and val != {}: + return val + return default + + +def extract_repo_id( + structure: dict | None, deps: dict | None, file_key: str, +) -> str: + """Best-effort extraction of owner/name repo_id.""" + rid = _get(structure, "repo_id", "repo") + if rid: + return str(rid) + rid = _get(deps, "repo_id", "repo") + if rid: + return str(rid) + return file_key + + +def extract_primary_language(structure: dict | None) -> str: + """Primary language from structure analysis.""" + lang = _get(structure, "primary_language", "language") + if lang and isinstance(lang, str): + return lang + # Fall back: pick language with highest LOC + langs = _get(structure, "languages", "language_stats", default={}) + if isinstance(langs, dict) and langs: + return max(langs, key=lambda k: langs[k] if isinstance(langs[k], (int, float)) else 0) + return "Unknown" + + +def extract_total_loc(structure: dict | None) -> int: + """Total lines of code.""" + loc = _get(structure, "total_loc", "loc", default=0) + if isinstance(loc, (int, float)): + return int(loc) + # Sum across languages + langs = _get(structure, "languages", "language_stats", default={}) + if isinstance(langs, dict): + return sum(v for v in langs.values() if isinstance(v, (int, float))) + return 0 + + +def extract_total_files(structure: dict | None) -> int: + """Total file count.""" + count = _get(structure, "total_files", "file_count", default=0) + if isinstance(count, (int, float)): + return int(count) + files = _get(structure, "files", "file_tree", default=[]) + if isinstance(files, list): + return len(files) + return 0 + + +def extract_stars(structure: dict | None) -> int: + """Star count.""" + return int(_get(structure, "stars", default=0) or 0) + + +def extract_ml_frameworks(deps: dict | None) -> list[str]: + """Detect ML frameworks from dependency data.""" + if deps is None: + return _get(deps, "ml_frameworks", default=[]) or [] # type: ignore[return-value] + + # If the deps file already has ml_frameworks, use it + existing = deps.get("ml_frameworks") + if existing and isinstance(existing, list): + return existing + + return [] + + +def extract_has_tests(structure: dict | None) -> bool: + """Whether the repo has test files.""" + if structure is None: + return False + # Direct field + test_files = _get(structure, "test_files", default=[]) + if isinstance(test_files, list) and test_files: + return True + # Check has_tests flag + if structure.get("has_tests"): + return True + # Scan file list for test indicators + files = _get(structure, "files", "file_tree", default=[]) + if isinstance(files, list): + for f in files: + name = f if isinstance(f, str) else ( + f.get("path", "") if isinstance(f, dict) else "" + ) + name_lower = name.lower() + if ("/test/" in name_lower or "/tests/" in name_lower + or name_lower.startswith("test") or "test_" in name_lower): + return True + # Check key directories + key_dirs = _get(structure, "key_dirs", "directories", default=[]) + if isinstance(key_dirs, list): + for d in key_dirs: + d_name = d if isinstance(d, str) else ( + d.get("name", "") if isinstance(d, dict) else "" + ) + if d_name.lower() in ("test", "tests", "testing", "test_suite"): + return True + return False + + +def extract_has_docker(structure: dict | None) -> bool: + """Whether the repo has Docker configuration.""" + if structure is None: + return False + config_files = _get(structure, "config_files", default=[]) + if isinstance(config_files, list): + for f in config_files: + name = f if isinstance(f, str) else ( + f.get("path", f.get("name", "")) if isinstance(f, dict) else "" + ) + name_lower = name.lower() + if name_lower in ("dockerfile", "docker-compose.yml", + "docker-compose.yaml", ".dockerignore"): + return True + # Also check files list + files = _get(structure, "files", "file_tree", default=[]) + if isinstance(files, list): + for f in files: + name = f if isinstance(f, str) else ( + f.get("path", "") if isinstance(f, dict) else "" + ) + basename = os.path.basename(name).lower() + if basename in ("dockerfile", "docker-compose.yml", + "docker-compose.yaml"): + return True + return False + + +def extract_license(structure: dict | None) -> str: + """License identifier.""" + return str(_get(structure, "license", default="") or "") + + +def extract_entry_points(structure: dict | None) -> list[str]: + """Main entry points.""" + entries = _get(structure, "entry_points", "main_files", default=[]) + if isinstance(entries, list): + return [str(e) for e in entries[:10]] + return [] + + +def extract_config_format(structure: dict | None) -> str: + """Dominant config file format.""" + config_files = _get(structure, "config_files", default=[]) + if not isinstance(config_files, list): + return "" + + format_counts: dict[str, int] = {} + ext_map: dict[str, str] = { + ".yaml": "yaml", ".yml": "yaml", + ".toml": "toml", + ".json": "json", + ".ini": "ini", ".cfg": "ini", + } + + for f in config_files: + name = f if isinstance(f, str) else ( + f.get("path", f.get("name", "")) if isinstance(f, dict) else "" + ) + _, ext = os.path.splitext(name.lower()) + fmt = ext_map.get(ext, "") + if fmt: + format_counts[fmt] = format_counts.get(fmt, 0) + 1 + + if not format_counts: + return "" + return max(format_counts, key=lambda k: format_counts[k]) + + +# --------------------------------------------------------------------------- +# Dependency overlap +# --------------------------------------------------------------------------- + +def collect_all_packages(deps: dict | None) -> set[str]: + """Collect a flat set of normalized package names from a deps record.""" + if deps is None: + return set() + + # Prefer the pre-computed all_packages field + all_pkgs = deps.get("all_packages") + if isinstance(all_pkgs, list) and all_pkgs: + return {p.lower() for p in all_pkgs} + + # Otherwise, gather from per-ecosystem sections + names: set[str] = set() + + # Python + py = deps.get("python") + if isinstance(py, dict): + for spec in py.get("requirements", []): + if isinstance(spec, str): + bare = spec.split(">=")[0].split("<=")[0].split("==")[0] + bare = bare.split("~=")[0].split("!=")[0].split("[")[0].strip() + if bare: + names.add(bare.lower()) + + # Node + node = deps.get("node") + if isinstance(node, dict): + for key in ("dependencies", "devDependencies"): + val = node.get(key, {}) + if isinstance(val, dict): + names.update(k.lower() for k in val) + + # Rust + rust = deps.get("rust") + if isinstance(rust, dict): + for key in ("dependencies",): + val = rust.get(key, {}) + if isinstance(val, dict): + names.update(k.lower() for k in val) + + # Go + go = deps.get("go") + if isinstance(go, dict): + for mod in go.get("modules", []): + if isinstance(mod, str): + parts = mod.rstrip("/").split("/") + names.add(parts[-1].lower()) + + return names + + +def compute_dependency_overlap( + repo_packages: dict[str, set[str]], + min_shared: int = 3, +) -> list[dict]: + """Compute shared packages for each pair of repos. + + Only includes pairs with at least min_shared common packages. + """ + overlaps: list[dict] = [] + repo_ids = sorted(repo_packages.keys()) + for a, b in combinations(repo_ids, 2): + shared = sorted(repo_packages[a] & repo_packages[b]) + if len(shared) >= min_shared: + overlaps.append({ + "repos": [a, b], + "shared": shared, + }) + return overlaps + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate comparison matrix across analyzed repositories", + ) + parser.add_argument( + "--input", required=True, + help="Directory containing *_structure.json and *_deps.json files", + ) + parser.add_argument( + "--output", required=True, + help="Output comparison JSON file path", + ) + parser.add_argument( + "--repo-db", default=None, + help="Optional repo_db.jsonl to enrich with stars/metadata", + ) + + args = parser.parse_args() + + if not os.path.isdir(args.input): + print(f"Error: analyses directory not found: {args.input}", + file=sys.stderr) + sys.exit(1) + + # Load optional repo_db for enrichment (stars, etc.) + repo_db_map: dict[str, dict] = {} + if args.repo_db and os.path.isfile(args.repo_db): + with open(args.repo_db, encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + rec = json.loads(line) + rid = rec.get("repo_id", "") + # Index by both full id and short name + repo_db_map[rid] = rec + repo_db_map[rid.split("/")[-1]] = rec + + # Discover and load analysis files + repos = find_analysis_files(args.input) + if not repos: + print("Error: no *_structure.json or *_deps.json files found", + file=sys.stderr) + sys.exit(1) + + print(f"Found analysis files for {len(repos)} repo(s)", file=sys.stderr) + + # --- Build dimension maps --- + repo_ids: list[str] = [] + dim_primary_language: dict[str, str] = {} + dim_total_loc: dict[str, int] = {} + dim_total_files: dict[str, int] = {} + dim_stars: dict[str, int] = {} + dim_ml_framework: dict[str, list[str]] = {} + dim_has_tests: dict[str, bool] = {} + dim_has_docker: dict[str, bool] = {} + dim_license: dict[str, str] = {} + dim_entry_points: dict[str, list[str]] = {} + dim_config_format: dict[str, str] = {} + + repo_packages: dict[str, set[str]] = {} + + for file_key, data in sorted(repos.items()): + structure = data.get("structure") + deps = data.get("deps") + rid = extract_repo_id(structure, deps, file_key) + repo_ids.append(rid) + + dim_primary_language[rid] = extract_primary_language(structure) + dim_total_loc[rid] = extract_total_loc(structure) + dim_total_files[rid] = extract_total_files(structure) + # Stars from structure.json, fallback to repo_db + stars = extract_stars(structure) + if stars == 0 and rid in repo_db_map: + stars = int(repo_db_map[rid].get("stars", 0)) + if stars == 0: + # Try matching by short name + short = rid.split("/")[-1] + if short in repo_db_map: + stars = int(repo_db_map[short].get("stars", 0)) + dim_stars[rid] = stars + dim_ml_framework[rid] = extract_ml_frameworks(deps) + dim_has_tests[rid] = extract_has_tests(structure) + dim_has_docker[rid] = extract_has_docker(structure) + dim_license[rid] = extract_license(structure) + dim_entry_points[rid] = extract_entry_points(structure) + dim_config_format[rid] = extract_config_format(structure) + + repo_packages[rid] = collect_all_packages(deps) + + # --- Compute pairwise dependency overlap --- + dependency_overlap = compute_dependency_overlap(repo_packages) + + # --- Assemble output --- + comparison: dict = { + "repos": repo_ids, + "dimensions": { + "primary_language": dim_primary_language, + "total_loc": dim_total_loc, + "total_files": dim_total_files, + "stars": dim_stars, + "ml_framework": dim_ml_framework, + "has_tests": dim_has_tests, + "has_docker": dim_has_docker, + "license": dim_license, + "entry_points": dim_entry_points, + "config_format": dim_config_format, + }, + "dependency_overlap": dependency_overlap, + } + + # --- Write output --- + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + json.dump(comparison, f, indent=2, ensure_ascii=False) + f.write("\n") + + # --- Summary to stderr --- + n_dims = len(comparison["dimensions"]) + lang_counts: dict[str, int] = {} + for lang in dim_primary_language.values(): + lang_counts[lang] = lang_counts.get(lang, 0) + 1 + lang_summary = ", ".join( + f"{l}({c})" for l, c in sorted(lang_counts.items(), key=lambda x: -x[1]) + ) + + fw_counts: dict[str, int] = {} + for fws in dim_ml_framework.values(): + for fw in fws: + fw_counts[fw] = fw_counts.get(fw, 0) + 1 + fw_summary = ", ".join( + f"{f}({c})" for f, c in sorted(fw_counts.items(), key=lambda x: -x[1]) + ) if fw_counts else "none" + + test_count = sum(1 for v in dim_has_tests.values() if v) + docker_count = sum(1 for v in dim_has_docker.values() if v) + + print(f"Compared {len(repo_ids)} repos across {n_dims} dimensions", + file=sys.stderr) + print(f" Languages: {lang_summary}", file=sys.stderr) + print(f" ML frameworks: {fw_summary}", file=sys.stderr) + print(f" With tests: {test_count}/{len(repo_ids)}", file=sys.stderr) + print(f" With Docker: {docker_count}/{len(repo_ids)}", file=sys.stderr) + print(f" Dependency overlaps: {len(dependency_overlap)} pair(s)", + file=sys.stderr) + print(f" Output: {args.output}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/skills/github-research/scripts/compile_github_report.py b/skills/github-research/scripts/compile_github_report.py new file mode 100755 index 0000000..69c1682 --- /dev/null +++ b/skills/github-research/scripts/compile_github_report.py @@ -0,0 +1,524 @@ +#!/usr/bin/env python3 +"""Assemble final GitHub research report from all phase outputs. + +Reads repo_db.jsonl and phase outputs to generate statistics and a comprehensive report. + +Usage: + python compile_github_report.py --topic-dir ./github-research-output/my-topic +""" + +import argparse +import json +import math +import os +import sys +from datetime import datetime, timezone +from pathlib import Path + + +# -- I/O helpers -------------------------------------------------------------- + +def load_jsonl(path: str) -> list[dict]: + """Load records from a JSONL file. Returns empty list if file missing.""" + records: list[dict] = [] + if not os.path.isfile(path): + return records + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + try: + records.append(json.loads(line)) + except json.JSONDecodeError: + continue + return records + + +def load_text(path: str) -> str: + """Load a text file. Returns empty string if file missing.""" + if not os.path.isfile(path): + return "" + try: + with open(path, encoding="utf-8") as f: + return f.read() + except OSError: + return "" + + +def format_table(headers: list[str], rows: list[list[str]]) -> str: + """Generate a markdown table from headers and row data.""" + if not headers: + return "" + lines: list[str] = [] + lines.append("| " + " | ".join(headers) + " |") + # Alignment: right-align columns that look numeric in first data row + seps: list[str] = [] + for i, h in enumerate(headers): + if rows and i < len(rows[0]): + try: + float(str(rows[0][i]).replace(",", "")) + seps.append("------:") + except (ValueError, IndexError): + seps.append("------") + else: + seps.append("------") + lines.append("| " + " | ".join(seps) + " |") + for row in rows: + cells = [str(c) for c in row] + while len(cells) < len(headers): + cells.append("") + lines.append("| " + " | ".join(cells) + " |") + return "\n".join(lines) + + +def truncate(text: str, max_chars: int = 200) -> str: + """Truncate text with ellipsis if it exceeds max_chars.""" + if len(text) <= max_chars: + return text + return text[: max_chars - 3].rstrip() + "..." + + +# -- Statistics --------------------------------------------------------------- + +def compute_stats(repos: list[dict]) -> dict: + """Compute all statistics from repo records.""" + if not repos: + return { + "total_discovered": 0, "total_filtered": 0, "total_analyzed": 0, + "sources": {}, "languages": {}, "stars": {}, + "activity": {}, "score_buckets": {}, "ml_frameworks": {}, "licenses": {}, + } + + now = datetime.now(timezone.utc) + + # -- By source -- + sources: dict[str, int] = {} + for rec in repos: + src = rec.get("source", "unknown") + sources[src] = sources.get(src, 0) + 1 + + # -- By language -- + languages: dict[str, int] = {} + for rec in repos: + lang = rec.get("language") or "Unknown" + languages[lang] = languages.get(lang, 0) + 1 + languages = dict(sorted(languages.items(), key=lambda x: -x[1])) + + # -- Stars distribution -- + star_values = [rec.get("stars", 0) or 0 for rec in repos] + star_values_sorted = sorted(star_values) + n = len(star_values_sorted) + stars_info: dict[str, float | int] = { + "min": star_values_sorted[0], + "max": star_values_sorted[-1], + "median": star_values_sorted[n // 2] if n % 2 else + (star_values_sorted[n // 2 - 1] + star_values_sorted[n // 2]) / 2, + "mean": round(sum(star_values) / n, 1), + "p25": star_values_sorted[max(n // 4 - 1, 0)], + "p75": star_values_sorted[min(3 * n // 4, n - 1)], + } + + # -- Activity -- + activity: dict[str, int] = {"last_90d": 0, "last_year": 0, "older": 0} + for rec in repos: + pushed_str = rec.get("pushed_at") or rec.get("updated_at") or "" + if not pushed_str: + activity["older"] += 1 + continue + try: + pushed = datetime.fromisoformat(pushed_str.replace("Z", "+00:00")) + days = (now - pushed).days + if days <= 90: + activity["last_90d"] += 1 + elif days <= 365: + activity["last_year"] += 1 + else: + activity["older"] += 1 + except (ValueError, TypeError): + activity["older"] += 1 + + # -- Composite score histogram -- + score_buckets: dict[str, int] = { + "0.0-0.2": 0, "0.2-0.4": 0, "0.4-0.6": 0, "0.6-0.8": 0, "0.8-1.0": 0, + } + for rec in repos: + cs = rec.get("composite_score", 0.0) or 0.0 + if cs >= 0.8: + score_buckets["0.8-1.0"] += 1 + elif cs >= 0.6: + score_buckets["0.6-0.8"] += 1 + elif cs >= 0.4: + score_buckets["0.4-0.6"] += 1 + elif cs >= 0.2: + score_buckets["0.2-0.4"] += 1 + else: + score_buckets["0.0-0.2"] += 1 + + # -- ML frameworks -- + ml_keywords = { + "torch": "PyTorch", "pytorch": "PyTorch", + "tensorflow": "TensorFlow", "tf-": "TensorFlow", + "jax": "JAX", "flax": "JAX", + "keras": "Keras", "sklearn": "scikit-learn", + "transformers": "HuggingFace Transformers", + "lightning": "PyTorch Lightning", "paddle": "PaddlePaddle", + } + ml_frameworks: dict[str, int] = {} + for rec in repos: + topics = rec.get("topics", []) or [] + desc = (rec.get("description") or "").lower() + readme = (rec.get("readme_excerpt") or "").lower() + combined = desc + " " + readme + " " + " ".join(str(t) for t in topics) + found_for_repo: set[str] = set() + for kw, framework in ml_keywords.items(): + if kw in combined: + found_for_repo.add(framework) + for fw in found_for_repo: + ml_frameworks[fw] = ml_frameworks.get(fw, 0) + 1 + ml_frameworks = dict(sorted(ml_frameworks.items(), key=lambda x: -x[1])) + + # -- Licenses -- + licenses: dict[str, int] = {} + for rec in repos: + lic = rec.get("license") or "None/Unknown" + if isinstance(lic, dict): + lic = lic.get("spdx_id") or lic.get("name") or "None/Unknown" + licenses[lic] = licenses.get(lic, 0) + 1 + licenses = dict(sorted(licenses.items(), key=lambda x: -x[1])) + + # -- Counts for filtered / analyzed -- + scored_repos = [r for r in repos if r.get("composite_score")] + analyzed_repos = [r for r in repos if r.get("tags") and "deep-dived" in r.get("tags", [])] + # Fallback: count repos with rank field as proxy for filtered + ranked_repos = [r for r in repos if r.get("rank")] + + return { + "total_discovered": len(repos), + "total_filtered": len(ranked_repos) if ranked_repos else len(scored_repos), + "total_analyzed": len(analyzed_repos), + "num_sources": len(sources), + "sources": sources, + "languages": languages, + "stars": stars_info, + "activity": activity, + "score_buckets": score_buckets, + "ml_frameworks": ml_frameworks, + "licenses": licenses, + } + + +# -- Report sections ---------------------------------------------------------- + +def build_header(topic_dir: str) -> str: + """Build the report header with topic name and date.""" + slug = os.path.basename(topic_dir.rstrip("/")) + topic = slug.replace("-", " ").title() + date = datetime.now().strftime("%Y-%m-%d %H:%M") + return f"# GitHub Research Report: {topic}\n\nGenerated: {date}\n" + + +def build_executive_summary(stats: dict, blueprint_summary: str) -> str: + """Build the executive summary section.""" + lines = ["## Executive Summary\n"] + if blueprint_summary: + lines.append(blueprint_summary.strip()) + else: + total = stats["total_discovered"] + ns = stats["num_sources"] + filtered = stats["total_filtered"] + analyzed = stats["total_analyzed"] + top_lang = next(iter(stats["languages"]), "N/A") + active = stats["activity"].get("last_90d", 0) + lines.append( + f"This report covers a systematic GitHub research effort that discovered " + f"**{total} repositories** from **{ns} source(s)**. " + f"After scoring and filtering, **{filtered}** repos were shortlisted " + f"and **{analyzed or 'several'}** received deep code analysis. " + f"The dominant language is **{top_lang}** and " + f"**{active}** repos have been active in the last 90 days." + ) + lines.append("") + return "\n".join(lines) + + +def build_discovery_stats(stats: dict) -> str: + """Build the discovery statistics section.""" + lines = ["## 1. Discovery Statistics\n"] + lines.append( + f"- Repos discovered: **{stats['total_discovered']}** " + f"from **{stats['num_sources']}** sources" + ) + lines.append( + f"- After filtering: **{stats['total_filtered']}** repos " + f"(top by composite score)" + ) + lines.append(f"- Deep-dived: **{stats['total_analyzed']}** repos\n") + + # Source table + if stats["sources"]: + lines.append("### Repos by Source\n") + rows = [[src, str(cnt)] for src, cnt in + sorted(stats["sources"].items(), key=lambda x: -x[1])] + lines.append(format_table(["Source", "Count"], rows)) + lines.append("") + + # Language table + if stats["languages"]: + lines.append("### Repos by Language\n") + rows = [[lang, str(cnt)] for lang, cnt in list(stats["languages"].items())[:15]] + lines.append(format_table(["Language", "Count"], rows)) + lines.append("") + + # Stars distribution + stars = stats.get("stars", {}) + if stars: + lines.append("### Stars Distribution\n") + lines.append(f"- Min: **{stars.get('min', 0)}** | " + f"Max: **{stars.get('max', 0)}** | " + f"Median: **{stars.get('median', 0)}** | " + f"Mean: **{stars.get('mean', 0)}**") + lines.append(f"- P25: **{stars.get('p25', 0)}** | P75: **{stars.get('p75', 0)}**\n") + + # Score buckets + if stats["score_buckets"]: + lines.append("### Composite Score Distribution\n") + rows = [[bucket, str(cnt)] for bucket, cnt in stats["score_buckets"].items()] + lines.append(format_table(["Score Range", "Count"], rows)) + lines.append("") + + # Activity + if stats["activity"]: + lines.append("### Activity\n") + act = stats["activity"] + lines.append(f"- Pushed in last 90 days: **{act.get('last_90d', 0)}**") + lines.append(f"- Pushed in last year: **{act.get('last_year', 0)}**") + lines.append(f"- Older / unknown: **{act.get('older', 0)}**\n") + + # ML frameworks + if stats["ml_frameworks"]: + lines.append("### ML Frameworks\n") + rows = [[fw, str(cnt)] for fw, cnt in stats["ml_frameworks"].items()] + lines.append(format_table(["Framework", "Repos"], rows)) + lines.append("") + + # Licenses + if stats["licenses"]: + lines.append("### License Distribution\n") + rows = [[lic, str(cnt)] for lic, cnt in list(stats["licenses"].items())[:10]] + lines.append(format_table(["License", "Count"], rows)) + lines.append("") + + return "\n".join(lines) + + +def build_top_repos(ranked_repos: list[dict]) -> str: + """Build the top repositories table from ranked_repos.jsonl.""" + lines = ["## 2. Top Repositories\n"] + if not ranked_repos: + lines.append("_No ranked repos available._\n") + return "\n".join(lines) + + headers = ["Rank", "Repo", "Stars", "Language", "Composite", "Description"] + rows: list[list[str]] = [] + for i, rec in enumerate(ranked_repos[:30], 1): + rid = rec.get("repo_id", "") + url = rec.get("url", f"https://github.com/{rid}") + stars = str(rec.get("stars", 0)) + lang = rec.get("language", "") + score = f"{(rec.get('composite_score', 0.0) or 0.0):.3f}" + desc = truncate((rec.get("description") or "").replace("|", "/"), 60) + rows.append([str(i), f"[{rid}]({url})", stars, lang, score, desc]) + + lines.append(format_table(headers, rows)) + lines.append("") + return "\n".join(lines) + + +def build_deep_analysis_summaries(analyses_dir: str) -> str: + """Build condensed summaries from per-repo analyses/*.md files.""" + lines = ["## 3. Deep Analysis Summaries\n"] + if not os.path.isdir(analyses_dir): + lines.append("_No deep analysis files found._\n") + return "\n".join(lines) + + analysis_files = sorted( + f for f in os.listdir(analyses_dir) if f.endswith("_analysis.md") + ) + if not analysis_files: + lines.append("_No per-repo analysis files found._\n") + return "\n".join(lines) + + for fname in analysis_files: + content = load_text(os.path.join(analyses_dir, fname)) + if not content: + continue + # Extract repo name from filename + repo_name = fname.removesuffix("_analysis.md") + lines.append(f"### {repo_name}\n") + # Include the first ~600 chars as a condensed summary + lines.append(truncate(content.strip(), 600)) + lines.append("") + + return "\n".join(lines) + + +def build_section(title: str, content: str) -> str: + """Wrap phase content in a section. Return empty section note if no content.""" + lines = [f"{title}\n"] + if content.strip(): + lines.append(content.strip()) + else: + heading_text = title.lstrip("#").strip() + lines.append(f"_{heading_text} not available._") + lines.append("") + return "\n".join(lines) + + +def build_appendix(repos: list[dict]) -> str: + """Build the full repo database appendix table.""" + lines = ["## Appendix A: Full Repo Database\n"] + if not repos: + lines.append("_No repos in database._\n") + return "\n".join(lines) + + sorted_repos = sorted(repos, key=lambda r: -(r.get("composite_score", 0) or 0)) + headers = ["Repo", "Stars", "Language", "Score", "Source", "Description"] + rows: list[list[str]] = [] + for rec in sorted_repos: + rid = rec.get("repo_id", "") + url = rec.get("url", f"https://github.com/{rid}") + stars = str(rec.get("stars", 0)) + lang = rec.get("language", "") + score = f"{(rec.get('composite_score', 0.0) or 0.0):.3f}" + src = rec.get("source", "") + desc = truncate((rec.get("description") or "").replace("|", "/"), 60) + rows.append([f"[{rid}]({url})", stars, lang, score, src, desc]) + + lines.append(format_table(headers, rows)) + lines.append("") + return "\n".join(lines) + + +def build_methodology() -> str: + """Build the methodology appendix.""" + return """## Appendix B: Methodology + +- **Phase 1: Intake** -- Parse deep-research output for GitHub URLs, paper references, and search keywords +- **Phase 2: Discovery** -- Multi-source search (GitHub repo search, Papers With Code, GitHub code search, direct URLs) +- **Phase 3: Scoring & Filtering** -- Composite score = relevance x 0.4 + quality x 0.35 + activity x 0.25; select top repos +- **Phase 4: Deep Dive** -- Shallow clone + deep code reading of top repos (architecture, algorithms, quality, reusability) +- **Phase 5: Cross-Repo Analysis** -- Comparison matrix, technique-to-code mapping, gap analysis +- **Phase 6: Integration Blueprint** -- Recommended architecture, reuse catalog, license compatibility, effort estimates +""" + + +# -- Main --------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser( + description="Assemble final GitHub research report from all phase outputs." + ) + parser.add_argument( + "--topic-dir", required=True, + help="Path to the github-research-output/{slug}/ directory", + ) + args = parser.parse_args() + + topic_dir = args.topic_dir.rstrip("/") + if not os.path.isdir(topic_dir): + print(f"[error] topic directory not found: {topic_dir}", file=sys.stderr) + sys.exit(1) + + print(f"[compile] loading data from {topic_dir}", file=sys.stderr) + + # -- Load repo database -- + repo_db_path = os.path.join(topic_dir, "repo_db.jsonl") + all_repos = load_jsonl(repo_db_path) + print(f" repo_db.jsonl: {len(all_repos)} repos", file=sys.stderr) + + # -- Load ranked repos -- + ranked_path = os.path.join(topic_dir, "phase3_filtering", "ranked_repos.jsonl") + ranked_repos = load_jsonl(ranked_path) + print(f" ranked_repos.jsonl: {len(ranked_repos)} repos", file=sys.stderr) + + # -- Load phase text outputs -- + phase_paths = { + "intake_summary": os.path.join(topic_dir, "phase1_intake", "intake_summary.md"), + "discovery_log": os.path.join(topic_dir, "phase2_discovery", "discovery_log.md"), + "filtering_report": os.path.join(topic_dir, "phase3_filtering", "filtering_report.md"), + "deep_dive_summary": os.path.join(topic_dir, "phase4_deep_dive", "deep_dive_summary.md"), + "comparison_matrix": os.path.join(topic_dir, "phase5_analysis", "comparison_matrix.md"), + "technique_map": os.path.join(topic_dir, "phase5_analysis", "technique_map.md"), + "analysis_report": os.path.join(topic_dir, "phase5_analysis", "analysis_report.md"), + "integration_plan": os.path.join(topic_dir, "phase6_blueprint", "integration_plan.md"), + "reuse_catalog": os.path.join(topic_dir, "phase6_blueprint", "reuse_catalog.md"), + "blueprint_summary": os.path.join(topic_dir, "phase6_blueprint", "blueprint_summary.md"), + } + + phases: dict[str, str] = {} + for key, path in phase_paths.items(): + phases[key] = load_text(path) + status = "loaded" if phases[key] else "not found" + print(f" {key}: {status}", file=sys.stderr) + + analyses_dir = os.path.join(topic_dir, "phase4_deep_dive", "analyses") + + # -- Compute statistics -- + stats = compute_stats(all_repos) + # Override total_filtered with actual ranked count if available + if ranked_repos: + stats["total_filtered"] = len(ranked_repos) + # Count analysis files for total_analyzed + if os.path.isdir(analyses_dir): + analysis_count = sum( + 1 for f in os.listdir(analyses_dir) if f.endswith("_analysis.md") + ) + if analysis_count > 0: + stats["total_analyzed"] = analysis_count + + print(f"[compile] assembling report...", file=sys.stderr) + + # -- Assemble report -- + sections: list[str] = [ + build_header(topic_dir), + "---\n", + build_executive_summary(stats, phases["blueprint_summary"]), + build_discovery_stats(stats), + build_top_repos(ranked_repos if ranked_repos else all_repos), + build_deep_analysis_summaries(analyses_dir), + build_section("## 4. Cross-Repository Comparison", phases["comparison_matrix"]), + build_section("## 5. Technique-to-Code Mapping", phases["technique_map"]), + build_section("## 6. Integration Blueprint", phases["integration_plan"]), + build_section("## 7. Reusable Components", phases["reuse_catalog"]), + build_appendix(all_repos), + build_methodology(), + ] + + report = "\n".join(sections) + + # -- Write outputs -- + output_dir = os.path.join(topic_dir, "phase6_blueprint") + os.makedirs(output_dir, exist_ok=True) + + report_path = os.path.join(output_dir, "final_report.md") + with open(report_path, "w", encoding="utf-8") as f: + f.write(report) + + stats_path = os.path.join(output_dir, "stats.json") + with open(stats_path, "w", encoding="utf-8") as f: + json.dump(stats, f, indent=2, ensure_ascii=False) + + # -- Summary -- + section_count = sum(1 for s in sections if s.strip().startswith("##")) + print( + f"\nReport compiled: {section_count} sections, " + f"{stats['total_discovered']} repos, " + f"{stats['total_analyzed']} analyzed", + file=sys.stderr, + ) + print(f" -> {report_path}", file=sys.stderr) + print(f" -> {stats_path}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/skills/github-research/scripts/extract_dependencies.py b/skills/github-research/scripts/extract_dependencies.py new file mode 100755 index 0000000..bc69e68 --- /dev/null +++ b/skills/github-research/scripts/extract_dependencies.py @@ -0,0 +1,543 @@ +#!/usr/bin/env python3 +"""Extract and parse dependency files from a cloned repository. + +Identifies Python, Node, Rust, Go, and system dependencies. +Self-contained: stdlib only (regex-based parsing, no toml/yaml libraries). + +Usage: + python extract_dependencies.py --repo-dir ./repos/owner_name --output deps.json +""" + +import argparse +import json +import os +import re +import sys +from pathlib import Path + + +# --------------------------------------------------------------------------- +# ML framework detection mapping +# --------------------------------------------------------------------------- + +ML_FRAMEWORK_MAP: dict[str, str] = { + # PyTorch ecosystem + "torch": "pytorch", "torchvision": "pytorch", "torchaudio": "pytorch", + "pytorch-lightning": "pytorch", "lightning": "pytorch", + # TensorFlow ecosystem + "tensorflow": "tensorflow", "tensorflow-gpu": "tensorflow", + "tf-nightly": "tensorflow", "tf-estimator-nightly": "tensorflow", + "keras": "tensorflow", + # JAX ecosystem + "jax": "jax", "jaxlib": "jax", "flax": "jax", "optax": "jax", + # HuggingFace ecosystem + "transformers": "huggingface", "datasets": "huggingface", + "tokenizers": "huggingface", "accelerate": "huggingface", + "diffusers": "huggingface", "peft": "huggingface", + # scikit-learn + "scikit-learn": "scikit-learn", "sklearn": "scikit-learn", + # Other ML tools + "onnx": "onnx", "onnxruntime": "onnx", + "tensorrt": "tensorrt", "triton": "triton", + "deepspeed": "deepspeed", "fairscale": "fairscale", +} + +SYSTEM_KEYWORDS: list[str] = [ + "cuda", "cudnn", "nvidia", "ffmpeg", "libsndfile", "sox", + "opencv", "libgl", "libglib", "cmake", "gcc", "g++", +] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _read_text(path: Path) -> str | None: + """Read a file as UTF-8, returning None on failure.""" + try: + return path.read_text(encoding="utf-8", errors="replace") + except OSError: + return None + + +def _strip_version(spec: str) -> str: + """Extract bare package name from a version specifier string.""" + return re.split(r"[><=!~;\[\s@]", spec.strip())[0].strip().lower() + + +def _extract_quoted_strings(text: str) -> list[str]: + """Extract all single- or double-quoted strings from text.""" + return re.findall(r"""['"]([^'"]+)['"]""", text) + + +# --------------------------------------------------------------------------- +# Python: requirements.txt +# --------------------------------------------------------------------------- + +def parse_requirements_txt(repo_dir: Path) -> dict | None: + """Parse requirements*.txt files.""" + candidates = [ + "requirements.txt", "requirements-dev.txt", "requirements_dev.txt", + "requirements-test.txt", "requirements_test.txt", + "requirements/base.txt", "requirements/main.txt", + ] + all_specs: list[str] = [] + source_files: list[str] = [] + + for name in candidates: + path = repo_dir / name + if not path.is_file(): + continue + text = _read_text(path) + if text is None: + continue + source_files.append(name) + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#") or line.startswith("-"): + continue + all_specs.append(line) + + if not all_specs: + return None + # Deduplicate while preserving order + seen: set[str] = set() + deduped: list[str] = [] + for spec in all_specs: + if spec not in seen: + seen.add(spec) + deduped.append(spec) + return {"requirements": deduped, "source_file": ", ".join(source_files)} + + +# --------------------------------------------------------------------------- +# Python: setup.py +# --------------------------------------------------------------------------- + +def parse_setup_py(repo_dir: Path) -> dict | None: + """Parse setup.py for install_requires and extras_require.""" + path = repo_dir / "setup.py" + if not path.is_file(): + return None + text = _read_text(path) + if text is None: + return None + + specs: list[str] = [] + + # install_requires = [...] + match = re.search(r"install_requires\s*=\s*\[([^\]]+)\]", text, re.DOTALL) + if match: + specs.extend(_extract_quoted_strings(match.group(1))) + + # extras_require = { ... } + match = re.search(r"extras_require\s*=\s*\{([^}]+)\}", text, re.DOTALL) + if match: + specs.extend(_extract_quoted_strings(match.group(1))) + + if not specs: + return None + return {"requirements": specs, "source_file": "setup.py"} + + +# --------------------------------------------------------------------------- +# Python: pyproject.toml +# --------------------------------------------------------------------------- + +def parse_pyproject_toml(repo_dir: Path) -> dict | None: + """Parse pyproject.toml for dependencies (regex-based, no toml lib).""" + path = repo_dir / "pyproject.toml" + if not path.is_file(): + return None + text = _read_text(path) + if text is None: + return None + + specs: list[str] = [] + + # [project] dependencies = [...] + match = re.search(r"dependencies\s*=\s*\[([^\]]*)\]", text, re.DOTALL) + if match: + specs.extend(_extract_quoted_strings(match.group(1))) + + # [project.optional-dependencies] section -- grab all arrays + section_match = re.search( + r"\[project\.optional-dependencies\](.*?)(?=\n\[|\Z)", text, re.DOTALL, + ) + if section_match: + for array_match in re.finditer(r"=\s*\[([^\]]*)\]", section_match.group(1)): + specs.extend(_extract_quoted_strings(array_match.group(1))) + + if not specs: + return None + return {"requirements": specs, "source_file": "pyproject.toml"} + + +# --------------------------------------------------------------------------- +# Python: environment.yml / environment.yaml +# --------------------------------------------------------------------------- + +def parse_environment_yml(repo_dir: Path) -> dict | None: + """Parse environment.yml / environment.yaml for conda dependencies.""" + source_name: str | None = None + for name in ("environment.yml", "environment.yaml"): + path = repo_dir / name + if path.is_file(): + source_name = name + break + if source_name is None: + return None + + text = _read_text(repo_dir / source_name) + if text is None: + return None + + specs: list[str] = [] + in_deps = False + in_pip = False + + for line in text.splitlines(): + stripped = line.strip() + + # Detect dependencies: section + if re.match(r"^dependencies\s*:", stripped): + in_deps = True + continue + + if not in_deps: + continue + + # End of section on non-indented, non-list line + if stripped and not stripped.startswith("-") and not stripped.startswith("#"): + if not in_pip: + in_deps = False + continue + + # pip sub-section + if stripped == "- pip:" or stripped == "- pip": + in_pip = True + continue + if in_pip: + if stripped.startswith("- "): + pkg = stripped[2:].strip() + if pkg: + specs.append(pkg) + continue + if not stripped.startswith("-") and stripped: + in_pip = False + + if stripped.startswith("- "): + pkg = stripped[2:].strip() + if pkg and pkg not in ("pip", "pip:"): + specs.append(pkg) + + if not specs: + return None + return {"requirements": specs, "source_file": source_name} + + +# --------------------------------------------------------------------------- +# Node: package.json +# --------------------------------------------------------------------------- + +def parse_package_json(repo_dir: Path) -> dict | None: + """Parse package.json for dependencies and devDependencies.""" + path = repo_dir / "package.json" + if not path.is_file(): + return None + text = _read_text(path) + if text is None: + return None + try: + data = json.loads(text) + except json.JSONDecodeError: + return None + + deps = data.get("dependencies") or {} + dev_deps = data.get("devDependencies") or {} + if not deps and not dev_deps: + return None + + result: dict = {"source_file": "package.json"} + if deps: + result["dependencies"] = deps + if dev_deps: + result["devDependencies"] = dev_deps + return result + + +# --------------------------------------------------------------------------- +# Rust: Cargo.toml +# --------------------------------------------------------------------------- + +def parse_cargo_toml(repo_dir: Path) -> dict | None: + """Parse Cargo.toml for [dependencies] section.""" + path = repo_dir / "Cargo.toml" + if not path.is_file(): + return None + text = _read_text(path) + if text is None: + return None + + deps: dict[str, str] = {} + match = re.search( + r"\[dependencies\](.*?)(?=\n\[|\Z)", text, re.DOTALL, + ) + if match: + for line in match.group(1).splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + # name = "version" + m = re.match(r'^(\S+)\s*=\s*"([^"]*)"', line) + if m: + deps[m.group(1)] = m.group(2) + else: + # name = { version = "...", ... } + m = re.match(r"^(\S+)\s*=\s*\{", line) + if m: + ver = re.search(r'version\s*=\s*"([^"]*)"', line) + deps[m.group(1)] = ver.group(1) if ver else "*" + + if not deps: + return None + return {"dependencies": deps, "source_file": "Cargo.toml"} + + +# --------------------------------------------------------------------------- +# Go: go.mod +# --------------------------------------------------------------------------- + +def parse_go_mod(repo_dir: Path) -> dict | None: + """Parse go.mod for require block.""" + path = repo_dir / "go.mod" + if not path.is_file(): + return None + text = _read_text(path) + if text is None: + return None + + modules: list[str] = [] + in_require = False + + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("require ("): + in_require = True + continue + if in_require: + if stripped == ")": + in_require = False + continue + parts = stripped.split() + if parts and not parts[0].startswith("//"): + modules.append(parts[0]) + elif stripped.startswith("require "): + parts = stripped.split() + if len(parts) >= 2: + modules.append(parts[1]) + + if not modules: + return None + return {"modules": modules, "source_file": "go.mod"} + + +# --------------------------------------------------------------------------- +# System dependency detection +# --------------------------------------------------------------------------- + +def detect_system_deps(repo_dir: Path) -> list[str]: + """Detect system-level dependencies from Dockerfile, README, setup.py.""" + found: set[str] = set() + scan_files = [ + "Dockerfile", "docker-compose.yml", "docker-compose.yaml", + "README.md", "README.rst", "readme.md", + "setup.py", "Makefile", "CMakeLists.txt", + ] + + for name in scan_files: + path = repo_dir / name + if not path.is_file(): + continue + text = _read_text(path) + if text is None: + continue + text_lower = text.lower() + + # Check for known system keywords + for kw in SYSTEM_KEYWORDS: + if kw in text_lower: + found.add(kw) + + # Extract apt-get install packages from Dockerfiles + if name.lower().startswith("dockerfile") or name.lower() == "dockerfile": + for m in re.finditer(r"apt-get\s+install[^&\n]*", text): + tokens = m.group(0).split() + for tok in tokens: + tok = tok.strip().rstrip("\\") + if tok and not tok.startswith("-") and tok not in ( + "apt-get", "install", "&&", "||", "RUN", + ): + found.add(tok) + + return sorted(found) + + +# --------------------------------------------------------------------------- +# ML framework detection +# --------------------------------------------------------------------------- + +def detect_ml_frameworks(all_packages: list[str]) -> list[str]: + """Identify ML frameworks from the combined package list.""" + frameworks: set[str] = set() + for pkg in all_packages: + pkg_lower = pkg.lower() + if pkg_lower in ML_FRAMEWORK_MAP: + frameworks.add(ML_FRAMEWORK_MAP[pkg_lower]) + # Handle tf-* prefix patterns + elif pkg_lower.startswith("tf-") or pkg_lower.startswith("tensorflow-"): + frameworks.add("tensorflow") + return sorted(frameworks) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser( + description="Extract and parse dependency files from a cloned repository", + ) + parser.add_argument( + "--repo-dir", required=True, + help="Path to cloned repository directory", + ) + parser.add_argument( + "--output", required=True, + help="Output JSON file path", + ) + parser.add_argument( + "--repo-id", default=None, + help="Repository identifier (owner/name); inferred from dir name if omitted", + ) + + args = parser.parse_args() + + repo_dir = Path(args.repo_dir).resolve() + if not repo_dir.is_dir(): + print(f"Error: repo directory not found: {repo_dir}", file=sys.stderr) + sys.exit(1) + + # Infer repo_id from directory name if not provided + repo_id: str = args.repo_id or "" + if not repo_id: + dir_name = repo_dir.name + repo_id = dir_name.replace("_", "/", 1) if "_" in dir_name else dir_name + + print(f"Extracting dependencies from {repo_dir} ...", file=sys.stderr) + + # --- Python: merge results from all parsers --- + python_info: dict | None = None + for parser_fn in (parse_requirements_txt, parse_setup_py, + parse_pyproject_toml, parse_environment_yml): + result = parser_fn(repo_dir) + if result is not None: + if python_info is None: + python_info = result + else: + # Merge: append specs, combine source_file + existing = set(python_info["requirements"]) + for spec in result["requirements"]: + if spec not in existing: + python_info["requirements"].append(spec) + existing.add(spec) + python_info["source_file"] += f", {result['source_file']}" + if python_info: + print(f" Python: {len(python_info['requirements'])} packages " + f"from {python_info['source_file']}", file=sys.stderr) + + # --- Node --- + node_info = parse_package_json(repo_dir) + if node_info: + n = len(node_info.get("dependencies", {})) + len(node_info.get("devDependencies", {})) + print(f" Node: {n} packages from {node_info['source_file']}", file=sys.stderr) + + # --- Rust --- + rust_info = parse_cargo_toml(repo_dir) + if rust_info: + print(f" Rust: {len(rust_info['dependencies'])} crates " + f"from {rust_info['source_file']}", file=sys.stderr) + + # --- Go --- + go_info = parse_go_mod(repo_dir) + if go_info: + print(f" Go: {len(go_info['modules'])} modules " + f"from {go_info['source_file']}", file=sys.stderr) + + # --- Collect all package names (flat, normalized) --- + all_packages: set[str] = set() + + if python_info: + for spec in python_info["requirements"]: + name = _strip_version(spec) + if name: + all_packages.add(name) + + if node_info: + for key in ("dependencies", "devDependencies"): + for pkg_name in node_info.get(key, {}): + all_packages.add(pkg_name.lower()) + + if rust_info: + for crate_name in rust_info["dependencies"]: + all_packages.add(crate_name.lower()) + + if go_info: + for mod_path in go_info["modules"]: + # Use last path segment as package name + parts = mod_path.rstrip("/").split("/") + all_packages.add(parts[-1].lower()) + + all_packages_sorted = sorted(all_packages) + + # --- System dependencies --- + system_deps = detect_system_deps(repo_dir) + if system_deps: + print(f" System: {len(system_deps)} system dependencies detected", + file=sys.stderr) + + # --- ML frameworks --- + ml_frameworks = detect_ml_frameworks(all_packages_sorted) + if ml_frameworks: + print(f" ML frameworks: {', '.join(ml_frameworks)}", file=sys.stderr) + + # --- Build output --- + output: dict = { + "repo_id": repo_id, + "python": python_info, + "node": node_info, + "rust": rust_info, + "go": go_info, + "system": system_deps, + "ml_frameworks": ml_frameworks, + "all_packages": all_packages_sorted, + "overlap_with": {}, + } + + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + json.dump(output, f, indent=2, ensure_ascii=False) + f.write("\n") + + ecosystems = sum(1 for x in (python_info, node_info, rust_info, go_info) + if x is not None) + print( + f"Extracted {len(all_packages_sorted)} packages across " + f"{ecosystems} ecosystem(s) -> {args.output}", + file=sys.stderr, + ) + + +if __name__ == "__main__": + main() diff --git a/skills/github-research/scripts/extract_research_refs.py b/skills/github-research/scripts/extract_research_refs.py new file mode 100755 index 0000000..82daa78 --- /dev/null +++ b/skills/github-research/scripts/extract_research_refs.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +"""Extract GitHub URLs, paper refs, and keywords from deep-research output. + +Scans markdown files for GitHub URLs (cleaning sub-paths to extract owner/name), +parses paper_db.jsonl for paper metadata, extracts keywords from titles and tags, +and outputs deduplicated JSONL records. + +Usage: + python extract_research_refs.py --research-dir ./deep-research-output/topic/ --output refs.jsonl +""" + +import argparse +import json +import re +import sys +from collections import Counter +from pathlib import Path + +GITHUB_URL_RE = re.compile(r'https://github\.com/[^\s\)\]\>\"\']+') + +# Sub-paths to strip to get base repo URL +REPO_SUBPATH_RE = re.compile( + r"/(?:tree|blob|issues|pull|releases|wiki|actions|commits|compare|archive|raw|discussions|pkgs|packages|security|stargazers|network)(?:/.*)?$" +) + +# Non-repo top-level GitHub pages +NON_REPO_OWNERS = frozenset({ + "topics", "explore", "trending", "settings", + "organizations", "sponsors", "features", "marketplace", + "notifications", "new", "login", "signup", "pricing", +}) + +# Common English stopwords for keyword filtering +STOPWORDS = frozenset({ + "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", + "of", "with", "by", "from", "is", "are", "was", "were", "be", "been", + "being", "have", "has", "had", "do", "does", "did", "will", "would", + "could", "should", "may", "might", "can", "shall", "not", "no", "nor", + "so", "if", "then", "than", "that", "this", "these", "those", "it", + "its", "we", "our", "they", "their", "them", "you", "your", "he", + "she", "his", "her", "as", "up", "out", "about", "into", "over", + "after", "before", "between", "through", "during", "each", "every", + "all", "both", "few", "more", "most", "other", "some", "such", "only", + "same", "also", "how", "what", "which", "who", "when", "where", "why", + "very", "just", "because", "while", "using", "via", "based", "new", + "one", "two", "first", "use", "used", "paper", "method", "approach", + "proposed", "show", "results", "model", "models", +}) + + +def clean_github_url(raw_url: str) -> tuple[str, str]: + """Clean a raw GitHub URL and extract owner/name repo_id. + + Returns (clean_url, repo_id) or ("", "") if invalid. + """ + url = raw_url.rstrip(".,;:!?)]\">'/") + url = url.rstrip("/") + + # Remove sub-paths (tree, blob, issues, etc.) to get base repo URL + url = REPO_SUBPATH_RE.sub("", url) + + # Extract owner/name + path = url.replace("https://github.com/", "") + parts = path.split("/") + if len(parts) >= 2 and parts[0] and parts[1]: + owner, name = parts[0], parts[1] + # Filter out non-repo pages + if owner.lower() in NON_REPO_OWNERS: + return "", "" + # Strip .git suffix + name = name.removesuffix(".git") + repo_id = f"{owner}/{name}" + clean_url = f"https://github.com/{repo_id}" + return clean_url, repo_id + return "", "" + + +def scan_md_files_for_urls(research_dir: Path) -> list[dict]: + """Scan all .md files recursively for GitHub URLs.""" + results: list[dict] = [] + seen_repos: set[str] = set() + + for md_file in sorted(research_dir.rglob("*.md")): + try: + text = md_file.read_text(encoding="utf-8", errors="replace") + except OSError as e: + print(f"[warn] cannot read {md_file}: {e}", file=sys.stderr) + continue + + rel_path = str(md_file.relative_to(research_dir)) + + for match in GITHUB_URL_RE.finditer(text): + raw_url = match.group(0) + clean_url, repo_id = clean_github_url(raw_url) + if not repo_id: + continue + + # Extract surrounding context (the line containing the URL) + line_start = text.rfind("\n", 0, match.start()) + 1 + line_end = text.find("\n", match.end()) + if line_end == -1: + line_end = len(text) + context = text[line_start:line_end].strip() + + # Deduplicate by repo_id (not raw URL) to merge /tree/... variants + if repo_id in seen_repos: + continue + seen_repos.add(repo_id) + + results.append({ + "type": "github_url", + "url": clean_url, + "repo_id": repo_id, + "source_file": rel_path, + "context": context[:200], + }) + + return results + + +def parse_paper_db(research_dir: Path) -> list[dict]: + """Parse paper_db.jsonl for paper titles, arxiv IDs, tags.""" + results: list[dict] = [] + paper_db = research_dir / "paper_db.jsonl" + if not paper_db.exists(): + print("[info] paper_db.jsonl not found, skipping paper extraction", file=sys.stderr) + return results + + try: + for line_num, line in enumerate(paper_db.read_text(encoding="utf-8").splitlines(), 1): + line = line.strip() + if not line: + continue + try: + paper = json.loads(line) + except json.JSONDecodeError as e: + print(f"[warn] paper_db.jsonl line {line_num}: {e}", file=sys.stderr) + continue + + title = paper.get("title", "") + arxiv_id = paper.get("arxiv_id", paper.get("id", "")) + tags = paper.get("tags", paper.get("keywords", paper.get("categories", []))) + paper_id = paper.get("paperId", arxiv_id) + + results.append({ + "type": "paper", + "title": title, + "arxiv_id": arxiv_id, + "tags": tags if isinstance(tags, list) else [], + "paper_id": paper_id, + }) + except OSError as e: + print(f"[warn] cannot read paper_db.jsonl: {e}", file=sys.stderr) + + return results + + +def extract_keywords(papers: list[dict]) -> list[dict]: + """Extract search keywords from paper titles and tags.""" + tag_counter: Counter[str] = Counter() + title_term_counter: Counter[str] = Counter() + + for paper in papers: + # Count tags + for tag in paper.get("tags", []): + if isinstance(tag, str): + tag_lower = tag.strip().lower() + if tag_lower and len(tag_lower) > 2: + tag_counter[tag_lower] += 1 + + # Extract terms from titles + title = paper.get("title", "") + if not title: + continue + words = re.findall(r"[A-Za-z][-A-Za-z]+", title) + filtered = [w.lower() for w in words + if w.lower() not in STOPWORDS and len(w) > 2] + + # Unigrams + for w in filtered: + title_term_counter[w] += 1 + + # Bigrams (multi-word technical terms) + for i in range(len(filtered) - 1): + bigram = f"{filtered[i]} {filtered[i + 1]}" + title_term_counter[bigram] += 1 + + keywords: list[dict] = [] + + # Tags with frequency + for tag, freq in tag_counter.most_common(): + keywords.append({ + "type": "keyword", + "value": tag, + "frequency": freq, + "source": "paper_tags", + }) + + # Title terms (2+ occurrences, or bigrams with 1+ occurrence) + for term, freq in title_term_counter.most_common(): + if freq >= 2 or (freq >= 1 and " " in term): + # Skip if already covered by tags + if term not in tag_counter: + keywords.append({ + "type": "keyword", + "value": term, + "frequency": freq, + "source": "paper_titles", + }) + + return keywords + + +def extract_synthesis_themes(research_dir: Path) -> list[dict]: + """Extract research themes from synthesis/report markdown headings.""" + results: list[dict] = [] + + # Check several possible locations for synthesis/report files + candidates = [ + research_dir / "phase5_synthesis" / "synthesis.md", + research_dir / "phase6_report" / "report.md", + research_dir / "phase3_deep_dive" / "deep_dive.md", + ] + + for filepath in candidates: + if not filepath.exists(): + continue + try: + text = filepath.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + + for match in re.finditer(r'^#{1,3}\s+(.+)$', text, re.MULTILINE): + heading = match.group(1).strip() + if len(heading) > 3: + results.append({ + "type": "keyword", + "value": heading.lower(), + "frequency": 1, + "source": "paper_titles", + }) + + return results + + +def main(): + parser = argparse.ArgumentParser( + description="Extract GitHub URLs, paper refs, and keywords from deep-research output." + ) + parser.add_argument("--research-dir", required=True, + help="Path to deep-research output directory") + parser.add_argument("--output", required=True, + help="Output JSONL file path") + args = parser.parse_args() + + research_dir = Path(args.research_dir).resolve() + if not research_dir.is_dir(): + print(f"[error] research dir not found: {research_dir}", file=sys.stderr) + sys.exit(1) + + print(f"[info] scanning {research_dir} ...", file=sys.stderr) + + # 1. Extract GitHub URLs from all markdown files + github_refs = scan_md_files_for_urls(research_dir) + print(f" GitHub URLs: {len(github_refs)}", file=sys.stderr) + + # 2. Parse paper_db.jsonl + papers = parse_paper_db(research_dir) + print(f" Papers: {len(papers)}", file=sys.stderr) + + # 3. Extract keywords from papers + keywords = extract_keywords(papers) + + # 4. Extract themes from synthesis/report headings + themes = extract_synthesis_themes(research_dir) + keywords.extend(themes) + + # Deduplicate keywords by value + seen_kw: set[str] = set() + deduped_kw: list[dict] = [] + for kw in keywords: + if kw["value"] not in seen_kw: + seen_kw.add(kw["value"]) + deduped_kw.append(kw) + keywords = deduped_kw + + print(f" Keywords: {len(keywords)}", file=sys.stderr) + + # Write output + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + for record in github_refs: + f.write(json.dumps(record, ensure_ascii=False) + "\n") + for record in papers: + f.write(json.dumps(record, ensure_ascii=False) + "\n") + for record in keywords: + f.write(json.dumps(record, ensure_ascii=False) + "\n") + + # Summary stats + unique_repos = len({r["repo_id"] for r in github_refs}) + print( + f"Extracted {unique_repos} GitHub URLs, " + f"{len(papers)} papers, {len(keywords)} keywords " + f"-> {output_path}", + file=sys.stderr, + ) + + +if __name__ == "__main__": + main() diff --git a/skills/github-research/scripts/find_implementations.py b/skills/github-research/scripts/find_implementations.py new file mode 100755 index 0000000..500274f --- /dev/null +++ b/skills/github-research/scripts/find_implementations.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +"""Search cloned repo for specific implementations (classes, functions, algorithms). + +Usage: + python find_implementations.py --repo-dir ./repos/owner_name --patterns "class Transformer" "def train" --output matches.jsonl + python find_implementations.py --repo-dir ./repos/owner_name --keywords "attention" "embedding" --output matches.jsonl +""" + +import argparse +import json +import os +import re +import sys +from pathlib import Path + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +SKIP_DIRS: frozenset[str] = frozenset({ + ".git", "node_modules", "__pycache__", ".venv", "venv", + ".tox", ".mypy_cache", ".pytest_cache", ".eggs", "dist", "build", + ".next", ".nuxt", +}) + +TEST_INDICATORS: tuple[str, ...] = ( + "/test/", "/tests/", "test_", "_test.py", ".test.", ".spec.", +) + +MAX_FILE_SIZE: int = 1_048_576 # 1 MB + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def is_binary(filepath: Path) -> bool: + """Check if a file is likely binary by reading the first 8 KB.""" + try: + with open(filepath, "rb") as f: + chunk = f.read(8192) + return b"\x00" in chunk + except OSError: + return True + + +def is_test_path(rel_path: str) -> bool: + """Return True if the relative path looks like a test file.""" + rel_lower = rel_path.lower() + for indicator in TEST_INDICATORS: + if indicator in rel_lower: + return True + # Also check basename prefix + basename = os.path.basename(rel_lower) + if basename.startswith("test_"): + return True + return False + + +def classify_pattern(pattern: str) -> tuple[str, re.Pattern[str]]: + """Classify a search pattern and return (match_type, compiled_regex). + + Patterns starting with "class " or "def " get special match types. + Other patterns are compiled as-is (user-supplied regex). + """ + if pattern.startswith("class "): + name = pattern[6:].strip() + return "class", re.compile(r"class\s+" + re.escape(name)) + elif pattern.startswith("def "): + name = pattern[4:].strip() + return "function", re.compile(r"def\s+" + re.escape(name)) + else: + try: + return "keyword", re.compile(pattern) + except re.error: + # Fall back to escaped literal if pattern is not valid regex + return "keyword", re.compile(re.escape(pattern)) + + +def read_lines(filepath: Path) -> list[str] | None: + """Read all lines from a text file. Returns None on decode failure.""" + try: + with open(filepath, "r", encoding="utf-8", errors="strict") as f: + return f.readlines() + except (OSError, UnicodeDecodeError): + return None + + +# --------------------------------------------------------------------------- +# Search logic +# --------------------------------------------------------------------------- + +def search_file( + filepath: Path, + rel_path: str, + patterns: list[tuple[str, re.Pattern[str]]], + keywords: list[str], + context: int, + repo_id: str, +) -> list[dict]: + """Search a single file for patterns and keywords. Returns match dicts.""" + lines = read_lines(filepath) + if lines is None: + return [] + + results: list[dict] = [] + + # --- Pattern search (regex) --- + for match_type, regex in patterns: + for i, line in enumerate(lines): + if regex.search(line): + start = max(0, i - context) + end = min(len(lines), i + context + 1) + results.append({ + "repo_id": repo_id, + "file_path": rel_path, + "line_number": i + 1, + "match_type": match_type, + "matched_text": line.rstrip("\n"), + "context_before": [ + l.rstrip("\n") for l in lines[start:i] + ], + "context_after": [ + l.rstrip("\n") for l in lines[i + 1:end] + ], + }) + + # --- Keyword search (case-insensitive substring) --- + for kw in keywords: + kw_lower = kw.lower() + for i, line in enumerate(lines): + if kw_lower in line.lower(): + start = max(0, i - context) + end = min(len(lines), i + context + 1) + results.append({ + "repo_id": repo_id, + "file_path": rel_path, + "line_number": i + 1, + "match_type": "keyword", + "matched_text": line.rstrip("\n"), + "context_before": [ + l.rstrip("\n") for l in lines[start:i] + ], + "context_after": [ + l.rstrip("\n") for l in lines[i + 1:end] + ], + }) + + return results + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser( + description="Search cloned repo for specific implementations " + "(classes, functions, algorithms)", + ) + parser.add_argument( + "--repo-dir", required=True, + help="Path to the cloned repository", + ) + parser.add_argument( + "--patterns", nargs="+", default=None, + help='Regex patterns (e.g. "class Transformer" "def train")', + ) + parser.add_argument( + "--keywords", nargs="+", default=None, + help="Simple keyword search terms (case-insensitive substring match)", + ) + parser.add_argument( + "--output", required=True, + help="Output JSONL file path", + ) + parser.add_argument( + "--context", type=int, default=5, + help="Lines of context before and after each match (default: 5)", + ) + parser.add_argument( + "--skip-tests", action="store_true", + help="Skip test files and test directories", + ) + parser.add_argument( + "--repo-id", default=None, + help="Repository identifier (owner/name); inferred from dir name if omitted", + ) + + args = parser.parse_args() + + if not args.patterns and not args.keywords: + parser.error("Provide at least one of --patterns or --keywords") + + repo_dir = Path(args.repo_dir).resolve() + if not repo_dir.is_dir(): + print(f"Error: repo directory not found: {repo_dir}", file=sys.stderr) + sys.exit(1) + + # Infer repo_id + repo_id: str = args.repo_id or "" + if not repo_id: + dir_name = repo_dir.name + repo_id = dir_name.replace("_", "/", 1) if "_" in dir_name else dir_name + + # Classify patterns + classified_patterns: list[tuple[str, re.Pattern[str]]] = [] + if args.patterns: + for p in args.patterns: + match_type, regex = classify_pattern(p) + classified_patterns.append((match_type, regex)) + print(f" Pattern: '{p}' -> type={match_type}", file=sys.stderr) + + keyword_list: list[str] = args.keywords or [] + if keyword_list: + print(f" Keywords: {keyword_list}", file=sys.stderr) + + # Walk and search + total_matches = 0 + files_with_matches = 0 + files_scanned = 0 + files_skipped = 0 + + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + + print(f"Searching {repo_dir} ...", file=sys.stderr) + + with open(output_path, "w", encoding="utf-8") as out_f: + for dirpath, dirnames, filenames in os.walk(repo_dir): + # Filter directories in-place to prune traversal + dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] + + for fname in filenames: + filepath = Path(dirpath) / fname + rel_path = str(filepath.relative_to(repo_dir)).replace(os.sep, "/") + + # Skip test files if requested + if args.skip_tests and is_test_path(rel_path): + files_skipped += 1 + continue + + # Skip files that are too large + try: + size = filepath.stat().st_size + except OSError: + continue + if size > MAX_FILE_SIZE: + files_skipped += 1 + continue + + # Skip binary files + if is_binary(filepath): + files_skipped += 1 + continue + + files_scanned += 1 + matches = search_file( + filepath, rel_path, + classified_patterns, keyword_list, + args.context, repo_id, + ) + + if matches: + files_with_matches += 1 + for m in matches: + out_f.write(json.dumps(m, ensure_ascii=False) + "\n") + total_matches += 1 + + print( + f"Found {total_matches} matches across {files_with_matches} files " + f"({files_scanned} scanned, {files_skipped} skipped) -> {args.output}", + file=sys.stderr, + ) + + +if __name__ == "__main__": + main() diff --git a/skills/github-research/scripts/repo_db.py b/skills/github-research/scripts/repo_db.py new file mode 100755 index 0000000..4f137b4 --- /dev/null +++ b/skills/github-research/scripts/repo_db.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python3 +"""JSONL GitHub repo database management. + +Subcommands: merge, filter, score, search, tag, stats, export, rank. +Deduplication by exact repo_id (owner/name) match. + +Usage: + python repo_db.py merge --inputs a.jsonl b.jsonl --output merged.jsonl + python repo_db.py filter --input db.jsonl --output filtered.jsonl --min-stars 100 + python repo_db.py score --input db.jsonl --output scored.jsonl + python repo_db.py search --input db.jsonl --query "transformer" + python repo_db.py tag --input db.jsonl --ids owner/name --tags impl baseline + python repo_db.py stats --input db.jsonl + python repo_db.py export --input db.jsonl --format markdown + python repo_db.py rank --input db.jsonl --output ranked.jsonl --by composite_score +""" + +import argparse +import csv +import io +import json +import math +import os +import re +import sys +from datetime import datetime, timezone + + +# -- I/O helpers -------------------------------------------------------------- + +def load_jsonl(path: str) -> list[dict]: + """Load records from a JSONL file.""" + records = [] + if not os.path.exists(path): + return records + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + records.append(json.loads(line)) + return records + + +def save_jsonl(records: list[dict], path: str): + """Save records to a JSONL file.""" + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + for rec in records: + f.write(json.dumps(rec, ensure_ascii=False) + "\n") + + +def deduplicate(records: list[dict]) -> list[dict]: + """Remove duplicate repos by exact repo_id match. Later entries win.""" + seen: dict[str, int] = {} + for i, rec in enumerate(records): + rid = rec.get("repo_id", "") + if rid: + seen[rid] = i + # Preserve order of last occurrence + indices = sorted(seen.values()) + return [records[i] for i in indices] + + +# -- Scoring ------------------------------------------------------------------ + +def _sigmoid(x: float) -> float: + """Sigmoid with steepness 5 centered at 0.5.""" + return 1.0 / (1.0 + math.exp(-5.0 * (x - 0.5))) + + +def _parse_iso(dt_str: str | None) -> datetime | None: + """Parse an ISO-8601 datetime string.""" + if not dt_str: + return None + # Handle trailing Z and optional fractional seconds + dt_str = dt_str.replace("Z", "+00:00") + try: + return datetime.fromisoformat(dt_str) + except (ValueError, TypeError): + return None + + +def compute_activity_score(rec: dict, now: datetime) -> float: + """Compute activity score in [0, 1].""" + pushed = _parse_iso(rec.get("pushed_at")) + if pushed is None: + return _sigmoid(0.0) + + days_since_push = max((now - pushed).days, 0) + recent_push = 1.0 if days_since_push < 90 else 0.0 + has_recent_commits = 1.0 if days_since_push < 180 else 0.0 + + stars = max(rec.get("stars", 0), 1) + open_issues = rec.get("open_issues", 0) + ratio = min(open_issues / stars, 1.0) + issues_component = 1.0 - ratio # lower ratio is better + + raw = recent_push * 0.4 + has_recent_commits * 0.3 + issues_component * 0.3 + return _sigmoid(raw) + + +def compute_quality_score(rec: dict) -> float: + """Compute quality score in [0, 1].""" + stars = rec.get("stars", 0) + forks = rec.get("forks", 0) + + # Normalize log components: log(x+1) / log(100001) gives ~0-1 for 0-100k + max_log = math.log(100001) + star_comp = min(math.log(stars + 1) / max_log, 1.0) * 0.3 + fork_comp = min(math.log(forks + 1) / max_log, 1.0) * 0.2 + + has_license = 0.15 if rec.get("license") else 0.0 + has_readme = 0.15 if rec.get("readme_excerpt") else 0.0 + not_archived = 0.2 if not rec.get("archived", False) else 0.0 + + raw = star_comp + fork_comp + has_license + has_readme + not_archived + return min(raw, 1.0) + + +def score_record(rec: dict, now: datetime) -> dict: + """Compute all scores for a single record and return updated copy.""" + rec = dict(rec) + rec["activity_score"] = round(compute_activity_score(rec, now), 4) + rec["quality_score"] = round(compute_quality_score(rec), 4) + + relevance = rec.get("relevance_score", 0.0) or 0.0 + quality = rec["quality_score"] + activity = rec["activity_score"] + rec["composite_score"] = round( + relevance * 0.4 + quality * 0.35 + activity * 0.25, 4 + ) + return rec + + +# -- Subcommand implementations ----------------------------------------------- + +def cmd_merge(args): + """Merge multiple JSONL files with deduplication by repo_id.""" + all_records = [] + for path in args.inputs: + records = load_jsonl(path) + print(f"Loaded {len(records)} from {path}", file=sys.stderr) + all_records.extend(records) + + merged = deduplicate(all_records) + save_jsonl(merged, args.output) + print(f"Merged: {len(all_records)} -> {len(merged)} unique repos -> {args.output}", + file=sys.stderr) + + +def cmd_filter(args): + """Filter repos by various criteria.""" + records = load_jsonl(args.input) + kept = [] + + for rec in records: + if args.min_stars is not None and rec.get("stars", 0) < args.min_stars: + continue + if args.min_score is not None and rec.get("composite_score", 0.0) < args.min_score: + continue + if args.language and rec.get("language", "").lower() != args.language.lower(): + continue + if args.not_archived and rec.get("archived", False): + continue + kept.append(rec) + + # Sort by composite_score descending + kept.sort(key=lambda r: -(r.get("composite_score", 0.0) or 0.0)) + + if args.max_repos and args.max_repos > 0 and len(kept) > args.max_repos: + kept = kept[:args.max_repos] + + save_jsonl(kept, args.output) + print(f"Filtered: {len(records)} -> {len(kept)} repos -> {args.output}", + file=sys.stderr) + + +def cmd_score(args): + """Compute composite scores for all repos.""" + records = load_jsonl(args.input) + now = datetime.now(timezone.utc) + + scored = [score_record(rec, now) for rec in records] + save_jsonl(scored, args.output) + print(f"Scored {len(scored)} repos -> {args.output}", file=sys.stderr) + + +def cmd_search(args): + """Search repos by keyword match in a field.""" + records = load_jsonl(args.input) + query_lower = args.query.lower() + results = [] + + for rec in records: + value = rec.get(args.field, "") + if isinstance(value, list): + value = " ".join(str(v) for v in value) + if query_lower in str(value).lower(): + results.append(rec) + + for rec in results: + print(json.dumps(rec, ensure_ascii=False)) + print(f"Found {len(results)} matches", file=sys.stderr) + + +def cmd_tag(args): + """Add tags to specific repos. Supports 'relevance:0.85' format.""" + records = load_jsonl(args.input) + id_set = set(args.ids) + tagged = 0 + + # Separate relevance assignments from plain tags + plain_tags = [] + relevance_val = None + for t in args.tags: + if t.startswith("relevance:"): + try: + relevance_val = float(t.split(":", 1)[1]) + except ValueError: + plain_tags.append(t) + else: + plain_tags.append(t) + + for rec in records: + rid = rec.get("repo_id", "") + if rid in id_set: + if plain_tags: + existing = rec.get("tags", []) + rec["tags"] = sorted(set(existing + plain_tags)) + if relevance_val is not None: + rec["relevance_score"] = relevance_val + tagged += 1 + + save_jsonl(records, args.input) + msg_parts = [] + if plain_tags: + msg_parts.append(f"tags={plain_tags}") + if relevance_val is not None: + msg_parts.append(f"relevance_score={relevance_val}") + print(f"Tagged {tagged} repos with {', '.join(msg_parts)}", file=sys.stderr) + + +def cmd_stats(args): + """Compute and print JSON summary statistics.""" + records = load_jsonl(args.input) + if not records: + print(json.dumps({"total": 0}, indent=2)) + return + + languages: dict[str, int] = {} + sources: dict[str, int] = {} + tags_dist: dict[str, int] = {} + total_stars = 0 + total_forks = 0 + archived_count = 0 + with_readme = 0 + with_papers = 0 + scored_count = 0 + score_sum = 0.0 + + for rec in records: + lang = rec.get("language") or "Unknown" + languages[lang] = languages.get(lang, 0) + 1 + + src = rec.get("source", "unknown") + sources[src] = sources.get(src, 0) + 1 + + total_stars += rec.get("stars", 0) + total_forks += rec.get("forks", 0) + + if rec.get("archived"): + archived_count += 1 + if rec.get("readme_excerpt"): + with_readme += 1 + if rec.get("paper_ids"): + with_papers += 1 + + cs = rec.get("composite_score", 0.0) or 0.0 + if cs > 0: + scored_count += 1 + score_sum += cs + + for tag in rec.get("tags", []): + tags_dist[tag] = tags_dist.get(tag, 0) + 1 + + stats = { + "total": len(records), + "archived": archived_count, + "with_readme": with_readme, + "with_papers": with_papers, + "total_stars": total_stars, + "total_forks": total_forks, + "avg_stars": round(total_stars / len(records), 1), + "avg_composite_score": round(score_sum / scored_count, 4) if scored_count else 0.0, + "languages": dict(sorted(languages.items(), key=lambda x: -x[1])[:15]), + "sources": sources, + "tags": tags_dist, + } + print(json.dumps(stats, indent=2)) + + +def cmd_export(args): + """Export database in csv, jsonl, or markdown format.""" + records = load_jsonl(args.input) + if not records: + print("No records to export.", file=sys.stderr) + return + + fmt = args.format + + if fmt == "csv": + output = _export_csv(records) + elif fmt == "jsonl": + output = "\n".join(json.dumps(r, ensure_ascii=False) for r in records) + "\n" + elif fmt == "markdown": + output = _export_markdown(records) + else: + print(f"Unknown format: {fmt}", file=sys.stderr) + return + + if args.output: + with open(args.output, "w", encoding="utf-8") as f: + f.write(output) + print(f"Exported {len(records)} repos to {args.output}", file=sys.stderr) + else: + print(output, end="") + + +def _export_csv(records: list[dict]) -> str: + fields = [ + "repo_id", "name", "owner", "stars", "forks", "language", "license", + "composite_score", "quality_score", "activity_score", "relevance_score", + "topics", "tags", "archived", "source", + ] + buf = io.StringIO() + writer = csv.DictWriter(buf, fieldnames=fields, extrasaction="ignore") + writer.writeheader() + for rec in records: + row = dict(rec) + if isinstance(row.get("topics"), list): + row["topics"] = "; ".join(row["topics"]) + if isinstance(row.get("tags"), list): + row["tags"] = "; ".join(row["tags"]) + writer.writerow(row) + return buf.getvalue() + + +def _export_markdown(records: list[dict]) -> str: + lines = ["| Repo | Stars | Language | Score | Description |", + "|------|------:|----------|------:|-------------|"] + for rec in records: + rid = rec.get("repo_id", "") + url = rec.get("url", f"https://github.com/{rid}") + stars = rec.get("stars", 0) + lang = rec.get("language", "") + score = rec.get("composite_score", 0.0) or 0.0 + desc = (rec.get("description") or "")[:80] + desc = desc.replace("|", "\\|") + lines.append(f"| [{rid}]({url}) | {stars} | {lang} | {score:.3f} | {desc} |") + return "\n".join(lines) + "\n" + + +def cmd_rank(args): + """Rank repos by a given field descending.""" + records = load_jsonl(args.input) + # Map CLI choice to actual record field name + field_map = { + "composite_score": "composite_score", + "stars": "stars", + "updated": "updated_at", + } + field = field_map.get(args.by, args.by) + + def sort_key(r): + val = r.get(field, 0) + if val is None: + return "" + # For date strings, lexicographic sort works with ISO format + if isinstance(val, str): + return val + return val + + records.sort(key=sort_key, reverse=True) + + for i, rec in enumerate(records): + rec["rank"] = i + 1 + + save_jsonl(records, args.output) + print(f"Ranked {len(records)} repos by {field} -> {args.output}", file=sys.stderr) + + +# -- CLI ----------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser( + description="JSONL GitHub repo database management tool" + ) + sub = parser.add_subparsers(dest="command", required=True) + + # merge + p = sub.add_parser("merge", help="Merge multiple JSONL files with dedup by repo_id") + p.add_argument("--inputs", nargs="+", required=True, help="Input JSONL files") + p.add_argument("--output", required=True, help="Output JSONL file") + + # filter + p = sub.add_parser("filter", help="Filter repos by stars, score, language, etc.") + p.add_argument("--input", required=True, help="Input JSONL file") + p.add_argument("--output", required=True, help="Output JSONL file") + p.add_argument("--min-stars", type=int, default=None, help="Minimum star count") + p.add_argument("--min-score", type=float, default=None, help="Minimum composite score") + p.add_argument("--max-repos", type=int, default=None, help="Max repos to keep (0=unlimited)") + p.add_argument("--language", default=None, help="Filter by primary language") + p.add_argument("--not-archived", action="store_true", help="Exclude archived repos") + + # score + p = sub.add_parser("score", help="Compute composite scores for all repos") + p.add_argument("--input", required=True, help="Input JSONL file") + p.add_argument("--output", required=True, help="Output scored JSONL file") + + # search + p = sub.add_parser("search", help="Search repos by keyword in a field") + p.add_argument("--input", required=True, help="Input JSONL file") + p.add_argument("--query", required=True, help="Search query string") + p.add_argument("--field", default="description", + choices=["description", "topics", "name"], + help="Field to search (default: description)") + + # tag + p = sub.add_parser("tag", help="Add tags to repos; supports 'relevance:0.85' format") + p.add_argument("--input", required=True, help="Input JSONL file (modified in-place)") + p.add_argument("--ids", nargs="+", required=True, help="Repo IDs (owner/name)") + p.add_argument("--tags", nargs="+", required=True, + help="Tags to add; use 'relevance:0.85' to set relevance_score") + + # stats + p = sub.add_parser("stats", help="Print JSON summary statistics") + p.add_argument("--input", required=True, help="Input JSONL file") + + # export + p = sub.add_parser("export", help="Export database to csv, jsonl, or markdown") + p.add_argument("--input", required=True, help="Input JSONL file") + p.add_argument("--format", choices=["csv", "jsonl", "markdown"], default="jsonl", + help="Output format (default: jsonl)") + p.add_argument("--output", "-o", default=None, help="Output file (default: stdout)") + + # rank + p = sub.add_parser("rank", help="Rank repos by a field descending") + p.add_argument("--input", required=True, help="Input JSONL file") + p.add_argument("--output", required=True, help="Output ranked JSONL file") + p.add_argument("--by", default="composite_score", + choices=["composite_score", "stars", "updated"], + help="Field to rank by (default: composite_score)") + + args = parser.parse_args() + + dispatch = { + "merge": cmd_merge, + "filter": cmd_filter, + "score": cmd_score, + "search": cmd_search, + "tag": cmd_tag, + "stats": cmd_stats, + "export": cmd_export, + "rank": cmd_rank, + } + dispatch[args.command](args) + + +if __name__ == "__main__": + main() diff --git a/skills/github-research/scripts/repo_metadata.py b/skills/github-research/scripts/repo_metadata.py new file mode 100755 index 0000000..20558ec --- /dev/null +++ b/skills/github-research/scripts/repo_metadata.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Fetch detailed metadata for GitHub repos via ``gh api``. + +Retrieves main repo info and language breakdown, then maps the response +to the repo_db schema. Can enrich existing JSONL records or create new +ones from scratch. +""" + +import argparse +import json +import subprocess +import sys +import time + + +# --------------------------------------------------------------------------- +# GitHub helpers +# --------------------------------------------------------------------------- + +def gh_api(endpoint: str) -> dict | None: + """Call ``gh api`` and return parsed JSON, or *None* on failure.""" + try: + proc = subprocess.run( + ["gh", "api", endpoint], + capture_output=True, text=True, timeout=30, + ) + if proc.returncode == 0: + return json.loads(proc.stdout) + stderr = proc.stderr.strip() + if "404" in stderr or "Not Found" in stderr: + print(f" 404 — repo not found", file=sys.stderr) + elif "rate limit" in stderr.lower() or "403" in stderr: + print(f" Rate limited. Consider increasing --delay.", file=sys.stderr) + else: + print(f" gh api error ({proc.returncode}): {stderr[:120]}", file=sys.stderr) + except FileNotFoundError: + print("Error: 'gh' CLI not found. Please install GitHub CLI.", file=sys.stderr) + sys.exit(1) + except (subprocess.TimeoutExpired, json.JSONDecodeError) as exc: + print(f" gh api exception: {exc}", file=sys.stderr) + return None + + +# --------------------------------------------------------------------------- +# Language breakdown +# --------------------------------------------------------------------------- + +def compute_languages_pct(lang_data: dict) -> dict[str, float]: + """Convert raw byte counts to percentages (0–100).""" + total = sum(lang_data.values()) + if total == 0: + return {} + return {lang: round(count / total * 100, 2) for lang, count in lang_data.items()} + + +# --------------------------------------------------------------------------- +# Record building +# --------------------------------------------------------------------------- + +def build_record(repo_id: str, meta: dict, lang_data: dict) -> dict: + """Build a repo_db-schema record from GitHub API responses.""" + license_info = meta.get("license") or {} + languages_pct = compute_languages_pct(lang_data) + + return { + "repo_id": repo_id, + "github_url": meta.get("html_url", f"https://github.com/{repo_id}"), + "description": meta.get("description", ""), + "homepage": meta.get("homepage", ""), + "stars": meta.get("stargazers_count", 0), + "forks_count": meta.get("forks_count", 0), + "open_issues_count": meta.get("open_issues_count", 0), + "watchers_count": meta.get("watchers_count", 0), + "default_branch": meta.get("default_branch", ""), + "license": license_info.get("spdx_id") or license_info.get("name", ""), + "language": meta.get("language", ""), + "languages_pct": languages_pct, + "topics": meta.get("topics", []), + "is_fork": meta.get("fork", False), + "is_archived": meta.get("archived", False), + "size_kb": meta.get("size", 0), + "created_at": meta.get("created_at", ""), + "updated_at": meta.get("updated_at", ""), + "pushed_at": meta.get("pushed_at", ""), + "owner_type": (meta.get("owner") or {}).get("type", ""), + "network_count": meta.get("network_count", 0), + "subscribers_count": meta.get("subscribers_count", 0), + } + + +def enrich_record(existing: dict, fresh: dict) -> dict: + """Merge *fresh* metadata into an *existing* record. + + Fresh values overwrite existing ones, except we preserve fields + that exist only in the original (e.g. ``source``, ``paper_ids``). + """ + merged = dict(existing) + merged.update(fresh) + # Preserve provenance fields from the original record. + for key in ("source", "paper_ids", "paper_titles", "is_official", "framework"): + if key in existing: + merged[key] = existing[key] + return merged + + +# --------------------------------------------------------------------------- +# I/O helpers +# --------------------------------------------------------------------------- + +def load_jsonl(path: str) -> list[dict]: + """Load a JSONL file into a list of dicts.""" + records: list[dict] = [] + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError: + continue + return records + + +def write_jsonl(records: list[dict], path: str) -> None: + """Write a list of dicts as JSONL.""" + with open(path, "w", encoding="utf-8") as fout: + for rec in records: + fout.write(json.dumps(rec, ensure_ascii=False) + "\n") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser( + description="Fetch detailed GitHub repo metadata via gh api.", + ) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--repos", nargs="+", metavar="OWNER/NAME", + help="One or more repos as owner/name") + source.add_argument("--input", metavar="FILE", + help="JSONL file with existing repo records to enrich") + parser.add_argument("--output", required=True, help="Output JSONL file path") + parser.add_argument("--delay", type=float, default=0.5, + help="Seconds between API requests per repo (default: 0.5)") + + args = parser.parse_args() + + # Determine which repos to process. + existing_records: dict[str, dict] = {} # keyed by repo_id + + if args.input: + for rec in load_jsonl(args.input): + rid = rec.get("repo_id") + if rid: + existing_records[rid] = rec + repo_ids = list(existing_records.keys()) + else: + repo_ids = args.repos or [] + + if not repo_ids: + print("No repos to process.", file=sys.stderr) + sys.exit(0) + + total = len(repo_ids) + results: list[dict] = [] + + for idx, repo_id in enumerate(repo_ids, 1): + print(f"Fetching metadata: {idx}/{total} — {repo_id}", file=sys.stderr) + + meta = gh_api(f"/repos/{repo_id}") + if meta is None: + print(f" Skipping {repo_id} (metadata unavailable)", file=sys.stderr) + # Keep the original record if enriching. + if repo_id in existing_records: + results.append(existing_records[repo_id]) + continue + + time.sleep(args.delay) + + lang_data = gh_api(f"/repos/{repo_id}/languages") or {} + + fresh = build_record(repo_id, meta, lang_data) + + if repo_id in existing_records: + record = enrich_record(existing_records[repo_id], fresh) + else: + record = fresh + + results.append(record) + + if idx < total: + time.sleep(args.delay) + + write_jsonl(results, args.output) + print(f"Done. {len(results)}/{total} record(s) written to {args.output}", + file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/skills/github-research/scripts/repo_readme_fetch.py b/skills/github-research/scripts/repo_readme_fetch.py new file mode 100755 index 0000000..ef41b9f --- /dev/null +++ b/skills/github-research/scripts/repo_readme_fetch.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Fetch README content for GitHub repos without cloning. + +Uses ``gh api`` to retrieve the README of one or more repositories, +decodes the base64 payload, and writes the results as JSONL. +""" + +import argparse +import base64 +import json +import subprocess +import sys +import time + + +def gh_api(endpoint: str) -> dict | None: + """Call ``gh api`` and return parsed JSON, or *None* on failure.""" + try: + proc = subprocess.run( + ["gh", "api", endpoint], + capture_output=True, text=True, timeout=30, + ) + if proc.returncode == 0: + return json.loads(proc.stdout) + # Distinguish 404 (no README) from other errors. + if "404" in proc.stderr or "Not Found" in proc.stderr: + return None + print(f" gh api error ({proc.returncode}): {proc.stderr.strip()[:120]}", + file=sys.stderr) + except FileNotFoundError: + print(" Error: 'gh' CLI not found. Please install GitHub CLI.", file=sys.stderr) + sys.exit(1) + except (subprocess.TimeoutExpired, json.JSONDecodeError) as exc: + print(f" gh api exception: {exc}", file=sys.stderr) + return None + + +def fetch_readme(repo_id: str, max_chars: int) -> dict | None: + """Return a dict with repo_id, readme_text, readme_length or *None*.""" + data = gh_api(f"/repos/{repo_id}/readme") + if data is None: + return None + + content_b64 = data.get("content", "") + try: + raw = base64.b64decode(content_b64).decode("utf-8", errors="replace") + except Exception: + raw = "" + + truncated = raw[:max_chars] + return { + "repo_id": repo_id, + "readme_text": truncated, + "readme_length": len(raw), + } + + +def load_repo_ids_from_jsonl(path: str) -> list[str]: + """Read repo_id values from a JSONL file.""" + ids: list[str] = [] + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + rid = obj.get("repo_id") + if rid: + ids.append(rid) + except json.JSONDecodeError: + continue + return ids + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Fetch README content for GitHub repos without cloning.", + ) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--repos", nargs="+", metavar="OWNER/NAME", + help="One or more repos as owner/name") + source.add_argument("--input", metavar="FILE", + help="JSONL file with repo_id fields") + parser.add_argument("--output", required=True, help="Output JSONL file path") + parser.add_argument("--max-chars", type=int, default=5000, + help="Max characters to keep from README (default: 5000)") + + args = parser.parse_args() + + repo_ids: list[str] = args.repos if args.repos else load_repo_ids_from_jsonl(args.input) + + if not repo_ids: + print("No repos to process.", file=sys.stderr) + sys.exit(0) + + total = len(repo_ids) + results: list[dict] = [] + + for idx, repo_id in enumerate(repo_ids, 1): + print(f"Fetching README: {idx}/{total} — {repo_id}", file=sys.stderr) + record = fetch_readme(repo_id, max_chars=args.max_chars) + if record: + results.append(record) + else: + print(f" No README found for {repo_id}", file=sys.stderr) + if idx < total: + time.sleep(0.5) + + with open(args.output, "w", encoding="utf-8") as fout: + for rec in results: + fout.write(json.dumps(rec, ensure_ascii=False) + "\n") + + print(f"Done. {len(results)}/{total} README(s) written to {args.output}", + file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/skills/github-research/scripts/search_github.py b/skills/github-research/scripts/search_github.py new file mode 100755 index 0000000..6eebfd5 --- /dev/null +++ b/skills/github-research/scripts/search_github.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +"""Search GitHub repositories via `gh api` with multiple query strategies. + +Builds qualified search queries and paginates through results, outputting +repo_db-compatible JSONL records. +""" + +import argparse +import json +import math +import shutil +import subprocess +import sys +import time +from pathlib import Path + +RESULTS_PER_PAGE = 30 +DEFAULT_MAX_RESULTS = 100 + + +def check_gh_installed(): + """Verify gh CLI is available.""" + if not shutil.which("gh"): + print("[error] gh CLI not found. Install it: https://cli.github.com/", file=sys.stderr) + sys.exit(1) + + +def gh_api(endpoint: str, params: dict | None = None) -> dict: + """Call gh api and return parsed JSON.""" + if params: + from urllib.parse import urlencode + endpoint = f"{endpoint}?{urlencode(params)}" + cmd = ["gh", "api", endpoint] + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=30, + ) + except FileNotFoundError: + print("[error] gh CLI not found", file=sys.stderr) + sys.exit(1) + except subprocess.TimeoutExpired: + print("[error] gh api call timed out", file=sys.stderr) + sys.exit(1) + + if result.returncode != 0: + stderr = result.stderr.strip() + # Check for rate limit + if "rate limit" in stderr.lower() or "403" in stderr: + print(f"[error] GitHub API rate limit hit: {stderr}", file=sys.stderr) + sys.exit(1) + print(f"[error] gh api failed (exit {result.returncode}): {stderr}", file=sys.stderr) + sys.exit(1) + + try: + return json.loads(result.stdout) + except json.JSONDecodeError as e: + print(f"[error] invalid JSON from gh api: {e}", file=sys.stderr) + sys.exit(1) + + +def build_query(query: str, language: str | None, min_stars: int | None, topic: str | None) -> str: + """Build GitHub search query string with qualifiers.""" + parts = [query] + if language: + parts.append(f"language:{language}") + if min_stars is not None and min_stars > 0: + parts.append(f"stars:>={min_stars}") + if topic: + parts.append(f"topic:{topic}") + return " ".join(parts) + + +def extract_license(license_obj) -> str: + """Extract license identifier from GitHub API license object.""" + if not license_obj or not isinstance(license_obj, dict): + return "" + return license_obj.get("spdx_id") or license_obj.get("name") or "" + + +def map_repo(item: dict) -> dict: + """Map a GitHub API repository item to repo_db schema.""" + owner_obj = item.get("owner") or {} + return { + "repo_id": item.get("full_name", ""), + "url": item.get("html_url", ""), + "name": item.get("name", ""), + "owner": owner_obj.get("login", ""), + "description": item.get("description") or "", + "stars": item.get("stargazers_count", 0), + "forks": item.get("forks_count", 0), + "language": item.get("language") or "", + "license": extract_license(item.get("license")), + "topics": item.get("topics", []), + "created_at": item.get("created_at", ""), + "updated_at": item.get("updated_at", ""), + "pushed_at": item.get("pushed_at", ""), + "open_issues": item.get("open_issues_count", 0), + "default_branch": item.get("default_branch", "main"), + "archived": item.get("archived", False), + "source": "search_github", + # Default fields for repo_db compatibility + "languages_pct": {}, + "readme_excerpt": "", + "paper_ids": [], + "paper_titles": [], + "relevance_score": 0.0, + "quality_score": 0.0, + "activity_score": 0.0, + "composite_score": 0.0, + "tags": [], + "analyzed": False, + "local_path": None, + } + + +def check_rate_limit(): + """Check GitHub API rate limit and warn if approaching it.""" + try: + result = subprocess.run( + ["gh", "api", "rate_limit"], + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode == 0: + data = json.loads(result.stdout) + search = data.get("resources", {}).get("search", {}) + remaining = search.get("remaining", -1) + limit = search.get("limit", -1) + reset_ts = search.get("reset", 0) + + if remaining >= 0: + if remaining <= 3: + reset_time = time.strftime("%H:%M:%S", time.localtime(reset_ts)) + print( + f"[warn] GitHub search API rate limit: {remaining}/{limit} remaining, " + f"resets at {reset_time}", + file=sys.stderr, + ) + else: + print(f"[info] rate limit: {remaining}/{limit} search requests remaining", file=sys.stderr) + except (subprocess.TimeoutExpired, json.JSONDecodeError, OSError): + pass # Non-critical, skip + + +def search_repos( + query: str, + sort: str, + max_results: int, +) -> list[dict]: + """Search GitHub repositories with pagination.""" + total_pages = math.ceil(max_results / RESULTS_PER_PAGE) + all_repos = [] + seen_ids = set() + + for page in range(1, total_pages + 1): + print(f"[info] fetching page {page}/{total_pages} ...", file=sys.stderr) + + params = { + "q": query, + "sort": sort, + "order": "desc", + "per_page": str(min(RESULTS_PER_PAGE, max_results - len(all_repos))), + "page": str(page), + } + + data = gh_api("/search/repositories", params) + + total_count = data.get("total_count", 0) + if page == 1: + print(f"[info] total matching repos: {total_count}", file=sys.stderr) + + items = data.get("items", []) + if not items: + print("[info] no more results", file=sys.stderr) + break + + for item in items: + repo = map_repo(item) + if repo["repo_id"] not in seen_ids: + seen_ids.add(repo["repo_id"]) + all_repos.append(repo) + + if len(all_repos) >= max_results: + break + + if len(all_repos) >= max_results: + break + + # Incomplete results means GitHub truncated + if data.get("incomplete_results", False): + print("[warn] GitHub returned incomplete results (query too broad?)", file=sys.stderr) + + # Respect rate limit: sleep 2s between pages for search API + if page < total_pages: + time.sleep(2) + + return all_repos + + +def main(): + parser = argparse.ArgumentParser( + description="Search GitHub repositories via gh API with flexible query strategies." + ) + parser.add_argument("--query", required=True, help="Search query string") + parser.add_argument("--language", default=None, help="Filter by programming language") + parser.add_argument("--min-stars", type=int, default=None, help="Minimum star count") + parser.add_argument( + "--sort", + choices=["stars", "updated", "best-match"], + default="best-match", + help="Sort order (default: best-match)", + ) + parser.add_argument("--max-results", type=int, default=DEFAULT_MAX_RESULTS, help="Max results to fetch (default: 100)") + parser.add_argument("--topic", default=None, help="Filter by GitHub topic") + parser.add_argument("--output", required=True, help="Output JSONL file path") + args = parser.parse_args() + + check_gh_installed() + + # Build the qualified query + full_query = build_query(args.query, args.language, args.min_stars, args.topic) + print(f"[info] search query: {full_query}", file=sys.stderr) + print(f"[info] sort: {args.sort}, max: {args.max_results}", file=sys.stderr) + + # Check rate limit before starting + check_rate_limit() + + # Handle "best-match" -> empty sort for GitHub API (relevance is default) + sort_param = "" if args.sort == "best-match" else args.sort + + repos = search_repos(full_query, sort_param, args.max_results) + + # Write output + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + for repo in repos: + f.write(json.dumps(repo, ensure_ascii=False) + "\n") + + # Summary stats + print(f"\n[info] results summary:", file=sys.stderr) + print(f" repos found: {len(repos)}", file=sys.stderr) + if repos: + languages = {} + for r in repos: + lang = r["language"] or "unknown" + languages[lang] = languages.get(lang, 0) + 1 + top_langs = sorted(languages.items(), key=lambda x: -x[1])[:5] + print(f" top languages: {', '.join(f'{l}({c})' for l, c in top_langs)}", file=sys.stderr) + + star_counts = [r["stars"] for r in repos] + print(f" stars range: {min(star_counts)} - {max(star_counts)}", file=sys.stderr) + + print(f"[info] written to {output_path}", file=sys.stderr) + + # Check rate limit after + check_rate_limit() + + +if __name__ == "__main__": + main() diff --git a/skills/github-research/scripts/search_github_code.py b/skills/github-research/scripts/search_github_code.py new file mode 100755 index 0000000..71c0897 --- /dev/null +++ b/skills/github-research/scripts/search_github_code.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""Search GitHub code for specific function/class implementations via `gh api`. + +Searches code using GitHub's code search API, groups results by repo, and +outputs both code-level matches (stderr) and repo-level JSONL (--output). +""" + +import argparse +import json +import math +import shutil +import subprocess +import sys +import time +from pathlib import Path + +RESULTS_PER_PAGE = 30 +DEFAULT_MAX_RESULTS = 50 + + +def check_gh_installed(): + """Verify gh CLI is available.""" + if not shutil.which("gh"): + print("[error] gh CLI not found. Install it: https://cli.github.com/", file=sys.stderr) + sys.exit(1) + + +def gh_api(endpoint: str, params: dict | None = None) -> dict: + """Call gh api and return parsed JSON.""" + if params: + from urllib.parse import urlencode + endpoint = f"{endpoint}?{urlencode(params)}" + cmd = ["gh", "api", endpoint] + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=30, + ) + except FileNotFoundError: + print("[error] gh CLI not found", file=sys.stderr) + sys.exit(1) + except subprocess.TimeoutExpired: + print("[error] gh api call timed out", file=sys.stderr) + sys.exit(1) + + if result.returncode != 0: + stderr = result.stderr.strip() + if "rate limit" in stderr.lower() or "403" in stderr: + print(f"[error] GitHub API rate limit hit: {stderr}", file=sys.stderr) + sys.exit(1) + print(f"[error] gh api failed (exit {result.returncode}): {stderr}", file=sys.stderr) + sys.exit(1) + + try: + return json.loads(result.stdout) + except json.JSONDecodeError as e: + print(f"[error] invalid JSON from gh api: {e}", file=sys.stderr) + sys.exit(1) + + +def build_code_query(query: str, language: str | None, filename: str | None) -> str: + """Build GitHub code search query with qualifiers.""" + parts = [query] + if language: + parts.append(f"language:{language}") + if filename: + parts.append(f"filename:{filename}") + return " ".join(parts) + + +def search_code(query: str, max_results: int) -> list[dict]: + """Search GitHub code with pagination.""" + total_pages = math.ceil(max_results / RESULTS_PER_PAGE) + all_items = [] + + for page in range(1, total_pages + 1): + print(f"[info] fetching page {page}/{total_pages} ...", file=sys.stderr) + + params = { + "q": query, + "per_page": str(min(RESULTS_PER_PAGE, max_results - len(all_items))), + "page": str(page), + } + + data = gh_api("/search/code", params) + + total_count = data.get("total_count", 0) + if page == 1: + print(f"[info] total matching files: {total_count}", file=sys.stderr) + + items = data.get("items", []) + if not items: + print("[info] no more results", file=sys.stderr) + break + + all_items.extend(items) + + if len(all_items) >= max_results: + all_items = all_items[:max_results] + break + + if data.get("incomplete_results", False): + print("[warn] GitHub returned incomplete results", file=sys.stderr) + + # Code search is heavily rate limited: 10 req/min + if page < total_pages: + time.sleep(6) + + return all_items + + +def extract_code_match(item: dict) -> dict: + """Extract a code match record from a search result item.""" + repo = item.get("repository", {}) + owner_obj = repo.get("owner", {}) + full_name = repo.get("full_name", "") + + # text_matches may be present if Accept header includes text-match + text_fragments = [] + for tm in item.get("text_matches", []): + fragment = tm.get("fragment", "") + if fragment: + text_fragments.append(fragment) + + return { + "repo_id": full_name, + "file_path": item.get("path", ""), + "file_url": item.get("html_url", ""), + "file_name": item.get("name", ""), + "matched_text": "\n---\n".join(text_fragments) if text_fragments else "", + "context": { + "sha": item.get("sha", ""), + "score": item.get("score", 0), + }, + } + + +def build_repo_record(repo_id: str, repo_data: dict) -> dict: + """Build a minimal repo_db-compatible record from code search results.""" + owner = repo_id.split("/")[0] if "/" in repo_id else "" + name = repo_id.split("/")[1] if "/" in repo_id else repo_id + + return { + "repo_id": repo_id, + "url": f"https://github.com/{repo_id}", + "name": name, + "owner": owner, + "description": repo_data.get("description", ""), + "stars": 0, + "forks": 0, + "language": "", + "license": "", + "topics": [], + "created_at": "", + "updated_at": "", + "pushed_at": "", + "open_issues": 0, + "default_branch": "main", + "archived": False, + "source": "code_search", + "code_matches": repo_data.get("files", []), + "languages_pct": {}, + "readme_excerpt": "", + "paper_ids": [], + "paper_titles": [], + "relevance_score": 0.0, + "quality_score": 0.0, + "activity_score": 0.0, + "composite_score": 0.0, + "tags": [], + "analyzed": False, + "local_path": None, + } + + +def main(): + parser = argparse.ArgumentParser( + description="Search GitHub code for function/class implementations via gh API." + ) + parser.add_argument("--query", required=True, help="Code search query string") + parser.add_argument("--language", default=None, help="Filter by programming language") + parser.add_argument("--filename", default=None, help="Filter by filename pattern") + parser.add_argument("--max-results", type=int, default=DEFAULT_MAX_RESULTS, help="Max code results to fetch (default: 50)") + parser.add_argument("--output", required=True, help="Output JSONL file for repo-level records") + args = parser.parse_args() + + check_gh_installed() + + full_query = build_code_query(args.query, args.language, args.filename) + print(f"[info] code search query: {full_query}", file=sys.stderr) + print(f"[info] max results: {args.max_results}", file=sys.stderr) + + items = search_code(full_query, args.max_results) + + # Process matches and group by repo + code_matches = [] + repos: dict[str, dict] = {} # repo_id -> {description, file_count, files} + + for item in items: + match = extract_code_match(item) + code_matches.append(match) + + rid = match["repo_id"] + if rid not in repos: + repo_obj = item.get("repository", {}) + repos[rid] = { + "description": repo_obj.get("description") or "", + "file_count": 0, + "files": [], + } + repos[rid]["file_count"] += 1 + repos[rid]["files"].append(match["file_path"]) + + # Print code matches to stderr, grouped by repo + print(f"\n{'=' * 60}", file=sys.stderr) + print(f"Code matches: {len(code_matches)} files in {len(repos)} repos", file=sys.stderr) + print(f"{'=' * 60}", file=sys.stderr) + + for rid in sorted(repos.keys()): + info = repos[rid] + print(f"\n {rid} ({info['file_count']} files)", file=sys.stderr) + if info["description"]: + print(f" {info['description'][:100]}", file=sys.stderr) + for fpath in info["files"][:5]: + print(f" - {fpath}", file=sys.stderr) + if len(info["files"]) > 5: + print(f" ... and {len(info['files']) - 5} more", file=sys.stderr) + + # Write repo-level JSONL to --output (for feeding into repo_db merge) + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + for rid, rdata in sorted(repos.items()): + record = build_repo_record(rid, rdata) + f.write(json.dumps(record, ensure_ascii=False) + "\n") + + print(f"\n[info] wrote {len(repos)} repo records to {output_path}", file=sys.stderr) + + # Also write code-level matches alongside (with .code suffix) + code_output = output_path.with_suffix(".code.jsonl") + with open(code_output, "w", encoding="utf-8") as f: + for match in code_matches: + f.write(json.dumps(match, ensure_ascii=False) + "\n") + + print(f"[info] wrote {len(code_matches)} code matches to {code_output}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/skills/github-research/scripts/search_paperswithcode.py b/skills/github-research/scripts/search_paperswithcode.py new file mode 100755 index 0000000..6da6f77 --- /dev/null +++ b/skills/github-research/scripts/search_paperswithcode.py @@ -0,0 +1,413 @@ +#!/usr/bin/env python3 +"""Search Papers With Code for paper-to-repo mappings. + +Uses the Papers With Code public API (https://paperswithcode.com/api/v1/) +to find GitHub repositories linked to research papers. Supports lookup by +single arXiv ID, multiple IDs, a file of IDs, or keyword search. + +Self-contained: stdlib only (urllib for HTTP). + +Usage: + python search_paperswithcode.py --arxiv-id 2301.12345 --output repos.jsonl + python search_paperswithcode.py --query "multi-agent" --output repos.jsonl + python search_paperswithcode.py --arxiv-ids-file papers.txt --output repos.jsonl + python search_paperswithcode.py --arxiv-ids 2301.12345 2305.67890 --output repos.jsonl +""" + +import argparse +import json +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + +PWC_API = "https://paperswithcode.com/api/v1" +USER_AGENT = "github-research/1.0" +REQUEST_DELAY = 1.0 # seconds between requests + + +# --------------------------------------------------------------------------- +# HTTP helpers +# --------------------------------------------------------------------------- + +def pwc_request(url: str) -> dict | list | None: + """Make a GET request to the Papers With Code API with retry logic. + + Returns parsed JSON or None on failure. + """ + headers = {"User-Agent": USER_AGENT, "Accept": "application/json"} + req = urllib.request.Request(url, headers=headers) + + for attempt in range(3): + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as e: + if e.code == 404: + return None + if e.code == 429: + wait = 2 ** (attempt + 1) + print(f"[warn] rate limited, waiting {wait}s ...", file=sys.stderr) + time.sleep(wait) + continue + print(f"[warn] HTTP {e.code} for {url}", file=sys.stderr) + if attempt < 2: + time.sleep(1) + continue + return None + except (urllib.error.URLError, OSError) as e: + print(f"[warn] request failed: {e}", file=sys.stderr) + if attempt < 2: + time.sleep(1) + continue + return None + return None + + +def paginate_results(url: str) -> list[dict]: + """Follow ``next`` links and collect all ``results`` items.""" + items: list[dict] = [] + current_url: str | None = url + while current_url: + data = pwc_request(current_url) + if not data or not isinstance(data, dict): + break + items.extend(data.get("results", [])) + current_url = data.get("next") + if current_url: + time.sleep(REQUEST_DELAY) + return items + + +# --------------------------------------------------------------------------- +# GitHub URL parsing +# --------------------------------------------------------------------------- + +def parse_repo_url(url: str) -> tuple[str, str]: + """Parse a GitHub URL into (owner, name). Returns ("", "") if not GitHub.""" + if not url or "github.com" not in url: + return "", "" + url = url.rstrip("/") + for prefix in ("https://github.com/", "http://github.com/"): + if url.startswith(prefix): + path = url[len(prefix):] + parts = path.split("/") + if len(parts) >= 2 and parts[0] and parts[1]: + name = parts[1].removesuffix(".git") + return parts[0], name + return "", "" + + +# --------------------------------------------------------------------------- +# Repo record mapping +# --------------------------------------------------------------------------- + +def map_to_repo_record( + repo: dict, + paper_title: str = "", + arxiv_id: str = "", +) -> dict | None: + """Map a PWC repository entry to the repo_db schema. + + Returns None if the URL is not a valid GitHub repo. + """ + url = repo.get("url", "") + owner, name = parse_repo_url(url) + if not owner or not name: + return None + + repo_id = f"{owner}/{name}" + paper_ids = [arxiv_id] if arxiv_id else [] + paper_titles = [paper_title] if paper_title else [] + + return { + "repo_id": repo_id, + "url": f"https://github.com/{repo_id}", + "name": name, + "owner": owner, + "description": repo.get("description") or "", + "stars": repo.get("stars", 0) or 0, + "forks": 0, + "language": repo.get("framework") or "", + "license": "", + "topics": [], + "created_at": "", + "updated_at": "", + "pushed_at": "", + "open_issues": 0, + "default_branch": "main", + "archived": False, + "source": "paperswithcode", + "paper_ids": paper_ids, + "paper_titles": paper_titles, + "is_official": repo.get("is_official", False), + # Default fields for repo_db compatibility + "languages_pct": {}, + "readme_excerpt": "", + "relevance_score": 0.0, + "quality_score": 0.0, + "activity_score": 0.0, + "composite_score": 0.0, + "tags": [], + "analyzed": False, + "local_path": None, + } + + +# --------------------------------------------------------------------------- +# Core lookup functions +# --------------------------------------------------------------------------- + +def lookup_paper_by_arxiv(arxiv_id: str) -> dict | None: + """Look up a paper on PWC by arXiv ID. Returns first matching paper or None.""" + clean_id = arxiv_id.strip() + # Remove version suffix if present (e.g. 2301.12345v2 -> 2301.12345) + if "v" in clean_id and clean_id[-1].isdigit(): + base, _, ver = clean_id.rpartition("v") + if ver.isdigit(): + clean_id = base + + url = f"{PWC_API}/papers/?arxiv_id={urllib.parse.quote(clean_id, safe='')}" + data = pwc_request(url) + if not data or not isinstance(data, dict): + return None + + results = data.get("results", []) + return results[0] if results else None + + +def fetch_paper_repos(paper_id: str) -> list[dict]: + """Fetch all repositories linked to a paper by its PWC paper ID.""" + url = f"{PWC_API}/papers/{urllib.parse.quote(paper_id, safe='')}/repositories/" + return paginate_results(url) + + +def search_papers_by_query(query: str, max_papers: int = 50) -> list[dict]: + """Search PWC papers by keyword query.""" + papers: list[dict] = [] + page = 1 + + while len(papers) < max_papers: + params = urllib.parse.urlencode({"q": query, "page": page}) + url = f"{PWC_API}/papers/?{params}" + data = pwc_request(url) + if not data or not isinstance(data, dict): + break + + results = data.get("results", []) + if not results: + break + + papers.extend(results) + + if data.get("next"): + page += 1 + time.sleep(REQUEST_DELAY) + else: + break + + return papers[:max_papers] + + +# --------------------------------------------------------------------------- +# Processing functions +# --------------------------------------------------------------------------- + +def process_arxiv_id(arxiv_id: str) -> tuple[list[dict], str]: + """Process a single arXiv ID: look up paper, fetch repos. + + Returns (repo_records, paper_title). + """ + clean_id = arxiv_id.strip() + if not clean_id: + return [], "" + + print(f" arXiv:{clean_id} ...", file=sys.stderr) + paper = lookup_paper_by_arxiv(clean_id) + if not paper: + print(f" not found on PWC", file=sys.stderr) + return [], "" + + paper_id = paper.get("id", "") + paper_title = paper.get("title", "") + repo_count = paper.get("repository_count", 0) + + print(f" {paper_title[:80]}", file=sys.stderr) + print(f" repos: {repo_count}", file=sys.stderr) + + if not paper_id or repo_count == 0: + return [], paper_title + + time.sleep(REQUEST_DELAY) + raw_repos = fetch_paper_repos(paper_id) + + records = [] + for repo in raw_repos: + record = map_to_repo_record(repo, paper_title=paper_title, arxiv_id=clean_id) + if record: + records.append(record) + + print(f" mapped {len(records)} GitHub repos", file=sys.stderr) + return records, paper_title + + +def process_query(query: str) -> list[dict]: + """Search PWC by keyword and fetch repos for matching papers.""" + print(f"[info] searching PWC for: {query}", file=sys.stderr) + papers = search_papers_by_query(query) + print(f"[info] found {len(papers)} papers", file=sys.stderr) + + all_records: list[dict] = [] + papers_with_repos = 0 + + for i, paper in enumerate(papers): + paper_id = paper.get("id", "") + paper_title = paper.get("title", "") + arxiv_id = paper.get("arxiv_id", "") or "" + repo_count = paper.get("repository_count", 0) + + if not paper_id or repo_count == 0: + continue + + print( + f" [{i + 1}/{len(papers)}] {paper_title[:60]} ({repo_count} repos)", + file=sys.stderr, + ) + + time.sleep(REQUEST_DELAY) + raw_repos = fetch_paper_repos(paper_id) + + for repo in raw_repos: + record = map_to_repo_record( + repo, paper_title=paper_title, arxiv_id=arxiv_id + ) + if record: + all_records.append(record) + + papers_with_repos += 1 + time.sleep(REQUEST_DELAY) + + print(f"[info] {papers_with_repos} papers had repos", file=sys.stderr) + return all_records + + +def deduplicate_repos(records: list[dict]) -> list[dict]: + """Deduplicate repos by repo_id, merging paper_ids and paper_titles.""" + seen: dict[str, dict] = {} + for record in records: + rid = record["repo_id"] + if rid in seen: + existing = seen[rid] + # Merge paper references + for pid in record.get("paper_ids", []): + if pid and pid not in existing["paper_ids"]: + existing["paper_ids"].append(pid) + for pt in record.get("paper_titles", []): + if pt and pt not in existing["paper_titles"]: + existing["paper_titles"].append(pt) + # Keep higher star count + if record.get("stars", 0) > existing.get("stars", 0): + existing["stars"] = record["stars"] + # Prefer official repos + if record.get("is_official", False): + existing["is_official"] = True + else: + seen[rid] = record + return list(seen.values()) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser( + description="Search Papers With Code for paper-to-repo mappings." + ) + parser.add_argument("--arxiv-id", default=None, + help="Single arXiv ID to look up") + parser.add_argument("--arxiv-ids", nargs="+", default=None, + help="Multiple arXiv IDs") + parser.add_argument("--arxiv-ids-file", default=None, + help="File with one arXiv ID per line") + parser.add_argument("--query", default=None, + help="Search papers by keyword") + parser.add_argument("--output", required=True, + help="Output JSONL file path") + args = parser.parse_args() + + # Validate: at least one input source required + if not any([args.arxiv_id, args.arxiv_ids, args.arxiv_ids_file, args.query]): + parser.error( + "At least one of --arxiv-id, --arxiv-ids, --arxiv-ids-file, " + "or --query is required" + ) + + # Collect all arXiv IDs from all sources + all_arxiv_ids: list[str] = [] + if args.arxiv_id: + all_arxiv_ids.append(args.arxiv_id) + if args.arxiv_ids: + all_arxiv_ids.extend(args.arxiv_ids) + if args.arxiv_ids_file: + ids_file = Path(args.arxiv_ids_file) + if not ids_file.exists(): + print(f"[error] arXiv IDs file not found: {ids_file}", file=sys.stderr) + sys.exit(1) + try: + for line in ids_file.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line and not line.startswith("#"): + all_arxiv_ids.append(line) + except OSError as e: + print(f"[error] cannot read {ids_file}: {e}", file=sys.stderr) + sys.exit(1) + + all_records: list[dict] = [] + papers_found = 0 + + # Process arXiv IDs + if all_arxiv_ids: + print( + f"[info] looking up {len(all_arxiv_ids)} arXiv IDs on Papers With Code ...", + file=sys.stderr, + ) + for arxiv_id in all_arxiv_ids: + records, title = process_arxiv_id(arxiv_id) + if title: + papers_found += 1 + all_records.extend(records) + time.sleep(REQUEST_DELAY) + + # Process keyword query + if args.query: + query_records = process_query(args.query) + all_records.extend(query_records) + + # Deduplicate + deduped = deduplicate_repos(all_records) + + # Write output + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + for record in deduped: + f.write(json.dumps(record, ensure_ascii=False) + "\n") + + # Summary + total_paper_links = sum(len(r.get("paper_ids", [])) for r in deduped) + official = sum(1 for r in deduped if r.get("is_official", False)) + + print(f"\n[info] results summary:", file=sys.stderr) + print(f" repos found: {len(deduped)}", file=sys.stderr) + print(f" linked to papers: {total_paper_links}", file=sys.stderr) + if official: + print(f" official implementations: {official}", file=sys.stderr) + print(f"Found {len(deduped)} repos linked to {papers_found} papers", file=sys.stderr) + print(f"[info] written to {output_path}", file=sys.stderr) + + +if __name__ == "__main__": + main()