diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..fe06955 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,35 @@ +name: CI + +on: + push: + branches: [main, v2] + pull_request: + +jobs: + build-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.25' + check-latest: true + + - name: Vet + run: go vet ./... + + - name: Test + run: go test ./... + + - name: Build (cgo-free, static single binary) + env: + CGO_ENABLED: 0 + run: go build -o github-export . + + - name: gofmt + run: | + unformatted=$(gofmt -l .) + if [ -n "$unformatted" ]; then + echo "These files need gofmt:"; echo "$unformatted"; exit 1 + fi diff --git a/.gitignore b/.gitignore index 6b2549b..4fdfdfb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ .claude /github-data/ -github-export \ No newline at end of file +github-export +*.sqlite +*.sqlite-wal +*.sqlite-shm \ No newline at end of file diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..ae1671a --- /dev/null +++ b/PLAN.md @@ -0,0 +1,560 @@ +# Plan: SQLite-backed store, on-demand Markdown export, OpenAPI + MCP, gh/GitHub-MCP parity + +## 1. Goals + +1. **SQLite as the primary store.** All synced GitHub content lives in a single + SQLite database as the source of truth. The sync engine writes to SQLite, not + to markdown. +2. **Markdown export becomes on demand and one-way.** Markdown files are a + *derived, one-way view* produced only when the user explicitly asks + (`github-export export`). SQLite is the sole source of truth; markdown is + never read back. The export format is free to evolve — it is not contractually + pinned to today's byte layout. +3. **OpenAPI spec.** Ship a machine-readable OpenAPI 3.1 description of a local + HTTP API that serves the synced data. +4. **MCP server.** Expose the data through a Model Context Protocol server whose + tool names/shapes match GitHub's official MCP server. +5. **Parity with `gh` CLI + GitHub MCP for reads/queries; proxy for writes.** + The local API mirrors GitHub's own REST/GraphQL surface closely enough that + reads can be served from SQLite offline, while any write (and any read we do + not yet mirror) is transparently proxied to `api.github.com`. + +### Non-goals (initial scope) + +- Re-implementing GitHub's *entire* REST surface by hand. We mirror the slices + we already sync; everything else falls through to the proxy. "100% match" is + achieved by the **proxy fallback**, not by hand-writing every endpoint. +- Becoming a write-through cache with conflict resolution. Writes go to GitHub; + we re-sync the affected entity afterward. No offline write queue. +- Multi-repo databases in v1 (one DB = one repo, matching today's model). The + schema leaves room for it (see §4). + +--- + +## 2. Where we are today + +- Go 1.22, single binary, only dep is `gopkg.in/yaml.v3`. +- `main.go` orchestrates `sync.Labels/Milestones/Projects/Issues/Releases/Discussions/Repo`. +- Each `sync.*` function fetches from REST/GraphQL (`internal/github/client.go`), + parses JSON into `map[string]any` (`internal/jsonutil`), and **writes markdown + directly** (`internal/document`). +- Incremental state is a single `synced_at` in `repo.yml`. +- Change detection (`internal/hooks`) diffs fresh API data against the previous + on-disk markdown frontmatter and emits `events/*.md`. + +The key structural fact: today **fetch, model, and render-to-markdown are fused** +inside each `sync.*` function. The core refactor is to split them into +**fetch → store (SQLite) → render**, so SQLite sits in the middle and markdown +becomes one of several renderers. + +--- + +## 3. Target architecture + +``` + GitHub REST + GraphQL (api.github.com) + ▲ pull (sync) ▲ push (write proxy) + │ │ + ┌───────────────┴──────────────────────┴────────────────┐ + │ core │ + │ internal/github – API client (unchanged) │ + │ internal/sync – fetch → UPSERT into store │ + │ internal/store – SQLite: schema, upserts, queries │ ← source of truth + │ internal/query – high-level read/query API │ + │ internal/gitbackend– repo content from a local git repo│ ← files/commits/code-search + └───┬───────────────┬───────────────┬───────────────┬────┘ + │ │ │ │ + exporter httpapi mcp writeproxy + (markdown, (REST+GraphQL+ (MCP tools, (POST/PATCH/… + on demand) OpenAPI, reads reads from → GitHub, + from query/git; query/git; then sync. + writes via proxy) writes proxy) re-sync) +``` + +**Centerpiece — the GitHub-API-compatible read proxy.** One HTTP handler speaks +GitHub's API shape (same paths/queries, same JSON field names) over three +read backends, with a proxy fallback: + +- **SQLite** (`internal/query`) for synced collaboration data — issues, PRs, + comments, reviews, timelines, labels, milestones, releases, **projects and + discussions** (the last two via GraphQL — see §7). +- **Local git** (`internal/gitbackend`) for repo *content* — file contents, + commits, blobs, branches/tags, diffs, blame, and code search — answered + straight from a local clone, no GitHub API needed (your decision on §6/4). +- **Proxy fallback** → `api.github.com` for anything neither backend can answer + (e.g. check runs, Actions, anything we don't mirror) **and all writes** + (response streamed back, then the affected entity re-synced into SQLite). + +This single component is simultaneously: the thing the **OpenAPI spec +describes**, the backend the **MCP server** calls, and the endpoint you point +**`gh`** at. We build the compatibility surface once and reuse it three ways. + +--- + +## 4. Component 1 — SQLite store (`internal/store`) + +### Library + +Use **`modernc.org/sqlite`** (pure-Go, cgo-free) to preserve today's +"`go build .` with no toolchain" property. (`mattn/go-sqlite3` is faster but +needs cgo — rejected for that reason; revisit only if perf demands it.) + +### Schema principles + +- **Dual storage per entity: typed columns + `raw_json`.** Typed columns + (`number`, `state`, `author`, timestamps, …) power querying and the API; + `raw_json` holds the full upstream payload so markdown export and API + responses are full-fidelity and round-trip-safe. This mirrors how `jsonutil` + already pulls fields off `map[string]any`. +- Use **SQLite JSON1** for flexible filtering and **FTS5** for body/comment text + search (this is what powers `gh search` / MCP `search_*` offline). +- Schema versioned via `meta` + a tiny migration runner (embedded `.sql` files, + `PRAGMA user_version`). +- `repo_id` column on every table from day one so a future multi-repo DB is an + additive change, not a rewrite. + +### Tables (initial) + +| Table | Key columns | Notes | +|---|---|---| +| `meta` | `key, value` | `owner`, `repo`, `synced_at`, `schema_version` (replaces `repo.yml` state) | +| `repositories` | `id`, owner, name, default_branch, visibility, … `raw_json` | repo metadata | +| `users` | `id`, `login`, type, `raw_json` | dedup actors/authors | +| `issues` | `number` PK, `is_pull_request`, title, state, state_reason, locked, author, milestone, created/updated/closed_at, body, `raw_json` | issues + PRs unified, as today | +| `pull_request_details` | `number` FK, draft, source/target_branch, source_repo, merged, merged_at, merged_by, merge_commit_sha | PR-only fields | +| `issue_labels` | `issue_number, label` | join | +| `issue_assignees` | `issue_number, login` | join | +| `issue_projects` | `issue_number, project_number` | drives the `projects:` frontmatter list | +| `comments` | `id` PK, issue_number, author, created/updated_at, body, `raw_json` | | +| `reviews` | `id` PK, pr_number, author, state, commit_sha, submitted_at, body | | +| `review_comments` | `id` PK, review_id, pr_number, author, path, line, side, commit_sha, created_at, body | | +| `timeline_events` | `id` PK, issue_number, event, actor, created_at, `extra_json` | labeled/assigned/renamed/… | +| `labels` | `name` PK, color, description | | +| `milestones` | `number` PK, title, state, description, due_on, closed_at | | +| `projects` | `number` PK, title, state, public, owner, url, description, created/updated_at, readme, `fields_json` | Projects v2 | +| `project_items` | `project_number, item_number`, type, title, repo, `fields_json` | | +| `discussions` | `number` PK, title, category, state, state_reason, author, created/updated/closed_at, answer_id, answer_chosen_at, answer_chosen_by, body | | +| `discussion_comments` | `id` PK, discussion_number, parent_id, author, created_at, is_answer, body | comment + reply, `parent_id` for nesting | +| `releases` | `tag` PK, name, draft, prerelease, author, created_at, published_at, target_commitish, body | | +| `release_assets` | `release_tag, name`, content_type, size_bytes, download_count | | +| `fts_content` | FTS5 virtual table | bodies of issues/comments/discussions for search | + +### Store API (sketch) + +```go +type Store struct{ db *sql.DB } + +func Open(path string) (*Store, error) // opens + runs migrations +func (s *Store) Meta() (owner, repo, syncedAt string, err error) +func (s *Store) Tx(fn func(*Txn) error) error // one transaction per sync pass + +// upserts take the raw map already parsed by jsonutil +func (t *Txn) UpsertIssue(m map[string]any) error +func (t *Txn) UpsertComment(issueNum int64, m map[string]any) error +func (t *Txn) ReplaceTimeline(issueNum int64, events []map[string]any) error +// … one per entity, each writing typed columns + raw_json +``` + +Each sync pass runs in a single transaction for atomicity and speed (SQLite is +much faster with batched writes; set `PRAGMA journal_mode=WAL`, +`synchronous=NORMAL`). + +--- + +## 5. Component 2 — Sync engine refactor (`internal/sync`) + +The refactor is mechanical and low-risk because the fetch + JSON-parse logic +stays; only the *sink* changes from "write markdown" to "upsert into store". + +1. Add a `*store.Txn` (or `*store.Store`) parameter to each `sync.*` function. +2. Replace `document.Writer` / file-writing tails with `txn.Upsert*` calls. +3. Move incremental state from `repo.yml` to `meta.synced_at`. +4. **Change detection moves to the store.** Today `hooks` diffs API data vs. + on-disk markdown. Instead, diff the incoming entity against its **current + SQLite row** (the previous `raw_json` / typed columns) *before* the upsert, + inside the transaction. This is cleaner and removes the markdown dependency + from event detection. Events are written to an `events` table (and rendered + to `events/*.md` only on export, or emitted live by `serve`/`mcp`). +5. `main.go`'s `sync` path becomes: open store → `Tx(func(txn){ Labels; Milestones; + Projects; Issues; Releases; Discussions; Repo })` → done. No markdown by default. + +The existing `--max-age` / `since` logic is unchanged — it still drives the +GitHub `?since=` filter; `synced_at` now comes from `meta`. + +--- + +## 6. Component 3 — Markdown exporter (`internal/exporter`) + +- New subcommand `github-export export [--out github-data/]`. +- **One-way and read-only against the store.** Export reads rows from SQLite and + renders files; nothing ever parses markdown back. The store is authoritative. +- Reuses `internal/document` as the rendering primitive. Writer functions are + fed by rows read back from the store (`raw_json` rehydrates the map shape, so + `writeIssueFile` etc. need minimal change). +- Produces a familiar layout (`issues/NNNN.md`, `projects/`, `discussions/`, + `releases/`, `labels.yml`, `milestones.yml`, `repo.yml`, `events/`), but the + format is **not** frozen — it may evolve as the store gains fields. We do not + maintain byte-for-byte compatibility with the pre-SQLite output. +- Optional `export --since` to write only entities changed since a timestamp. + +--- + +## 7. Component 4 — Read/query layer + HTTP API + OpenAPI + +### `internal/query` + +Backend-agnostic read functions over the store, shaped to serve all three +consumers (HTTP, MCP, exporter): + +```go +ListIssues(ListIssuesOpts) ([]Issue, error) // state, labels, author, since, sort, page +GetIssue(number) (Issue, error) +ListPullRequests(...) / GetPullRequest(...) +SearchIssues(query) (...) // FTS5-backed +ListDiscussions / GetDiscussion / ListReleases / ListProjects / ... +``` + +Return values serialize to **GitHub's REST JSON shape** (straight from +`raw_json` where possible) so the HTTP layer is a thin adapter. + +### `internal/httpapi` — `github-export serve [--addr :8080]` + +A `net/http` server. **No auth: it binds localhost only and trusts any local +caller** (your decision). The proxy fallback and writes spend the server's own +`GITHUB_TOKEN`; clients never present a token. + +It serves three read backends plus a fallback: + +1. **REST mirror from SQLite:** + - `GET /repos/{owner}/{repo}/issues` (+ `state`, `labels`, `since`, `sort`, `per_page`, `page`) + - `GET /repos/{owner}/{repo}/issues/{n}` and `/comments`, `/timeline` + - `GET /repos/{owner}/{repo}/pulls`, `/pulls/{n}`, `/pulls/{n}/reviews`, `/comments` + - `GET /repos/{owner}/{repo}/labels`, `/milestones`, `/releases` + - `GET /search/issues`, `GET /search/code` (FTS5 + git, see search below) + - `GET /repos/{owner}/{repo}` (metadata) + - Pagination via `Link` headers, matching GitHub. +2. **GraphQL mirror from SQLite** (`POST /graphql`) — **required for read + parity**, because Projects v2 and Discussions are GraphQL-only and `gh` / + GitHub MCP issue many reads as GraphQL. See "GraphQL read parity" below. +3. **Repo content from local git** (`internal/gitbackend`): + - `GET /repos/{o}/{r}/contents/{path}`, `/git/blobs/{sha}`, `/git/trees/{sha}` + - `GET /repos/{o}/{r}/commits`, `/commits/{sha}`, `/compare/{base}...{head}` + - `GET /repos/{o}/{r}/branches`, `/tags` + - answered from a local clone (`--repo-path`, default: the cwd's git repo); + no GitHub API call. +4. **Proxy fallback** (Component 5) for everything else (check runs, Actions, + unmirrored paths) **and all writes**. + +Response includes a freshness header (`X-GitHub-Export-Synced-At`) so callers +know how stale the local copy is. + +### GraphQL read parity (`internal/graphqlmirror`) + +`gh` and the GitHub MCP lean heavily on GraphQL, so a REST-only mirror is not +enough. Approach: + +- Parse incoming GraphQL queries (use a Go GraphQL parser, e.g. + `github.com/vektah/gqlparser`) against a **schema subset** covering the + fields we actually store (repository → issues/pullRequests/discussions/ + projectsV2 and their nested connections, plus `search`). +- Resolve fields from SQLite via `internal/query`; honor `first`/`after` + cursor pagination by encoding offsets the way the GraphQL connection spec + expects. +- Any query touching a field outside the supported schema **falls through to the + proxy** (`POST /graphql` → `api.github.com`), so coverage is "everything", + with the mirrored subset served offline and the rest forwarded. +- This is the larger build item in the API phase; scope it to the exact queries + `gh` and the GitHub MCP emit first (capture them with the shadow-compare mode + below), then widen. + +### OpenAPI 3.1 (`api/openapi.yaml`) + +- Describes exactly the **mirrored** read endpoints (the offline-serveable + subset) plus the proxied write endpoints we officially support. +- Author by hand initially (the surface is small and stable), reusing + component schemas lifted from GitHub's published OpenAPI for fidelity. +- Validate in CI with a linter (e.g. `vacuum`/`redocly`) and, ideally, generate + request/response model structs from it so handlers and spec cannot drift. +- Serve the spec at `GET /openapi.yaml` and a Swagger/Redoc page at `/docs`. + +### Local git backend (`internal/gitbackend`) + +Repo *content* endpoints (file contents, blobs, trees, commits, compare/diffs, +branches, tags, blame, **code search**) are answered from a **local git clone** +rather than the GitHub API or SQLite — your decision that "point 4 can be +answered by git directly." + +- `--repo-path` selects the working tree (default: discover the git repo of the + cwd; fall back to a managed clone under the DB's directory if absent). +- Shell out to `git` (or use `go-git`) for: `git show :` (contents), + `git cat-file` (blobs), `git log`/`rev-list` (commits), `git diff`/`merge-base` + (compare), `git for-each-ref` (branches/tags), `git grep` (code search). +- Map results into GitHub's REST JSON shapes so `gh` / MCP see no difference. +- `serve --git-fetch` optionally `git fetch` before answering, to stay current; + otherwise content is as fresh as the local checkout. +- **Not git-answerable** (check runs, Actions, deployments) stay proxy-only. + +### Search — full coverage + +Search must "support everything" (your decision): + +- Issue/PR/discussion search → FTS5 over bodies/titles + a qualifier parser + (`is:`, `state:`, `label:`, `author:`, `assignee:`, `milestone:`, `in:`, + date ranges, `sort:`) translated to SQL. +- Code search → `git grep` via the git backend. +- Any query using a qualifier we don't yet translate **falls through to the + proxy**, so the surface is complete even before every qualifier is local. + The shadow-compare mode (below) is how we find the gaps to close. + +### Shadow-compare debug mode (`--debug-compare`) + +A self-verifying parity harness, on `serve`/`mcp`/`api`: + +- For each **read**, answer it **both** locally (SQLite/git) **and** remotely + (proxy to `api.github.com`), then **diff the two JSON results**. +- On a meaningful difference (after normalizing volatile fields — rate-limit + headers, ephemeral URLs, ordering where GitHub is unordered), **automatically + open an issue on this project's own repo** (`mevdschee/github-export`) with the + request, both responses, and the diff — deduplicated by a fingerprint of + (endpoint + diff shape) so the same gap files once, not every request. +- The locally-computed answer is still what's returned to the caller; remote is + only for comparison. Off by default (it doubles request cost and needs a + token with issue-write scope on the project repo). +- Doubles as the data source for the §9 gh-parity matrix and the §7 GraphQL + "capture the queries gh actually emits" step. + +--- + +## 8. Component 5 — Write proxy (`internal/writeproxy`) + +- Any non-GET request (and unmirrored GETs) is forwarded to `api.github.com` + with the `GITHUB_TOKEN`, preserving method, path, query, body, and relevant + headers. Response (status, body, rate-limit headers) is streamed back + verbatim. +- **Read-after-write consistency (synchronous):** when a write targets a known + entity (`POST .../issues`, `PATCH .../issues/{n}`, `POST .../issues/{n}/comments`, + merges, label edits, …), run a **synchronous targeted re-sync** of just that + entity into SQLite *before returning to the caller* (your decision). The local + store is therefore guaranteed consistent the moment a write call returns — at + the cost of one extra GET on the write path. Map write paths → re-sync action + in a small table. If the re-sync GET fails, the write still succeeded upstream; + return success with a warning header and mark the entity stale. +- Configurable modes: `--proxy=on` (default), `--proxy=off` (offline-only; + writes and unmirrored reads return `501`), so the tool can run fully air-gapped. +- Surface this in `serve`, `mcp`, and a `github-export api ` + passthrough command (a `gh api` work-alike). + +--- + +## 9. Component 6 — `gh` CLI parity strategy + +"100% match the `gh` read/query API" is delivered in two complementary ways, +because `gh` itself is just a client of GitHub's API: + +1. **Proxy redirection (the 100% mechanism).** Document and support pointing + `gh` at our local server: + - `gh api` and most commands honor `GH_HOST` / a base-URL override; with the + compatibility proxy, every `gh api ` is served from SQLite when + mirrored and transparently proxied otherwise. Writes proxy through. This + gives full surface coverage without re-implementing `gh`. + - Ship a wrapper/env snippet (`GH_HOST=localhost:8080` or + `gh --hostname`) and verify against a matrix of common commands + (`gh issue list/view`, `gh pr list/view/diff`, `gh search issues`, + `gh release list`, `gh api ...`). +2. **Native convenience subcommands (ergonomic subset).** A handful of + first-class read commands for offline use without `gh` installed: + `github-export issue list/view`, `pr list/view`, `search`, `release list`, + mirroring `gh`'s flags and (optionally) its `--json` field selection. These + are thin wrappers over `internal/query`. + +A parity test matrix (Phase 4) runs the same query against (a) real `gh` → +GitHub and (b) `gh` → our proxy, and diffs the JSON to measure and track +coverage. + +--- + +## 10. Component 7 — MCP server (`internal/mcp` / `github-export mcp`) + +- Use the **official Go MCP SDK** (`github.com/modelcontextprotocol/go-sdk`). +- Serve over stdio by default (`github-export mcp`), optional `--http` for the + streamable-HTTP transport. +- **Tool names/shapes mirror GitHub's official MCP server** so existing agent + configs work unchanged. Read tools resolve against `internal/query` (SQLite); + write tools call `internal/writeproxy`: + + | Tool (GitHub MCP name) | Backed by | + |---|---| + | `get_issue`, `list_issues`, `search_issues` | SQLite (`query`) | + | `get_pull_request`, `list_pull_requests`, `get_pull_request_files`, `get_pull_request_reviews` | SQLite | + | `list_commits`, `get_file_contents`, `search_code` | proxy (not synced) → GitHub | + | `create_issue`, `add_issue_comment`, `update_issue`, `create_pull_request`, `merge_pull_request` | write proxy → GitHub + re-sync | + +- Each read tool returns the same JSON the GitHub MCP server returns (from + `raw_json`), plus a `synced_at` note so agents can reason about staleness. +- A capability/config flag chooses **read-only mode** (no write tools + registered) vs. **read-write** (write tools proxy out). + +--- + +## 11. CLI surface redesign + +Move from "flags only" to subcommands (keep the old bare form working for one +release with a deprecation notice): + +``` +github-export sync [--max-age=…] [owner/repo] [--db github.sqlite] # → SQLite (default action) +github-export export [--out github-data/] [--since=…] # SQLite → markdown (one-way) +github-export serve [--addr :8080] [--proxy=on|off] [--repo-path .] # HTTP API + OpenAPI + [--git-fetch] [--debug-compare] [--auto-sync=15m] +github-export mcp [--http] [--read-only] [--proxy=on|off] [--repo-path .] [--debug-compare] +github-export api [--input -] [--debug-compare] # gh-api-style passthrough +github-export issue|pr|search|release … # native read subcommands +``` + +- Default DB path: `./github.sqlite` (or `--db`). One repo per DB. +- `serve`/`mcp` bind **localhost and require no token from clients** (§7); the + server's `GITHUB_TOKEN` is used only for proxy fallback and writes. +- `--repo-path` points the git backend at a local clone (default: cwd's repo). +- **Fold in the TODO UX win:** when no `owner/repo` and no DB/`repo.yml` is + present, auto-detect from `git remote get-url origin` (parse + `github.com:OWNER/REPO(.git)`). Removes the most common first-run failure. +- Bare invocation with an existing DB ⇒ `sync` (incremental), matching today's + "run in the export dir to update" behavior. One repo per DB. + +--- + +## 12. Consistency & freshness model + +- The store is a **read cache of GitHub**, explicitly possibly-stale. +- Every API/MCP read advertises `synced_at`. A `GET /status` (and MCP `status` + tool) reports last sync time and per-entity counts. +- Optional `serve --auto-sync=15m` background incremental sync; otherwise the + data is exactly as fresh as the last `sync`. +- Writes trigger targeted re-sync (§8) so the entity you just changed is + immediately consistent locally. + +--- + +## 13. Migration & backward compatibility + +- **No markdown import.** Markdown is a one-way output; we never parse it back. + Existing users get a fresh SQLite store by re-running `sync` against GitHub + (the API is the source, not their old `github-data/` folder). +- **Sync state:** the only state worth carrying over is `synced_at`. On first + `sync`, if a legacy `repo.yml` is present in the working dir, read its + `synced_at` into the `meta` table so the first run stays incremental rather + than re-pulling everything. Otherwise a full sync runs as today. +- `export` output is allowed to differ from the pre-SQLite layout (§6); we do + not promise byte-stability to downstream markdown consumers. + +--- + +## 14. Testing strategy + +- **Refactor guard (Phase 1):** reframe `e2e_test.go` as sync→SQLite, asserting + on **store contents** (row counts, key fields, timeline ordering, event + detection) rather than markdown bytes. Since export is one-way and the format + may evolve, data integrity in the store — not file layout — is the contract. + A lighter `export` smoke test checks files render and parse as valid + markdown/YAML, without pinning exact bytes. +- **Store unit tests:** upsert idempotency, incremental diffing, FTS queries, + migration runner. +- **API tests:** golden JSON for mirrored endpoints; pagination/`Link` headers; + proxy fallback (httptest fake upstream); freshness headers. +- **OpenAPI:** lint spec in CI; contract-test that every mirrored handler + validates against the spec. +- **gh parity matrix (Phase 4):** diff `gh→GitHub` vs `gh→proxy` JSON for a + command list; track coverage %. +- **MCP:** tool-list snapshot vs. GitHub MCP names; per-tool I/O tests against + the store; write tools hit a fake upstream. +- Keep cgo-free: CI builds with `CGO_ENABLED=0`. + +--- + +## 15. Phased roadmap + +**Phase 0 — Foundations** +- Add `modernc.org/sqlite`; `internal/store` skeleton, migration runner, schema + v1, `Open`/`Meta`/`Tx`. + +**Phase 1 — SQLite as source of truth (highest value, de-risks everything)** +- Refactor `sync.*` to upsert into the store; move `synced_at` to `meta` + (seeding from a legacy `repo.yml` if present). +- Move change detection to store-diff; events → `events` table. +- Build `internal/exporter` + one-way `export` subcommand reusing `internal/document`. +- Reframe e2e test as sync→SQLite asserting on store contents; add an `export` + smoke test (valid markdown/YAML, not byte-pinned). +- *Deliverable:* default run populates SQLite; `export` renders markdown on demand. + +**Phase 2 — Read/query + REST mirror + OpenAPI** +- `internal/query`; `serve` with the REST mirror; `api/openapi.yaml`; `/docs` + + spec endpoint; freshness headers; `/status`. (REST only; GraphQL in Phase 3.) + +**Phase 3 — GraphQL mirror + git backend + write proxy** +- `internal/graphqlmirror` (`POST /graphql` served from SQLite, proxy + fallthrough) — needed for Projects/Discussions read parity. +- `internal/gitbackend` (files/commits/blobs/compare/code-search from a local + clone); decide `go-git` vs shell-out. +- `internal/writeproxy`; proxy fallback in `serve`; `github-export api` + passthrough; **synchronous** re-sync after writes; `--proxy` modes. + +**Phase 4 — gh parity + shadow-compare** +- Document `gh`→proxy redirection; native `issue/pr/search/release` subcommands + with `gh`-style `--json`. +- `--debug-compare` shadow mode (local vs remote diff → auto-file issue on + `mevdschee/github-export`); use it to capture the real `gh`/MCP query set and + drive the parity matrix + coverage report. + +**Phase 5 — MCP server** +- `internal/mcp` with GitHub-MCP-matching tool names; read tools over `query`, + write tools over `writeproxy`; stdio + HTTP transports; read-only flag. + +**Phase 6 — Polish** +- `serve --auto-sync`; multi-repo DB (optional); perf pass (indexes, batched + upserts); docs (`README`, `docs/` updates), and update + `docs/data-model.md`/`export-plan.md` to describe the SQLite schema. + +--- + +## 16. Decisions (settled) + +1. **SQLite driver:** `modernc.org/sqlite` (cgo-free), to keep single-binary, + no-toolchain builds. (Revisit `mattn/go-sqlite3` only if perf forces it.) +2. **Markdown is on-demand and one-way:** `sync` never writes markdown; `export` + is explicit and reads only from the store. Export format may evolve — no + byte-for-byte compatibility promise. +3. **One repo per DB**, default `./github.sqlite`. Schema still carries `repo_id` + so a multi-repo DB would be additive, not a rewrite. +4. **No markdown import:** the store is rebuilt from GitHub, not from old files; + only `synced_at` is carried over from a legacy `repo.yml` if present. +5. **MCP SDK:** official `github.com/modelcontextprotocol/go-sdk`. +6. **OpenAPI scope = mirrored reads + supported writes**, not all of GitHub. +7. **"100% parity"** is realized by the transparent proxy fallback (any + read/write we don't mirror is forwarded), with mirrored endpoints serving the + hot path offline — not by re-implementing every endpoint by hand. +8. **GraphQL is mirrored for reads**, not just proxied — required because + Projects/Discussions and much of `gh`/MCP are GraphQL (§7). Unsupported + fields fall through to the proxy. +9. **Local auth: none.** `serve`/`mcp` bind localhost and trust local callers; + only the server's `GITHUB_TOKEN` reaches GitHub. +10. **Repo content (files/commits/blobs/code-search) comes from local git**, not + the API; check runs/Actions remain proxy-only. +11. **Search supports everything** — local for parsed qualifiers + code search + via git, proxy fallthrough for the rest. +12. **Writes re-sync synchronously** before returning (read-after-write + consistency). +13. **Shadow-compare debug mode** answers locally + remotely, diffs, and + auto-files an issue on `mevdschee/github-export` on divergence (off by + default; also drives the parity matrix and GraphQL query capture). + +## 17. New dependencies + +- `modernc.org/sqlite` (store) +- `github.com/modelcontextprotocol/go-sdk` (MCP server) +- A GraphQL parser for the read mirror (`github.com/vektah/gqlparser`) +- `go-git` (`github.com/go-git/go-git/v5`) *or* shell out to the `git` binary + for the git backend — decide in Phase 3 (shell-out is simpler and matches + today's zero-heavy-deps spirit; `go-git` avoids a runtime `git` dependency). +- Dev/CI only: an OpenAPI linter (`redocly`/`vacuum`) +- (Still no cgo; `net/http`, `database/sql`, `yaml.v3` cover the rest.) +``` diff --git a/README.md b/README.md index 34cca9a..2f81ecc 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,147 @@ # github-export -Export all GitHub issues, pull requests, discussions, releases, labels, -milestones, and Projects (v2) from a repository into a local `github-data/` -folder as plain markdown files. Runs incrementally on subsequent invocations. +Sync all GitHub issues, pull requests, discussions, releases, labels, +milestones, and Projects (v2) from a repository into a local SQLite database +(`github.sqlite`), the primary store. Render that store to plain markdown files +on demand. Runs incrementally on subsequent invocations. -Designed so an agent can read, grep, and reason about GitHub data without API -access. +Designed so an agent can read, grep, query, and reason about GitHub data without +API access. Blog post: https://www.tqdev.com/2026-github-export-open-source-maintenance-ai/ ## Usage +The core subcommands are `sync` (GitHub → SQLite) and `export` (SQLite → +markdown). Running with no subcommand defaults to `sync`. On top of the store +sit a GitHub-compatible HTTP API (`serve`), an MCP server (`mcp`), a `gh api` +work-alike (`api`), and native read subcommands (`issue`/`pr`/`search`/`release`). + ```bash export GITHUB_TOKEN=$(gh auth token) -./github-export [flags] [owner/repo] [output-dir] + +# 1. Sync into ./github.sqlite (the source of truth) +./github-export sync [flags] [owner/repo] + +# 2. Render markdown from the database into ./github-data/ when you want it +./github-export export [--out github-data] ``` -The output directory defaults to `github-data/`. On the first run, all data is -fetched. On subsequent runs, only issues and PRs updated since the last sync are -re-fetched. +`owner/repo` is resolved from, in order: the argument, the existing database, or +the `origin` git remote of the current directory — so inside a checked-out +GitHub repo you can just run `github-export` with no arguments. On the first run +all data is fetched; on later runs only issues and PRs updated since the last +sync are re-fetched (the `synced_at` cursor lives in the database). + +`export` is one-way: it reads the database and writes files, never the reverse. +The database, not the markdown, is authoritative. + +### Flags + +`sync`: `--max-age` (see below), `--db` (database path, default +`github.sqlite`). +`export`: `--db` (database to read), `--out` (output directory, default +`github-data`). -Run with no arguments to sync the export in the current directory — owner/repo -are read from `./repo.yml`, so this is the easiest way to update an existing -export: +#### `--max-age` + +`--max-age=2y` (also `6mo`, `4w`, `30d`, `12h`) only fetches issues/PRs/projects +updated within this window — useful for the first sync of very large repos +(cli/cli, kubernetes/kubernetes). It is a floor on the effective `since`: on +later runs the more-recent `synced_at` recorded in the database takes over +automatically, so you can leave the flag in your sync command. Months count as +30 days and years as 365 days (approximate, not calendar-correct). + +## Serving the data (HTTP API, MCP, `gh`) + +Once synced, the same store can be served three ways. All of them answer +mirrored reads from SQLite (and repo *content* — files, commits, branches — from +a local git clone), and transparently proxy anything unmirrored, plus every +write, to `api.github.com` using the server's `GITHUB_TOKEN`. They bind +localhost and trust local callers (clients present no token). + +### `serve` — GitHub-compatible HTTP API + OpenAPI ```bash -cd path/to/github-data && github-export +./github-export serve --addr localhost:8080 +# API docs: http://localhost:8080/docs (Redoc over the OpenAPI 3.1 spec) +# Spec: http://localhost:8080/openapi.yaml +# Status: http://localhost:8080/status (freshness + per-entity counts) ``` -### Flags +Paths and JSON field names match GitHub's REST API, so you can point `gh` at it: + +```bash +GH_HOST=localhost:8080 gh issue list # served from SQLite, offline +``` + +Every mirrored response carries an `X-GitHub-Export-Synced-At` freshness header. +Writes (and unmirrored reads) proxy to GitHub and then synchronously re-sync the +touched entity, so a subsequent local read is immediately consistent. Use +`--proxy=off` to run fully offline (writes/unmirrored reads then return `501`). + +Flags: `--proxy on|off`, `--repo-path .` (git clone for content endpoints), +`--git-fetch`, `--auto-sync 15m` (background incremental sync), and +`--debug-compare` (answer reads locally *and* remotely, diff, and log +divergences; add `--debug-compare-file` to file deduplicated gap issues on +`mevdschee/github-export`). + +GraphQL (`POST /graphql`) is mirrored from SQLite for the common +`repository(owner,name)` reads — `issue`/`pullRequest`/`discussion`, +the `issues`/`pullRequests`/`discussions` connections (with `first`/`after` +cursor pagination, `states`, `totalCount`, `nodes`/`edges`/`pageInfo`), nested +`author`/`labels`/`assignees`/`comments`, and fragments/variables. The rule is +strict: a query is served locally only if **every** requested field is one the +mirror resolves correctly; the moment anything is unsupported (an unmapped +field, a mutation, a repo we have not synced) the whole request is forwarded to +`api.github.com`, so a response is never half-correct. Widening the covered +field set is the main place this mirror grows. + +### `mcp` — Model Context Protocol server + +```bash +./github-export mcp # stdio transport (default) +./github-export mcp --http localhost:8081 # streamable-HTTP transport +./github-export mcp --read-only # register read tools only +``` + +Tool names mirror GitHub's official MCP server so existing agent configs work +unchanged. Read tools resolve against the store/git clone; write tools proxy to +GitHub and re-sync. + +- Issues/PRs: `get_issue`, `get_issue_comments`, `list_issues`, + `get_pull_request`, `list_pull_requests`, `get_pull_request_reviews`, + `get_pull_request_comments`, `search_issues` +- Discussions: `list_discussions`, `get_discussion`, `get_discussion_comments` +- Content (local git): `get_file_contents`, `list_commits`, `get_commit`, + `list_branches`, `list_tags`, `search_code` +- Releases / status: `list_releases`, `status` +- Writes (proxy + re-sync): `create_issue`, `add_issue_comment`, `update_issue`, + `update_pull_request`, `merge_pull_request` -| Flag | Example | What it does | -| ----------- | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | -| `--max-age` | `--max-age=2y`, `6mo`, `4w`, `30d`, `12h` | Only fetch issues/PRs/projects updated within this window. Useful for the first sync of very large repos (cli/cli, kubernetes/kubernetes). | +### `api` — one-shot `gh api` work-alike -`--max-age` is a floor on the effective `since`: on later runs the more-recent -`synced_at` from `repo.yml` takes over automatically, so you can leave the flag -in your sync command. Months count as 30 days and years as 365 days -(approximate, not calendar-correct). +```bash +./github-export api '/repos/OWNER/REPO/issues?state=all' # local when mirrored +./github-export api --method PATCH --input - /repos/OWNER/REPO/issues/1 +``` + +### Native read subcommands (no `gh` needed) + +```bash +./github-export issue list --state all +./github-export issue view 42 --json +./github-export pr list +./github-export search "is:pr label:bug broken" +./github-export release list +``` ## What it syncs -| Data | Output | Sync mode | +Everything is stored in the SQLite database during `sync`; the paths below are +where `export` renders each entity. + +| Data | Exported to | Sync mode | | ------------- | --------------------------------- | ---------------- | | Labels | `github-data/labels.yml` | Always full sync | | Milestones | `github-data/milestones.yml` | Always full sync | @@ -101,10 +201,11 @@ are exported. ### Incremental sync -On second run, the program reads `synced_at` from `repo.yml` and passes it as +On second run, the program reads `synced_at` from the database and passes it as `?since=` to the GitHub Issues API. Only issues with `updated_at >= synced_at` -are re-fetched. Each touched issue file is fully rebuilt from the timeline -endpoint. +are re-fetched. Each touched issue is fully rebuilt in the store from the +timeline endpoint. Change detection diffs the incoming data against the existing +row, so events are emitted without re-reading any markdown. When `--max-age` is set on the first sync, the same `?since=` filter is used with the cutoff timestamp. The PR list is fetched sorted newest-first and @@ -218,8 +319,11 @@ GITHUB_TOKEN=$(gh auth token) go test ./... # full end-to-end suite The end-to-end suite (`e2e_test.go`) runs a real sync of this repository's TEST-prefixed fixtures (issues `#1–#3`, PRs `#4–#6`, discussions `#7–#8`, -project, releases, milestones) and asserts the exported files match. It -auto-skips when `GITHUB_TOKEN` is unset or `-short` is passed. +project, releases, milestones) into a SQLite store, asserts on the store +contents, then renders a markdown export and asserts on the rendered files. It +auto-skips when `GITHUB_TOKEN` is unset or `-short` is passed; project +assertions are skipped when the token lacks the `read:project` scope. The +`internal/store` package has standalone unit tests that need no token. ## Installation diff --git a/api.go b/api.go new file mode 100644 index 0000000..214b869 --- /dev/null +++ b/api.go @@ -0,0 +1,127 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "net/http/httptest" + "os" + "strings" + + "github.com/mevdschee/github-export/internal/gitbackend" + "github.com/mevdschee/github-export/internal/github" + "github.com/mevdschee/github-export/internal/graphqlmirror" + "github.com/mevdschee/github-export/internal/httpapi" + "github.com/mevdschee/github-export/internal/query" + "github.com/mevdschee/github-export/internal/shadow" + "github.com/mevdschee/github-export/internal/store" + "github.com/mevdschee/github-export/internal/sync" + "github.com/mevdschee/github-export/internal/writeproxy" +) + +// runAPI is a `gh api`-style passthrough: it answers GET reads from the local +// store/git when mirrored and proxies everything else to GitHub. It reuses the +// exact serve handler in-process, so behaviour matches `serve`. +func runAPI(args []string) { + fs := flag.NewFlagSet("api", flag.ExitOnError) + dbPath := fs.String("db", defaultDB, "path to the SQLite database") + method := fs.String("method", "GET", "HTTP method (or pass --method shorthand -X)") + input := fs.String("input", "", "request body file ('-' for stdin)") + proxyMode := fs.String("proxy", "on", "proxy fallback: on|off") + repoPath := fs.String("repo-path", ".", "local git clone for content endpoints") + debugCompare := fs.Bool("debug-compare", false, "also fetch the path from GitHub and log any divergence from the local answer") + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s api [flags] \n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Makes a single GitHub-API request, served locally when mirrored and proxied\n") + fmt.Fprintf(os.Stderr, "otherwise. Example: %s api /repos/OWNER/REPO/issues?state=all\n\n", os.Args[0]) + fs.PrintDefaults() + } + fs.Parse(args) + + path := fs.Arg(0) + if path == "" { + fs.Usage() + os.Exit(2) + } + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + + var body io.Reader + if *input == "-" { + body = os.Stdin + if *method == "GET" { + *method = "POST" + } + } else if *input != "" { + f, err := os.Open(*input) + if err != nil { + log.Fatalf("opening --input %s: %v", *input, err) + } + defer f.Close() + body = f + if *method == "GET" { + *method = "POST" + } + } + + s, err := store.Open(*dbPath) + if err != nil { + log.Fatalf("opening database %s: %v", *dbPath, err) + } + defer s.Close() + owner, repo, _ := s.OwnerRepo() + syncedAt, _ := s.SyncedAt() + + token := os.Getenv("GITHUB_TOKEN") + var client *github.Client + if token != "" { + client = github.NewClient(token) + } + proxy := writeproxy.New(token, func(kind string, number int64) error { + if client != nil && kind == "issue" && number > 0 { + return sync.ResyncIssue(client, s, owner, repo, number) + } + return nil + }) + if *proxyMode == "off" { + proxy.Disabled = true + } + + var git *gitbackend.Backend + if *repoPath != "" { + git = gitbackend.New(*repoPath, owner, repo, false) + if !git.Available() { + git = nil + } + } + + q := query.New(s) + srv := httpapi.New(httpapi.Config{ + Query: q, Proxy: proxy, Git: git, + GraphQL: graphqlmirror.New(q, owner, repo), + Owner: owner, Repo: repo, SyncedAt: syncedAt, + }) + + req := httptest.NewRequest(*method, path, body) + rec := httptest.NewRecorder() + srv.Handler().ServeHTTP(rec, req) + + res := rec.Result() + out, _ := io.ReadAll(res.Body) + os.Stdout.Write(out) + + // One-shot parity check: compare the local answer with GitHub synchronously + // (so the process does not exit before the diff is logged). + if *debugCompare && *method == "GET" && client != nil && res.StatusCode < 400 { + fetch := func(ctx context.Context, p string) (int, []byte, error) { + return proxy.Request(ctx, "GET", p, nil) + } + shadow.New(fetch, nil, owner+"/"+repo).Check(path, out) + } + if res.StatusCode >= 400 { + os.Exit(1) + } +} diff --git a/api/openapi.yaml b/api/openapi.yaml new file mode 100644 index 0000000..cf50946 --- /dev/null +++ b/api/openapi.yaml @@ -0,0 +1,261 @@ +openapi: 3.1.0 +info: + title: github-export local API + version: "1.0.0" + description: > + A GitHub-compatible read mirror served from a local SQLite store and a local + git clone. Mirrored reads are answered offline; any unmirrored read and all + writes fall through to api.github.com via the built-in proxy. Paths and JSON + field names match GitHub's REST API so existing clients (including `gh`) work + unchanged. The server binds localhost and requires no token from clients. +servers: + - url: http://localhost:8080 + description: Local serve address (default) +tags: + - name: issues + - name: pulls + - name: repository + - name: search + - name: content + - name: meta +paths: + /repos/{owner}/{repo}: + get: + tags: [repository] + summary: Get repository metadata + parameters: + - $ref: '#/components/parameters/owner' + - $ref: '#/components/parameters/repo' + responses: + '200': + description: Repository object (GitHub shape) + content: + application/json: + schema: { type: object } + /repos/{owner}/{repo}/issues: + get: + tags: [issues] + summary: List issues and pull requests + parameters: + - $ref: '#/components/parameters/owner' + - $ref: '#/components/parameters/repo' + - { name: state, in: query, schema: { type: string, enum: [open, closed, all], default: open } } + - { name: labels, in: query, schema: { type: string }, description: Comma-separated label names } + - { name: creator, in: query, schema: { type: string } } + - { name: assignee, in: query, schema: { type: string } } + - { name: milestone, in: query, schema: { type: string } } + - { name: since, in: query, schema: { type: string, format: date-time } } + - { name: sort, in: query, schema: { type: string, enum: [created, updated], default: created } } + - { name: direction, in: query, schema: { type: string, enum: [asc, desc], default: desc } } + - $ref: '#/components/parameters/perPage' + - $ref: '#/components/parameters/page' + responses: + '200': + description: Array of issue objects + headers: + Link: { $ref: '#/components/headers/Link' } + X-GitHub-Export-Synced-At: { $ref: '#/components/headers/SyncedAt' } + content: + application/json: + schema: { type: array, items: { type: object } } + /repos/{owner}/{repo}/issues/{number}: + get: + tags: [issues] + summary: Get an issue + parameters: + - $ref: '#/components/parameters/owner' + - $ref: '#/components/parameters/repo' + - $ref: '#/components/parameters/number' + responses: + '200': { description: Issue object, content: { application/json: { schema: { type: object } } } } + /repos/{owner}/{repo}/issues/{number}/comments: + get: + tags: [issues] + summary: List issue comments + parameters: + - $ref: '#/components/parameters/owner' + - $ref: '#/components/parameters/repo' + - $ref: '#/components/parameters/number' + responses: + '200': { description: Array of comment objects, content: { application/json: { schema: { type: array, items: { type: object } } } } } + /repos/{owner}/{repo}/issues/{number}/timeline: + get: + tags: [issues] + summary: List the issue/PR timeline + parameters: + - $ref: '#/components/parameters/owner' + - $ref: '#/components/parameters/repo' + - $ref: '#/components/parameters/number' + responses: + '200': { description: Array of timeline events, content: { application/json: { schema: { type: array, items: { type: object } } } } } + /repos/{owner}/{repo}/pulls: + get: + tags: [pulls] + summary: List pull requests + parameters: + - $ref: '#/components/parameters/owner' + - $ref: '#/components/parameters/repo' + - { name: state, in: query, schema: { type: string, enum: [open, closed, all], default: open } } + - $ref: '#/components/parameters/perPage' + - $ref: '#/components/parameters/page' + responses: + '200': + description: Array of pull request objects + headers: + Link: { $ref: '#/components/headers/Link' } + content: { application/json: { schema: { type: array, items: { type: object } } } } + /repos/{owner}/{repo}/pulls/{number}: + get: + tags: [pulls] + summary: Get a pull request + parameters: + - $ref: '#/components/parameters/owner' + - $ref: '#/components/parameters/repo' + - $ref: '#/components/parameters/number' + responses: + '200': { description: Pull request object, content: { application/json: { schema: { type: object } } } } + /repos/{owner}/{repo}/pulls/{number}/reviews: + get: + tags: [pulls] + summary: List reviews on a pull request + parameters: + - $ref: '#/components/parameters/owner' + - $ref: '#/components/parameters/repo' + - $ref: '#/components/parameters/number' + responses: + '200': { description: Array of review objects, content: { application/json: { schema: { type: array, items: { type: object } } } } } + /repos/{owner}/{repo}/pulls/{number}/comments: + get: + tags: [pulls] + summary: List review comments on a pull request + parameters: + - $ref: '#/components/parameters/owner' + - $ref: '#/components/parameters/repo' + - $ref: '#/components/parameters/number' + responses: + '200': { description: Array of review comment objects, content: { application/json: { schema: { type: array, items: { type: object } } } } } + /repos/{owner}/{repo}/labels: + get: + tags: [repository] + summary: List labels + parameters: [ { $ref: '#/components/parameters/owner' }, { $ref: '#/components/parameters/repo' } ] + responses: { '200': { description: Array of label objects, content: { application/json: { schema: { type: array, items: { type: object } } } } } } + /repos/{owner}/{repo}/milestones: + get: + tags: [repository] + summary: List milestones + parameters: [ { $ref: '#/components/parameters/owner' }, { $ref: '#/components/parameters/repo' } ] + responses: { '200': { description: Array of milestone objects, content: { application/json: { schema: { type: array, items: { type: object } } } } } } + /repos/{owner}/{repo}/releases: + get: + tags: [repository] + summary: List releases + parameters: [ { $ref: '#/components/parameters/owner' }, { $ref: '#/components/parameters/repo' } ] + responses: { '200': { description: Array of release objects, content: { application/json: { schema: { type: array, items: { type: object } } } } } } + /repos/{owner}/{repo}/releases/tags/{tag}: + get: + tags: [repository] + summary: Get a release by tag + parameters: + - $ref: '#/components/parameters/owner' + - $ref: '#/components/parameters/repo' + - { name: tag, in: path, required: true, schema: { type: string } } + responses: { '200': { description: Release object, content: { application/json: { schema: { type: object } } } } } + /repos/{owner}/{repo}/branches: + get: + tags: [content] + summary: List branches (from the local git clone) + parameters: [ { $ref: '#/components/parameters/owner' }, { $ref: '#/components/parameters/repo' } ] + responses: { '200': { description: Array of branch objects, content: { application/json: { schema: { type: array, items: { type: object } } } } } } + /repos/{owner}/{repo}/tags: + get: + tags: [content] + summary: List tags (from the local git clone) + parameters: [ { $ref: '#/components/parameters/owner' }, { $ref: '#/components/parameters/repo' } ] + responses: { '200': { description: Array of tag objects, content: { application/json: { schema: { type: array, items: { type: object } } } } } } + /repos/{owner}/{repo}/commits: + get: + tags: [content] + summary: List commits (from the local git clone) + parameters: + - $ref: '#/components/parameters/owner' + - $ref: '#/components/parameters/repo' + - { name: sha, in: query, schema: { type: string } } + - $ref: '#/components/parameters/perPage' + - $ref: '#/components/parameters/page' + responses: { '200': { description: Array of commit objects, content: { application/json: { schema: { type: array, items: { type: object } } } } } } + /repos/{owner}/{repo}/commits/{sha}: + get: + tags: [content] + summary: Get a commit (from the local git clone) + parameters: + - $ref: '#/components/parameters/owner' + - $ref: '#/components/parameters/repo' + - { name: sha, in: path, required: true, schema: { type: string } } + responses: { '200': { description: Commit object, content: { application/json: { schema: { type: object } } } } } + /repos/{owner}/{repo}/contents/{path}: + get: + tags: [content] + summary: Get file or directory contents (from the local git clone) + parameters: + - $ref: '#/components/parameters/owner' + - $ref: '#/components/parameters/repo' + - { name: path, in: path, required: true, schema: { type: string } } + - { name: ref, in: query, schema: { type: string } } + responses: { '200': { description: File object or directory listing, content: { application/json: { schema: { type: object } } } } } + /search/issues: + get: + tags: [search] + summary: Search issues and pull requests + parameters: + - { name: q, in: query, required: true, schema: { type: string }, description: "GitHub search query (is:, label:, author:, state:, milestone:, free text)" } + - $ref: '#/components/parameters/perPage' + - $ref: '#/components/parameters/page' + responses: + '200': + description: Search results + content: + application/json: + schema: + type: object + properties: + total_count: { type: integer } + incomplete_results: { type: boolean } + items: { type: array, items: { type: object } } + /status: + get: + tags: [meta] + summary: Local store status and freshness + responses: + '200': + description: Status object + content: + application/json: + schema: + type: object + properties: + owner: { type: string } + repo: { type: string } + synced_at: { type: string, format: date-time } + proxy: { type: boolean } + git: { type: boolean } + counts: { type: object, additionalProperties: { type: integer } } + /openapi.yaml: + get: + tags: [meta] + summary: This OpenAPI document + responses: { '200': { description: OpenAPI 3.1 spec, content: { application/yaml: {} } } } +components: + parameters: + owner: { name: owner, in: path, required: true, schema: { type: string } } + repo: { name: repo, in: path, required: true, schema: { type: string } } + number: { name: number, in: path, required: true, schema: { type: integer } } + perPage: { name: per_page, in: query, schema: { type: integer, default: 30, maximum: 100 } } + page: { name: page, in: query, schema: { type: integer, default: 1 } } + headers: + Link: + description: GitHub-style pagination links + schema: { type: string } + SyncedAt: + description: Timestamp of the last sync (freshness indicator) + schema: { type: string, format: date-time } diff --git a/api/spec.go b/api/spec.go new file mode 100644 index 0000000..0b54aa3 --- /dev/null +++ b/api/spec.go @@ -0,0 +1,9 @@ +// Package api embeds the OpenAPI 3.1 description of the locally-mirrored HTTP +// surface so the serve command can publish it at /openapi.yaml without depending +// on the file being present at runtime. +package api + +import _ "embed" + +//go:embed openapi.yaml +var OpenAPISpec []byte diff --git a/api/spec_test.go b/api/spec_test.go new file mode 100644 index 0000000..e7db746 --- /dev/null +++ b/api/spec_test.go @@ -0,0 +1,91 @@ +package api + +import ( + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// openapiDoc is a minimal view of the structure we assert on. It intentionally +// avoids a heavyweight OpenAPI library to keep the project's dependency set +// small; it validates the invariants that matter for our hand-written spec. +type openapiDoc struct { + OpenAPI string `yaml:"openapi"` + Info struct { + Title string `yaml:"title"` + Version string `yaml:"version"` + } `yaml:"info"` + Paths map[string]map[string]struct { + Responses map[string]any `yaml:"responses"` + } `yaml:"paths"` +} + +func loadSpec(t *testing.T) openapiDoc { + t.Helper() + var doc openapiDoc + if err := yaml.Unmarshal(OpenAPISpec, &doc); err != nil { + t.Fatalf("openapi.yaml is not valid YAML: %v", err) + } + return doc +} + +func TestSpecIsOpenAPI31(t *testing.T) { + doc := loadSpec(t) + if !strings.HasPrefix(doc.OpenAPI, "3.1") { + t.Errorf("openapi version = %q, want 3.1.x", doc.OpenAPI) + } + if doc.Info.Title == "" || doc.Info.Version == "" { + t.Errorf("info.title/version must be set: %+v", doc.Info) + } + if len(doc.Paths) == 0 { + t.Fatal("spec declares no paths") + } +} + +func TestEveryPathHasAnOperationWithResponses(t *testing.T) { + doc := loadSpec(t) + methods := map[string]bool{"get": true, "post": true, "put": true, "patch": true, "delete": true} + for path, item := range doc.Paths { + ops := 0 + for method, op := range item { + if !methods[method] { + continue + } + ops++ + if len(op.Responses) == 0 { + t.Errorf("%s %s declares no responses", strings.ToUpper(method), path) + } + } + if ops == 0 { + t.Errorf("path %s has no HTTP operations", path) + } + } +} + +// TestDocumentedMirrorPathsPresent guards against the spec drifting away from +// the endpoints the server actually mirrors. If a handler path is renamed, this +// list and the spec must be updated together. +func TestDocumentedMirrorPathsPresent(t *testing.T) { + doc := loadSpec(t) + want := []string{ + "/repos/{owner}/{repo}", + "/repos/{owner}/{repo}/issues", + "/repos/{owner}/{repo}/issues/{number}", + "/repos/{owner}/{repo}/issues/{number}/comments", + "/repos/{owner}/{repo}/pulls", + "/repos/{owner}/{repo}/pulls/{number}", + "/repos/{owner}/{repo}/pulls/{number}/reviews", + "/repos/{owner}/{repo}/labels", + "/repos/{owner}/{repo}/releases", + "/repos/{owner}/{repo}/commits", + "/repos/{owner}/{repo}/contents/{path}", + "/search/issues", + "/status", + } + for _, p := range want { + if _, ok := doc.Paths[p]; !ok { + t.Errorf("OpenAPI spec is missing documented path %q", p) + } + } +} diff --git a/e2e_test.go b/e2e_test.go index da2233d..f27411e 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -16,7 +16,9 @@ import ( "testing" "time" + "github.com/mevdschee/github-export/internal/exporter" "github.com/mevdschee/github-export/internal/github" + "github.com/mevdschee/github-export/internal/store" "github.com/mevdschee/github-export/internal/sync" "gopkg.in/yaml.v3" ) @@ -26,6 +28,15 @@ const ( testRepo = "github-export" ) +// projectsSynced records whether Projects v2 data made it into the store. It is +// false when the token lacks the read:project scope, in which case the +// project-coupled assertions are skipped rather than reported as failures. +var projectsSynced bool + +// TestE2E runs a real full sync of the TEST fixtures into a SQLite store, then +// asserts on the store contents (the source of truth) and on a one-way markdown +// export of that store. The export assertions parse the files structurally, so +// they tolerate format evolution. func TestE2E(t *testing.T) { token := os.Getenv("GITHUB_TOKEN") if token == "" { @@ -36,36 +47,29 @@ func TestE2E(t *testing.T) { } out := t.TempDir() - for _, sub := range []string{"issues", "releases", "projects"} { - if err := os.MkdirAll(filepath.Join(out, sub), 0o755); err != nil { - t.Fatalf("mkdir %s: %v", sub, err) - } + dbPath := filepath.Join(t.TempDir(), "e2e.sqlite") + s, err := store.Open(dbPath) + if err != nil { + t.Fatalf("store.Open: %v", err) } + defer s.Close() c := github.NewClient(token) - if err := sync.Labels(c, testOwner, testRepo, out); err != nil { - t.Fatalf("sync.Labels: %v", err) - } - if err := sync.Milestones(c, testOwner, testRepo, out); err != nil { - t.Fatalf("sync.Milestones: %v", err) - } - issueProjects, _, err := sync.Projects(c, testOwner, testRepo, out, "") - if err != nil { - t.Fatalf("sync.Projects: %v", err) + syncStart := time.Now().UTC().Format(time.RFC3339) + if _, err := sync.Run(c, s, testOwner, testRepo, "", syncStart); err != nil { + t.Fatalf("sync.Run: %v", err) } - if _, err := sync.Issues(c, testOwner, testRepo, out, "", issueProjects); err != nil { - t.Fatalf("sync.Issues: %v", err) + if err := exporter.Export(s, out); err != nil { + t.Fatalf("exporter.Export: %v", err) } - if _, err := sync.Releases(c, testOwner, testRepo, out); err != nil { - t.Fatalf("sync.Releases: %v", err) - } - if _, err := sync.Discussions(c, testOwner, testRepo, out, ""); err != nil { - t.Fatalf("sync.Discussions: %v", err) - } - if err := sync.Repo(c, testOwner, testRepo, out, time.Now().UTC().Format(time.RFC3339)); err != nil { - t.Fatalf("sync.Repo: %v", err) + + if projs, _ := s.AllProjects(); len(projs) > 0 { + projectsSynced = true + } else { + t.Log("no projects synced (token likely lacks read:project scope); skipping project assertions") } + t.Run("Store", func(t *testing.T) { checkStore(t, s) }) t.Run("Labels", func(t *testing.T) { checkLabels(t, out) }) t.Run("Milestones", func(t *testing.T) { checkMilestones(t, out) }) t.Run("Issue1_OpenWithFullMetadata", func(t *testing.T) { checkIssue1(t, out) }) @@ -524,6 +528,9 @@ func checkPR6(t *testing.T, out string) { } func checkProject1(t *testing.T, out string) { + if !projectsSynced { + t.Skip("projects not synced (token lacks read:project scope)") + } docs := parseDocs(t, filepath.Join(out, "projects", "0001.md")) if len(docs) == 0 { t.Fatal("no docs in projects/0001.md") @@ -625,11 +632,106 @@ func checkCommonScalars(t *testing.T, fr map[string]any) { if !containsString(toStringList(fr["labels"]), "TEST") { t.Errorf("labels missing TEST: %v", fr["labels"]) } - if !containsString(toStringList(fr["projects"]), "TEST exporter fixtures") { + if projectsSynced && !containsString(toStringList(fr["projects"]), "TEST exporter fixtures") { t.Errorf("projects missing %q: %v", "TEST exporter fixtures", fr["projects"]) } } +// checkStore asserts the store holds the expected fixtures. The store is the +// source of truth, so these checks are independent of the markdown format. +func checkStore(t *testing.T, s *store.Store) { + owner, repo, err := s.OwnerRepo() + if err != nil || owner != testOwner || repo != testRepo { + t.Errorf("OwnerRepo = %s/%s (err %v), want %s/%s", owner, repo, err, testOwner, testRepo) + } + if at, _ := s.SyncedAt(); !strings.Contains(at, "T") { + t.Errorf("synced_at malformed or empty: %q", at) + } + + issues, err := s.AllIssues() + if err != nil { + t.Fatalf("AllIssues: %v", err) + } + byNum := map[int64]store.IssueRow{} + for _, r := range issues { + byNum[r.Number] = r + } + for n := int64(1); n <= 6; n++ { + if _, ok := byNum[n]; !ok { + t.Errorf("issue #%d missing from store", n) + } + } + // #1 is a plain open issue; #4..#6 are PRs. + if r := byNum[1]; r.IsPR { + t.Error("#1 stored as PR, want issue") + } + for _, n := range []int64{4, 5, 6} { + if r, ok := byNum[n]; ok && !r.IsPR { + t.Errorf("#%d stored as issue, want PR", n) + } + } + + // State transitions via typed columns. + if exists, state, isPR, merged, _ := s.IssueState(4); !exists || !isPR || state != "closed" || !merged { + t.Errorf("#4 state: exists=%v isPR=%v state=%q merged=%v, want merged closed PR", exists, isPR, state, merged) + } + if exists, state, _, _, _ := s.IssueState(1); !exists || state != "open" { + t.Errorf("#1 state=%q, want open", state) + } + + if max, _ := s.MaxIssueNumber(); max < 6 { + t.Errorf("MaxIssueNumber=%d, want >=6", max) + } + + // Discussions, projects, releases present. + discussions, _ := s.AllDiscussions() + dseen := map[int64]bool{} + for _, d := range discussions { + dseen[d.Number] = true + } + if !dseen[7] || !dseen[8] { + t.Errorf("discussions 7 and 8 expected in store; got %v", dseen) + } + + if projectsSynced { + projects, _ := s.AllProjects() + found := false + for _, p := range projects { + if p.Number == 1 { + found = true + } + } + if !found { + t.Error("project #1 missing from store") + } + } + + releases, _ := s.AllReleases() + relFound := false + for _, m := range releases { + if tag, _ := m["tag_name"].(string); tag == "TEST-v0.0.1" { + relFound = true + } + } + if !relFound { + t.Error("release TEST-v0.0.1 missing from store") + } + + // A full sync records *_created events; verify at least the issue/PR ones. + var eventCount int + if err := s.DB().QueryRow("SELECT COUNT(*) FROM events").Scan(&eventCount); err != nil { + t.Fatalf("counting events: %v", err) + } + if eventCount == 0 { + t.Error("no events recorded by full sync") + } + var created int + s.DB().QueryRow("SELECT COUNT(*) FROM events WHERE type IN ('issue_created','pr_created')").Scan(&created) + if created < 6 { + t.Errorf("issue/pr_created events = %d, want >=6", created) + } +} + // --- multi-document markdown parser --- type doc struct { diff --git a/go.mod b/go.mod index 934214d..42c5941 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,32 @@ module github.com/mevdschee/github-export -go 1.22 +go 1.25.0 -require gopkg.in/yaml.v3 v3.0.1 +require ( + github.com/modelcontextprotocol/go-sdk v1.6.1 + github.com/vektah/gqlparser/v2 v2.5.33 + gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.34.4 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/jsonschema-go v0.4.3 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/segmentio/asm v1.1.3 // indirect + github.com/segmentio/encoding v0.5.4 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + golang.org/x/oauth2 v0.35.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.41.0 // indirect + modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect + modernc.org/strutil v1.2.0 // indirect + modernc.org/token v1.1.0 // indirect +) diff --git a/go.sum b/go.sum index a62c313..345b14e 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,81 @@ +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= +github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modelcontextprotocol/go-sdk v1.6.1 h1:0zOSupjKUxPKSocPT1Wtago+mUHU2/uZ4xSOY0FGReU= +github.com/modelcontextprotocol/go-sdk v1.6.1/go.mod h1:kzm3kzFL1/+AziGOE0nUs3gvPoNxMCvkxokMkuFapXQ= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= +github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= +github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= +github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/vektah/gqlparser/v2 v2.5.33 h1:lRp8aIeNUNbimf/axZd7ETg24q06hBtPaas+TcvI/7E= +github.com/vektah/gqlparser/v2 v2.5.33/go.mod h1:c1I28gSOVNzlfc4WuDlqU7voQnsqI6OG2amkBAFmgts= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= +golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.34.4 h1:sjdARozcL5KJBvYQvLlZEmctRgW9xqIZc2ncN7PU0P8= +modernc.org/sqlite v1.34.4/go.mod h1:3QQFCG2SEMtc2nv+Wq4cQCH7Hjcg+p/RMlS1XK+zwbk= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/exporter/exporter.go b/internal/exporter/exporter.go new file mode 100644 index 0000000..b5e9ec2 --- /dev/null +++ b/internal/exporter/exporter.go @@ -0,0 +1,262 @@ +// Package exporter renders the store's contents to the markdown export layout +// on demand. It is one-way: it reads from the store and writes files, and never +// parses markdown back. The store remains the source of truth. +package exporter + +import ( + "fmt" + "log" + "os" + "path/filepath" + "time" + + "github.com/mevdschee/github-export/internal/config" + "github.com/mevdschee/github-export/internal/ghmodel" + "github.com/mevdschee/github-export/internal/hooks" + "github.com/mevdschee/github-export/internal/jsonutil" + "github.com/mevdschee/github-export/internal/render" + "github.com/mevdschee/github-export/internal/store" + + "gopkg.in/yaml.v3" +) + +// dataDirs are recreated on each export so the output reflects exactly what the +// store holds (e.g. a closed project's file does not linger). The events/ dir is +// deliberately preserved — it is a handoff inbox the caller may have partly +// consumed. +var dataDirs = []string{"issues", "projects", "discussions", "releases"} + +// Export writes the full markdown view of the store into outDir. +func Export(s *store.Store, outDir string) error { + if err := os.MkdirAll(outDir, 0o755); err != nil { + return fmt.Errorf("creating %s: %w", outDir, err) + } + for _, d := range dataDirs { + p := filepath.Join(outDir, d) + if err := os.RemoveAll(p); err != nil { + return fmt.Errorf("clearing %s: %w", p, err) + } + if err := os.MkdirAll(p, 0o755); err != nil { + return fmt.Errorf("creating %s: %w", p, err) + } + } + + if err := exportRepoMeta(s, outDir); err != nil { + return err + } + if err := exportLabels(s, outDir); err != nil { + return err + } + if err := exportMilestones(s, outDir); err != nil { + return err + } + if err := exportIssues(s, outDir); err != nil { + return err + } + if err := exportProjects(s, outDir); err != nil { + return err + } + if err := exportDiscussions(s, outDir); err != nil { + return err + } + if err := exportReleases(s, outDir); err != nil { + return err + } + return exportEvents(s, outDir) +} + +func exportRepoMeta(s *store.Store, outDir string) error { + owner, repo, m, syncedAt, err := s.ExportRepo() + if err != nil { + return fmt.Errorf("reading repo metadata: %w", err) + } + if m == nil { + return nil + } + cfg := &config.RepoConfig{ + Owner: owner, + Repo: repo, + DefaultBranch: jsonutil.Str(m, "default_branch"), + Description: jsonutil.Str(m, "description"), + Homepage: jsonutil.Str(m, "homepage"), + Visibility: jsonutil.Str(m, "visibility"), + Language: jsonutil.Str(m, "language"), + License: jsonutil.Str(jsonutil.Map(m, "license"), "name"), + Topics: stringList(jsonutil.List(m, "topics")), + Archived: jsonutil.Bool(m, "archived"), + HasIssues: jsonutil.Bool(m, "has_issues"), + HasProjects: jsonutil.Bool(m, "has_projects"), + HasWiki: jsonutil.Bool(m, "has_wiki"), + HasPages: jsonutil.Bool(m, "has_pages"), + HasDiscussions: jsonutil.Bool(m, "has_discussions"), + CreatedAt: jsonutil.Str(m, "created_at"), + UpdatedAt: jsonutil.Str(m, "updated_at"), + PushedAt: jsonutil.Str(m, "pushed_at"), + SyncedAt: syncedAt, + } + return config.WriteRepoConfig(filepath.Join(outDir, "repo.yml"), cfg) +} + +func exportLabels(s *store.Store, outDir string) error { + maps, err := s.AllLabels() + if err != nil { + return fmt.Errorf("reading labels: %w", err) + } + type label struct { + Name string `yaml:"name"` + Color string `yaml:"color"` + Description string `yaml:"description,omitempty"` + } + out := make([]label, 0, len(maps)) + for _, m := range maps { + out = append(out, label{ + Name: jsonutil.Str(m, "name"), + Color: jsonutil.Str(m, "color"), + Description: jsonutil.Str(m, "description"), + }) + } + return writeYAML(filepath.Join(outDir, "labels.yml"), out) +} + +func exportMilestones(s *store.Store, outDir string) error { + maps, err := s.AllMilestones() + if err != nil { + return fmt.Errorf("reading milestones: %w", err) + } + type milestone struct { + Title string `yaml:"title"` + State string `yaml:"state"` + Description string `yaml:"description,omitempty"` + DueOn string `yaml:"due_on,omitempty"` + ClosedAt string `yaml:"closed_at,omitempty"` + } + out := make([]milestone, 0, len(maps)) + for _, m := range maps { + out = append(out, milestone{ + Title: jsonutil.Str(m, "title"), + State: jsonutil.Str(m, "state"), + Description: jsonutil.Str(m, "description"), + DueOn: jsonutil.Str(m, "due_on"), + ClosedAt: jsonutil.Str(m, "closed_at"), + }) + } + return writeYAML(filepath.Join(outDir, "milestones.yml"), out) +} + +func exportIssues(s *store.Store, outDir string) error { + rows, err := s.AllIssues() + if err != nil { + return fmt.Errorf("reading issues: %w", err) + } + var maxNum int64 + for _, r := range rows { + if r.Number > maxNum { + maxNum = r.Number + } + } + pad := ghmodel.PadWidth(maxNum) + dir := filepath.Join(outDir, "issues") + for _, r := range rows { + path := filepath.Join(dir, ghmodel.IssueFilename(r.Number, pad)) + if err := render.IssueFile(path, r.Issue, r.IsPR, r.PR, r.Timeline, r.Projects); err != nil { + log.Printf("Warning: exporting issue #%d: %v", r.Number, err) + } + } + return nil +} + +func exportProjects(s *store.Store, outDir string) error { + rows, err := s.AllProjects() + if err != nil { + return fmt.Errorf("reading projects: %w", err) + } + dir := filepath.Join(outDir, "projects") + for _, r := range rows { + path := filepath.Join(dir, fmt.Sprintf("%04d.md", r.Number)) + if err := render.ProjectFile(path, r.Project, r.Items); err != nil { + log.Printf("Warning: exporting project #%d: %v", r.Number, err) + } + } + return nil +} + +func exportDiscussions(s *store.Store, outDir string) error { + nodes, err := s.AllDiscussions() + if err != nil { + return fmt.Errorf("reading discussions: %w", err) + } + dir := filepath.Join(outDir, "discussions") + for _, d := range nodes { + path := filepath.Join(dir, fmt.Sprintf("%04d.md", d.Number)) + if err := render.DiscussionFile(path, d); err != nil { + log.Printf("Warning: exporting discussion #%d: %v", d.Number, err) + } + } + return nil +} + +func exportReleases(s *store.Store, outDir string) error { + maps, err := s.AllReleases() + if err != nil { + return fmt.Errorf("reading releases: %w", err) + } + dir := filepath.Join(outDir, "releases") + for _, m := range maps { + tag := jsonutil.Str(m, "tag_name") + if tag == "" { + continue + } + path := filepath.Join(dir, ghmodel.SafeTag(tag)+".md") + if err := render.ReleaseFile(path, m); err != nil { + log.Printf("Warning: exporting release %q: %v", tag, err) + } + } + return nil +} + +// exportEvents renders not-yet-exported events to events/*.md and stamps them so +// re-running export does not re-create files the caller may have consumed. +func exportEvents(s *store.Store, outDir string) error { + pending, err := s.PendingEvents() + if err != nil { + return fmt.Errorf("reading events: %w", err) + } + if len(pending) == 0 { + return nil + } + events := make([]hooks.Event, 0, len(pending)) + ids := make([]int64, 0, len(pending)) + for _, pe := range pending { + ev := pe.Event + // The stored file path is repo-relative (e.g. issues/0042.md); point it + // at this export's output location. + if ev.File != "" { + ev.File = filepath.Join(outDir, ev.File) + } + events = append(events, ev) + ids = append(ids, pe.ID) + } + eventsDir := filepath.Join(outDir, "events") + if err := hooks.Export(eventsDir, events); err != nil { + return fmt.Errorf("writing events: %w", err) + } + return s.MarkEventsExported(ids, time.Now().UTC().Format(time.RFC3339)) +} + +func writeYAML(path string, v any) error { + data, err := yaml.Marshal(v) + if err != nil { + return err + } + return os.WriteFile(path, data, 0o644) +} + +func stringList(xs []any) []string { + out := make([]string, 0, len(xs)) + for _, x := range xs { + if s, ok := x.(string); ok { + out = append(out, s) + } + } + return out +} diff --git a/internal/ghmodel/ghmodel.go b/internal/ghmodel/ghmodel.go new file mode 100644 index 0000000..f239f84 --- /dev/null +++ b/internal/ghmodel/ghmodel.go @@ -0,0 +1,173 @@ +// Package ghmodel holds GitHub domain types and pure helpers shared between the +// sync layer (which detects changes) and the render layer (which writes +// markdown). Keeping them here avoids an import cycle between the two and gives +// the discussion GraphQL shapes a single home that both the fetcher and the +// exporter can unmarshal into. +package ghmodel + +import ( + "fmt" + "strings" + + "github.com/mevdschee/github-export/internal/jsonutil" +) + +// --- Discussion GraphQL shapes (fetched by sync, rendered by exporter) --- + +type DiscussionAuthor struct { + Login string `json:"login"` +} + +type DiscussionCategory struct { + Name string `json:"name"` + Emoji string `json:"emoji"` + IsAnswerable bool `json:"isAnswerable"` +} + +type DiscussionReply struct { + DatabaseID int64 `json:"databaseId"` + CreatedAt string `json:"createdAt"` + Body string `json:"body"` + Author DiscussionAuthor `json:"author"` +} + +type DiscussionComment struct { + DatabaseID int64 `json:"databaseId"` + CreatedAt string `json:"createdAt"` + Body string `json:"body"` + Author DiscussionAuthor `json:"author"` + Replies struct { + PageInfo struct { + HasNextPage bool `json:"hasNextPage"` + } `json:"pageInfo"` + Nodes []DiscussionReply `json:"nodes"` + } `json:"replies"` +} + +type DiscussionNode struct { + Number int64 `json:"number"` + Title string `json:"title"` + Body string `json:"body"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + Closed bool `json:"closed"` + ClosedAt string `json:"closedAt"` + StateReason string `json:"stateReason"` + Locked bool `json:"locked"` + URL string `json:"url"` + Author DiscussionAuthor `json:"author"` + Category DiscussionCategory `json:"category"` + Labels struct { + Nodes []struct { + Name string `json:"name"` + } `json:"nodes"` + } `json:"labels"` + Answer *struct { + DatabaseID int64 `json:"databaseId"` + } `json:"answer"` + AnswerChosenAt string `json:"answerChosenAt"` + AnswerChosenBy *DiscussionAuthor `json:"answerChosenBy"` + Comments struct { + PageInfo struct { + HasNextPage bool `json:"hasNextPage"` + } `json:"pageInfo"` + Nodes []DiscussionComment `json:"nodes"` + } `json:"comments"` +} + +// DiscussionState maps the boolean closed flag to the open/closed string used +// across the export. +func DiscussionState(d DiscussionNode) string { + if d.Closed { + return "closed" + } + return "open" +} + +// AnswerID returns the chosen-answer comment ID, or 0 if the discussion has no +// accepted answer. +func (d DiscussionNode) AnswerID() int64 { + if d.Answer == nil { + return 0 + } + return d.Answer.DatabaseID +} + +// --- Projects v2 helpers --- + +// FieldValueString flattens a single Projects v2 field value node into its +// string representation, keyed off the GraphQL __typename. +func FieldValueString(fv map[string]any) string { + switch jsonutil.Str(fv, "__typename") { + case "ProjectV2ItemFieldTextValue": + return jsonutil.Str(fv, "text") + case "ProjectV2ItemFieldNumberValue": + return jsonutil.Str(fv, "number") + case "ProjectV2ItemFieldDateValue": + return jsonutil.Str(fv, "date") + case "ProjectV2ItemFieldSingleSelectValue": + return jsonutil.Str(fv, "name") + case "ProjectV2ItemFieldIterationValue": + return jsonutil.Str(fv, "title") + } + return "" +} + +// OwnerLogin returns the login of a project's owner object. +func OwnerLogin(p map[string]any) string { + o := jsonutil.Map(p, "owner") + if o == nil { + return "" + } + return jsonutil.Str(o, "login") +} + +// ExtractItemFields flattens a project item's fieldValues into name → string +// pairs (empty values dropped). +func ExtractItemFields(item map[string]any) map[string]string { + out := map[string]string{} + fvMap := jsonutil.Map(item, "fieldValues") + for _, fv := range jsonutil.List(fvMap, "nodes") { + fvm, _ := fv.(map[string]any) + if fvm == nil { + continue + } + fld := jsonutil.Map(fvm, "field") + if fld == nil { + continue + } + name := jsonutil.Str(fld, "name") + if name == "" { + continue + } + if v := FieldValueString(fvm); v != "" { + out[name] = v + } + } + return out +} + +// --- Filename helpers --- + +// PadWidth returns the zero-pad width for issue/PR/discussion/project filenames +// given the largest number in play. +func PadWidth(maxNumber int64) int { + switch { + case maxNumber >= 100000: + return 6 + case maxNumber >= 10000: + return 5 + default: + return 4 + } +} + +// IssueFilename renders the markdown filename for a numbered entity. +func IssueFilename(number int64, pad int) string { + return fmt.Sprintf("%0*d.md", pad, number) +} + +// SafeTag turns a release tag into a filesystem-safe base name. +func SafeTag(tag string) string { + return strings.ReplaceAll(tag, "/", "-") +} diff --git a/internal/gitbackend/gitbackend.go b/internal/gitbackend/gitbackend.go new file mode 100644 index 0000000..14b7288 --- /dev/null +++ b/internal/gitbackend/gitbackend.go @@ -0,0 +1,277 @@ +// Package gitbackend answers GitHub "repo content" reads — branches, tags, +// commits, and file contents — straight from a local git clone, with no call to +// api.github.com. Results are mapped into GitHub's REST JSON shapes so callers +// (gh, the MCP server) see no difference. Content-type endpoints that are not +// answerable from git (check runs, Actions) are left to the proxy fallback. +package gitbackend + +import ( + "encoding/base64" + "fmt" + "os/exec" + "strconv" + "strings" +) + +// Backend runs git commands against a working tree. +type Backend struct { + dir string + owner, repo string + fetch bool +} + +// New builds a Backend rooted at dir. owner/repo are used to populate URL-ish +// fields in responses. If fetch is true, Fetch() should be called before serving. +func New(dir, owner, repo string, fetch bool) *Backend { + return &Backend{dir: dir, owner: owner, repo: repo, fetch: fetch} +} + +// Available reports whether dir is inside a git work tree. +func (b *Backend) Available() bool { + return b.git("rev-parse", "--is-inside-work-tree") == "true" +} + +// Fetch runs `git fetch --all` to refresh refs (best-effort). +func (b *Backend) Fetch() error { + if !b.fetch { + return nil + } + _, err := b.run("fetch", "--all", "--quiet") + return err +} + +// run executes git in the backend's directory and returns trimmed stdout. +func (b *Backend) run(args ...string) (string, error) { + cmd := exec.Command("git", args...) + cmd.Dir = b.dir + out, err := cmd.Output() + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + return "", fmt.Errorf("git %s: %s", strings.Join(args, " "), strings.TrimSpace(string(ee.Stderr))) + } + return "", err + } + return strings.TrimRight(string(out), "\n"), nil +} + +// git is run() that swallows errors (returns "" on failure) for probe calls. +func (b *Backend) git(args ...string) string { + out, err := b.run(args...) + if err != nil { + return "" + } + return out +} + +// Branches returns the local/remote branch refs in the /repos/.../branches shape. +func (b *Backend) Branches() ([]map[string]any, error) { + out, err := b.run("for-each-ref", "--format=%(refname:short) %(objectname)", "refs/heads") + if err != nil { + return nil, err + } + var branches []map[string]any + for _, line := range nonEmptyLines(out) { + name, sha, _ := strings.Cut(line, " ") + branches = append(branches, map[string]any{ + "name": name, + "commit": map[string]any{"sha": sha}, + "protected": false, + }) + } + return branches, nil +} + +// Tags returns tag refs in the /repos/.../tags shape. +func (b *Backend) Tags() ([]map[string]any, error) { + out, err := b.run("for-each-ref", "--format=%(refname:short) %(objectname)", "refs/tags") + if err != nil { + return nil, err + } + var tags []map[string]any + for _, line := range nonEmptyLines(out) { + name, sha, _ := strings.Cut(line, " ") + tags = append(tags, map[string]any{ + "name": name, + "commit": map[string]any{"sha": sha}, + }) + } + return tags, nil +} + +// commitFormat is a unit-separator-delimited git log format we can parse safely. +const commitFormat = "%H%x1f%an%x1f%ae%x1f%aI%x1f%cn%x1f%ce%x1f%cI%x1f%P%x1f%s%x1f%b%x1e" + +// Commits lists commits (newest first) in the /repos/.../commits shape. sha is +// an optional starting ref; perPage/page paginate. +func (b *Backend) Commits(sha string, perPage, page int) ([]map[string]any, error) { + if perPage <= 0 { + perPage = 30 + } + if page <= 0 { + page = 1 + } + args := []string{"log", "--no-color", "--pretty=format:" + commitFormat, + "--skip=" + strconv.Itoa((page-1)*perPage), "-n", strconv.Itoa(perPage)} + if sha != "" { + args = append(args, sha) + } + out, err := b.run(args...) + if err != nil { + return nil, err + } + var commits []map[string]any + for _, rec := range strings.Split(out, "\x1e") { + rec = strings.Trim(rec, "\n") + if rec == "" { + continue + } + commits = append(commits, b.parseCommit(rec)) + } + return commits, nil +} + +// Commit returns a single commit by ref, or ok=false if it does not resolve. +func (b *Backend) Commit(ref string) (map[string]any, bool, error) { + out, err := b.run("log", "--no-color", "-n", "1", "--pretty=format:"+commitFormat, ref) + if err != nil { + return nil, false, nil // unknown ref: let the caller proxy + } + rec := strings.Trim(out, "\n\x1e") + if rec == "" { + return nil, false, nil + } + return b.parseCommit(rec), true, nil +} + +func (b *Backend) parseCommit(rec string) map[string]any { + f := strings.Split(rec, "\x1f") + get := func(i int) string { + if i < len(f) { + return f[i] + } + return "" + } + message := get(8) + if body := get(9); body != "" { + message += "\n\n" + body + } + var parents []map[string]any + for _, p := range strings.Fields(get(7)) { + parents = append(parents, map[string]any{"sha": p}) + } + sha := get(0) + return map[string]any{ + "sha": sha, + "commit": map[string]any{ + "author": map[string]any{"name": get(1), "email": get(2), "date": get(3)}, + "committer": map[string]any{"name": get(4), "email": get(5), "date": get(6)}, + "message": message, + }, + "parents": parents, + } +} + +// Contents returns the /repos/.../contents/{path} shape for ref (default HEAD). +// A file returns a single object with base64 content; a directory returns a +// listing array (returned as []map). ok=false means the path/ref is unknown. +func (b *Backend) Contents(path, ref string) (any, bool, error) { + if ref == "" { + ref = "HEAD" + } + path = strings.Trim(path, "/") + objType := b.git("cat-file", "-t", ref+":"+path) + switch objType { + case "blob": + raw, err := b.run("show", ref+":"+path) + if err != nil { + return nil, false, nil + } + name := path + if i := strings.LastIndex(path, "/"); i >= 0 { + name = path[i+1:] + } + return map[string]any{ + "type": "file", + "name": name, + "path": path, + "size": len(raw), + "encoding": "base64", + "content": base64.StdEncoding.EncodeToString([]byte(raw)), + }, true, nil + case "tree": + out, err := b.run("ls-tree", "--full-tree", ref, ensureTrailingSlash(path)) + if err != nil { + return nil, false, nil + } + var entries []map[string]any + for _, line := range nonEmptyLines(out) { + // \t + meta, file, ok := strings.Cut(line, "\t") + if !ok { + continue + } + parts := strings.Fields(meta) + if len(parts) < 3 { + continue + } + etype := "file" + if parts[1] == "tree" { + etype = "dir" + } + name := file + if i := strings.LastIndex(file, "/"); i >= 0 { + name = file[i+1:] + } + entries = append(entries, map[string]any{ + "type": etype, "name": name, "path": file, "sha": parts[2], + }) + } + return entries, true, nil + } + return nil, false, nil +} + +func ensureTrailingSlash(p string) string { + if p == "" { + return "" + } + return p + "/" +} + +func nonEmptyLines(s string) []string { + var out []string + for _, l := range strings.Split(s, "\n") { + if strings.TrimSpace(l) != "" { + out = append(out, l) + } + } + return out +} + +// Grep runs `git grep` for code search, returning matches as a slice of +// {path, line_number, line} maps. ref defaults to HEAD. +func (b *Backend) Grep(pattern, ref string) ([]map[string]any, error) { + if ref == "" { + ref = "HEAD" + } + out, err := b.run("grep", "-n", "-I", "--fixed-strings", pattern, ref) + if err != nil { + // git grep exits non-zero when there are no matches; treat as empty. + return nil, nil + } + var results []map[string]any + for _, line := range nonEmptyLines(out) { + // ref:path:lineno:content + rest := strings.TrimPrefix(line, ref+":") + path, after, ok := strings.Cut(rest, ":") + if !ok { + continue + } + lineno, content, _ := strings.Cut(after, ":") + n, _ := strconv.Atoi(lineno) + results = append(results, map[string]any{ + "path": path, "line_number": n, "line": content, + }) + } + return results, nil +} diff --git a/internal/gitbackend/gitbackend_test.go b/internal/gitbackend/gitbackend_test.go new file mode 100644 index 0000000..22d7102 --- /dev/null +++ b/internal/gitbackend/gitbackend_test.go @@ -0,0 +1,110 @@ +package gitbackend + +import ( + "encoding/base64" + "strings" + "testing" +) + +// These tests run against the github-export repo itself (tests execute inside +// the package directory, which is within the work tree). + +func newBackend() *Backend { return New(".", "mevdschee", "github-export", false) } + +func TestAvailable(t *testing.T) { + if !newBackend().Available() { + t.Skip("not in a git work tree") + } +} + +func TestBranches(t *testing.T) { + b := newBackend() + if !b.Available() { + t.Skip("no git") + } + branches, err := b.Branches() + if err != nil { + t.Fatal(err) + } + if len(branches) == 0 { + t.Fatal("expected at least one branch") + } + if _, ok := branches[0]["commit"].(map[string]any); !ok { + t.Errorf("branch missing commit object: %v", branches[0]) + } +} + +func TestCommits(t *testing.T) { + b := newBackend() + if !b.Available() { + t.Skip("no git") + } + commits, err := b.Commits("", 5, 1) + if err != nil { + t.Fatal(err) + } + if len(commits) == 0 { + t.Fatal("expected commits") + } + c := commits[0] + if c["sha"] == "" { + t.Errorf("commit missing sha: %v", c) + } + commit, _ := c["commit"].(map[string]any) + if commit == nil || commit["message"] == "" { + t.Errorf("commit missing message: %v", c) + } +} + +func TestContentsFile(t *testing.T) { + b := newBackend() + if !b.Available() { + t.Skip("no git") + } + doc, ok, err := b.Contents("go.mod", "") + if err != nil || !ok { + t.Fatalf("Contents go.mod ok=%v err=%v", ok, err) + } + m := doc.(map[string]any) + if m["type"] != "file" || m["encoding"] != "base64" { + t.Errorf("unexpected file shape: %v", m) + } + raw, err := base64.StdEncoding.DecodeString(m["content"].(string)) + if err != nil || !strings.Contains(string(raw), "module github.com/mevdschee/github-export") { + t.Errorf("decoded content wrong: err=%v", err) + } +} + +func TestContentsDir(t *testing.T) { + b := newBackend() + if !b.Available() { + t.Skip("no git") + } + doc, ok, err := b.Contents("internal", "") + if err != nil || !ok { + t.Fatalf("Contents internal ok=%v err=%v", ok, err) + } + entries, isList := doc.([]map[string]any) + if !isList || len(entries) == 0 { + t.Fatalf("expected directory listing, got %T", doc) + } + foundDir := false + for _, e := range entries { + if e["type"] == "dir" { + foundDir = true + } + } + if !foundDir { + t.Errorf("expected at least one subdir in internal/: %v", entries) + } +} + +func TestContentsMiss(t *testing.T) { + b := newBackend() + if !b.Available() { + t.Skip("no git") + } + if _, ok, _ := b.Contents("does/not/exist.xyz", ""); ok { + t.Error("expected miss for nonexistent path") + } +} diff --git a/internal/github/client.go b/internal/github/client.go index 30f3ee0..0e16741 100644 --- a/internal/github/client.go +++ b/internal/github/client.go @@ -21,6 +21,10 @@ const ( type Client struct { token string client *http.Client + + scopes []string + scopesKnown bool + scopesFetched bool } func NewClient(token string) *Client { @@ -30,6 +34,44 @@ func NewClient(token string) *Client { } } +// Scopes returns the OAuth scopes granted to the token, as reported by the +// X-OAuth-Scopes header on an authenticated request. The boolean is false when +// the token does not report scopes — fine-grained PATs and GitHub App +// installation tokens omit the header — in which case the caller must not +// infer that any scope is missing. The result is fetched once and cached. +func (c *Client) Scopes() ([]string, bool) { + if c.scopesFetched { + return c.scopes, c.scopesKnown + } + c.scopesFetched = true + + req, err := http.NewRequest("GET", API+"/", nil) + if err != nil { + return nil, false + } + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("Authorization", "Bearer "+c.token) + + resp, err := c.client.Do(req) + if err != nil { + return nil, false + } + defer resp.Body.Close() + io.Copy(io.Discard, resp.Body) + + header, ok := resp.Header["X-Oauth-Scopes"] + if !ok { + return nil, false + } + c.scopesKnown = true + for _, part := range strings.Split(strings.Join(header, ","), ",") { + if s := strings.TrimSpace(part); s != "" { + c.scopes = append(c.scopes, s) + } + } + return c.scopes, true +} + var linkNextRe = regexp.MustCompile(`<([^>]+)>;\s*rel="next"`) func apiError(url string, status int, body []byte) error { diff --git a/internal/graphqlmirror/graphqlmirror.go b/internal/graphqlmirror/graphqlmirror.go new file mode 100644 index 0000000..f632446 --- /dev/null +++ b/internal/graphqlmirror/graphqlmirror.go @@ -0,0 +1,826 @@ +// Package graphqlmirror answers a subset of GitHub's GraphQL API from the local +// SQLite store. gh and the GitHub MCP issue many reads as GraphQL (and Projects +// v2 / Discussions are GraphQL-only), so a REST-only mirror is not enough. +// +// The design rule is strict: a query is served locally only if EVERY requested +// field is one the mirror knows how to resolve correctly. The moment any field, +// argument, or top-level shape is unsupported, Execute reports servedLocally +// false and the caller forwards the original request to api.github.com. We never +// return a half-correct GraphQL response. +package graphqlmirror + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/vektah/gqlparser/v2/ast" + "github.com/vektah/gqlparser/v2/parser" + + "github.com/mevdschee/github-export/internal/query" +) + +// errUnsupported signals that some part of the query cannot be served locally; +// the caller proxies the whole request. +var errUnsupported = errors.New("graphql: unsupported field for local mirror") + +// Mirror resolves GraphQL queries against the store. +type Mirror struct { + q *query.Query + owner string + repo string +} + +// New builds a Mirror. +func New(q *query.Query, owner, repo string) *Mirror { + return &Mirror{q: q, owner: owner, repo: repo} +} + +// exec carries per-request state (fragments + variables) through resolution. +type exec struct { + m *Mirror + fragments map[string]*ast.FragmentDefinition + vars map[string]any +} + +// Execute resolves a GraphQL request. servedLocally is false when any part of +// the query is unsupported (the caller must then proxy to GitHub). +func (m *Mirror) Execute(queryStr string, variables map[string]any) (response json.RawMessage, servedLocally bool) { + doc, err := parser.ParseQuery(&ast.Source{Input: queryStr}) + if err != nil || len(doc.Operations) != 1 { + return nil, false + } + op := doc.Operations[0] + if op.Operation != ast.Query { + return nil, false // mutations/subscriptions always proxy + } + e := &exec{ + m: m, + fragments: map[string]*ast.FragmentDefinition{}, + vars: variables, + } + for _, f := range doc.Fragments { + e.fragments[f.Name] = f + } + + data, err := e.resolveObject(op.SelectionSet, e.resolveRoot) + if err != nil { + return nil, false + } + out, err := json.Marshal(map[string]any{"data": data}) + if err != nil { + return nil, false + } + return out, true +} + +// resolveObject walks a selection set, resolving each field with fn and keying +// the result by alias-or-name. Fragment spreads and inline fragments are +// flattened. Any unsupported field aborts the whole resolution. +func (e *exec) resolveObject(sel ast.SelectionSet, fn func(*ast.Field) (any, error)) (map[string]any, error) { + out := map[string]any{} + for _, s := range sel { + switch f := s.(type) { + case *ast.Field: + if f.Name == "__typename" { + continue // tolerated; value omitted + } + v, err := fn(f) + if err != nil { + return nil, err + } + out[aliasOf(f)] = v + case *ast.FragmentSpread: + frag := e.fragments[f.Name] + if frag == nil { + return nil, errUnsupported + } + sub, err := e.resolveObject(frag.SelectionSet, fn) + if err != nil { + return nil, err + } + for k, v := range sub { + out[k] = v + } + case *ast.InlineFragment: + sub, err := e.resolveObject(f.SelectionSet, fn) + if err != nil { + return nil, err + } + for k, v := range sub { + out[k] = v + } + default: + return nil, errUnsupported + } + } + return out, nil +} + +// --- root: only `repository(owner,name)` (and a tolerated rateLimit) --- + +func (e *exec) resolveRoot(f *ast.Field) (any, error) { + switch f.Name { + case "repository": + owner := e.argStr(f, "owner") + name := e.argStr(f, "name") + // A mismatch with the synced repo means we have no local data: proxy. + if owner != e.m.owner || name != e.m.repo { + return nil, errUnsupported + } + var repoMap map[string]any + if doc, ok, err := e.m.q.Repo(); err == nil && ok { + json.Unmarshal(doc, &repoMap) + } + return e.resolveObject(f.SelectionSet, e.repositoryResolver(repoMap)) + default: + return nil, errUnsupported + } +} + +// repositoryResolver resolves repository fields, with scalar fields backed by +// the stored REST repo payload (translated to GraphQL names). +func (e *exec) repositoryResolver(repo map[string]any) func(*ast.Field) (any, error) { + return func(f *ast.Field) (any, error) { + switch f.Name { + case "name": + return e.m.repo, nil + case "nameWithOwner": + return e.m.owner + "/" + e.m.repo, nil + case "owner": + return e.resolveObject(f.SelectionSet, loginFieldFn(e.m.owner)) + case "description": + return nullable(repo["description"]), nil + case "url", "resourcePath": + return nullable(repo["html_url"]), nil + case "homepageUrl": + return nullable(repo["homepage"]), nil + case "isPrivate": + return boolOf(repo["private"]), nil + case "isArchived": + return boolOf(repo["archived"]), nil + case "isFork": + return boolOf(repo["fork"]), nil + case "databaseId", "id": + return jsonNumber(repo["id"]), nil + case "createdAt": + return nullable(repo["created_at"]), nil + case "updatedAt": + return nullable(repo["updated_at"]), nil + case "pushedAt": + return nullable(repo["pushed_at"]), nil + case "stargazerCount": + return jsonNumber(repo["stargazers_count"]), nil + case "forkCount": + return jsonNumber(repo["forks_count"]), nil + case "defaultBranchRef": + branch := str(repo, "default_branch") + if branch == "" { + return nil, nil + } + return e.resolveObject(f.SelectionSet, func(rf *ast.Field) (any, error) { + if rf.Name == "name" { + return branch, nil + } + return nil, errUnsupported + }) + case "issue": + return e.resolveSingleIssue(f, false) + case "pullRequest": + return e.resolveSingleIssue(f, true) + case "issueOrPullRequest": + return e.resolveSingleIssue(f, false) + case "discussion": + return e.resolveSingleDiscussion(f) + case "projectV2": + return e.resolveSingleProject(f) + case "issues": + return e.resolveIssueConnection(f, false) + case "pullRequests": + return e.resolveIssueConnection(f, true) + case "discussions": + return e.resolveDiscussionConnection(f) + case "projectsV2": + return e.resolveProjectConnection(f) + default: + return nil, errUnsupported + } + } +} + +// --- single issue / PR --- + +func (e *exec) resolveSingleIssue(f *ast.Field, wantPR bool) (any, error) { + number := e.argInt(f, "number") + if number == 0 { + return nil, errUnsupported + } + var doc json.RawMessage + var ok bool + var err error + if wantPR { + doc, ok, err = e.m.q.GetPull(number) + } else { + doc, ok, err = e.m.q.GetIssue(number) + } + if err != nil { + return nil, errUnsupported + } + if !ok { + return nil, nil // GraphQL null for a missing node is valid + } + var issue map[string]any + if err := json.Unmarshal(doc, &issue); err != nil { + return nil, errUnsupported + } + return e.resolveObject(f.SelectionSet, e.issueFieldResolver(number, issue)) +} + +// issueFieldResolver maps GraphQL issue/PR fields onto the stored REST payload. +func (e *exec) issueFieldResolver(number int64, issue map[string]any) func(*ast.Field) (any, error) { + return func(f *ast.Field) (any, error) { + switch f.Name { + case "number": + return jsonNumber(issue["number"]), nil + case "title": + return issue["title"], nil + case "body": + return issue["body"], nil + case "url", "permalink", "resourcePath": + return issue["html_url"], nil + case "state": + return upperState(str(issue, "state")), nil + case "closed": + return str(issue, "state") == "closed", nil + case "createdAt": + return issue["created_at"], nil + case "updatedAt": + return issue["updated_at"], nil + case "closedAt": + return nullable(issue["closed_at"]), nil + case "locked": + return boolOf(issue["locked"]), nil + case "isDraft": + return boolOf(issue["draft"]), nil + case "merged": + return boolOf(issue["merged"]), nil + case "mergedAt": + return nullable(issue["merged_at"]), nil + case "databaseId", "id": + return jsonNumber(issue["id"]), nil + case "author", "user": + return e.resolveObject(f.SelectionSet, loginResolver(userLogin(issue, "user"))) + case "labels": + return e.resolveLabelConnection(f, labelList(issue)) + case "assignees": + return e.resolveLoginConnection(f, logins(issue, "assignees")) + case "comments": + return e.resolveCommentConnection(f, number) + default: + return nil, errUnsupported + } + } +} + +// --- discussions (stored already in GraphQL shape) --- + +func (e *exec) resolveSingleDiscussion(f *ast.Field) (any, error) { + number := e.argInt(f, "number") + if number == 0 { + return nil, errUnsupported + } + doc, ok, err := e.m.q.GetDiscussion(number) + if err != nil { + return nil, errUnsupported + } + if !ok { + return nil, nil + } + var node map[string]any + if err := json.Unmarshal(doc, &node); err != nil { + return nil, errUnsupported + } + return e.resolveObject(f.SelectionSet, e.genericResolver(node)) +} + +// --- projects v2 (stored already in GraphQL shape) --- + +func (e *exec) resolveSingleProject(f *ast.Field) (any, error) { + number := e.argInt(f, "number") + if number == 0 { + return nil, errUnsupported + } + doc, ok, err := e.m.q.GetProject(number) + if err != nil { + return nil, errUnsupported + } + if !ok { + return nil, nil + } + var node map[string]any + if err := json.Unmarshal(doc, &node); err != nil { + return nil, errUnsupported + } + return e.resolveObject(f.SelectionSet, e.genericResolver(node)) +} + +func (e *exec) resolveProjectConnection(f *ast.Field) (any, error) { + docs, err := e.m.q.ListProjects() + if err != nil { + return nil, errUnsupported + } + first := e.argInt(f, "first") + if first <= 0 { + first = 20 + } + offset := decodeCursor(e.argStr(f, "after")) + total := len(docs) + start, end := clampRange(offset, int(first), total) + pageDocs := docs[start:end] + return e.resolveConnection(f, total, offset, len(pageDocs), func(nodeField *ast.Field) (any, error) { + var nodes []any + for _, d := range pageDocs { + var node map[string]any + if err := json.Unmarshal(d, &node); err != nil { + return nil, errUnsupported + } + r, err := e.resolveObject(nodeField.SelectionSet, e.genericResolver(node)) + if err != nil { + return nil, err + } + nodes = append(nodes, r) + } + return nodes, nil + }) +} + +// genericResolver resolves fields by matching the GraphQL field name to the same +// key in an already-GraphQL-shaped source map (discussions, projects). Nested +// objects and connection-like {nodes} sub-structures are recursed into. +func (e *exec) genericResolver(src map[string]any) func(*ast.Field) (any, error) { + return func(f *ast.Field) (any, error) { + val, present := src[f.Name] + if !present { + // Common computed scalars we can derive. + switch f.Name { + case "id", "databaseId": + return nullable(src["databaseId"]), nil + } + return nil, errUnsupported + } + if len(f.SelectionSet) == 0 { + return val, nil + } + switch child := val.(type) { + case map[string]any: + return e.resolveObject(f.SelectionSet, e.genericResolver(child)) + case []any: + var nodes []any + for _, it := range child { + m, _ := it.(map[string]any) + if m == nil { + return nil, errUnsupported + } + r, err := e.resolveObject(f.SelectionSet, e.genericResolver(m)) + if err != nil { + return nil, err + } + nodes = append(nodes, r) + } + return nodes, nil + case nil: + return nil, nil + default: + return nil, errUnsupported + } + } +} + +// --- connections --- + +func (e *exec) resolveIssueConnection(f *ast.Field, wantPR bool) (any, error) { + state, err := e.statesArg(f) + if err != nil { + return nil, err + } + first := e.argInt(f, "first") + if first <= 0 { + first = 30 + } + offset := decodeCursor(e.argStr(f, "after")) + page := offset/int(first) + 1 + + opts := query.ListIssuesOpts{ + State: state, OnlyPulls: wantPR, OnlyIssues: !wantPR, + PerPage: int(first), Page: page, Sort: "created", Direction: "desc", + } + items, total, err := e.m.q.ListIssues(opts) + if err != nil { + return nil, errUnsupported + } + return e.resolveConnection(f, total, offset, len(items), func(nodeField *ast.Field) (any, error) { + var nodes []any + for i, it := range items { + var issue map[string]any + if err := json.Unmarshal(it, &issue); err != nil { + return nil, errUnsupported + } + r, err := e.resolveObject(nodeField.SelectionSet, + e.issueFieldResolver(int64(jsonInt(issue["number"])), issue)) + if err != nil { + return nil, err + } + _ = i + nodes = append(nodes, r) + } + return nodes, nil + }) +} + +func (e *exec) resolveDiscussionConnection(f *ast.Field) (any, error) { + docs, err := e.m.q.ListDiscussions() + if err != nil { + return nil, errUnsupported + } + first := e.argInt(f, "first") + if first <= 0 { + first = 30 + } + offset := decodeCursor(e.argStr(f, "after")) + total := len(docs) + start, end := clampRange(offset, int(first), total) + pageDocs := docs[start:end] + return e.resolveConnection(f, total, offset, len(pageDocs), func(nodeField *ast.Field) (any, error) { + var nodes []any + for _, d := range pageDocs { + var node map[string]any + if err := json.Unmarshal(d, &node); err != nil { + return nil, errUnsupported + } + r, err := e.resolveObject(nodeField.SelectionSet, e.genericResolver(node)) + if err != nil { + return nil, err + } + nodes = append(nodes, r) + } + return nodes, nil + }) +} + +// clampRange returns the [start,end) slice bounds for a page of size first at +// the given offset into a total-length list. +func clampRange(offset, first, total int) (start, end int) { + start = offset + if start > total { + start = total + } + end = start + first + if end > total { + end = total + } + return start, end +} + +// resolveConnection assembles the standard {totalCount, nodes, edges, pageInfo} +// connection envelope. nodeBuilder produces the nodes array for a given nodes/ +// edges field. +func (e *exec) resolveConnection(f *ast.Field, total, offset, pageLen int, nodeBuilder func(*ast.Field) (any, error)) (any, error) { + hasNext := offset+pageLen < total + endCursor := encodeCursor(offset + pageLen) + return e.resolveObject(f.SelectionSet, func(cf *ast.Field) (any, error) { + switch cf.Name { + case "totalCount": + return total, nil + case "nodes": + return nodeBuilder(cf) + case "edges": + nodesAny, err := nodeBuilder(cf) + if err != nil { + return nil, err + } + nodes, _ := nodesAny.([]any) + var edges []any + for i, n := range nodes { + edge, err := e.resolveObject(cf.SelectionSet, func(ef *ast.Field) (any, error) { + switch ef.Name { + case "node": + return n, nil + case "cursor": + return encodeCursor(offset + i + 1), nil + } + return nil, errUnsupported + }) + if err != nil { + return nil, err + } + edges = append(edges, edge) + } + return edges, nil + case "pageInfo": + return e.resolveObject(cf.SelectionSet, func(pf *ast.Field) (any, error) { + switch pf.Name { + case "hasNextPage": + return hasNext, nil + case "hasPreviousPage": + return offset > 0, nil + case "endCursor": + return endCursor, nil + case "startCursor": + return encodeCursor(offset), nil + } + return nil, errUnsupported + }) + default: + return nil, errUnsupported + } + }) +} + +func (e *exec) resolveLabelConnection(f *ast.Field, labels []map[string]any) (any, error) { + return e.resolveObject(f.SelectionSet, func(cf *ast.Field) (any, error) { + switch cf.Name { + case "totalCount": + return len(labels), nil + case "nodes": + var nodes []any + for _, l := range labels { + r, err := e.resolveObject(cf.SelectionSet, func(lf *ast.Field) (any, error) { + switch lf.Name { + case "name": + return l["name"], nil + case "color": + return l["color"], nil + case "description": + return l["description"], nil + } + return nil, errUnsupported + }) + if err != nil { + return nil, err + } + nodes = append(nodes, r) + } + return nodes, nil + case "pageInfo": + return e.resolveObject(cf.SelectionSet, falsePageInfo) + } + return nil, errUnsupported + }) +} + +func (e *exec) resolveLoginConnection(f *ast.Field, logins []string) (any, error) { + return e.resolveObject(f.SelectionSet, func(cf *ast.Field) (any, error) { + switch cf.Name { + case "totalCount": + return len(logins), nil + case "nodes": + var nodes []any + for _, login := range logins { + r, err := e.resolveObject(cf.SelectionSet, loginFieldFn(login)) + if err != nil { + return nil, err + } + nodes = append(nodes, r) + } + return nodes, nil + case "pageInfo": + return e.resolveObject(cf.SelectionSet, falsePageInfo) + } + return nil, errUnsupported + }) +} + +func (e *exec) resolveCommentConnection(f *ast.Field, number int64) (any, error) { + comments, err := e.m.q.IssueComments(number) + if err != nil { + return nil, errUnsupported + } + return e.resolveObject(f.SelectionSet, func(cf *ast.Field) (any, error) { + switch cf.Name { + case "totalCount": + return len(comments), nil + case "nodes": + var nodes []any + for _, c := range comments { + var cm map[string]any + if err := json.Unmarshal(c, &cm); err != nil { + return nil, errUnsupported + } + r, err := e.resolveObject(cf.SelectionSet, func(commentField *ast.Field) (any, error) { + switch commentField.Name { + case "body": + return cm["body"], nil + case "createdAt": + return cm["created_at"], nil + case "author", "user": + return e.resolveObject(commentField.SelectionSet, loginResolver(userLogin(cm, "user"))) + } + return nil, errUnsupported + }) + if err != nil { + return nil, err + } + nodes = append(nodes, r) + } + return nodes, nil + case "pageInfo": + return e.resolveObject(cf.SelectionSet, falsePageInfo) + } + return nil, errUnsupported + }) +} + +// --- small resolvers / helpers --- + +func loginResolver(login string) func(*ast.Field) (any, error) { + return loginFieldFn(login) +} + +func loginFieldFn(login string) func(*ast.Field) (any, error) { + return func(f *ast.Field) (any, error) { + switch f.Name { + case "login": + return login, nil + case "name": + return login, nil + } + return nil, errUnsupported + } +} + +func falsePageInfo(f *ast.Field) (any, error) { + switch f.Name { + case "hasNextPage", "hasPreviousPage": + return false, nil + case "endCursor", "startCursor": + return nil, nil + } + return nil, errUnsupported +} + +func (e *exec) statesArg(f *ast.Field) (string, error) { + v := e.arg(f, "states") + if v == nil { + return "all", nil + } + list, ok := v.([]any) + if !ok { + return "", errUnsupported + } + hasOpen, hasClosed, hasMerged := false, false, false + for _, s := range list { + switch strings.ToUpper(fmt.Sprint(s)) { + case "OPEN": + hasOpen = true + case "CLOSED": + hasClosed = true + case "MERGED": + hasMerged = true + default: + return "", errUnsupported + } + } + switch { + case hasOpen && (hasClosed || hasMerged): + return "all", nil + case hasOpen: + return "open", nil + case hasClosed || hasMerged: + return "closed", nil + } + return "all", nil +} + +func aliasOf(f *ast.Field) string { + if f.Alias != "" { + return f.Alias + } + return f.Name +} + +func (e *exec) arg(f *ast.Field, name string) any { + a := f.Arguments.ForName(name) + if a == nil { + return nil + } + v, err := a.Value.Value(e.vars) + if err != nil { + return nil + } + return v +} + +func (e *exec) argStr(f *ast.Field, name string) string { + v := e.arg(f, name) + if v == nil { + return "" + } + return fmt.Sprint(v) +} + +func (e *exec) argInt(f *ast.Field, name string) int64 { + v := e.arg(f, name) + switch n := v.(type) { + case int64: + return n + case int: + return int64(n) + case float64: + return int64(n) + case json.Number: + i, _ := n.Int64() + return i + case string: + i, _ := strconv.ParseInt(n, 10, 64) + return i + } + return 0 +} + +func encodeCursor(offset int) string { + return base64.StdEncoding.EncodeToString([]byte("cursor:" + strconv.Itoa(offset))) +} + +func decodeCursor(s string) int { + if s == "" { + return 0 + } + b, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return 0 + } + _, num, ok := strings.Cut(string(b), ":") + if !ok { + return 0 + } + n, _ := strconv.Atoi(num) + return n +} + +func upperState(s string) string { return strings.ToUpper(s) } + +func str(m map[string]any, k string) string { + s, _ := m[k].(string) + return s +} + +func nullable(v any) any { return v } + +func boolOf(v any) bool { + b, _ := v.(bool) + return b +} + +func jsonNumber(v any) any { + switch n := v.(type) { + case float64: + return int64(n) + default: + return v + } +} + +func jsonInt(v any) int64 { + switch n := v.(type) { + case float64: + return int64(n) + case int64: + return n + } + return 0 +} + +func userLogin(m map[string]any, key string) string { + u, _ := m[key].(map[string]any) + if u == nil { + return "" + } + return str(u, "login") +} + +func labelList(issue map[string]any) []map[string]any { + raw, _ := issue["labels"].([]any) + var out []map[string]any + for _, l := range raw { + if m, ok := l.(map[string]any); ok { + out = append(out, m) + } + } + return out +} + +func logins(issue map[string]any, key string) []string { + raw, _ := issue[key].([]any) + var out []string + for _, u := range raw { + if m, ok := u.(map[string]any); ok { + if l := str(m, "login"); l != "" { + out = append(out, l) + } + } + } + return out +} diff --git a/internal/graphqlmirror/graphqlmirror_test.go b/internal/graphqlmirror/graphqlmirror_test.go new file mode 100644 index 0000000..02e9d16 --- /dev/null +++ b/internal/graphqlmirror/graphqlmirror_test.go @@ -0,0 +1,283 @@ +package graphqlmirror + +import ( + "encoding/json" + "path/filepath" + "testing" + + "github.com/mevdschee/github-export/internal/ghmodel" + "github.com/mevdschee/github-export/internal/query" + "github.com/mevdschee/github-export/internal/store" +) + +func newMirror(t *testing.T) *Mirror { + t.Helper() + s, err := store.Open(filepath.Join(t.TempDir(), "t.sqlite")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + s.UpsertRepo("octocat", "hello", map[string]any{ + "full_name": "octocat/hello", "description": "a test repo", + "html_url": "https://github.com/octocat/hello", "default_branch": "main", + "private": false, + }) + s.UpsertIssue(1, false, map[string]any{ + "number": float64(1), "title": "Bug", "state": "open", + "user": map[string]any{"login": "alice"}, "body": "broken", + "labels": []any{map[string]any{"name": "bug", "color": "red"}}, + "created_at": "2024-01-01T00:00:00Z", + }, nil, []map[string]any{ + {"event": "commented", "body": "thanks", "user": map[string]any{"login": "bob"}, "created_at": "2024-01-03T00:00:00Z"}, + }, nil) + s.UpsertIssue(2, true, map[string]any{ + "number": float64(2), "title": "Feature", "state": "closed", + "user": map[string]any{"login": "carol"}, "created_at": "2024-02-01T00:00:00Z", + }, map[string]any{"number": float64(2), "title": "Feature", "state": "closed", "merged": true, "user": map[string]any{"login": "carol"}}, nil, nil) + + d := ghmodel.DiscussionNode{Number: 5, Title: "How?", Body: "halp"} + d.Author.Login = "dave" + d.Category.Name = "Q&A" + s.UpsertDiscussion(d) + + s.UpsertProject(3, map[string]any{ + "id": "PVT_x", "number": float64(3), "title": "Roadmap", "closed": false, + "shortDescription": "the plan", + "fields": map[string]any{"nodes": []any{ + map[string]any{"name": "Status", "dataType": "SINGLE_SELECT"}, + }}, + }, nil) + + return New(query.New(s), "octocat", "hello") +} + +func run(t *testing.T, m *Mirror, q string, vars map[string]any) (map[string]any, bool) { + t.Helper() + resp, ok := m.Execute(q, vars) + if !ok { + return nil, false + } + var out struct { + Data map[string]any `json:"data"` + } + if err := json.Unmarshal(resp, &out); err != nil { + t.Fatalf("unmarshal: %v\n%s", err, resp) + } + return out.Data, true +} + +func TestSingleIssue(t *testing.T) { + m := newMirror(t) + data, ok := run(t, m, `{ + repository(owner:"octocat", name:"hello") { + issue(number:1) { number title state author { login } } + } + }`, nil) + if !ok { + t.Fatal("expected local resolution") + } + issue := data["repository"].(map[string]any)["issue"].(map[string]any) + if issue["title"] != "Bug" || issue["state"] != "OPEN" { + t.Errorf("issue=%v", issue) + } + if issue["author"].(map[string]any)["login"] != "alice" { + t.Errorf("author=%v", issue["author"]) + } +} + +func TestIssueWithVariables(t *testing.T) { + m := newMirror(t) + data, ok := run(t, m, `query($n:Int!){ + repository(owner:"octocat", name:"hello") { issue(number:$n) { title } } + }`, map[string]any{"n": 1}) + if !ok { + t.Fatal("expected local resolution") + } + if data["repository"].(map[string]any)["issue"].(map[string]any)["title"] != "Bug" { + t.Errorf("data=%v", data) + } +} + +func TestIssueLabelsAndComments(t *testing.T) { + m := newMirror(t) + data, ok := run(t, m, `{ + repository(owner:"octocat", name:"hello") { + issue(number:1) { + labels(first:10){ totalCount nodes { name color } } + comments(first:10){ totalCount nodes { body author { login } } } + } + } + }`, nil) + if !ok { + t.Fatal("expected local resolution") + } + issue := data["repository"].(map[string]any)["issue"].(map[string]any) + labels := issue["labels"].(map[string]any) + if labels["totalCount"].(float64) != 1 { + t.Errorf("labels=%v", labels) + } + comments := issue["comments"].(map[string]any) + nodes := comments["nodes"].([]any) + if len(nodes) != 1 || nodes[0].(map[string]any)["body"] != "thanks" { + t.Errorf("comments=%v", comments) + } +} + +func TestIssuesConnection(t *testing.T) { + m := newMirror(t) + data, ok := run(t, m, `{ + repository(owner:"octocat", name:"hello") { + issues(first:10, states:[OPEN,CLOSED]) { + totalCount + nodes { number } + pageInfo { hasNextPage endCursor } + } + } + }`, nil) + if !ok { + t.Fatal("expected local resolution") + } + conn := data["repository"].(map[string]any)["issues"].(map[string]any) + if conn["totalCount"].(float64) != 1 { // only the non-PR issue + t.Errorf("issues totalCount=%v, want 1", conn["totalCount"]) + } +} + +func TestPullRequestsConnection(t *testing.T) { + m := newMirror(t) + data, ok := run(t, m, `{ + repository(owner:"octocat", name:"hello") { + pullRequests(first:10, states:[MERGED,CLOSED]) { totalCount nodes { number merged } } + } + }`, nil) + if !ok { + t.Fatal("expected local resolution") + } + conn := data["repository"].(map[string]any)["pullRequests"].(map[string]any) + if conn["totalCount"].(float64) != 1 { + t.Errorf("pulls totalCount=%v", conn["totalCount"]) + } +} + +func TestDiscussion(t *testing.T) { + m := newMirror(t) + data, ok := run(t, m, `{ + repository(owner:"octocat", name:"hello") { + discussion(number:5) { number title author { login } category { name } } + } + }`, nil) + if !ok { + t.Fatal("expected local resolution") + } + d := data["repository"].(map[string]any)["discussion"].(map[string]any) + if d["title"] != "How?" || d["author"].(map[string]any)["login"] != "dave" { + t.Errorf("discussion=%v", d) + } + if d["category"].(map[string]any)["name"] != "Q&A" { + t.Errorf("category=%v", d["category"]) + } +} + +func TestProjectV2(t *testing.T) { + m := newMirror(t) + data, ok := run(t, m, `{ + repository(owner:"octocat", name:"hello") { + projectV2(number:3) { number title closed fields(first:10){ nodes { name dataType } } } + } + }`, nil) + if !ok { + t.Fatal("expected local resolution") + } + p := data["repository"].(map[string]any)["projectV2"].(map[string]any) + if p["title"] != "Roadmap" || p["closed"] != false { + t.Errorf("project=%v", p) + } + fields := p["fields"].(map[string]any)["nodes"].([]any) + if len(fields) != 1 || fields[0].(map[string]any)["name"] != "Status" { + t.Errorf("fields=%v", fields) + } +} + +func TestProjectsV2Connection(t *testing.T) { + m := newMirror(t) + data, ok := run(t, m, `{ + repository(owner:"octocat", name:"hello") { + projectsV2(first:10){ totalCount nodes { number title } } + } + }`, nil) + if !ok { + t.Fatal("expected local resolution") + } + conn := data["repository"].(map[string]any)["projectsV2"].(map[string]any) + if conn["totalCount"].(float64) != 1 { + t.Errorf("projectsV2 totalCount=%v", conn["totalCount"]) + } +} + +func TestRepositoryScalars(t *testing.T) { + m := newMirror(t) + data, ok := run(t, m, `{ + repository(owner:"octocat", name:"hello") { + name nameWithOwner description isPrivate + owner { login } + defaultBranchRef { name } + } + }`, nil) + if !ok { + t.Fatal("expected local resolution") + } + repo := data["repository"].(map[string]any) + if repo["name"] != "hello" || repo["description"] != "a test repo" || repo["isPrivate"] != false { + t.Errorf("repo scalars=%v", repo) + } + if repo["owner"].(map[string]any)["login"] != "octocat" { + t.Errorf("owner=%v", repo["owner"]) + } + if repo["defaultBranchRef"].(map[string]any)["name"] != "main" { + t.Errorf("defaultBranchRef=%v", repo["defaultBranchRef"]) + } +} + +func TestUnsupportedFieldFallsThrough(t *testing.T) { + m := newMirror(t) + // `mergeStateStatus` is not mapped → whole query must proxy. + _, ok := m.Execute(`{ + repository(owner:"octocat", name:"hello") { + issue(number:1) { title mergeStateStatus } + } + }`, nil) + if ok { + t.Error("expected fallthrough (servedLocally=false) for unsupported field") + } +} + +func TestUnknownRepoFallsThrough(t *testing.T) { + m := newMirror(t) + _, ok := m.Execute(`{ repository(owner:"other", name:"repo") { name } }`, nil) + if ok { + t.Error("expected fallthrough for a repo we have not synced") + } +} + +func TestMutationFallsThrough(t *testing.T) { + m := newMirror(t) + _, ok := m.Execute(`mutation { addComment(input:{}) { clientMutationId } }`, nil) + if ok { + t.Error("expected mutations to proxy") + } +} + +func TestFragmentSupport(t *testing.T) { + m := newMirror(t) + data, ok := run(t, m, ` + fragment IssueFields on Issue { number title } + { repository(owner:"octocat", name:"hello") { issue(number:1) { ...IssueFields } } } + `, nil) + if !ok { + t.Fatal("expected fragments to resolve locally") + } + issue := data["repository"].(map[string]any)["issue"].(map[string]any) + if issue["title"] != "Bug" { + t.Errorf("fragment issue=%v", issue) + } +} diff --git a/internal/httpapi/docs.go b/internal/httpapi/docs.go new file mode 100644 index 0000000..cab3013 --- /dev/null +++ b/internal/httpapi/docs.go @@ -0,0 +1,32 @@ +package httpapi + +import ( + "net/http" + + "github.com/mevdschee/github-export/api" +) + +func (s *Server) handleOpenAPI(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/yaml") + w.Write(api.OpenAPISpec) +} + +// handleDocs serves a minimal Redoc page that renders the embedded spec. +func (s *Server) handleDocs(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Write([]byte(docsHTML)) +} + +const docsHTML = ` + + + github-export API + + + + + + + + +` diff --git a/internal/httpapi/git.go b/internal/httpapi/git.go new file mode 100644 index 0000000..2bd52c4 --- /dev/null +++ b/internal/httpapi/git.go @@ -0,0 +1,72 @@ +package httpapi + +import ( + "encoding/json" + "net/http" +) + +// Git content handlers. Each falls through to the proxy on a miss so unknown +// refs/paths still resolve against GitHub. + +func (s *Server) handleBranches(w http.ResponseWriter, r *http.Request) { + items, err := s.git.Branches() + if err != nil { + s.handleProxy(w, r) + return + } + s.freshness(w) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(orEmpty(items)) +} + +func (s *Server) handleTags(w http.ResponseWriter, r *http.Request) { + items, err := s.git.Tags() + if err != nil { + s.handleProxy(w, r) + return + } + s.freshness(w) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(orEmpty(items)) +} + +func (s *Server) handleCommits(w http.ResponseWriter, r *http.Request) { + v := r.URL.Query() + items, err := s.git.Commits(v.Get("sha"), atoiDefault(v.Get("per_page"), 30), atoiDefault(v.Get("page"), 1)) + if err != nil { + s.handleProxy(w, r) + return + } + s.freshness(w) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(orEmpty(items)) +} + +func (s *Server) handleCommit(w http.ResponseWriter, r *http.Request) { + doc, ok, err := s.git.Commit(r.PathValue("sha")) + if err != nil || !ok { + s.handleProxy(w, r) + return + } + s.freshness(w) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(doc) +} + +func (s *Server) handleContents(w http.ResponseWriter, r *http.Request) { + doc, ok, err := s.git.Contents(r.PathValue("path"), r.URL.Query().Get("ref")) + if err != nil || !ok { + s.handleProxy(w, r) + return + } + s.freshness(w) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(doc) +} + +func orEmpty(items []map[string]any) []map[string]any { + if items == nil { + return []map[string]any{} + } + return items +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go new file mode 100644 index 0000000..beb58d8 --- /dev/null +++ b/internal/httpapi/server.go @@ -0,0 +1,413 @@ +// Package httpapi serves the synced data over a GitHub-compatible HTTP surface. +// Reads that the store mirrors are answered from SQLite (and, for repo content, +// from a local git clone); everything else — unmirrored reads and all writes — +// falls through to api.github.com via the write proxy. The server binds +// localhost and trusts any local caller: clients present no token, and the +// server's own GITHUB_TOKEN is used only for the proxy fallback. +package httpapi + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + + "github.com/mevdschee/github-export/internal/gitbackend" + "github.com/mevdschee/github-export/internal/graphqlmirror" + "github.com/mevdschee/github-export/internal/query" + "github.com/mevdschee/github-export/internal/shadow" + "github.com/mevdschee/github-export/internal/writeproxy" +) + +// Server holds the read query layer, the proxy fallback, and request routing. +type Server struct { + q *query.Query + proxy *writeproxy.Proxy + git *gitbackend.Backend // optional; nil disables local content endpoints + gql *graphqlmirror.Mirror // optional; nil forwards all GraphQL to proxy + compare *shadow.Comparator // optional; --debug-compare parity harness + owner string + repo string + syncedAt string + mux *http.ServeMux +} + +// Config configures a Server. +type Config struct { + Query *query.Query + Proxy *writeproxy.Proxy + Git *gitbackend.Backend + GraphQL *graphqlmirror.Mirror + Compare *shadow.Comparator + Owner string + Repo string + SyncedAt string +} + +// New builds a Server and registers its routes. +func New(cfg Config) *Server { + s := &Server{ + q: cfg.Query, proxy: cfg.Proxy, git: cfg.Git, gql: cfg.GraphQL, compare: cfg.Compare, + owner: cfg.Owner, repo: cfg.Repo, syncedAt: cfg.SyncedAt, + mux: http.NewServeMux(), + } + s.routes() + return s +} + +// maybeCompare fires the shadow-compare harness for a mirrored GET response. +func (s *Server) maybeCompare(r *http.Request, body []byte) { + if s.compare == nil || r.Method != http.MethodGet { + return + } + path := r.URL.RequestURI() + b := append([]byte(nil), body...) + go s.compare.Check(path, b) +} + +// Handler returns the root handler. +func (s *Server) Handler() http.Handler { return s.mux } + +func (s *Server) routes() { + rp := "/repos/" + s.owner + "/" + s.repo + m := s.mux + + m.HandleFunc("GET "+rp, s.handleRepo) + m.HandleFunc("GET "+rp+"/issues", s.handleListIssues) + m.HandleFunc("GET "+rp+"/issues/{number}", s.handleGetIssue) + m.HandleFunc("GET "+rp+"/issues/{number}/comments", s.handleIssueComments) + m.HandleFunc("GET "+rp+"/issues/{number}/timeline", s.handleIssueTimeline) + m.HandleFunc("GET "+rp+"/pulls", s.handleListPulls) + m.HandleFunc("GET "+rp+"/pulls/{number}", s.handleGetPull) + m.HandleFunc("GET "+rp+"/pulls/{number}/reviews", s.handlePullReviews) + m.HandleFunc("GET "+rp+"/pulls/{number}/comments", s.handlePullReviewComments) + m.HandleFunc("GET "+rp+"/labels", s.handleLabels) + m.HandleFunc("GET "+rp+"/milestones", s.handleMilestones) + m.HandleFunc("GET "+rp+"/releases", s.handleReleases) + m.HandleFunc("GET "+rp+"/releases/tags/{tag}", s.handleReleaseByTag) + m.HandleFunc("GET "+rp+"/discussions", s.handleDiscussions) + m.HandleFunc("GET "+rp+"/discussions/{number}", s.handleGetDiscussion) + m.HandleFunc("GET /search/issues", s.handleSearchIssues) + + // GraphQL: Projects v2 and Discussions are GraphQL-only, and gh/MCP issue + // many reads as GraphQL. Supported queries are served from SQLite; anything + // the mirror does not fully cover falls through to api.github.com so coverage + // stays complete. + m.HandleFunc("POST /graphql", s.handleGraphQL) + + // Local git content endpoints (only when a backend is configured). + if s.git != nil { + m.HandleFunc("GET "+rp+"/branches", s.handleBranches) + m.HandleFunc("GET "+rp+"/tags", s.handleTags) + m.HandleFunc("GET "+rp+"/commits", s.handleCommits) + m.HandleFunc("GET "+rp+"/commits/{sha}", s.handleCommit) + m.HandleFunc("GET "+rp+"/contents/{path...}", s.handleContents) + } + + // Meta endpoints. + m.HandleFunc("GET /status", s.handleStatus) + m.HandleFunc("GET /openapi.yaml", s.handleOpenAPI) + m.HandleFunc("GET /docs", s.handleDocs) + + // Everything else (writes, unmirrored reads) falls through to the proxy. + m.HandleFunc("/", s.handleProxy) +} + +// --- mirror handlers --- + +func (s *Server) handleRepo(w http.ResponseWriter, r *http.Request) { + doc, ok, err := s.q.Repo() + s.one(w, r, doc, ok, err) +} + +func (s *Server) handleListIssues(w http.ResponseWriter, r *http.Request) { + o := issueOptsFromQuery(r) + items, total, err := s.q.ListIssues(o) + s.list(w, r, items, total, o.Page, o.PerPage, err) +} + +func (s *Server) handleListPulls(w http.ResponseWriter, r *http.Request) { + o := issueOptsFromQuery(r) + o.OnlyPulls = true + items, total, err := s.q.ListIssues(o) + s.list(w, r, items, total, o.Page, o.PerPage, err) +} + +func (s *Server) handleGetIssue(w http.ResponseWriter, r *http.Request) { + doc, ok, err := s.q.GetIssue(numFromPath(r, "number")) + s.one(w, r, doc, ok, err) +} + +func (s *Server) handleGetPull(w http.ResponseWriter, r *http.Request) { + doc, ok, err := s.q.GetPull(numFromPath(r, "number")) + s.one(w, r, doc, ok, err) +} + +func (s *Server) handleIssueComments(w http.ResponseWriter, r *http.Request) { + items, err := s.q.IssueComments(numFromPath(r, "number")) + s.array(w, r, items, err) +} + +func (s *Server) handleIssueTimeline(w http.ResponseWriter, r *http.Request) { + items, err := s.q.IssueTimeline(numFromPath(r, "number")) + s.array(w, r, items, err) +} + +func (s *Server) handlePullReviews(w http.ResponseWriter, r *http.Request) { + items, err := s.q.PullReviews(numFromPath(r, "number")) + s.array(w, r, items, err) +} + +func (s *Server) handlePullReviewComments(w http.ResponseWriter, r *http.Request) { + items, err := s.q.PullReviewComments(numFromPath(r, "number")) + s.array(w, r, items, err) +} + +func (s *Server) handleLabels(w http.ResponseWriter, r *http.Request) { + items, err := s.q.ListLabels() + s.array(w, r, items, err) +} + +func (s *Server) handleMilestones(w http.ResponseWriter, r *http.Request) { + items, err := s.q.ListMilestones() + s.array(w, r, items, err) +} + +func (s *Server) handleReleases(w http.ResponseWriter, r *http.Request) { + items, err := s.q.ListReleases() + s.array(w, r, items, err) +} + +func (s *Server) handleReleaseByTag(w http.ResponseWriter, r *http.Request) { + doc, ok, err := s.q.GetRelease(r.PathValue("tag")) + s.one(w, r, doc, ok, err) +} + +func (s *Server) handleDiscussions(w http.ResponseWriter, r *http.Request) { + items, err := s.q.ListDiscussions() + s.array(w, r, items, err) +} + +func (s *Server) handleGetDiscussion(w http.ResponseWriter, r *http.Request) { + doc, ok, err := s.q.GetDiscussion(numFromPath(r, "number")) + s.one(w, r, doc, ok, err) +} + +func (s *Server) handleSearchIssues(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("q") + page := atoiDefault(r.URL.Query().Get("page"), 1) + perPage := atoiDefault(r.URL.Query().Get("per_page"), 30) + items, total, err := s.q.SearchIssues(q, perPage, page) + if err != nil { + s.fail(w, err) + return + } + s.freshness(w) + w.Header().Set("Content-Type", "application/json") + resp := map[string]any{ + "total_count": total, + "incomplete_results": false, + "items": items, + } + json.NewEncoder(w).Encode(resp) +} + +func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { + counts, err := s.q.Counts() + if err != nil { + s.fail(w, err) + return + } + s.freshness(w) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]any{ + "owner": s.owner, + "repo": s.repo, + "synced_at": s.syncedAt, + "counts": counts, + "proxy": !s.proxy.Disabled, + "git": s.git != nil, + }) +} + +func (s *Server) handleProxy(w http.ResponseWriter, r *http.Request) { + s.proxy.Forward(w, r) +} + +// handleGraphQL tries the local mirror first and forwards to GitHub when the +// query is not fully supported. The request body is buffered so it can be +// replayed to the proxy on fallthrough. +func (s *Server) handleGraphQL(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + r.Body.Close() + if err != nil { + s.fail(w, err) + return + } + replayBody := func() { + r.Body = io.NopCloser(bytes.NewReader(body)) + r.ContentLength = int64(len(body)) + } + if s.gql == nil { + replayBody() + s.handleProxy(w, r) + return + } + var req struct { + Query string `json:"query"` + Variables map[string]any `json:"variables"` + } + if err := json.Unmarshal(body, &req); err != nil || req.Query == "" { + replayBody() + s.handleProxy(w, r) + return + } + if resp, ok := s.gql.Execute(req.Query, req.Variables); ok { + s.freshness(w) + w.Header().Set("Content-Type", "application/json") + w.Write(resp) + return + } + replayBody() + s.handleProxy(w, r) +} + +// --- response helpers --- + +func (s *Server) freshness(w http.ResponseWriter) { + if s.syncedAt != "" { + w.Header().Set("X-GitHub-Export-Synced-At", s.syncedAt) + } + w.Header().Set("X-GitHub-Export-Source", "local") +} + +func (s *Server) fail(w http.ResponseWriter, err error) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(w).Encode(map[string]string{"message": err.Error()}) +} + +// one writes a single object, or falls through to the proxy on a miss so the +// caller still gets the upstream answer (and we stay "100% compatible"). +func (s *Server) one(w http.ResponseWriter, r *http.Request, doc json.RawMessage, ok bool, err error) { + if err != nil { + s.fail(w, err) + return + } + if !ok { + s.handleProxy(w, r) + return + } + s.freshness(w) + w.Header().Set("Content-Type", "application/json") + w.Write(doc) + s.maybeCompare(r, doc) +} + +// array writes a JSON array of pre-serialized documents. +func (s *Server) array(w http.ResponseWriter, r *http.Request, items []json.RawMessage, err error) { + if err != nil { + s.fail(w, err) + return + } + s.freshness(w) + w.Header().Set("Content-Type", "application/json") + body := writeJSONArray(w, items) + s.maybeCompare(r, body) +} + +// list writes a paginated JSON array with a GitHub-style Link header. +func (s *Server) list(w http.ResponseWriter, r *http.Request, items []json.RawMessage, total, page, perPage int, err error) { + if err != nil { + s.fail(w, err) + return + } + if page <= 0 { + page = 1 + } + if perPage <= 0 { + perPage = 30 + } + if link := linkHeader(r, page, perPage, total); link != "" { + w.Header().Set("Link", link) + } + s.freshness(w) + w.Header().Set("Content-Type", "application/json") + body := writeJSONArray(w, items) + s.maybeCompare(r, body) +} + +func writeJSONArray(w http.ResponseWriter, items []json.RawMessage) []byte { + if items == nil { + items = []json.RawMessage{} + } + b, err := json.Marshal(items) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return nil + } + w.Write(b) + return b +} + +// linkHeader builds a GitHub-compatible pagination Link header. +func linkHeader(r *http.Request, page, perPage, total int) string { + last := (total + perPage - 1) / perPage + if last <= 1 { + return "" + } + mk := func(p int, rel string) string { + u := *r.URL + qv := u.Query() + qv.Set("page", strconv.Itoa(p)) + qv.Set("per_page", strconv.Itoa(perPage)) + u.RawQuery = qv.Encode() + return fmt.Sprintf(`<%s>; rel="%s"`, u.RequestURI(), rel) + } + var parts []string + if page < last { + parts = append(parts, mk(page+1, "next"), mk(last, "last")) + } + if page > 1 { + parts = append(parts, mk(page-1, "prev"), mk(1, "first")) + } + return strings.Join(parts, ", ") +} + +func issueOptsFromQuery(r *http.Request) query.ListIssuesOpts { + v := r.URL.Query() + o := query.ListIssuesOpts{ + State: v.Get("state"), + Creator: v.Get("creator"), + Assignee: v.Get("assignee"), + Milestone: v.Get("milestone"), + Since: v.Get("since"), + Sort: v.Get("sort"), + Direction: v.Get("direction"), + PerPage: atoiDefault(v.Get("per_page"), 30), + Page: atoiDefault(v.Get("page"), 1), + } + if lbls := v.Get("labels"); lbls != "" { + o.Labels = strings.Split(lbls, ",") + } + return o +} + +func numFromPath(r *http.Request, key string) int64 { + n, _ := strconv.ParseInt(r.PathValue(key), 10, 64) + return n +} + +func atoiDefault(s string, def int) int { + if s == "" { + return def + } + n, err := strconv.Atoi(s) + if err != nil { + return def + } + return n +} diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go new file mode 100644 index 0000000..4ec8123 --- /dev/null +++ b/internal/httpapi/server_test.go @@ -0,0 +1,209 @@ +package httpapi + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/mevdschee/github-export/internal/query" + "github.com/mevdschee/github-export/internal/store" + "github.com/mevdschee/github-export/internal/writeproxy" +) + +func newTestServer(t *testing.T) (*Server, *store.Store) { + t.Helper() + s, err := store.Open(filepath.Join(t.TempDir(), "t.sqlite")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + if err := s.UpsertRepo("octocat", "hello", map[string]any{"full_name": "octocat/hello", "default_branch": "main"}); err != nil { + t.Fatal(err) + } + s.SetMeta("synced_at", "2024-05-01T00:00:00Z") + + // One open issue with a comment in its timeline, one closed PR. + issue := map[string]any{ + "number": float64(1), "title": "Bug report", "state": "open", + "user": map[string]any{"login": "alice"}, "body": "something is broken", + "labels": []any{map[string]any{"name": "bug"}}, "created_at": "2024-01-01T00:00:00Z", + "updated_at": "2024-01-02T00:00:00Z", + } + timeline := []map[string]any{ + {"event": "commented", "id": float64(99), "body": "thanks", "user": map[string]any{"login": "bob"}, "created_at": "2024-01-03T00:00:00Z"}, + } + if err := s.UpsertIssue(1, false, issue, nil, timeline, []string{"Roadmap"}); err != nil { + t.Fatal(err) + } + pr := map[string]any{ + "number": float64(2), "title": "Add feature", "state": "closed", + "user": map[string]any{"login": "carol"}, "merged": true, "created_at": "2024-02-01T00:00:00Z", + "updated_at": "2024-02-02T00:00:00Z", + } + prDetail := map[string]any{"number": float64(2), "merged": true, "draft": false, "title": "Add feature", "state": "closed"} + prTimeline := []map[string]any{ + {"event": "reviewed", "id": float64(7), "state": "APPROVED", "body": "lgtm", "user": map[string]any{"login": "dave"}, "submitted_at": "2024-02-01T12:00:00Z"}, + } + if err := s.UpsertIssue(2, true, pr, prDetail, prTimeline, nil); err != nil { + t.Fatal(err) + } + s.ReplaceLabels([]map[string]any{{"name": "bug", "color": "red"}}) + + proxy := writeproxy.New("", nil) + proxy.Disabled = true + srv := New(Config{Query: query.New(s), Proxy: proxy, Owner: "octocat", Repo: "hello", SyncedAt: "2024-05-01T00:00:00Z"}) + return srv, s +} + +func get(t *testing.T, srv *Server, path string) *http.Response { + t.Helper() + req := httptest.NewRequest("GET", path, nil) + rec := httptest.NewRecorder() + srv.Handler().ServeHTTP(rec, req) + return rec.Result() +} + +func TestRepoEndpoint(t *testing.T) { + srv, _ := newTestServer(t) + res := get(t, srv, "/repos/octocat/hello") + if res.StatusCode != 200 { + t.Fatalf("status=%d", res.StatusCode) + } + if res.Header.Get("X-GitHub-Export-Synced-At") != "2024-05-01T00:00:00Z" { + t.Errorf("missing freshness header: %v", res.Header) + } + var m map[string]any + json.NewDecoder(res.Body).Decode(&m) + if m["full_name"] != "octocat/hello" { + t.Errorf("repo=%v", m) + } +} + +func TestListIssuesDefaultOpenOnly(t *testing.T) { + srv, _ := newTestServer(t) + res := get(t, srv, "/repos/octocat/hello/issues") + var items []map[string]any + json.NewDecoder(res.Body).Decode(&items) + if len(items) != 1 || items[0]["number"].(float64) != 1 { + t.Errorf("default issues listing = %v, want only open #1", items) + } +} + +func TestListIssuesStateAll(t *testing.T) { + srv, _ := newTestServer(t) + res := get(t, srv, "/repos/octocat/hello/issues?state=all") + var items []map[string]any + json.NewDecoder(res.Body).Decode(&items) + if len(items) != 2 { + t.Errorf("state=all len=%d, want 2", len(items)) + } +} + +func TestListIssuesLabelFilter(t *testing.T) { + srv, _ := newTestServer(t) + res := get(t, srv, "/repos/octocat/hello/issues?state=all&labels=bug") + var items []map[string]any + json.NewDecoder(res.Body).Decode(&items) + if len(items) != 1 || items[0]["number"].(float64) != 1 { + t.Errorf("label filter = %v", items) + } +} + +func TestPullsOnly(t *testing.T) { + srv, _ := newTestServer(t) + res := get(t, srv, "/repos/octocat/hello/pulls?state=all") + var items []map[string]any + json.NewDecoder(res.Body).Decode(&items) + if len(items) != 1 || items[0]["number"].(float64) != 2 { + t.Errorf("pulls = %v, want only #2", items) + } +} + +func TestIssueComments(t *testing.T) { + srv, _ := newTestServer(t) + res := get(t, srv, "/repos/octocat/hello/issues/1/comments") + var items []map[string]any + json.NewDecoder(res.Body).Decode(&items) + if len(items) != 1 || items[0]["body"] != "thanks" { + t.Errorf("comments = %v", items) + } + if _, hasEvent := items[0]["event"]; hasEvent { + t.Errorf("synthetic event marker leaked into comment: %v", items[0]) + } +} + +func TestPullReviews(t *testing.T) { + srv, _ := newTestServer(t) + res := get(t, srv, "/repos/octocat/hello/pulls/2/reviews") + var items []map[string]any + json.NewDecoder(res.Body).Decode(&items) + if len(items) != 1 || items[0]["state"] != "APPROVED" { + t.Errorf("reviews = %v", items) + } +} + +func TestSearchIssues(t *testing.T) { + srv, _ := newTestServer(t) + res := get(t, srv, "/search/issues?q=broken") + var out struct { + TotalCount int `json:"total_count"` + Items []map[string]any `json:"items"` + } + json.NewDecoder(res.Body).Decode(&out) + if out.TotalCount != 1 || len(out.Items) != 1 || out.Items[0]["number"].(float64) != 1 { + t.Errorf("search = %+v", out) + } +} + +func TestSearchQualifier(t *testing.T) { + srv, _ := newTestServer(t) + res := get(t, srv, "/search/issues?q=is:pr") + var out struct { + TotalCount int `json:"total_count"` + } + json.NewDecoder(res.Body).Decode(&out) + if out.TotalCount != 1 { + t.Errorf("is:pr search total=%d, want 1", out.TotalCount) + } +} + +func TestStatus(t *testing.T) { + srv, _ := newTestServer(t) + res := get(t, srv, "/status") + var st struct { + Owner string `json:"owner"` + Counts map[string]int `json:"counts"` + } + json.NewDecoder(res.Body).Decode(&st) + if st.Owner != "octocat" || st.Counts["issues"] != 1 || st.Counts["pulls"] != 1 { + t.Errorf("status=%+v", st) + } +} + +func TestOpenAPIServed(t *testing.T) { + srv, _ := newTestServer(t) + res := get(t, srv, "/openapi.yaml") + if res.StatusCode != 200 || res.Header.Get("Content-Type") != "application/yaml" { + t.Errorf("openapi status=%d ct=%s", res.StatusCode, res.Header.Get("Content-Type")) + } +} + +func TestPaginationLinkHeader(t *testing.T) { + srv, _ := newTestServer(t) + res := get(t, srv, "/repos/octocat/hello/issues?state=all&per_page=1&page=1") + link := res.Header.Get("Link") + if link == "" { + t.Errorf("expected Link header for paginated result, got none") + } +} + +func TestProxyDisabledFallthrough(t *testing.T) { + srv, _ := newTestServer(t) + // Unmirrored path → proxy, which is disabled → 501. + res := get(t, srv, "/repos/octocat/hello/actions/runs") + if res.StatusCode != http.StatusNotImplemented { + t.Errorf("status=%d, want 501 (proxy disabled)", res.StatusCode) + } +} diff --git a/internal/mcp/mcp.go b/internal/mcp/mcp.go new file mode 100644 index 0000000..f905158 --- /dev/null +++ b/internal/mcp/mcp.go @@ -0,0 +1,438 @@ +// Package mcp exposes the synced store and local git clone through a Model +// Context Protocol server whose tool names and argument shapes mirror GitHub's +// official MCP server, so existing agent configs work unchanged. Read tools +// resolve against internal/query (and the git backend); write tools proxy to +// api.github.com via internal/writeproxy and re-sync the touched entity. +package mcp + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/mevdschee/github-export/internal/gitbackend" + "github.com/mevdschee/github-export/internal/query" + "github.com/mevdschee/github-export/internal/writeproxy" +) + +// Deps bundles what the tools need. +type Deps struct { + Query *query.Query + Git *gitbackend.Backend + Proxy *writeproxy.Proxy + Owner string + Repo string + SyncedAt string + ReadOnly bool +} + +// NewServer builds an MCP server with GitHub-MCP-compatible tools registered. +func NewServer(d Deps) *mcp.Server { + s := mcp.NewServer(&mcp.Implementation{ + Name: "github-export", + Title: "github-export (local GitHub mirror)", + Version: "1.0.0", + }, nil) + + t := &tools{d: d} + t.registerReads(s) + if !d.ReadOnly { + t.registerWrites(s) + } + return s +} + +type tools struct{ d Deps } + +// --- helpers --- + +func textResult(v any) *mcp.CallToolResult { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return errResult(err) + } + return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: string(b)}}} +} + +// rawResult wraps already-serialized JSON plus a freshness note. +func (t *tools) rawResult(doc json.RawMessage) *mcp.CallToolResult { + text := string(doc) + if t.d.SyncedAt != "" { + text = fmt.Sprintf("// synced_at: %s\n%s", t.d.SyncedAt, text) + } + return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: text}}} +} + +func (t *tools) rawArrayResult(items []json.RawMessage) *mcp.CallToolResult { + if items == nil { + items = []json.RawMessage{} + } + b, _ := json.Marshal(items) + return t.rawResult(b) +} + +func errResult(err error) *mcp.CallToolResult { + return &mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{&mcp.TextContent{Text: err.Error()}}, + } +} + +func notFound(what string) *mcp.CallToolResult { + return errResult(fmt.Errorf("%s not found in local store (try `github-export sync`)", what)) +} + +func argInt(args json.RawMessage, key string) int64 { + var m map[string]any + json.Unmarshal(args, &m) + switch v := m[key].(type) { + case float64: + return int64(v) + case string: + var n int64 + fmt.Sscanf(v, "%d", &n) + return n + } + return 0 +} + +func argStr(args json.RawMessage, key string) string { + var m map[string]any + json.Unmarshal(args, &m) + s, _ := m[key].(string) + return s +} + +// schema builds a minimal JSON schema object for a tool's inputs. +func schema(props map[string]string, required ...string) json.RawMessage { + p := map[string]any{} + for name, typ := range props { + p[name] = map[string]any{"type": typ} + } + obj := map[string]any{"type": "object", "properties": p} + if len(required) > 0 { + obj["required"] = required + } + b, _ := json.Marshal(obj) + return b +} + +func (t *tools) registerReads(s *mcp.Server) { + s.AddTool(&mcp.Tool{ + Name: "get_issue", + Description: "Get a single issue by number from the local mirror.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "issue_number": "integer"}, "issue_number"), + }, t.getIssue) + + s.AddTool(&mcp.Tool{ + Name: "list_issues", + Description: "List issues (state: open|closed|all). Served from the local mirror.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "state": "string", "labels": "string", "perPage": "integer", "page": "integer"}), + }, t.listIssues) + + s.AddTool(&mcp.Tool{ + Name: "get_pull_request", + Description: "Get a single pull request by number from the local mirror.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "pullNumber": "integer"}, "pullNumber"), + }, t.getPull) + + s.AddTool(&mcp.Tool{ + Name: "list_pull_requests", + Description: "List pull requests (state: open|closed|all). Served from the local mirror.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "state": "string", "perPage": "integer", "page": "integer"}), + }, t.listPulls) + + s.AddTool(&mcp.Tool{ + Name: "get_issue_comments", + Description: "List comments on an issue or PR from the local mirror.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "issue_number": "integer"}, "issue_number"), + }, t.issueComments) + + s.AddTool(&mcp.Tool{ + Name: "get_pull_request_reviews", + Description: "List reviews on a pull request from the local mirror.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "pullNumber": "integer"}, "pullNumber"), + }, t.pullReviews) + + s.AddTool(&mcp.Tool{ + Name: "get_pull_request_comments", + Description: "List inline review comments on a pull request from the local mirror.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "pullNumber": "integer"}, "pullNumber"), + }, t.pullReviewComments) + + s.AddTool(&mcp.Tool{ + Name: "list_discussions", + Description: "List discussions from the local mirror.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string"}), + }, t.listDiscussions) + + s.AddTool(&mcp.Tool{ + Name: "get_discussion", + Description: "Get a discussion by number from the local mirror.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "discussionNumber": "integer"}, "discussionNumber"), + }, t.getDiscussion) + + s.AddTool(&mcp.Tool{ + Name: "get_discussion_comments", + Description: "List top-level comments on a discussion from the local mirror.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "discussionNumber": "integer"}, "discussionNumber"), + }, t.getDiscussionComments) + + s.AddTool(&mcp.Tool{ + Name: "search_issues", + Description: "Search issues/PRs with GitHub qualifiers (is:, label:, author:, state:) plus free text, served from the local mirror.", + InputSchema: schema(map[string]string{"query": "string", "perPage": "integer", "page": "integer"}, "query"), + }, t.searchIssues) + + s.AddTool(&mcp.Tool{ + Name: "list_releases", + Description: "List releases from the local mirror.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string"}), + }, t.listReleases) + + s.AddTool(&mcp.Tool{ + Name: "status", + Description: "Report local store freshness (synced_at) and per-entity counts.", + InputSchema: schema(map[string]string{}), + }, t.status) + + if t.d.Git != nil { + s.AddTool(&mcp.Tool{ + Name: "get_file_contents", + Description: "Get a file or directory from the local git clone.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "path": "string", "ref": "string"}, "path"), + }, t.getFileContents) + + s.AddTool(&mcp.Tool{ + Name: "list_commits", + Description: "List commits from the local git clone.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "sha": "string", "perPage": "integer", "page": "integer"}), + }, t.listCommits) + + s.AddTool(&mcp.Tool{ + Name: "get_commit", + Description: "Get a single commit by SHA/ref from the local git clone.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "sha": "string"}, "sha"), + }, t.getCommit) + + s.AddTool(&mcp.Tool{ + Name: "list_branches", + Description: "List branches from the local git clone.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string"}), + }, t.listBranches) + + s.AddTool(&mcp.Tool{ + Name: "list_tags", + Description: "List tags from the local git clone.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string"}), + }, t.listTags) + + s.AddTool(&mcp.Tool{ + Name: "search_code", + Description: "Search code in the local git clone (git grep).", + InputSchema: schema(map[string]string{"query": "string", "ref": "string"}, "query"), + }, t.searchCode) + } +} + +func (t *tools) getIssue(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + doc, ok, err := t.d.Query.GetIssue(argInt(req.Params.Arguments, "issue_number")) + if err != nil { + return errResult(err), nil + } + if !ok { + return notFound("issue"), nil + } + return t.rawResult(doc), nil +} + +func (t *tools) listIssues(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + o := query.ListIssuesOpts{ + State: argStr(req.Params.Arguments, "state"), + PerPage: int(argInt(req.Params.Arguments, "perPage")), + Page: int(argInt(req.Params.Arguments, "page")), + } + if lbls := argStr(req.Params.Arguments, "labels"); lbls != "" { + o.Labels = splitCSV(lbls) + } + items, _, err := t.d.Query.ListIssues(o) + if err != nil { + return errResult(err), nil + } + return t.rawArrayResult(items), nil +} + +func (t *tools) getPull(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + doc, ok, err := t.d.Query.GetPull(argInt(req.Params.Arguments, "pullNumber")) + if err != nil { + return errResult(err), nil + } + if !ok { + return notFound("pull request"), nil + } + return t.rawResult(doc), nil +} + +func (t *tools) listPulls(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + o := query.ListIssuesOpts{ + State: argStr(req.Params.Arguments, "state"), OnlyPulls: true, + PerPage: int(argInt(req.Params.Arguments, "perPage")), Page: int(argInt(req.Params.Arguments, "page")), + } + items, _, err := t.d.Query.ListIssues(o) + if err != nil { + return errResult(err), nil + } + return t.rawArrayResult(items), nil +} + +func (t *tools) issueComments(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + items, err := t.d.Query.IssueComments(argInt(req.Params.Arguments, "issue_number")) + if err != nil { + return errResult(err), nil + } + return t.rawArrayResult(items), nil +} + +func (t *tools) pullReviews(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + items, err := t.d.Query.PullReviews(argInt(req.Params.Arguments, "pullNumber")) + if err != nil { + return errResult(err), nil + } + return t.rawArrayResult(items), nil +} + +func (t *tools) pullReviewComments(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + items, err := t.d.Query.PullReviewComments(argInt(req.Params.Arguments, "pullNumber")) + if err != nil { + return errResult(err), nil + } + return t.rawArrayResult(items), nil +} + +func (t *tools) listDiscussions(_ context.Context, _ *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + items, err := t.d.Query.ListDiscussions() + if err != nil { + return errResult(err), nil + } + return t.rawArrayResult(items), nil +} + +func (t *tools) getDiscussion(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + doc, ok, err := t.d.Query.GetDiscussion(argInt(req.Params.Arguments, "discussionNumber")) + if err != nil { + return errResult(err), nil + } + if !ok { + return notFound("discussion"), nil + } + return t.rawResult(doc), nil +} + +func (t *tools) getDiscussionComments(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + items, err := t.d.Query.DiscussionComments(argInt(req.Params.Arguments, "discussionNumber")) + if err != nil { + return errResult(err), nil + } + return t.rawArrayResult(items), nil +} + +func (t *tools) getCommit(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + doc, ok, err := t.d.Git.Commit(argStr(req.Params.Arguments, "sha")) + if err != nil || !ok { + return notFound("commit"), nil + } + return textResult(doc), nil +} + +func (t *tools) listBranches(_ context.Context, _ *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + items, err := t.d.Git.Branches() + if err != nil { + return errResult(err), nil + } + return textResult(items), nil +} + +func (t *tools) listTags(_ context.Context, _ *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + items, err := t.d.Git.Tags() + if err != nil { + return errResult(err), nil + } + return textResult(items), nil +} + +func (t *tools) searchIssues(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + items, total, err := t.d.Query.SearchIssues(argStr(req.Params.Arguments, "query"), + int(argInt(req.Params.Arguments, "perPage")), int(argInt(req.Params.Arguments, "page"))) + if err != nil { + return errResult(err), nil + } + raws := make([]any, len(items)) + for i, it := range items { + raws[i] = json.RawMessage(it) + } + return textResult(map[string]any{"total_count": total, "items": raws}), nil +} + +func (t *tools) listReleases(_ context.Context, _ *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + items, err := t.d.Query.ListReleases() + if err != nil { + return errResult(err), nil + } + return t.rawArrayResult(items), nil +} + +func (t *tools) status(_ context.Context, _ *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + counts, err := t.d.Query.Counts() + if err != nil { + return errResult(err), nil + } + return textResult(map[string]any{ + "owner": t.d.Owner, "repo": t.d.Repo, "synced_at": t.d.SyncedAt, "counts": counts, + }), nil +} + +func (t *tools) getFileContents(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + doc, ok, err := t.d.Git.Contents(argStr(req.Params.Arguments, "path"), argStr(req.Params.Arguments, "ref")) + if err != nil || !ok { + return notFound("path"), nil + } + return textResult(doc), nil +} + +func (t *tools) listCommits(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + commits, err := t.d.Git.Commits(argStr(req.Params.Arguments, "sha"), + int(argInt(req.Params.Arguments, "perPage")), int(argInt(req.Params.Arguments, "page"))) + if err != nil { + return errResult(err), nil + } + return textResult(commits), nil +} + +func (t *tools) searchCode(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + results, err := t.d.Git.Grep(argStr(req.Params.Arguments, "query"), argStr(req.Params.Arguments, "ref")) + if err != nil { + return errResult(err), nil + } + return textResult(results), nil +} + +func splitCSV(s string) []string { + var out []string + cur := "" + for _, r := range s { + if r == ',' { + if cur != "" { + out = append(out, cur) + } + cur = "" + continue + } + cur += string(r) + } + if cur != "" { + out = append(out, cur) + } + return out +} diff --git a/internal/mcp/mcp_test.go b/internal/mcp/mcp_test.go new file mode 100644 index 0000000..2c04108 --- /dev/null +++ b/internal/mcp/mcp_test.go @@ -0,0 +1,159 @@ +package mcp + +import ( + "context" + "encoding/json" + "path/filepath" + "strings" + "testing" + + sdk "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/mevdschee/github-export/internal/query" + "github.com/mevdschee/github-export/internal/store" + "github.com/mevdschee/github-export/internal/writeproxy" +) + +func setup(t *testing.T, readOnly bool) *sdk.ClientSession { + t.Helper() + s, err := store.Open(filepath.Join(t.TempDir(), "t.sqlite")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + s.UpsertRepo("octocat", "hello", map[string]any{"full_name": "octocat/hello"}) + s.SetMeta("synced_at", "2024-05-01T00:00:00Z") + s.UpsertIssue(1, false, map[string]any{ + "number": float64(1), "title": "Bug", "state": "open", + "user": map[string]any{"login": "alice"}, "body": "broken", + }, nil, []map[string]any{ + {"event": "commented", "body": "me too", "user": map[string]any{"login": "bob"}, "created_at": "2024-01-03T00:00:00Z"}, + }, nil) + + proxy := writeproxy.New("", nil) + proxy.Disabled = true + srv := NewServer(Deps{ + Query: query.New(s), Proxy: proxy, Owner: "octocat", Repo: "hello", + SyncedAt: "2024-05-01T00:00:00Z", ReadOnly: readOnly, + }) + + ctx := context.Background() + st, ct := sdk.NewInMemoryTransports() + if _, err := srv.Connect(ctx, st, nil); err != nil { + t.Fatal(err) + } + client := sdk.NewClient(&sdk.Implementation{Name: "test", Version: "0"}, nil) + cs, err := client.Connect(ctx, ct, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { cs.Close() }) + return cs +} + +func callText(t *testing.T, cs *sdk.ClientSession, name string, args map[string]any) string { + t.Helper() + res, err := cs.CallTool(context.Background(), &sdk.CallToolParams{Name: name, Arguments: args}) + if err != nil { + t.Fatalf("CallTool %s: %v", name, err) + } + if len(res.Content) == 0 { + t.Fatalf("%s returned no content", name) + } + tc, ok := res.Content[0].(*sdk.TextContent) + if !ok { + t.Fatalf("%s content not text: %T", name, res.Content[0]) + } + return tc.Text +} + +func TestToolNamesMatchGitHubMCP(t *testing.T) { + cs := setup(t, false) + res, err := cs.ListTools(context.Background(), nil) + if err != nil { + t.Fatal(err) + } + got := map[string]bool{} + for _, tool := range res.Tools { + got[tool.Name] = true + } + for _, want := range []string{ + "get_issue", "get_issue_comments", "list_issues", "get_pull_request", + "list_pull_requests", "get_pull_request_reviews", "get_pull_request_comments", + "list_discussions", "get_discussion", "get_discussion_comments", + "search_issues", "create_issue", "add_issue_comment", "update_issue", + "update_pull_request", "merge_pull_request", "status", + } { + if !got[want] { + t.Errorf("missing tool %q (have %v)", want, keys(got)) + } + } +} + +func TestGetIssueTool(t *testing.T) { + cs := setup(t, false) + text := callText(t, cs, "get_issue", map[string]any{"issue_number": 1}) + if !strings.Contains(text, "synced_at: 2024-05-01") { + t.Errorf("missing freshness note: %s", text) + } + // Strip the synced_at comment line before parsing. + _, jsonPart, _ := strings.Cut(text, "\n") + var m map[string]any + if err := json.Unmarshal([]byte(jsonPart), &m); err != nil { + t.Fatalf("parse: %v\n%s", err, jsonPart) + } + if m["title"] != "Bug" { + t.Errorf("issue title=%v", m["title"]) + } +} + +func TestListIssuesTool(t *testing.T) { + cs := setup(t, false) + text := callText(t, cs, "list_issues", map[string]any{"state": "all"}) + if !strings.Contains(text, "Bug") { + t.Errorf("list_issues missing issue: %s", text) + } +} + +func TestIssueCommentsTool(t *testing.T) { + cs := setup(t, false) + text := callText(t, cs, "get_issue_comments", map[string]any{"issue_number": 1}) + if !strings.Contains(text, "me too") { + t.Errorf("get_issue_comments missing comment: %s", text) + } +} + +func TestReadOnlyHidesWrites(t *testing.T) { + cs := setup(t, true) + res, err := cs.ListTools(context.Background(), nil) + if err != nil { + t.Fatal(err) + } + for _, tool := range res.Tools { + if tool.Name == "create_issue" || tool.Name == "update_issue" { + t.Errorf("read-only server exposed write tool %q", tool.Name) + } + } +} + +func TestWriteToolProxyDisabled(t *testing.T) { + cs := setup(t, false) + // proxy is disabled in the test setup → create_issue surfaces an error result. + res, err := cs.CallTool(context.Background(), &sdk.CallToolParams{ + Name: "create_issue", Arguments: map[string]any{"title": "x"}, + }) + if err != nil { + t.Fatalf("CallTool: %v", err) + } + if !res.IsError { + t.Errorf("expected IsError for disabled proxy write") + } +} + +func keys(m map[string]bool) []string { + var out []string + for k := range m { + out = append(out, k) + } + return out +} diff --git a/internal/mcp/writes.go b/internal/mcp/writes.go new file mode 100644 index 0000000..d801ff3 --- /dev/null +++ b/internal/mcp/writes.go @@ -0,0 +1,121 @@ +package mcp + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// registerWrites adds the write tools (only when not in read-only mode). They +// proxy to api.github.com and then re-sync the touched entity into the store. +func (t *tools) registerWrites(s *mcp.Server) { + s.AddTool(&mcp.Tool{ + Name: "create_issue", + Description: "Create an issue. Proxies to GitHub and re-syncs the repo.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "title": "string", "body": "string"}, "title"), + }, t.createIssue) + + s.AddTool(&mcp.Tool{ + Name: "add_issue_comment", + Description: "Add a comment to an issue or PR. Proxies to GitHub and re-syncs the issue.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "issue_number": "integer", "body": "string"}, "issue_number", "body"), + }, t.addIssueComment) + + s.AddTool(&mcp.Tool{ + Name: "update_issue", + Description: "Update an issue (title/body/state). Proxies to GitHub and re-syncs the issue.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "issue_number": "integer", "title": "string", "body": "string", "state": "string"}, "issue_number"), + }, t.updateIssue) + + s.AddTool(&mcp.Tool{ + Name: "update_pull_request", + Description: "Update a pull request (title/body/state). Proxies to GitHub and re-syncs the PR.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "pullNumber": "integer", "title": "string", "body": "string", "state": "string"}, "pullNumber"), + }, t.updatePull) + + s.AddTool(&mcp.Tool{ + Name: "merge_pull_request", + Description: "Merge a pull request. Proxies to GitHub and re-syncs the PR.", + InputSchema: schema(map[string]string{"owner": "string", "repo": "string", "pullNumber": "integer", "merge_method": "string", "commit_title": "string"}, "pullNumber"), + }, t.mergePull) +} + +func (t *tools) ownerRepo(req *mcp.CallToolRequest) (string, string) { + owner := argStr(req.Params.Arguments, "owner") + repo := argStr(req.Params.Arguments, "repo") + if owner == "" { + owner = t.d.Owner + } + if repo == "" { + repo = t.d.Repo + } + return owner, repo +} + +func (t *tools) post(ctx context.Context, method, path string, payload map[string]any) (*mcp.CallToolResult, error) { + body, _ := json.Marshal(payload) + status, resp, err := t.d.Proxy.Request(ctx, method, path, bytes.NewReader(body)) + if err != nil { + return errResult(err), nil + } + if status >= 400 { + return errResult(fmt.Errorf("GitHub returned %d: %s", status, string(resp))), nil + } + return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: string(resp)}}}, nil +} + +func (t *tools) createIssue(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + owner, repo := t.ownerRepo(req) + payload := map[string]any{"title": argStr(req.Params.Arguments, "title")} + if b := argStr(req.Params.Arguments, "body"); b != "" { + payload["body"] = b + } + return t.post(ctx, "POST", fmt.Sprintf("/repos/%s/%s/issues", owner, repo), payload) +} + +func (t *tools) addIssueComment(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + owner, repo := t.ownerRepo(req) + n := argInt(req.Params.Arguments, "issue_number") + payload := map[string]any{"body": argStr(req.Params.Arguments, "body")} + return t.post(ctx, "POST", fmt.Sprintf("/repos/%s/%s/issues/%d/comments", owner, repo, n), payload) +} + +func (t *tools) updateIssue(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + owner, repo := t.ownerRepo(req) + n := argInt(req.Params.Arguments, "issue_number") + payload := map[string]any{} + for _, k := range []string{"title", "body", "state"} { + if v := argStr(req.Params.Arguments, k); v != "" { + payload[k] = v + } + } + return t.post(ctx, "PATCH", fmt.Sprintf("/repos/%s/%s/issues/%d", owner, repo, n), payload) +} + +func (t *tools) updatePull(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + owner, repo := t.ownerRepo(req) + n := argInt(req.Params.Arguments, "pullNumber") + payload := map[string]any{} + for _, k := range []string{"title", "body", "state"} { + if v := argStr(req.Params.Arguments, k); v != "" { + payload[k] = v + } + } + return t.post(ctx, "PATCH", fmt.Sprintf("/repos/%s/%s/pulls/%d", owner, repo, n), payload) +} + +func (t *tools) mergePull(ctx context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + owner, repo := t.ownerRepo(req) + n := argInt(req.Params.Arguments, "pullNumber") + payload := map[string]any{} + if v := argStr(req.Params.Arguments, "merge_method"); v != "" { + payload["merge_method"] = v + } + if v := argStr(req.Params.Arguments, "commit_title"); v != "" { + payload["commit_title"] = v + } + return t.post(ctx, "PUT", fmt.Sprintf("/repos/%s/%s/pulls/%d/merge", owner, repo, n), payload) +} diff --git a/internal/query/query.go b/internal/query/query.go new file mode 100644 index 0000000..944ddea --- /dev/null +++ b/internal/query/query.go @@ -0,0 +1,572 @@ +// Package query is the backend-agnostic read layer over the SQLite store. It +// returns data in GitHub's REST JSON shape (straight from the stored raw_json +// where possible) so the HTTP API, MCP server, and native CLI subcommands can +// all be thin adapters over it. +package query + +import ( + "database/sql" + "encoding/json" + "sort" + "strings" + + "github.com/mevdschee/github-export/internal/store" +) + +// Query wraps the store's database handle with read-only, GitHub-shaped queries. +type Query struct { + db *sql.DB + s *store.Store +} + +// New builds a Query over an open store. +func New(s *store.Store) *Query { + return &Query{db: s.DB(), s: s} +} + +// Store returns the underlying store (for meta reads). +func (q *Query) Store() *store.Store { return q.s } + +// raw is a pre-serialized JSON document read from the store. +type raw = json.RawMessage + +// ListIssuesOpts filters and paginates an issue/PR listing. Zero values mean +// "unset" and fall back to GitHub's defaults. +type ListIssuesOpts struct { + State string // open (default), closed, all + Labels []string + Creator string // author login + Assignee string // assignee login, or "none"/"*" + Milestone string // milestone title, or "none"/"*" + Since string // ISO8601; updated_at >= since + Sort string // created (default), updated, comments + Direction string // asc, desc (default desc) + PerPage int + Page int + // OnlyPulls/OnlyIssues narrow the set; both false means GitHub's /issues + // behaviour (issues and PRs together). + OnlyPulls bool + OnlyIssues bool +} + +func (o ListIssuesOpts) perPage() int { + if o.PerPage <= 0 { + return 30 + } + if o.PerPage > 100 { + return 100 + } + return o.PerPage +} + +func (o ListIssuesOpts) page() int { + if o.Page <= 0 { + return 1 + } + return o.Page +} + +// ListIssues returns issues/PRs matching opts, plus the total count of matches +// (before pagination) for Link-header construction. +func (q *Query) ListIssues(o ListIssuesOpts) ([]raw, int, error) { + var where []string + var args []any + + switch o.State { + case "", "open": + where = append(where, "state = 'open'") + case "closed": + where = append(where, "state = 'closed'") + case "all": + // no filter + default: + where = append(where, "state = ?") + args = append(args, o.State) + } + if o.OnlyPulls { + where = append(where, "is_pull_request = 1") + } + if o.OnlyIssues { + where = append(where, "is_pull_request = 0") + } + if o.Creator != "" { + where = append(where, "author = ?") + args = append(args, o.Creator) + } + if o.Milestone != "" && o.Milestone != "*" { + if o.Milestone == "none" { + where = append(where, "(milestone IS NULL OR milestone = '')") + } else { + where = append(where, "milestone = ?") + args = append(args, o.Milestone) + } + } + if o.Since != "" { + where = append(where, "updated_at >= ?") + args = append(args, o.Since) + } + for _, lbl := range o.Labels { + where = append(where, "number IN (SELECT issue_number FROM issue_labels WHERE label = ?)") + args = append(args, lbl) + } + if o.Assignee != "" && o.Assignee != "*" { + if o.Assignee == "none" { + where = append(where, "number NOT IN (SELECT issue_number FROM issue_assignees)") + } else { + where = append(where, "number IN (SELECT issue_number FROM issue_assignees WHERE login = ?)") + args = append(args, o.Assignee) + } + } + + clause := "" + if len(where) > 0 { + clause = " WHERE " + strings.Join(where, " AND ") + } + + var total int + if err := q.db.QueryRow("SELECT COUNT(*) FROM issues"+clause, args...).Scan(&total); err != nil { + return nil, 0, err + } + + order := orderBy(o.Sort, o.Direction) + limit := o.perPage() + offset := (o.page() - 1) * limit + // For pulls-only listings prefer the richer pr_json (has merged/draft/etc). + col := "raw_json" + if o.OnlyPulls { + col = "COALESCE(pr_json, raw_json)" + } + rows, err := q.db.Query( + "SELECT "+col+" FROM issues"+clause+order+" LIMIT ? OFFSET ?", + append(args, limit, offset)...) + if err != nil { + return nil, 0, err + } + defer rows.Close() + out, err := scanRaw(rows) + return out, total, err +} + +func orderBy(sortField, dir string) string { + col := "created_at" + switch sortField { + case "updated": + col = "updated_at" + case "created", "": + col = "created_at" + case "number": + col = "number" + } + d := "DESC" + if strings.EqualFold(dir, "asc") { + d = "ASC" + } + return " ORDER BY " + col + " " + d +} + +// GetIssue returns the stored issue/PR by number in the /issues/{n} shape. +func (q *Query) GetIssue(number int64) (raw, bool, error) { + return q.scalarRaw("SELECT raw_json FROM issues WHERE number = ?", number) +} + +// GetPull returns the PR detail (/pulls/{n} shape) for a number, falling back to +// the issue raw_json if PR details were not synced. +func (q *Query) GetPull(number int64) (raw, bool, error) { + return q.scalarRaw("SELECT COALESCE(pr_json, raw_json) FROM issues WHERE number = ? AND is_pull_request = 1", number) +} + +// timelineEntries returns the merged timeline for an issue/PR. +func (q *Query) timelineEntries(number int64) ([]map[string]any, error) { + var tj sql.NullString + err := q.db.QueryRow("SELECT timeline_json FROM issues WHERE number = ?", number).Scan(&tj) + if err == sql.ErrNoRows || !tj.Valid || tj.String == "" { + return nil, nil + } + if err != nil { + return nil, err + } + var out []map[string]any + if err := json.Unmarshal([]byte(tj.String), &out); err != nil { + return nil, err + } + return out, nil +} + +// IssueComments returns the comment objects on an issue/PR in the GitHub +// /issues/{n}/comments shape (the synthetic "event" marker is stripped). +func (q *Query) IssueComments(number int64) ([]raw, error) { + tl, err := q.timelineEntries(number) + if err != nil { + return nil, err + } + var out []raw + for _, e := range tl { + if e["event"] != "commented" { + continue + } + m := cloneWithout(e, "event") + out = append(out, mustMarshal(m)) + } + return out, nil +} + +// IssueTimeline returns the full merged timeline for an issue/PR. +func (q *Query) IssueTimeline(number int64) ([]raw, error) { + tl, err := q.timelineEntries(number) + if err != nil { + return nil, err + } + out := make([]raw, 0, len(tl)) + for _, e := range tl { + out = append(out, mustMarshal(e)) + } + return out, nil +} + +// PullReviews returns PR reviews (the /pulls/{n}/reviews shape) extracted from +// the stored timeline. +func (q *Query) PullReviews(number int64) ([]raw, error) { + tl, err := q.timelineEntries(number) + if err != nil { + return nil, err + } + var out []raw + for _, e := range tl { + if e["event"] != "reviewed" { + continue + } + out = append(out, mustMarshal(cloneWithout(e, "event"))) + } + return out, nil +} + +// PullReviewComments returns the inline review comments (/pulls/{n}/comments +// shape) extracted from the stored timeline's line-commented groups. +func (q *Query) PullReviewComments(number int64) ([]raw, error) { + tl, err := q.timelineEntries(number) + if err != nil { + return nil, err + } + var out []raw + for _, e := range tl { + if e["event"] != "line-commented" { + continue + } + for _, c := range toList(e["comments"]) { + if cm, ok := c.(map[string]any); ok { + out = append(out, mustMarshal(cm)) + } + } + } + return out, nil +} + +// --- simple full-collection reads --- + +func (q *Query) ListLabels() ([]raw, error) { + return q.queryRaw("SELECT raw_json FROM labels ORDER BY ord") +} + +func (q *Query) ListMilestones() ([]raw, error) { + return q.queryRaw("SELECT raw_json FROM milestones ORDER BY ord") +} + +func (q *Query) ListReleases() ([]raw, error) { + return q.queryRaw("SELECT raw_json FROM releases ORDER BY ord") +} + +func (q *Query) GetRelease(tag string) (raw, bool, error) { + return q.scalarRaw("SELECT raw_json FROM releases WHERE tag = ?", tag) +} + +func (q *Query) ListDiscussions() ([]raw, error) { + return q.queryRaw("SELECT raw_json FROM discussions ORDER BY number") +} + +func (q *Query) GetDiscussion(number int64) (raw, bool, error) { + return q.scalarRaw("SELECT raw_json FROM discussions WHERE number = ?", number) +} + +// DiscussionComments returns the top-level comment nodes of a discussion (the +// stored discussion raw_json is already GraphQL-shaped). +func (q *Query) DiscussionComments(number int64) ([]raw, error) { + doc, ok, err := q.GetDiscussion(number) + if err != nil || !ok { + return nil, err + } + var node map[string]any + if err := json.Unmarshal(doc, &node); err != nil { + return nil, err + } + comments, _ := node["comments"].(map[string]any) + nodes, _ := comments["nodes"].([]any) + out := make([]raw, 0, len(nodes)) + for _, n := range nodes { + out = append(out, mustMarshal(n)) + } + return out, nil +} + +func (q *Query) ListProjects() ([]raw, error) { + return q.queryRaw("SELECT raw_json FROM projects ORDER BY number") +} + +func (q *Query) GetProject(number int64) (raw, bool, error) { + return q.scalarRaw("SELECT raw_json FROM projects WHERE number = ?", number) +} + +// Repo returns stored repository metadata. +func (q *Query) Repo() (raw, bool, error) { + return q.scalarRaw("SELECT raw_json FROM repository WHERE id = 1") +} + +// --- search --- + +// SearchIssues runs a GitHub-style issue search. The query string may contain +// qualifiers (is:, state:, label:, author:, assignee:, milestone:, in:) plus +// free text matched against title/body. Returns matches and total count. +func (q *Query) SearchIssues(queryStr string, perPage, page int) ([]raw, int, error) { + o := ListIssuesOpts{State: "all", PerPage: perPage, Page: page} + var terms []string + for _, tok := range strings.Fields(queryStr) { + if k, v, ok := splitQualifier(tok); ok { + switch k { + case "is", "type": + switch v { + case "pr", "pull-request": + o.OnlyPulls = true + case "issue": + o.OnlyIssues = true + case "open", "closed": + o.State = v + case "merged": + o.OnlyPulls = true + o.State = "all" + } + case "state": + o.State = v + case "label": + o.Labels = append(o.Labels, strings.Trim(v, `"`)) + case "author": + o.Creator = v + case "assignee": + o.Assignee = v + case "milestone": + o.Milestone = strings.Trim(v, `"`) + case "sort": + o.Sort = v + case "in": + // in:title / in:body — narrowing handled implicitly by free text + default: + // unknown qualifier: treat as free text + terms = append(terms, tok) + } + continue + } + terms = append(terms, tok) + } + + // Build the base filtered set, then apply free-text matching in SQL. + matches, _, err := q.ListIssues(ListIssuesOpts{ + State: o.State, Labels: o.Labels, Creator: o.Creator, Assignee: o.Assignee, + Milestone: o.Milestone, OnlyPulls: o.OnlyPulls, OnlyIssues: o.OnlyIssues, + Sort: o.Sort, PerPage: 100000, Page: 1, + }) + if err != nil { + return nil, 0, err + } + + var filtered []raw + switch { + case len(terms) == 0: + filtered = matches + default: + // Try the FTS5 index first (relevance-ranked); intersect its ranked + // numbers with the qualifier-filtered set, preserving FTS order. + if nums, ok, err := q.s.SearchFTS(buildFTSQuery(terms)); err != nil { + return nil, 0, err + } else if ok { + byNum := map[int64]raw{} + for _, m := range matches { + var doc map[string]any + if json.Unmarshal(m, &doc) == nil { + byNum[int64(jsonInt(doc["number"]))] = m + } + } + for _, n := range nums { + if r, present := byNum[n]; present { + filtered = append(filtered, r) + } + } + } else { + // Malformed FTS expression: degrade to a substring scan. + text := strings.ToLower(strings.Join(terms, " ")) + for _, m := range matches { + var doc map[string]any + if err := json.Unmarshal(m, &doc); err != nil { + continue + } + hay := strings.ToLower(str(doc, "title") + "\n" + str(doc, "body")) + if strings.Contains(hay, text) { + filtered = append(filtered, m) + } + } + } + } + + total := len(filtered) + pp := perPage + if pp <= 0 { + pp = 30 + } + pg := page + if pg <= 0 { + pg = 1 + } + start := (pg - 1) * pp + if start > total { + start = total + } + end := start + pp + if end > total { + end = total + } + return filtered[start:end], total, nil +} + +// --- status --- + +// Counts returns row counts per major entity for the /status endpoint. +func (q *Query) Counts() (map[string]int, error) { + out := map[string]int{} + for label, table := range map[string]string{ + "issues": "issues WHERE is_pull_request = 0", + "pulls": "issues WHERE is_pull_request = 1", + "labels": "labels", + "milestones": "milestones", + "releases": "releases", + "projects": "projects", + "discussions": "discussions", + "events": "events", + } { + var n int + if err := q.db.QueryRow("SELECT COUNT(*) FROM " + table).Scan(&n); err != nil { + return nil, err + } + out[label] = n + } + return out, nil +} + +// --- helpers --- + +func (q *Query) queryRaw(sqlStr string, args ...any) ([]raw, error) { + rows, err := q.db.Query(sqlStr, args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanRaw(rows) +} + +func (q *Query) scalarRaw(sqlStr string, args ...any) (raw, bool, error) { + var s string + switch err := q.db.QueryRow(sqlStr, args...).Scan(&s); err { + case sql.ErrNoRows: + return nil, false, nil + case nil: + return raw(s), true, nil + default: + return nil, false, err + } +} + +func scanRaw(rows *sql.Rows) ([]raw, error) { + var out []raw + for rows.Next() { + var s string + if err := rows.Scan(&s); err != nil { + return nil, err + } + out = append(out, raw(s)) + } + return out, rows.Err() +} + +// buildFTSQuery turns free-text terms into an FTS5 MATCH expression. Each term +// is wrapped as a quoted string (embedded quotes doubled) and joined by spaces, +// which FTS5 treats as an implicit AND — every term must appear. +func buildFTSQuery(terms []string) string { + parts := make([]string, 0, len(terms)) + for _, t := range terms { + t = strings.ReplaceAll(t, `"`, `""`) + parts = append(parts, `"`+t+`"`) + } + return strings.Join(parts, " ") +} + +func jsonInt(v any) int64 { + switch n := v.(type) { + case float64: + return int64(n) + case int64: + return n + } + return 0 +} + +func splitQualifier(tok string) (key, val string, ok bool) { + i := strings.IndexByte(tok, ':') + if i <= 0 || i == len(tok)-1 { + return "", "", false + } + return strings.ToLower(tok[:i]), tok[i+1:], true +} + +func cloneWithout(m map[string]any, drop ...string) map[string]any { + out := make(map[string]any, len(m)) + for k, v := range m { + out[k] = v + } + for _, d := range drop { + delete(out, d) + } + return out +} + +func mustMarshal(v any) raw { + b, err := json.Marshal(v) + if err != nil { + return raw("null") + } + return raw(b) +} + +func toList(v any) []any { + l, _ := v.([]any) + return l +} + +func str(m map[string]any, k string) string { + s, _ := m[k].(string) + return s +} + +// SortedTags returns release tags sorted for stable output (used by tests). +func (q *Query) SortedTags() ([]string, error) { + rows, err := q.db.Query("SELECT tag FROM releases") + if err != nil { + return nil, err + } + defer rows.Close() + var tags []string + for rows.Next() { + var t string + if err := rows.Scan(&t); err != nil { + return nil, err + } + tags = append(tags, t) + } + sort.Strings(tags) + return tags, rows.Err() +} diff --git a/internal/render/render.go b/internal/render/render.go new file mode 100644 index 0000000..8edddda --- /dev/null +++ b/internal/render/render.go @@ -0,0 +1,541 @@ +// Package render writes the on-demand markdown views from data already held in +// the store. It is pure presentation: every function takes rehydrated maps or +// domain structs and produces the same YAML-frontmatter markdown the tool has +// always emitted. It performs no network or database access. +package render + +import ( + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/mevdschee/github-export/internal/document" + "github.com/mevdschee/github-export/internal/ghmodel" + "github.com/mevdschee/github-export/internal/jsonutil" +) + +// IssueFile writes a single issue or PR markdown file: frontmatter, body, then +// one sub-document per timeline entry. +func IssueFile(path string, issue map[string]any, isPR bool, pr map[string]any, timeline []map[string]any, projects []string) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + d := &document.Writer{} + d.KV("number", jsonutil.Int(issue, "number")) + d.KV("title", jsonutil.Str(issue, "title")) + if isPR { + d.KV("type", "pull_request") + } + d.KV("state", jsonutil.Str(issue, "state")) + if reason := jsonutil.Str(issue, "state_reason"); reason != "" { + d.KV("state_reason", reason) + } + if isPR && pr != nil && jsonutil.Bool(pr, "draft") { + d.KV("draft", true) + } + if jsonutil.Bool(issue, "locked") { + d.KV("locked", true) + } + d.KV("created_at", jsonutil.Str(issue, "created_at")) + d.KV("updated_at", jsonutil.Str(issue, "updated_at")) + d.KV("closed_at", jsonutil.Str(issue, "closed_at")) + d.KV("author", jsonutil.UserLogin(issue, "user")) + d.List("assignees", jsonutil.Logins(issue, "assignees")) + d.List("labels", jsonutil.LabelNames(issue, "labels")) + if ms := jsonutil.Map(issue, "milestone"); ms != nil { + d.KV("milestone", jsonutil.Str(ms, "title")) + } + if len(projects) > 0 { + d.List("projects", projects) + } + + if isPR && pr != nil { + if head := jsonutil.Map(pr, "head"); head != nil { + d.KV("source_branch", jsonutil.Str(head, "ref")) + } + if base := jsonutil.Map(pr, "base"); base != nil { + d.KV("target_branch", jsonutil.Str(base, "ref")) + } + head := jsonutil.Map(pr, "head") + base := jsonutil.Map(pr, "base") + if head != nil && base != nil { + headRepo := jsonutil.Map(head, "repo") + baseRepo := jsonutil.Map(base, "repo") + if headRepo != nil && baseRepo != nil { + hf := jsonutil.Str(headRepo, "full_name") + bf := jsonutil.Str(baseRepo, "full_name") + if hf != "" && bf != "" && hf != bf { + d.KV("source_repo", hf) + } + } + } + + merged := jsonutil.Bool(pr, "merged") + fmt.Fprint(d.Buf(), "merge:\n") + d.KVIndent(" ", "merged", merged) + if merged { + d.KVIndent(" ", "merged_at", jsonutil.Str(pr, "merged_at")) + mergedBy := jsonutil.UserLogin(pr, "merged_by") + if mergedBy == "" { + for _, ev := range timeline { + if jsonutil.Str(ev, "event") == "merged" { + mergedBy = jsonutil.UserLogin(ev, "actor") + break + } + } + } + if mergedBy != "" { + d.KVIndent(" ", "merged_by", mergedBy) + } + d.KVIndent(" ", "commit_sha", jsonutil.Str(pr, "merge_commit_sha")) + } + + reviewerSet := map[string]bool{} + for _, ev := range timeline { + if jsonutil.Str(ev, "event") == "reviewed" { + if login := jsonutil.UserLogin(ev, "user"); login != "" { + reviewerSet[login] = true + } + } + } + if len(reviewerSet) > 0 { + var reviewers []string + for login := range reviewerSet { + reviewers = append(reviewers, login) + } + sort.Strings(reviewers) + d.List("reviewers", reviewers) + } + d.List("requested_reviewers", jsonutil.Logins(pr, "requested_reviewers")) + } + + if reactions := buildReactions(issue); len(reactions) > 0 { + fmt.Fprint(d.Buf(), "reactions:\n") + for _, r := range reactions { + fmt.Fprintf(d.Buf(), " %s: %d\n", document.YamlScalar(r.key), r.count) + } + } + + document.WriteFirstDoc(f, d.String(), jsonutil.Str(issue, "body")) + + for _, event := range timeline { + writeTimelineDoc(f, event) + } + return nil +} + +type reaction struct { + key string + count int +} + +func buildReactions(issue map[string]any) []reaction { + r := jsonutil.Map(issue, "reactions") + if r == nil { + return nil + } + keys := []string{"+1", "-1", "laugh", "hooray", "confused", "heart", "rocket", "eyes"} + var out []reaction + for _, k := range keys { + if count := int(jsonutil.Int(r, k)); count > 0 { + out = append(out, reaction{k, count}) + } + } + return out +} + +func writeTimelineDoc(w io.Writer, event map[string]any) { + eventType := jsonutil.Str(event, "event") + + switch eventType { + case "commented": + d := &document.Writer{} + d.KV("document", "comment") + d.KV("id", jsonutil.Int(event, "id")) + d.KV("author", jsonutil.UserLogin(event, "user")) + d.KV("created_at", jsonutil.Str(event, "created_at")) + document.WriteSubDoc(w, d.String(), jsonutil.Str(event, "body")) + + case "reviewed": + d := &document.Writer{} + d.KV("document", "review") + d.KV("id", jsonutil.Int(event, "id")) + d.KV("author", jsonutil.UserLogin(event, "user")) + d.KV("state", strings.ToLower(jsonutil.Str(event, "state"))) + d.KV("commit_sha", jsonutil.Str(event, "commit_id")) + d.KV("submitted_at", jsonutil.Str(event, "submitted_at")) + document.WriteSubDoc(w, d.String(), jsonutil.Str(event, "body")) + + case "line-commented": + comments := jsonutil.List(event, "comments") + for _, c := range comments { + cm, ok := c.(map[string]any) + if !ok { + continue + } + d := &document.Writer{} + d.KV("document", "review_comment") + d.KV("id", jsonutil.Int(cm, "id")) + if rid := jsonutil.Int(cm, "pull_request_review_id"); rid > 0 { + d.KV("review_id", rid) + } + d.KV("author", jsonutil.UserLogin(cm, "user")) + d.KV("created_at", jsonutil.Str(cm, "created_at")) + d.KV("path", jsonutil.Str(cm, "path")) + if line := jsonutil.Int(cm, "line"); line > 0 { + d.KV("line", line) + } else if origLine := jsonutil.Int(cm, "original_line"); origLine > 0 { + d.KV("line", origLine) + } + if side := jsonutil.Str(cm, "side"); side != "" { + d.KV("side", side) + } + d.KV("commit_sha", jsonutil.Str(cm, "commit_id")) + document.WriteSubDoc(w, d.String(), jsonutil.Str(cm, "body")) + } + + case "labeled", "unlabeled": + label := jsonutil.Map(event, "label") + if label == nil { + return + } + d := &document.Writer{} + d.KV("document", "event") + d.KV("event", eventType) + d.KV("actor", jsonutil.UserLogin(event, "actor")) + d.KV("created_at", jsonutil.Str(event, "created_at")) + d.KV("label", jsonutil.Str(label, "name")) + document.WriteSubDoc(w, d.String(), "") + + case "assigned", "unassigned": + d := &document.Writer{} + d.KV("document", "event") + d.KV("event", eventType) + d.KV("actor", jsonutil.UserLogin(event, "actor")) + d.KV("created_at", jsonutil.Str(event, "created_at")) + d.KV("assignee", jsonutil.UserLogin(event, "assignee")) + document.WriteSubDoc(w, d.String(), "") + + case "milestoned", "demilestoned": + d := &document.Writer{} + d.KV("document", "event") + d.KV("event", eventType) + d.KV("actor", jsonutil.UserLogin(event, "actor")) + d.KV("created_at", jsonutil.Str(event, "created_at")) + if ms := jsonutil.Map(event, "milestone"); ms != nil { + d.KV("milestone", jsonutil.Str(ms, "title")) + } + document.WriteSubDoc(w, d.String(), "") + + case "renamed": + d := &document.Writer{} + d.KV("document", "event") + d.KV("event", eventType) + d.KV("actor", jsonutil.UserLogin(event, "actor")) + d.KV("created_at", jsonutil.Str(event, "created_at")) + if rename := jsonutil.Map(event, "rename"); rename != nil { + d.KV("from", jsonutil.Str(rename, "from")) + d.KV("to", jsonutil.Str(rename, "to")) + } + document.WriteSubDoc(w, d.String(), "") + + case "cross-referenced": + d := &document.Writer{} + d.KV("document", "event") + d.KV("event", eventType) + d.KV("actor", jsonutil.UserLogin(event, "actor")) + d.KV("created_at", jsonutil.Str(event, "created_at")) + if source := jsonutil.Map(event, "source"); source != nil { + if si := jsonutil.Map(source, "issue"); si != nil { + if n := jsonutil.Int(si, "number"); n > 0 { + d.KV("source_number", n) + } + if sr := jsonutil.Map(si, "repository"); sr != nil { + d.KV("source_repo", jsonutil.Str(sr, "full_name")) + } + } + } + document.WriteSubDoc(w, d.String(), "") + + case "committed": + return + + case "closed", "reopened", "merged", "referenced", + "locked", "unlocked", + "head_ref_force_pushed", "head_ref_deleted", "base_ref_changed", + "converted_to_draft", "ready_for_review", + "review_requested", "review_request_removed", + "review_dismissed", + "pinned", "unpinned", "transferred", + "connected", "disconnected", + "marked_as_duplicate", "unmarked_as_duplicate": + d := &document.Writer{} + d.KV("document", "event") + d.KV("event", eventType) + d.KV("actor", jsonutil.UserLogin(event, "actor")) + d.KV("created_at", jsonutil.Str(event, "created_at")) + if sha := jsonutil.Str(event, "commit_id"); sha != "" { + d.KV("commit_sha", sha) + } + if reviewer := jsonutil.UserLogin(event, "requested_reviewer"); reviewer != "" { + d.KV("reviewer", reviewer) + } + if reason := jsonutil.Str(event, "lock_reason"); reason != "" { + d.KV("lock_reason", reason) + } + if dismissal := jsonutil.Str(event, "dismissal_message"); dismissal != "" { + d.KV("dismissal_message", dismissal) + } + document.WriteSubDoc(w, d.String(), "") + + default: + if eventType != "" { + d := &document.Writer{} + d.KV("document", "event") + d.KV("event", eventType) + d.KV("actor", jsonutil.UserLogin(event, "actor")) + d.KV("created_at", jsonutil.Str(event, "created_at")) + document.WriteSubDoc(w, d.String(), "") + } + } +} + +// ReleaseFile writes a single release markdown file. +func ReleaseFile(path string, m map[string]any) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + d := &document.Writer{} + d.KV("tag", jsonutil.Str(m, "tag_name")) + d.KV("name", jsonutil.Str(m, "name")) + d.KV("draft", jsonutil.Bool(m, "draft")) + d.KV("prerelease", jsonutil.Bool(m, "prerelease")) + d.KV("author", jsonutil.UserLogin(m, "author")) + d.KV("created_at", jsonutil.Str(m, "created_at")) + d.KV("published_at", jsonutil.Str(m, "published_at")) + d.KV("target_commitish", jsonutil.Str(m, "target_commitish")) + + assets := jsonutil.List(m, "assets") + if len(assets) > 0 { + fmt.Fprint(d.Buf(), "assets:\n") + for _, a := range assets { + asset, _ := a.(map[string]any) + if asset == nil { + continue + } + fmt.Fprintf(d.Buf(), " - name: %s\n", document.YamlScalar(jsonutil.Str(asset, "name"))) + if ct := jsonutil.Str(asset, "content_type"); ct != "" { + fmt.Fprintf(d.Buf(), " content_type: %s\n", document.YamlScalar(ct)) + } + if sz := jsonutil.Int(asset, "size"); sz > 0 { + fmt.Fprintf(d.Buf(), " size_bytes: %d\n", sz) + } + if dc := jsonutil.Int(asset, "download_count"); dc > 0 { + fmt.Fprintf(d.Buf(), " download_count: %d\n", dc) + } + } + } + + document.WriteFirstDoc(f, d.String(), jsonutil.Str(m, "body")) + return nil +} + +// ProjectFile writes a project markdown file with its item sub-documents. +func ProjectFile(path string, p map[string]any, items []map[string]any) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + d := &document.Writer{} + d.KV("number", jsonutil.Int(p, "number")) + d.KV("title", jsonutil.Str(p, "title")) + d.KV("state", "open") + if jsonutil.Bool(p, "public") { + d.KV("public", true) + } + d.KV("url", jsonutil.Str(p, "url")) + d.KV("owner", ghmodel.OwnerLogin(p)) + if desc := jsonutil.Str(p, "shortDescription"); desc != "" { + d.KV("description", desc) + } + d.KV("created_at", jsonutil.Str(p, "createdAt")) + d.KV("updated_at", jsonutil.Str(p, "updatedAt")) + + fieldsMap := jsonutil.Map(p, "fields") + fieldNodes := jsonutil.List(fieldsMap, "nodes") + if len(fieldNodes) > 0 { + fmt.Fprint(d.Buf(), "fields:\n") + for _, fn := range fieldNodes { + fm, _ := fn.(map[string]any) + if fm == nil { + continue + } + name := jsonutil.Str(fm, "name") + if name == "" { + continue + } + fmt.Fprintf(d.Buf(), " - name: %s\n", document.YamlScalar(name)) + if dt := jsonutil.Str(fm, "dataType"); dt != "" { + fmt.Fprintf(d.Buf(), " type: %s\n", document.YamlScalar(dt)) + } + if opts := jsonutil.List(fm, "options"); len(opts) > 0 { + fmt.Fprint(d.Buf(), " options:\n") + for _, o := range opts { + om, _ := o.(map[string]any) + if om == nil { + continue + } + if optName := jsonutil.Str(om, "name"); optName != "" { + fmt.Fprintf(d.Buf(), " - %s\n", document.YamlScalar(optName)) + } + } + } + } + } + + document.WriteFirstDoc(f, d.String(), jsonutil.Str(p, "readme")) + + for _, item := range items { + writeProjectItem(f, item) + } + return nil +} + +func writeProjectItem(f io.Writer, item map[string]any) { + content := jsonutil.Map(item, "content") + if content == nil { + return + } + var itemType string + switch jsonutil.Str(content, "__typename") { + case "Issue": + itemType = "issue" + case "PullRequest": + itemType = "pull_request" + default: + return + } + + d := &document.Writer{} + d.KV("document", "item") + d.KV("type", itemType) + d.KV("number", jsonutil.Int(content, "number")) + d.KV("title", jsonutil.Str(content, "title")) + if r := jsonutil.Map(content, "repository"); r != nil { + d.KV("repo", jsonutil.Str(r, "nameWithOwner")) + } + + fvMap := jsonutil.Map(item, "fieldValues") + fvNodes := jsonutil.List(fvMap, "nodes") + var pairs [][2]string + for _, fv := range fvNodes { + fvm, _ := fv.(map[string]any) + if fvm == nil { + continue + } + fieldName := "" + if fld := jsonutil.Map(fvm, "field"); fld != nil { + fieldName = jsonutil.Str(fld, "name") + } + if fieldName == "" { + continue + } + value := ghmodel.FieldValueString(fvm) + if value == "" { + continue + } + pairs = append(pairs, [2]string{fieldName, value}) + } + if len(pairs) > 0 { + sort.Slice(pairs, func(i, j int) bool { return pairs[i][0] < pairs[j][0] }) + fmt.Fprint(d.Buf(), "fields:\n") + for _, kv := range pairs { + fmt.Fprintf(d.Buf(), " %s: %s\n", document.YamlScalar(kv[0]), document.YamlScalar(kv[1])) + } + } + + document.WriteSubDoc(f, d.String(), "") +} + +// DiscussionFile writes a discussion markdown file with comment and reply +// sub-documents. +func DiscussionFile(path string, d ghmodel.DiscussionNode) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + w := &document.Writer{} + w.KV("number", d.Number) + w.KV("title", d.Title) + w.KV("type", "discussion") + w.KV("state", ghmodel.DiscussionState(d)) + if d.StateReason != "" { + w.KV("state_reason", strings.ToLower(d.StateReason)) + } + if d.Locked { + w.KV("locked", true) + } + w.KV("created_at", d.CreatedAt) + w.KV("updated_at", d.UpdatedAt) + if d.ClosedAt != "" { + w.KV("closed_at", d.ClosedAt) + } + w.KV("author", d.Author.Login) + if d.Category.Name != "" { + w.KV("category", d.Category.Name) + } + var labels []string + for _, l := range d.Labels.Nodes { + if l.Name != "" { + labels = append(labels, l.Name) + } + } + w.List("labels", labels) + if d.Answer != nil { + w.KV("answer_id", d.Answer.DatabaseID) + if d.AnswerChosenAt != "" { + w.KV("answer_chosen_at", d.AnswerChosenAt) + } + if d.AnswerChosenBy != nil && d.AnswerChosenBy.Login != "" { + w.KV("answer_chosen_by", d.AnswerChosenBy.Login) + } + } + + document.WriteFirstDoc(f, w.String(), d.Body) + + answerID := d.AnswerID() + for _, c := range d.Comments.Nodes { + cw := &document.Writer{} + cw.KV("document", "comment") + cw.KV("id", c.DatabaseID) + cw.KV("author", c.Author.Login) + cw.KV("created_at", c.CreatedAt) + if answerID != 0 && c.DatabaseID == answerID { + cw.KV("is_answer", true) + } + document.WriteSubDoc(f, cw.String(), c.Body) + + for _, r := range c.Replies.Nodes { + rw := &document.Writer{} + rw.KV("document", "reply") + rw.KV("id", r.DatabaseID) + rw.KV("parent_id", c.DatabaseID) + rw.KV("author", r.Author.Login) + rw.KV("created_at", r.CreatedAt) + document.WriteSubDoc(f, rw.String(), r.Body) + } + } + return nil +} diff --git a/internal/shadow/shadow.go b/internal/shadow/shadow.go new file mode 100644 index 0000000..f3e3c22 --- /dev/null +++ b/internal/shadow/shadow.go @@ -0,0 +1,171 @@ +// Package shadow implements the --debug-compare parity harness. For each +// mirrored read it also fetches the same path from api.github.com, normalizes +// volatile fields out of both JSON payloads, and diffs them. On a meaningful +// divergence it logs the gap and (optionally) files a deduplicated issue on the +// project's own repo so coverage holes get tracked. The locally-computed answer +// is always what the caller receives; the remote fetch is comparison-only. +package shadow + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "sort" + "sync" + "time" +) + +// Fetcher is the minimal upstream-GET dependency (a function keeps the proxy +// coupling loose). +type Fetcher func(ctx context.Context, path string) (status int, body []byte, err error) + +// Comparator diffs local responses against upstream and tracks reported gaps. +type Comparator struct { + fetch Fetcher + fileIssue func(title, body string) error + repoSlug string + mu sync.Mutex + seen map[string]bool // fingerprint → already reported + volatile map[string]bool +} + +// New builds a Comparator. fileIssue may be nil to only log divergences. +func New(fetch Fetcher, fileIssue func(title, body string) error, repoSlug string) *Comparator { + return &Comparator{ + fetch: fetch, + fileIssue: fileIssue, + repoSlug: repoSlug, + seen: map[string]bool{}, + volatile: map[string]bool{ + // Fields GitHub fills that we cannot (or need not) mirror byte-for-byte. + "node_id": true, "url": true, "html_url": true, "events_url": true, + "labels_url": true, "comments_url": true, "timeline_url": true, + "repository_url": true, "avatar_url": true, "gravatar_id": true, + "followers_url": true, "following_url": true, "gists_url": true, + "starred_url": true, "subscriptions_url": true, "organizations_url": true, + "repos_url": true, "received_events_url": true, "performed_via_github_app": true, + "author_association": true, "reactions": true, "score": true, + }, + } +} + +// Check compares localBody against the upstream answer for path. It runs the +// comparison synchronously in a goroutine-friendly way; callers typically invoke +// it in a goroutine so the response is not delayed. +func (c *Comparator) Check(path string, localBody []byte) { + status, remoteBody, err := c.fetch(context.Background(), path) + if err != nil || status != http.StatusOK { + return // upstream unavailable or not a 200: nothing meaningful to diff + } + localNorm := c.normalize(localBody) + remoteNorm := c.normalize(remoteBody) + if bytes.Equal(localNorm, remoteNorm) { + return + } + fp := fingerprint(path, localNorm, remoteNorm) + c.mu.Lock() + already := c.seen[fp] + c.seen[fp] = true + c.mu.Unlock() + if already { + return + } + + log.Printf("shadow-compare: divergence on %s (fingerprint %s)", path, fp) + if c.fileIssue == nil { + return + } + title := fmt.Sprintf("[shadow-compare] divergence on %s", path) + body := fmt.Sprintf("The local mirror diverged from api.github.com.\n\n"+ + "**Path:** `%s`\n\n**Local (normalized):**\n```json\n%s\n```\n\n"+ + "**Remote (normalized):**\n```json\n%s\n```\n\n_Filed automatically at %s._", + path, truncate(localNorm), truncate(remoteNorm), time.Now().UTC().Format(time.RFC3339)) + if err := c.fileIssue(title, body); err != nil { + log.Printf("shadow-compare: filing issue failed: %v", err) + } +} + +// normalize parses JSON, strips volatile keys recursively, and re-marshals with +// sorted keys so the diff is order-independent. +func (c *Comparator) normalize(b []byte) []byte { + var v any + if err := json.Unmarshal(b, &v); err != nil { + return b + } + v = c.strip(v) + out, err := marshalSorted(v) + if err != nil { + return b + } + return out +} + +func (c *Comparator) strip(v any) any { + switch t := v.(type) { + case map[string]any: + out := map[string]any{} + for k, val := range t { + if c.volatile[k] { + continue + } + out[k] = c.strip(val) + } + return out + case []any: + for i := range t { + t[i] = c.strip(t[i]) + } + return t + default: + return v + } +} + +// marshalSorted marshals with map keys sorted (encoding/json already sorts map +// keys, so a plain Marshal suffices; kept as a seam for future tuning). +func marshalSorted(v any) ([]byte, error) { + return json.Marshal(v) +} + +func fingerprint(path string, local, remote []byte) string { + // A coarse fingerprint: the path plus the sorted set of top-level keys that + // differ. This collapses "same endpoint, same shape of gap" into one issue. + lk := topKeys(local) + rk := topKeys(remote) + all := map[string]bool{} + for _, k := range lk { + all[k] = true + } + for _, k := range rk { + all[k] = true + } + var keys []string + for k := range all { + keys = append(keys, k) + } + sort.Strings(keys) + return path + "|" + fmt.Sprint(keys) +} + +func topKeys(b []byte) []string { + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + return nil + } + var keys []string + for k := range m { + keys = append(keys, k) + } + return keys +} + +func truncate(b []byte) string { + const max = 4000 + if len(b) <= max { + return string(b) + } + return string(b[:max]) + "\n… (truncated)" +} diff --git a/internal/shadow/shadow_test.go b/internal/shadow/shadow_test.go new file mode 100644 index 0000000..749eaee --- /dev/null +++ b/internal/shadow/shadow_test.go @@ -0,0 +1,53 @@ +package shadow + +import ( + "context" + "sync" + "testing" +) + +func TestNormalizeStripsVolatileAndOrders(t *testing.T) { + c := New(nil, nil, "o/r") + a := c.normalize([]byte(`{"number":1,"url":"https://x","title":"hi"}`)) + b := c.normalize([]byte(`{"title":"hi","number":1,"url":"https://y"}`)) + if string(a) != string(b) { + t.Errorf("normalize not order/volatile-insensitive:\n a=%s\n b=%s", a, b) + } +} + +func TestCheckDedupAndFiles(t *testing.T) { + var mu sync.Mutex + filed := 0 + fetch := func(_ context.Context, _ string) (int, []byte, error) { + return 200, []byte(`{"number":1,"title":"REMOTE"}`), nil + } + file := func(title, body string) error { + mu.Lock() + filed++ + mu.Unlock() + return nil + } + c := New(fetch, file, "o/r") + local := []byte(`{"number":1,"title":"LOCAL"}`) + c.Check("/repos/o/r/issues/1", local) + c.Check("/repos/o/r/issues/1", local) // same fingerprint → deduped + mu.Lock() + defer mu.Unlock() + if filed != 1 { + t.Errorf("filed=%d, want 1 (deduped)", filed) + } +} + +func TestCheckNoDivergenceNoFile(t *testing.T) { + filed := 0 + fetch := func(_ context.Context, _ string) (int, []byte, error) { + return 200, []byte(`{"number":1,"url":"https://remote","title":"same"}`), nil + } + file := func(_, _ string) error { filed++; return nil } + c := New(fetch, file, "o/r") + // Differs only by the volatile url field → no divergence after normalize. + c.Check("/x", []byte(`{"number":1,"url":"https://local","title":"same"}`)) + if filed != 0 { + t.Errorf("filed=%d, want 0 (only volatile diff)", filed) + } +} diff --git a/internal/store/read.go b/internal/store/read.go new file mode 100644 index 0000000..477ef86 --- /dev/null +++ b/internal/store/read.go @@ -0,0 +1,364 @@ +package store + +import ( + "database/sql" + "encoding/json" + "fmt" + + "github.com/mevdschee/github-export/internal/ghmodel" + "github.com/mevdschee/github-export/internal/hooks" +) + +// --- prior-state reads for change detection (called before upsert) --- + +// IssueState returns the stored state for an issue/PR. exists is false when the +// issue has never been synced. +func (s *Store) IssueState(number int64) (exists bool, state string, isPR, merged bool, err error) { + var st string + var pr, mg int + row := s.db.QueryRow("SELECT state, is_pull_request, merged FROM issues WHERE number = ?", number) + switch err = row.Scan(&st, &pr, &mg); err { + case sql.ErrNoRows: + return false, "", false, false, nil + case nil: + return true, st, pr != 0, mg != 0, nil + default: + return false, "", false, false, err + } +} + +// ReleaseState reports whether a release with the given tag exists and, if so, +// whether it was previously a prerelease. +func (s *Store) ReleaseState(tag string) (exists, prerelease bool, err error) { + var pr int + switch err = s.db.QueryRow("SELECT prerelease FROM releases WHERE tag = ?", tag).Scan(&pr); err { + case sql.ErrNoRows: + return false, false, nil + case nil: + return true, pr != 0, nil + default: + return false, false, err + } +} + +// DiscussionNode returns the previously stored discussion, or exists=false. +func (s *Store) DiscussionNode(number int64) (d ghmodel.DiscussionNode, exists bool, err error) { + var raw string + switch err = s.db.QueryRow("SELECT raw_json FROM discussions WHERE number = ?", number).Scan(&raw); err { + case sql.ErrNoRows: + return d, false, nil + case nil: + if err = json.Unmarshal([]byte(raw), &d); err != nil { + return d, false, fmt.Errorf("decoding stored discussion #%d: %w", number, err) + } + return d, true, nil + default: + return d, false, err + } +} + +// ProjectItems returns the previously stored item maps for a project, or +// exists=false if the project has never been synced. +func (s *Store) ProjectItems(number int64) (items []map[string]any, exists bool, err error) { + var raw sql.NullString + switch err = s.db.QueryRow("SELECT items_json FROM projects WHERE number = ?", number).Scan(&raw); err { + case sql.ErrNoRows: + return nil, false, nil + case nil: + if raw.Valid && raw.String != "" { + if err = json.Unmarshal([]byte(raw.String), &items); err != nil { + return nil, true, fmt.Errorf("decoding stored items for project #%d: %w", number, err) + } + } + return items, true, nil + default: + return nil, false, err + } +} + +// SearchFTS returns the issue/PR numbers whose title or body match the FTS5 +// query, ordered by relevance (best first). An empty or malformed query returns +// (nil, false) so the caller can fall back to a non-FTS path. +func (s *Store) SearchFTS(ftsQuery string) (numbers []int64, ok bool, err error) { + if ftsQuery == "" { + return nil, false, nil + } + rows, err := s.db.Query( + "SELECT rowid FROM fts_issues WHERE fts_issues MATCH ? ORDER BY rank", ftsQuery) + if err != nil { + // A malformed MATCH expression is a user-input issue, not a store error: + // signal "no FTS result" so the caller can degrade gracefully. + return nil, false, nil + } + defer rows.Close() + for rows.Next() { + var n int64 + if err := rows.Scan(&n); err != nil { + return nil, false, err + } + numbers = append(numbers, n) + } + return numbers, true, rows.Err() +} + +// ProjectsForIssue returns the stored project cross-links for an issue/PR in +// sync order (used to preserve them on a targeted re-sync). +func (s *Store) ProjectsForIssue(number int64) ([]string, error) { + rows, err := s.db.Query("SELECT project FROM issue_projects WHERE issue_number = ? ORDER BY ord", number) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var p string + if err := rows.Scan(&p); err != nil { + return nil, err + } + out = append(out, p) + } + return out, rows.Err() +} + +// MaxIssueNumber returns the largest stored issue/PR number (0 if none). +func (s *Store) MaxIssueNumber() (int64, error) { + var n sql.NullInt64 + if err := s.db.QueryRow("SELECT MAX(number) FROM issues").Scan(&n); err != nil { + return 0, err + } + return n.Int64, nil +} + +// --- export reads --- + +// IssueRow is a fully-rehydrated issue/PR ready for rendering. +type IssueRow struct { + Number int64 + IsPR bool + Issue map[string]any + PR map[string]any + Timeline []map[string]any + Projects []string +} + +// ProjectRow is a rehydrated project plus its items. +type ProjectRow struct { + Number int64 + Project map[string]any + Items []map[string]any +} + +// PendingEvent pairs an event with its row ID so it can be marked exported. +type PendingEvent struct { + ID int64 + Event hooks.Event +} + +// ExportRepo returns the stored repo metadata and the last sync timestamp. +func (s *Store) ExportRepo() (owner, repo string, m map[string]any, syncedAt string, err error) { + var raw string + err = s.db.QueryRow("SELECT owner, name, raw_json FROM repository WHERE id = 1").Scan(&owner, &repo, &raw) + if err == sql.ErrNoRows { + return "", "", nil, "", nil + } + if err != nil { + return "", "", nil, "", err + } + if err = json.Unmarshal([]byte(raw), &m); err != nil { + return "", "", nil, "", err + } + syncedAt, err = s.SyncedAt() + return owner, repo, m, syncedAt, err +} + +func (s *Store) rawMaps(query string) ([]map[string]any, error) { + rows, err := s.db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + var out []map[string]any + for rows.Next() { + var raw string + if err := rows.Scan(&raw); err != nil { + return nil, err + } + var m map[string]any + if err := json.Unmarshal([]byte(raw), &m); err != nil { + return nil, err + } + out = append(out, m) + } + return out, rows.Err() +} + +// AllLabels returns label maps in sync order. +func (s *Store) AllLabels() ([]map[string]any, error) { + return s.rawMaps("SELECT raw_json FROM labels ORDER BY ord") +} + +// AllMilestones returns milestone maps in sync order. +func (s *Store) AllMilestones() ([]map[string]any, error) { + return s.rawMaps("SELECT raw_json FROM milestones ORDER BY ord") +} + +// AllReleases returns release maps in sync order. +func (s *Store) AllReleases() ([]map[string]any, error) { + return s.rawMaps("SELECT raw_json FROM releases ORDER BY ord") +} + +// AllIssues returns every issue/PR with its timeline and project cross-links, +// ordered by number. +func (s *Store) AllIssues() ([]IssueRow, error) { + projects, err := s.issueProjects() + if err != nil { + return nil, err + } + rows, err := s.db.Query("SELECT number, is_pull_request, raw_json, pr_json, timeline_json FROM issues ORDER BY number") + if err != nil { + return nil, err + } + defer rows.Close() + var out []IssueRow + for rows.Next() { + var ( + number int64 + isPR int + rawJSON string + prJSON sql.NullString + timeJSON sql.NullString + ) + if err := rows.Scan(&number, &isPR, &rawJSON, &prJSON, &timeJSON); err != nil { + return nil, err + } + ir := IssueRow{Number: number, IsPR: isPR != 0, Projects: projects[number]} + if err := json.Unmarshal([]byte(rawJSON), &ir.Issue); err != nil { + return nil, err + } + if prJSON.Valid && prJSON.String != "" { + if err := json.Unmarshal([]byte(prJSON.String), &ir.PR); err != nil { + return nil, err + } + } + if timeJSON.Valid && timeJSON.String != "" { + if err := json.Unmarshal([]byte(timeJSON.String), &ir.Timeline); err != nil { + return nil, err + } + } + out = append(out, ir) + } + return out, rows.Err() +} + +func (s *Store) issueProjects() (map[int64][]string, error) { + rows, err := s.db.Query("SELECT issue_number, project FROM issue_projects ORDER BY issue_number, ord") + if err != nil { + return nil, err + } + defer rows.Close() + out := map[int64][]string{} + for rows.Next() { + var n int64 + var p string + if err := rows.Scan(&n, &p); err != nil { + return nil, err + } + out[n] = append(out[n], p) + } + return out, rows.Err() +} + +// AllProjects returns every stored (open) project with its items, ordered by number. +func (s *Store) AllProjects() ([]ProjectRow, error) { + rows, err := s.db.Query("SELECT number, raw_json, items_json FROM projects ORDER BY number") + if err != nil { + return nil, err + } + defer rows.Close() + var out []ProjectRow + for rows.Next() { + var ( + number int64 + rawJSON string + itemsJSON sql.NullString + ) + if err := rows.Scan(&number, &rawJSON, &itemsJSON); err != nil { + return nil, err + } + pr := ProjectRow{Number: number} + if err := json.Unmarshal([]byte(rawJSON), &pr.Project); err != nil { + return nil, err + } + if itemsJSON.Valid && itemsJSON.String != "" { + if err := json.Unmarshal([]byte(itemsJSON.String), &pr.Items); err != nil { + return nil, err + } + } + out = append(out, pr) + } + return out, rows.Err() +} + +// AllDiscussions returns every stored discussion node, ordered by number. +func (s *Store) AllDiscussions() ([]ghmodel.DiscussionNode, error) { + rows, err := s.db.Query("SELECT raw_json FROM discussions ORDER BY number") + if err != nil { + return nil, err + } + defer rows.Close() + var out []ghmodel.DiscussionNode + for rows.Next() { + var raw string + if err := rows.Scan(&raw); err != nil { + return nil, err + } + var d ghmodel.DiscussionNode + if err := json.Unmarshal([]byte(raw), &d); err != nil { + return nil, err + } + out = append(out, d) + } + return out, rows.Err() +} + +// PendingEvents returns events that have not yet been written to an events/ file. +func (s *Store) PendingEvents() ([]PendingEvent, error) { + rows, err := s.db.Query(` + SELECT id, type, number, title, author, state, labels_json, file, repo, body, url, extra_json + FROM events WHERE exported_at IS NULL ORDER BY id`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []PendingEvent + for rows.Next() { + var ( + pe PendingEvent + number sql.NullInt64 + labelsJSON, extraJSON sql.NullString + typ, title, author, state, file, repo, body string + url string + ) + if err := rows.Scan(&pe.ID, &typ, &number, &title, &author, &state, + &labelsJSON, &file, &repo, &body, &url, &extraJSON); err != nil { + return nil, err + } + ev := hooks.Event{ + Type: typ, Number: number.Int64, Title: title, Author: author, + State: state, File: file, Repo: repo, Body: body, URL: url, + } + if labelsJSON.Valid && labelsJSON.String != "" { + if err := json.Unmarshal([]byte(labelsJSON.String), &ev.Labels); err != nil { + return nil, err + } + } + if extraJSON.Valid && extraJSON.String != "" { + if err := json.Unmarshal([]byte(extraJSON.String), &ev.Extra); err != nil { + return nil, err + } + } + pe.Event = ev + out = append(out, pe) + } + return out, rows.Err() +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 0000000..38f0941 --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,293 @@ +// Package store is the primary source of truth for synced GitHub data. The sync +// layer upserts parsed API payloads here; the exporter reads them back to render +// markdown on demand. Each entity keeps typed columns (for querying and change +// detection) alongside a raw_json blob (for full-fidelity rendering). +package store + +import ( + "database/sql" + "encoding/json" + "fmt" + + _ "modernc.org/sqlite" +) + +// schemaVersion is bumped whenever the schema changes; migrate() applies the +// base schema for a fresh DB and then runs each incremental migration in +// migrations up to this version. +const schemaVersion = 2 + +const schemaSQL = ` +CREATE TABLE meta ( + key TEXT PRIMARY KEY, + value TEXT +); + +CREATE TABLE repository ( + id INTEGER PRIMARY KEY CHECK (id = 1), + owner TEXT NOT NULL, + name TEXT NOT NULL, + raw_json TEXT NOT NULL +); + +CREATE TABLE labels ( + name TEXT PRIMARY KEY, + color TEXT, + description TEXT, + raw_json TEXT NOT NULL, + ord INTEGER +); + +CREATE TABLE milestones ( + number INTEGER PRIMARY KEY, + title TEXT, + state TEXT, + description TEXT, + due_on TEXT, + closed_at TEXT, + raw_json TEXT NOT NULL, + ord INTEGER +); + +CREATE TABLE issues ( + number INTEGER PRIMARY KEY, + is_pull_request INTEGER NOT NULL DEFAULT 0, + title TEXT, + state TEXT, + state_reason TEXT, + draft INTEGER NOT NULL DEFAULT 0, + locked INTEGER NOT NULL DEFAULT 0, + merged INTEGER NOT NULL DEFAULT 0, + created_at TEXT, + updated_at TEXT, + closed_at TEXT, + author TEXT, + milestone TEXT, + body TEXT, + raw_json TEXT NOT NULL, + pr_json TEXT, + timeline_json TEXT +); + +CREATE TABLE issue_labels ( + issue_number INTEGER NOT NULL, + label TEXT NOT NULL, + ord INTEGER, + PRIMARY KEY (issue_number, label) +); + +CREATE TABLE issue_assignees ( + issue_number INTEGER NOT NULL, + login TEXT NOT NULL, + ord INTEGER, + PRIMARY KEY (issue_number, login) +); + +CREATE TABLE issue_projects ( + issue_number INTEGER NOT NULL, + project TEXT NOT NULL, + ord INTEGER, + PRIMARY KEY (issue_number, project) +); + +CREATE TABLE releases ( + tag TEXT PRIMARY KEY, + name TEXT, + draft INTEGER NOT NULL DEFAULT 0, + prerelease INTEGER NOT NULL DEFAULT 0, + author TEXT, + created_at TEXT, + published_at TEXT, + target_commitish TEXT, + body TEXT, + raw_json TEXT NOT NULL, + ord INTEGER +); + +CREATE TABLE projects ( + number INTEGER PRIMARY KEY, + title TEXT, + state TEXT, + public INTEGER NOT NULL DEFAULT 0, + owner TEXT, + url TEXT, + description TEXT, + created_at TEXT, + updated_at TEXT, + raw_json TEXT NOT NULL, + items_json TEXT +); + +CREATE TABLE discussions ( + number INTEGER PRIMARY KEY, + title TEXT, + category TEXT, + state TEXT, + state_reason TEXT, + author TEXT, + created_at TEXT, + updated_at TEXT, + closed_at TEXT, + answer_id INTEGER, + body TEXT, + raw_json TEXT NOT NULL +); + +CREATE TABLE events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + type TEXT NOT NULL, + number INTEGER, + title TEXT, + author TEXT, + state TEXT, + labels_json TEXT, + file TEXT, + repo TEXT, + body TEXT, + url TEXT, + extra_json TEXT, + detected_at TEXT, + exported_at TEXT +); +` + +// migrations holds incremental schema upgrades applied after the base schema. +// Index i upgrades the DB from version i+1 to i+2 (migrations[0]: v1→v2). Adding +// a migration means appending here and bumping schemaVersion. +var migrations = []string{ + // v1 → v2: full-text search index over issue/PR titles and bodies. The issue + // number is used as the FTS rowid so matches join back to the issues table. + `CREATE VIRTUAL TABLE fts_issues USING fts5(title, body);`, +} + +// Store wraps the SQLite connection. Methods come in three families: Upsert* +// (write during sync), Prev*/*State (read prior state for change detection), +// and All*/Pending* (read for export). +type Store struct { + db *sql.DB +} + +// Open opens (creating if needed) the SQLite database at path, configures it for +// fast bulk writes, and runs migrations. +func Open(path string) (*Store, error) { + db, err := sql.Open("sqlite", path) + if err != nil { + return nil, fmt.Errorf("opening %s: %w", path, err) + } + // WAL + NORMAL gives us durable-enough, much faster bulk inserts. Single + // connection keeps the modernc driver's write path simple. + db.SetMaxOpenConns(1) + for _, pragma := range []string{ + "PRAGMA journal_mode=WAL", + "PRAGMA synchronous=NORMAL", + "PRAGMA foreign_keys=ON", + "PRAGMA busy_timeout=5000", + } { + if _, err := db.Exec(pragma); err != nil { + db.Close() + return nil, fmt.Errorf("%s: %w", pragma, err) + } + } + s := &Store{db: db} + if err := s.migrate(); err != nil { + db.Close() + return nil, err + } + return s, nil +} + +// Close closes the underlying database. +func (s *Store) Close() error { return s.db.Close() } + +// DB exposes the raw handle for the query/API layers built in later phases. +func (s *Store) DB() *sql.DB { return s.db } + +func (s *Store) migrate() error { + var v int + if err := s.db.QueryRow("PRAGMA user_version").Scan(&v); err != nil { + return fmt.Errorf("reading user_version: %w", err) + } + if v > schemaVersion { + return fmt.Errorf("database schema version %d is newer than supported %d; upgrade the tool", v, schemaVersion) + } + if v == schemaVersion { + return nil + } + // Fresh database: apply the base schema (version 1), then fall through to run + // any incremental migrations up to schemaVersion. + if v == 0 { + if _, err := s.db.Exec(schemaSQL); err != nil { + return fmt.Errorf("applying schema: %w", err) + } + v = 1 + } + // Apply incremental migrations. migrations[i] upgrades v=(i+1) to v=(i+2). + for v < schemaVersion { + stmt := migrations[v-1] + if _, err := s.db.Exec(stmt); err != nil { + return fmt.Errorf("migration to v%d: %w", v+1, err) + } + v++ + } + // Backfill the FTS index when it was just created on an existing DB. + if err := s.rebuildFTS(); err != nil { + return fmt.Errorf("backfilling search index: %w", err) + } + if _, err := s.db.Exec(fmt.Sprintf("PRAGMA user_version=%d", schemaVersion)); err != nil { + return fmt.Errorf("setting user_version: %w", err) + } + return nil +} + +// rebuildFTS repopulates fts_issues from the issues table. It is idempotent: +// safe to run on a fresh (empty) DB and on an upgrade that backfills history. +func (s *Store) rebuildFTS() error { + if _, err := s.db.Exec("DELETE FROM fts_issues"); err != nil { + return err + } + _, err := s.db.Exec( + "INSERT INTO fts_issues(rowid, title, body) SELECT number, COALESCE(title,''), COALESCE(body,'') FROM issues") + return err +} + +// --- meta --- + +// GetMeta returns the value for key, or "" if absent. +func (s *Store) GetMeta(key string) (string, error) { + var v string + err := s.db.QueryRow("SELECT value FROM meta WHERE key = ?", key).Scan(&v) + if err == sql.ErrNoRows { + return "", nil + } + return v, err +} + +// SetMeta upserts a meta key/value. +func (s *Store) SetMeta(key, value string) error { + _, err := s.db.Exec( + "INSERT INTO meta(key, value) VALUES(?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", + key, value) + return err +} + +// OwnerRepo reads the synced owner/repo recorded in meta. +func (s *Store) OwnerRepo() (owner, repo string, err error) { + if owner, err = s.GetMeta("owner"); err != nil { + return + } + repo, err = s.GetMeta("repo") + return +} + +// SyncedAt returns the timestamp of the last completed sync, or "". +func (s *Store) SyncedAt() (string, error) { return s.GetMeta("synced_at") } + +func mustJSON(v any) string { + b, err := json.Marshal(v) + if err != nil { + // The inputs are always JSON-decoded maps/slices, so this is unreachable + // in practice; fall back to an empty object rather than crash a sync. + return "{}" + } + return string(b) +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 0000000..355daad --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,325 @@ +package store + +import ( + "path/filepath" + "testing" + + "github.com/mevdschee/github-export/internal/ghmodel" + "github.com/mevdschee/github-export/internal/hooks" +) + +func openTemp(t *testing.T) *Store { + t.Helper() + s, err := Open(filepath.Join(t.TempDir(), "test.sqlite")) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { s.Close() }) + return s +} + +func TestMigrateSetsVersion(t *testing.T) { + s := openTemp(t) + var v int + if err := s.db.QueryRow("PRAGMA user_version").Scan(&v); err != nil { + t.Fatal(err) + } + if v != schemaVersion { + t.Errorf("user_version=%d, want %d", v, schemaVersion) + } + // Re-opening an existing DB must be a no-op, not an error. + if err := s.migrate(); err != nil { + t.Errorf("second migrate: %v", err) + } +} + +func TestMeta(t *testing.T) { + s := openTemp(t) + if v, err := s.GetMeta("missing"); err != nil || v != "" { + t.Errorf("GetMeta(missing)=%q,%v want \"\",nil", v, err) + } + if err := s.SetMeta("owner", "octocat"); err != nil { + t.Fatal(err) + } + if err := s.SetMeta("owner", "octocat2"); err != nil { // upsert + t.Fatal(err) + } + if v, _ := s.GetMeta("owner"); v != "octocat2" { + t.Errorf("owner=%q, want octocat2", v) + } +} + +func TestUpsertIssueRoundTripAndState(t *testing.T) { + s := openTemp(t) + issue := map[string]any{ + "number": float64(42), + "title": "Crash on empty input", + "state": "open", + "user": map[string]any{"login": "octocat"}, + "labels": []any{map[string]any{"name": "bug"}, map[string]any{"name": "p1"}}, + "assignees": []any{map[string]any{"login": "hubot"}}, + "milestone": map[string]any{"title": "v2.1"}, + "body": "boom", + "created_at": "2024-01-01T00:00:00Z", + } + if err := s.UpsertIssue(42, false, issue, nil, nil, []string{"Roadmap"}); err != nil { + t.Fatalf("UpsertIssue: %v", err) + } + + exists, state, isPR, merged, err := s.IssueState(42) + if err != nil { + t.Fatal(err) + } + if !exists || state != "open" || isPR || merged { + t.Errorf("IssueState = exists=%v state=%q isPR=%v merged=%v", exists, state, isPR, merged) + } + + rows, err := s.AllIssues() + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 { + t.Fatalf("AllIssues len=%d, want 1", len(rows)) + } + r := rows[0] + if r.Number != 42 || r.IsPR { + t.Errorf("row = number=%d isPR=%v", r.Number, r.IsPR) + } + if got := r.Issue["title"]; got != "Crash on empty input" { + t.Errorf("title round-trip = %v", got) + } + if len(r.Projects) != 1 || r.Projects[0] != "Roadmap" { + t.Errorf("projects = %v, want [Roadmap]", r.Projects) + } + + // Idempotency + state transition: re-upsert as closed/merged PR. + issue["state"] = "closed" + pr := map[string]any{"merged": true, "draft": false} + if err := s.UpsertIssue(42, true, issue, pr, nil, nil); err != nil { + t.Fatalf("re-upsert: %v", err) + } + if rows, _ := s.AllIssues(); len(rows) != 1 { + t.Errorf("after re-upsert AllIssues len=%d, want 1 (idempotent)", len(rows)) + } + exists, state, isPR, merged, _ = s.IssueState(42) + if !exists || state != "closed" || !isPR || !merged { + t.Errorf("after transition: state=%q isPR=%v merged=%v", state, isPR, merged) + } + // projects join should have been cleared (passed nil this time). + if rows, _ := s.AllIssues(); len(rows[0].Projects) != 0 { + t.Errorf("projects not cleared on re-upsert: %v", rows[0].Projects) + } + + if max, _ := s.MaxIssueNumber(); max != 42 { + t.Errorf("MaxIssueNumber=%d, want 42", max) + } +} + +func TestFTSSearch(t *testing.T) { + s := openTemp(t) + s.UpsertIssue(1, false, map[string]any{ + "number": float64(1), "title": "Crash on startup", "body": "segfault when launching", + }, nil, nil, nil) + s.UpsertIssue(2, false, map[string]any{ + "number": float64(2), "title": "Docs typo", "body": "fix the readme", + }, nil, nil, nil) + + nums, ok, err := s.SearchFTS(`"segfault"`) + if err != nil || !ok { + t.Fatalf("SearchFTS ok=%v err=%v", ok, err) + } + if len(nums) != 1 || nums[0] != 1 { + t.Errorf("SearchFTS(segfault)=%v, want [1]", nums) + } + + // Title is indexed too. + if nums, _, _ := s.SearchFTS(`"typo"`); len(nums) != 1 || nums[0] != 2 { + t.Errorf("SearchFTS(typo)=%v, want [2]", nums) + } + + // Updating the body must update the index (delete-then-insert by rowid). + s.UpsertIssue(1, false, map[string]any{ + "number": float64(1), "title": "Crash on startup", "body": "now fixed, no more crash", + }, nil, nil, nil) + if nums, _, _ := s.SearchFTS(`"segfault"`); len(nums) != 0 { + t.Errorf("stale FTS row after update: %v", nums) + } +} + +func TestFTSMigrationBackfill(t *testing.T) { + path := filepath.Join(t.TempDir(), "legacy.sqlite") + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + s.UpsertIssue(7, false, map[string]any{ + "number": float64(7), "title": "Legacy", "body": "indexable content here", + }, nil, nil, nil) + // Simulate a pre-FTS (v1) database: drop the index and roll the version back. + if _, err := s.db.Exec("DROP TABLE fts_issues"); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec("PRAGMA user_version=1"); err != nil { + t.Fatal(err) + } + s.Close() + + // Reopening must run the v1→v2 migration and backfill the existing issue. + s2, err := Open(path) + if err != nil { + t.Fatalf("reopen/migrate: %v", err) + } + defer s2.Close() + nums, ok, err := s2.SearchFTS(`"indexable"`) + if err != nil || !ok || len(nums) != 1 || nums[0] != 7 { + t.Errorf("backfill search = %v ok=%v err=%v, want [7]", nums, ok, err) + } +} + +func TestLabelsAndMilestonesReplace(t *testing.T) { + s := openTemp(t) + if err := s.ReplaceLabels([]map[string]any{ + {"name": "bug", "color": "red", "description": "a bug"}, + {"name": "docs", "color": "blue"}, + }); err != nil { + t.Fatal(err) + } + // Replace again with fewer — must not accumulate. + if err := s.ReplaceLabels([]map[string]any{{"name": "bug", "color": "red"}}); err != nil { + t.Fatal(err) + } + labels, err := s.AllLabels() + if err != nil { + t.Fatal(err) + } + if len(labels) != 1 || labels[0]["name"] != "bug" { + t.Errorf("labels=%v, want single bug", labels) + } + + if err := s.ReplaceMilestones([]map[string]any{ + {"number": float64(1), "title": "v1", "state": "closed"}, + }); err != nil { + t.Fatal(err) + } + ms, _ := s.AllMilestones() + if len(ms) != 1 || ms[0]["title"] != "v1" { + t.Errorf("milestones=%v", ms) + } +} + +func TestReleaseState(t *testing.T) { + s := openTemp(t) + if exists, _, _ := mustReleaseState(t, s, "v1"); exists { + t.Error("release should not exist yet") + } + if err := s.UpsertRelease(map[string]any{"tag_name": "v1", "prerelease": true}); err != nil { + t.Fatal(err) + } + exists, pre, _ := mustReleaseState(t, s, "v1") + if !exists || !pre { + t.Errorf("state exists=%v prerelease=%v, want true,true", exists, pre) + } + // Promote to full release. + if err := s.UpsertRelease(map[string]any{"tag_name": "v1", "prerelease": false}); err != nil { + t.Fatal(err) + } + _, pre, _ = mustReleaseState(t, s, "v1") + if pre { + t.Error("prerelease should be false after promotion") + } + if rels, _ := s.AllReleases(); len(rels) != 1 { + t.Errorf("AllReleases len=%d, want 1", len(rels)) + } +} + +func mustReleaseState(t *testing.T, s *Store, tag string) (bool, bool, bool) { + t.Helper() + exists, pre, err := s.ReleaseState(tag) + if err != nil { + t.Fatal(err) + } + return exists, pre, true +} + +func TestProjectUpsertDeleteAndItems(t *testing.T) { + s := openTemp(t) + if _, exists, _ := s.ProjectItems(1); exists { + t.Error("project 1 should not exist") + } + items := []map[string]any{ + {"type": "ISSUE", "content": map[string]any{"__typename": "Issue", "number": float64(5), "title": "T"}}, + } + p := map[string]any{"number": float64(1), "title": "Board", "closed": false} + if err := s.UpsertProject(1, p, items); err != nil { + t.Fatal(err) + } + got, exists, err := s.ProjectItems(1) + if err != nil || !exists { + t.Fatalf("ProjectItems exists=%v err=%v", exists, err) + } + if len(got) != 1 { + t.Errorf("items len=%d, want 1", len(got)) + } + if rows, _ := s.AllProjects(); len(rows) != 1 || rows[0].Number != 1 { + t.Errorf("AllProjects=%v", rows) + } + if err := s.DeleteProject(1); err != nil { + t.Fatal(err) + } + if _, exists, _ := s.ProjectItems(1); exists { + t.Error("project 1 should be gone after delete") + } +} + +func TestDiscussionRoundTrip(t *testing.T) { + s := openTemp(t) + d := ghmodel.DiscussionNode{Number: 7, Title: "Q", Closed: false} + d.Author.Login = "octocat" + d.Category.Name = "Q&A" + d.Comments.Nodes = []ghmodel.DiscussionComment{{DatabaseID: 100}} + if err := s.UpsertDiscussion(d); err != nil { + t.Fatal(err) + } + got, exists, err := s.DiscussionNode(7) + if err != nil || !exists { + t.Fatalf("DiscussionNode exists=%v err=%v", exists, err) + } + if got.Title != "Q" || got.Category.Name != "Q&A" || len(got.Comments.Nodes) != 1 { + t.Errorf("round-trip mismatch: %+v", got) + } + if all, _ := s.AllDiscussions(); len(all) != 1 { + t.Errorf("AllDiscussions len=%d, want 1", len(all)) + } +} + +func TestEventsLifecycle(t *testing.T) { + s := openTemp(t) + evs := []hooks.Event{ + {Type: "issue_created", Number: 1, Title: "A", Labels: []string{"bug"}, File: "issues/0001.md", Repo: "o/r"}, + {Type: "comment_created", Number: 1, Extra: map[string]string{"k": "v"}}, + } + if err := s.InsertEvents("2024-01-01T00:00:00Z", evs); err != nil { + t.Fatal(err) + } + pending, err := s.PendingEvents() + if err != nil { + t.Fatal(err) + } + if len(pending) != 2 { + t.Fatalf("pending=%d, want 2", len(pending)) + } + if pending[0].Event.Type != "issue_created" || len(pending[0].Event.Labels) != 1 { + t.Errorf("event0=%+v", pending[0].Event) + } + if pending[1].Event.Extra["k"] != "v" { + t.Errorf("event1 extra=%v", pending[1].Event.Extra) + } + + ids := []int64{pending[0].ID, pending[1].ID} + if err := s.MarkEventsExported(ids, "2024-01-02T00:00:00Z"); err != nil { + t.Fatal(err) + } + if again, _ := s.PendingEvents(); len(again) != 0 { + t.Errorf("pending after mark=%d, want 0", len(again)) + } +} diff --git a/internal/store/write.go b/internal/store/write.go new file mode 100644 index 0000000..03d0b81 --- /dev/null +++ b/internal/store/write.go @@ -0,0 +1,265 @@ +package store + +import ( + "database/sql" + + "github.com/mevdschee/github-export/internal/ghmodel" + "github.com/mevdschee/github-export/internal/hooks" + "github.com/mevdschee/github-export/internal/jsonutil" +) + +// UpsertRepo records repository metadata (single row) and the owner/repo in meta. +func (s *Store) UpsertRepo(owner, repo string, m map[string]any) error { + if err := s.SetMeta("owner", owner); err != nil { + return err + } + if err := s.SetMeta("repo", repo); err != nil { + return err + } + _, err := s.db.Exec(` + INSERT INTO repository(id, owner, name, raw_json) VALUES(1, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET owner=excluded.owner, name=excluded.name, raw_json=excluded.raw_json`, + owner, repo, mustJSON(m)) + return err +} + +// ReplaceLabels replaces the full label set (labels are always full-synced). +func (s *Store) ReplaceLabels(items []map[string]any) error { + return s.tx(func(tx *sql.Tx) error { + if _, err := tx.Exec("DELETE FROM labels"); err != nil { + return err + } + for i, m := range items { + if _, err := tx.Exec( + "INSERT INTO labels(name, color, description, raw_json, ord) VALUES(?, ?, ?, ?, ?)", + jsonutil.Str(m, "name"), jsonutil.Str(m, "color"), jsonutil.Str(m, "description"), + mustJSON(m), i); err != nil { + return err + } + } + return nil + }) +} + +// ReplaceMilestones replaces the full milestone set. +func (s *Store) ReplaceMilestones(items []map[string]any) error { + return s.tx(func(tx *sql.Tx) error { + if _, err := tx.Exec("DELETE FROM milestones"); err != nil { + return err + } + for i, m := range items { + if _, err := tx.Exec( + `INSERT INTO milestones(number, title, state, description, due_on, closed_at, raw_json, ord) + VALUES(?, ?, ?, ?, ?, ?, ?, ?)`, + jsonutil.Int(m, "number"), jsonutil.Str(m, "title"), jsonutil.Str(m, "state"), + jsonutil.Str(m, "description"), jsonutil.Str(m, "due_on"), jsonutil.Str(m, "closed_at"), + mustJSON(m), i); err != nil { + return err + } + } + return nil + }) +} + +// UpsertIssue stores an issue or PR with its composed timeline and project +// cross-links. issue is the issues-API map, pr is the pulls-API detail map (nil +// for plain issues), timeline is the merged chronological timeline. +func (s *Store) UpsertIssue(number int64, isPR bool, issue, pr map[string]any, timeline []map[string]any, projects []string) error { + var prJSON, timelineJSON any + if pr != nil { + prJSON = mustJSON(pr) + } + if timeline != nil { + timelineJSON = mustJSON(timeline) + } + milestone := "" + if ms := jsonutil.Map(issue, "milestone"); ms != nil { + milestone = jsonutil.Str(ms, "title") + } + return s.tx(func(tx *sql.Tx) error { + _, err := tx.Exec(` + INSERT INTO issues(number, is_pull_request, title, state, state_reason, draft, locked, merged, + created_at, updated_at, closed_at, author, milestone, body, raw_json, pr_json, timeline_json) + VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(number) DO UPDATE SET + is_pull_request=excluded.is_pull_request, title=excluded.title, state=excluded.state, + state_reason=excluded.state_reason, draft=excluded.draft, locked=excluded.locked, + merged=excluded.merged, created_at=excluded.created_at, updated_at=excluded.updated_at, + closed_at=excluded.closed_at, author=excluded.author, milestone=excluded.milestone, + body=excluded.body, raw_json=excluded.raw_json, pr_json=excluded.pr_json, + timeline_json=excluded.timeline_json`, + number, boolInt(isPR), jsonutil.Str(issue, "title"), jsonutil.Str(issue, "state"), + jsonutil.Str(issue, "state_reason"), boolInt(pr != nil && jsonutil.Bool(pr, "draft")), + boolInt(jsonutil.Bool(issue, "locked")), boolInt(pr != nil && jsonutil.Bool(pr, "merged")), + jsonutil.Str(issue, "created_at"), jsonutil.Str(issue, "updated_at"), jsonutil.Str(issue, "closed_at"), + jsonutil.UserLogin(issue, "user"), milestone, jsonutil.Str(issue, "body"), + mustJSON(issue), prJSON, timelineJSON) + if err != nil { + return err + } + if err := replaceJoin(tx, "issue_labels", "label", number, jsonutil.LabelNames(issue, "labels")); err != nil { + return err + } + if err := replaceJoin(tx, "issue_assignees", "login", number, jsonutil.Logins(issue, "assignees")); err != nil { + return err + } + if err := replaceJoin(tx, "issue_projects", "project", number, projects); err != nil { + return err + } + // Keep the full-text index in sync (delete-then-insert by rowid=number). + if _, err := tx.Exec("DELETE FROM fts_issues WHERE rowid = ?", number); err != nil { + return err + } + _, err = tx.Exec("INSERT INTO fts_issues(rowid, title, body) VALUES(?, ?, ?)", + number, jsonutil.Str(issue, "title"), jsonutil.Str(issue, "body")) + return err + }) +} + +// UpsertRelease stores a release row. +func (s *Store) UpsertRelease(m map[string]any) error { + tag := jsonutil.Str(m, "tag_name") + if tag == "" { + return nil + } + _, err := s.db.Exec(` + INSERT INTO releases(tag, name, draft, prerelease, author, created_at, published_at, target_commitish, body, raw_json, ord) + VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, COALESCE((SELECT ord FROM releases WHERE tag=?), (SELECT COUNT(*) FROM releases))) + ON CONFLICT(tag) DO UPDATE SET + name=excluded.name, draft=excluded.draft, prerelease=excluded.prerelease, author=excluded.author, + created_at=excluded.created_at, published_at=excluded.published_at, + target_commitish=excluded.target_commitish, body=excluded.body, raw_json=excluded.raw_json`, + tag, jsonutil.Str(m, "name"), boolInt(jsonutil.Bool(m, "draft")), boolInt(jsonutil.Bool(m, "prerelease")), + jsonutil.UserLogin(m, "author"), jsonutil.Str(m, "created_at"), jsonutil.Str(m, "published_at"), + jsonutil.Str(m, "target_commitish"), jsonutil.Str(m, "body"), mustJSON(m), tag) + return err +} + +// UpsertProject stores a project (open) with its items. +func (s *Store) UpsertProject(number int64, p map[string]any, items []map[string]any) error { + var itemsJSON any + if items != nil { + itemsJSON = mustJSON(items) + } + state := "open" + if jsonutil.Bool(p, "closed") { + state = "closed" + } + _, err := s.db.Exec(` + INSERT INTO projects(number, title, state, public, owner, url, description, created_at, updated_at, raw_json, items_json) + VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(number) DO UPDATE SET + title=excluded.title, state=excluded.state, public=excluded.public, owner=excluded.owner, + url=excluded.url, description=excluded.description, created_at=excluded.created_at, + updated_at=excluded.updated_at, raw_json=excluded.raw_json, items_json=excluded.items_json`, + number, jsonutil.Str(p, "title"), state, boolInt(jsonutil.Bool(p, "public")), ghmodel.OwnerLogin(p), + jsonutil.Str(p, "url"), jsonutil.Str(p, "shortDescription"), jsonutil.Str(p, "createdAt"), + jsonutil.Str(p, "updatedAt"), mustJSON(p), itemsJSON) + return err +} + +// DeleteProject removes a project row (used when a project is closed). +func (s *Store) DeleteProject(number int64) error { + _, err := s.db.Exec("DELETE FROM projects WHERE number = ?", number) + return err +} + +// UpsertDiscussion stores a discussion node (re-marshaled to raw_json for +// faithful rendering). +func (s *Store) UpsertDiscussion(d ghmodel.DiscussionNode) error { + stateReason := "" + if d.StateReason != "" { + stateReason = d.StateReason + } + _, err := s.db.Exec(` + INSERT INTO discussions(number, title, category, state, state_reason, author, created_at, updated_at, closed_at, answer_id, body, raw_json) + VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(number) DO UPDATE SET + title=excluded.title, category=excluded.category, state=excluded.state, + state_reason=excluded.state_reason, author=excluded.author, created_at=excluded.created_at, + updated_at=excluded.updated_at, closed_at=excluded.closed_at, answer_id=excluded.answer_id, + body=excluded.body, raw_json=excluded.raw_json`, + d.Number, d.Title, d.Category.Name, ghmodel.DiscussionState(d), stateReason, d.Author.Login, + d.CreatedAt, d.UpdatedAt, d.ClosedAt, d.AnswerID(), d.Body, mustJSON(d)) + return err +} + +// InsertEvents appends detected change events, stamped with detectedAt. +func (s *Store) InsertEvents(detectedAt string, events []hooks.Event) error { + if len(events) == 0 { + return nil + } + return s.tx(func(tx *sql.Tx) error { + for _, ev := range events { + var labelsJSON, extraJSON any + if len(ev.Labels) > 0 { + labelsJSON = mustJSON(ev.Labels) + } + if len(ev.Extra) > 0 { + extraJSON = mustJSON(ev.Extra) + } + if _, err := tx.Exec(` + INSERT INTO events(type, number, title, author, state, labels_json, file, repo, body, url, extra_json, detected_at) + VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ev.Type, ev.Number, ev.Title, ev.Author, ev.State, labelsJSON, ev.File, ev.Repo, + ev.Body, ev.URL, extraJSON, detectedAt); err != nil { + return err + } + } + return nil + }) +} + +// MarkEventsExported stamps the given event IDs as exported at the given time. +func (s *Store) MarkEventsExported(ids []int64, at string) error { + return s.tx(func(tx *sql.Tx) error { + for _, id := range ids { + if _, err := tx.Exec("UPDATE events SET exported_at = ? WHERE id = ?", at, id); err != nil { + return err + } + } + return nil + }) +} + +// --- helpers --- + +func (s *Store) tx(fn func(*sql.Tx) error) error { + tx, err := s.db.Begin() + if err != nil { + return err + } + if err := fn(tx); err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + +func replaceJoin(tx *sql.Tx, table, col string, issueNumber int64, values []string) error { + if _, err := tx.Exec("DELETE FROM "+table+" WHERE issue_number = ?", issueNumber); err != nil { + return err + } + seen := map[string]bool{} + ord := 0 + for _, v := range values { + if v == "" || seen[v] { + continue + } + seen[v] = true + if _, err := tx.Exec( + "INSERT INTO "+table+"(issue_number, "+col+", ord) VALUES(?, ?, ?)", + issueNumber, v, ord); err != nil { + return err + } + ord++ + } + return nil +} + +func boolInt(b bool) int { + if b { + return 1 + } + return 0 +} diff --git a/internal/sync/discussions.go b/internal/sync/discussions.go index 4cf9f1d..862e58b 100644 --- a/internal/sync/discussions.go +++ b/internal/sync/discussions.go @@ -1,19 +1,15 @@ package sync import ( - "bufio" "encoding/json" "fmt" "log" - "os" "path/filepath" - "strings" - "github.com/mevdschee/github-export/internal/document" + "github.com/mevdschee/github-export/internal/ghmodel" "github.com/mevdschee/github-export/internal/github" "github.com/mevdschee/github-export/internal/hooks" - - "gopkg.in/yaml.v3" + "github.com/mevdschee/github-export/internal/store" ) // listDiscussionsQuery fetches discussions sorted by updated_at descending so @@ -64,25 +60,14 @@ query($owner: String!, $name: String!, $cursor: String) { } }` -// Discussions exports GitHub Discussions linked to the repository. Mirrors the -// PR-bulk-reviews behavior: paginates updated_at-descending and stops once a -// page's oldest item is older than `since`. -// -// File layout: one markdown file per discussion at -// `github-data/discussions/.md`. Top-level comments are emitted as -// `document: comment` sub-documents; nested replies as `document: reply` with -// a `parent_id` pointing at the comment they reply to. -// -// Hook events: emits `discussion_created` for any discussion whose file did -// not previously exist on disk. -func Discussions(c *github.Client, owner, repo, outDir, since string) ([]hooks.Event, error) { +// Discussions syncs GitHub Discussions into the store. Mirrors the PR-bulk +// behavior: paginates updated_at-descending and stops once a page's oldest item +// is older than `since`. Emits discussion_created / _closed / _answered / +// _comment_created events by diffing against the stored discussion. +func Discussions(c *github.Client, s *store.Store, owner, repo, since string) ([]hooks.Event, error) { log.Println("Syncing discussions...") repoSlug := owner + "/" + repo - dir := filepath.Join(outDir, "discussions") - if err := os.MkdirAll(dir, 0o755); err != nil { - return nil, fmt.Errorf("creating discussions dir: %w", err) - } var events []hooks.Event var processed int @@ -104,7 +89,7 @@ func Discussions(c *github.Client, owner, repo, outDir, since string) ([]hooks.E HasNextPage bool `json:"hasNextPage"` EndCursor string `json:"endCursor"` } `json:"pageInfo"` - Nodes []discussionNode `json:"nodes"` + Nodes []ghmodel.DiscussionNode `json:"nodes"` } `json:"discussions"` } `json:"repository"` } @@ -119,16 +104,22 @@ func Discussions(c *github.Client, owner, repo, outDir, since string) ([]hooks.E hitCutoff = true continue } - path := filepath.Join(dir, fmt.Sprintf("%04d.md", d.Number)) - prev := readPrevDiscussion(path) - if err := writeDiscussionFile(path, d); err != nil { - log.Printf(" Warning: writing discussion #%d: %v", d.Number, err) + relPath := filepath.Join("discussions", fmt.Sprintf("%04d.md", d.Number)) + prev, existed, err := s.DiscussionNode(d.Number) + if err != nil { + log.Printf(" Warning: reading discussion #%d: %v", d.Number, err) + } + + events = append(events, detectDiscussionEvents(d, existed, prev, relPath, repoSlug)...) + + if err := s.UpsertDiscussion(d); err != nil { + log.Printf(" Warning: storing discussion #%d: %v", d.Number, err) continue } processed++ - events = append(events, detectDiscussionEvents(d, prev, path, repoSlug)...) + warnDiscussionTruncation(d) } if hitCutoff || !resp.Repository.Discussions.PageInfo.HasNextPage { @@ -141,109 +132,48 @@ func Discussions(c *github.Client, owner, repo, outDir, since string) ([]hooks.E return events, nil } -type discussionAuthor struct { - Login string `json:"login"` -} - -type discussionCategory struct { - Name string `json:"name"` - Emoji string `json:"emoji"` - IsAnswerable bool `json:"isAnswerable"` -} - -type discussionReply struct { - DatabaseID int64 `json:"databaseId"` - CreatedAt string `json:"createdAt"` - Body string `json:"body"` - Author discussionAuthor `json:"author"` -} - -type discussionComment struct { - DatabaseID int64 `json:"databaseId"` - CreatedAt string `json:"createdAt"` - Body string `json:"body"` - Author discussionAuthor `json:"author"` - Replies struct { - PageInfo struct { - HasNextPage bool `json:"hasNextPage"` - } `json:"pageInfo"` - Nodes []discussionReply `json:"nodes"` - } `json:"replies"` -} - -type discussionNode struct { - Number int64 `json:"number"` - Title string `json:"title"` - Body string `json:"body"` - CreatedAt string `json:"createdAt"` - UpdatedAt string `json:"updatedAt"` - Closed bool `json:"closed"` - ClosedAt string `json:"closedAt"` - StateReason string `json:"stateReason"` - Locked bool `json:"locked"` - URL string `json:"url"` - Author discussionAuthor `json:"author"` - Category discussionCategory `json:"category"` - Labels struct { - Nodes []struct { - Name string `json:"name"` - } `json:"nodes"` - } `json:"labels"` - Answer *struct { - DatabaseID int64 `json:"databaseId"` - } `json:"answer"` - AnswerChosenAt string `json:"answerChosenAt"` - AnswerChosenBy *discussionAuthor `json:"answerChosenBy"` - Comments struct { - PageInfo struct { - HasNextPage bool `json:"hasNextPage"` - } `json:"pageInfo"` - Nodes []discussionComment `json:"nodes"` - } `json:"comments"` -} - -// prevDiscussion captures state from a previously-written discussion file used -// to diff against the latest data. -type prevDiscussion struct { - exists bool - state string - answerID int64 - commentIDs map[int64]bool +func warnDiscussionTruncation(d ghmodel.DiscussionNode) { + if d.Comments.PageInfo.HasNextPage { + log.Printf(" Warning: discussion #%d has more than 100 top-level comments — only first 100 stored", d.Number) + } + for _, c := range d.Comments.Nodes { + if c.Replies.PageInfo.HasNextPage { + log.Printf(" Warning: discussion #%d comment %d has more than 50 replies — only first 50 stored", d.Number, c.DatabaseID) + } + } } -func detectDiscussionEvents(d discussionNode, prev *prevDiscussion, path, repoSlug string) []hooks.Event { - state := discussionState(d) +func detectDiscussionEvents(d ghmodel.DiscussionNode, existed bool, prev ghmodel.DiscussionNode, relPath, repoSlug string) []hooks.Event { + state := ghmodel.DiscussionState(d) base := hooks.Event{ Number: d.Number, Title: d.Title, Author: d.Author.Login, State: state, - File: path, + File: relPath, Repo: repoSlug, URL: d.URL, } var events []hooks.Event - if prev == nil || !prev.exists { + if !existed { created := base created.Type = hooks.DiscussionCreated created.Body = d.Body return append(events, created) } - if state == "closed" && prev.state == "open" { + if state == "closed" && ghmodel.DiscussionState(prev) == "open" { ev := base ev.Type = hooks.DiscussionClosed events = append(events, ev) } - if d.Answer != nil && d.Answer.DatabaseID != 0 && prev.answerID == 0 { + if d.AnswerID() != 0 && prev.AnswerID() == 0 { ev := base ev.Type = hooks.DiscussionAnswered - extra := map[string]string{ - "answer_id": fmt.Sprintf("%d", d.Answer.DatabaseID), - } + extra := map[string]string{"answer_id": fmt.Sprintf("%d", d.AnswerID())} if d.AnswerChosenAt != "" { extra["answer_chosen_at"] = d.AnswerChosenAt } @@ -254,8 +184,12 @@ func detectDiscussionEvents(d discussionNode, prev *prevDiscussion, path, repoSl events = append(events, ev) } + prevComments := map[int64]bool{} + for _, c := range prev.Comments.Nodes { + prevComments[c.DatabaseID] = true + } for _, c := range d.Comments.Nodes { - if prev.commentIDs[c.DatabaseID] { + if prevComments[c.DatabaseID] { continue } ev := base @@ -271,158 +205,3 @@ func detectDiscussionEvents(d discussionNode, prev *prevDiscussion, path, repoSl return events } - -// readPrevDiscussion parses the frontmatter and comment IDs from a previously -// written discussion file. Returns a struct with exists=false if the file is -// missing. -func readPrevDiscussion(path string) *prevDiscussion { - out := &prevDiscussion{commentIDs: map[int64]bool{}} - f, err := os.Open(path) - if err != nil { - return out - } - defer f.Close() - out.exists = true - - scanner := bufio.NewScanner(f) - scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) - - var current []string - inFrontmatter := false - first := true - flush := func() { - if len(current) == 0 { - return - } - raw := strings.Join(current, "\n") - if first { - first = false - var fm struct { - State string `yaml:"state"` - AnswerID int64 `yaml:"answer_id"` - } - if err := yaml.Unmarshal([]byte(raw), &fm); err == nil { - out.state = fm.State - out.answerID = fm.AnswerID - } - } else { - var doc struct { - Document string `yaml:"document"` - ID int64 `yaml:"id"` - } - if err := yaml.Unmarshal([]byte(raw), &doc); err == nil { - if doc.Document == "comment" && doc.ID > 0 { - out.commentIDs[doc.ID] = true - } - } - } - current = nil - } - for scanner.Scan() { - line := scanner.Text() - if strings.TrimSpace(line) == "---" { - if inFrontmatter { - flush() - inFrontmatter = false - } else { - inFrontmatter = true - } - continue - } - if inFrontmatter { - current = append(current, line) - } - } - return out -} - -func discussionState(d discussionNode) string { - if d.Closed { - return "closed" - } - return "open" -} - -func writeDiscussionFile(path string, d discussionNode) error { - f, err := os.Create(path) - if err != nil { - return err - } - defer f.Close() - - w := &document.Writer{} - w.KV("number", d.Number) - w.KV("title", d.Title) - w.KV("type", "discussion") - w.KV("state", discussionState(d)) - if d.StateReason != "" { - w.KV("state_reason", strings.ToLower(d.StateReason)) - } - if d.Locked { - w.KV("locked", true) - } - w.KV("created_at", d.CreatedAt) - w.KV("updated_at", d.UpdatedAt) - if d.ClosedAt != "" { - w.KV("closed_at", d.ClosedAt) - } - w.KV("author", d.Author.Login) - if d.Category.Name != "" { - w.KV("category", d.Category.Name) - } - var labels []string - for _, l := range d.Labels.Nodes { - if l.Name != "" { - labels = append(labels, l.Name) - } - } - w.List("labels", labels) - if d.Answer != nil { - w.KV("answer_id", d.Answer.DatabaseID) - if d.AnswerChosenAt != "" { - w.KV("answer_chosen_at", d.AnswerChosenAt) - } - if d.AnswerChosenBy != nil && d.AnswerChosenBy.Login != "" { - w.KV("answer_chosen_by", d.AnswerChosenBy.Login) - } - } - - document.WriteFirstDoc(f, w.String(), d.Body) - - // Warn on truncation rather than fall back (PR-reviews policy). - if d.Comments.PageInfo.HasNextPage { - log.Printf(" Warning: discussion #%d has more than 100 top-level comments — only first 100 exported", d.Number) - } - - var answerID int64 - if d.Answer != nil { - answerID = d.Answer.DatabaseID - } - - for _, c := range d.Comments.Nodes { - cw := &document.Writer{} - cw.KV("document", "comment") - cw.KV("id", c.DatabaseID) - cw.KV("author", c.Author.Login) - cw.KV("created_at", c.CreatedAt) - if answerID != 0 && c.DatabaseID == answerID { - cw.KV("is_answer", true) - } - document.WriteSubDoc(f, cw.String(), c.Body) - - if c.Replies.PageInfo.HasNextPage { - log.Printf(" Warning: discussion #%d comment %d has more than 50 replies — only first 50 exported", d.Number, c.DatabaseID) - } - for _, r := range c.Replies.Nodes { - rw := &document.Writer{} - rw.KV("document", "reply") - rw.KV("id", r.DatabaseID) - rw.KV("parent_id", c.DatabaseID) - rw.KV("author", r.Author.Login) - rw.KV("created_at", r.CreatedAt) - document.WriteSubDoc(f, rw.String(), r.Body) - } - } - - return nil -} diff --git a/internal/sync/issues.go b/internal/sync/issues.go index 40b71ba..e431229 100644 --- a/internal/sync/issues.go +++ b/internal/sync/issues.go @@ -3,36 +3,30 @@ package sync import ( "encoding/json" "fmt" - "io" "log" - "os" "path/filepath" - "sort" - "strconv" "strings" - "github.com/mevdschee/github-export/internal/document" + "github.com/mevdschee/github-export/internal/ghmodel" "github.com/mevdschee/github-export/internal/github" "github.com/mevdschee/github-export/internal/hooks" "github.com/mevdschee/github-export/internal/jsonutil" + "github.com/mevdschee/github-export/internal/store" ) -func padWidth(maxNumber int64) int { - switch { - case maxNumber >= 100000: - return 6 - case maxNumber >= 10000: - return 5 - default: - return 4 - } +// prevIssue is the prior on-store state of an issue/PR needed to detect +// transitions. +type prevIssue struct { + exists bool + state string + merged bool } -func issueFilename(number int64, pad int) string { - return fmt.Sprintf("%0*d.md", pad, number) +func issueRelPath(number int64, pad int) string { + return filepath.Join("issues", ghmodel.IssueFilename(number, pad)) } -func Issues(c *github.Client, owner, repo, outDir, since string, issueProjects map[int64][]string) ([]hooks.Event, error) { +func Issues(c *github.Client, s *store.Store, owner, repo, since string, issueProjects map[int64][]string) ([]hooks.Event, error) { log.Println("Syncing issues and pull requests...") url := fmt.Sprintf("%s/repos/%s/%s/issues?state=all&sort=updated&direction=asc&per_page=%d", @@ -46,10 +40,6 @@ func Issues(c *github.Client, owner, repo, outDir, since string, issueProjects m return nil, fmt.Errorf("fetching issues: %w", err) } - issueDir := filepath.Join(outDir, "issues") - os.MkdirAll(issueDir, 0755) - - // Parse issue items var issues []map[string]any var maxNum int64 for _, raw := range items { @@ -60,16 +50,12 @@ func Issues(c *github.Client, owner, repo, outDir, since string, issueProjects m maxNum = n } } - // Also check existing files for pad width - if entries, err := os.ReadDir(issueDir); err == nil { - for _, e := range entries { - name := strings.TrimSuffix(e.Name(), ".md") - if n, err := strconv.ParseInt(name, 10, 64); err == nil && n > maxNum { - maxNum = n - } - } + // Existing stored issues may have a higher number than this (incremental) + // page; keep the pad width stable across runs. + if storeMax, err := s.MaxIssueNumber(); err == nil && storeMax > maxNum { + maxNum = storeMax } - pad := padWidth(maxNum) + pad := ghmodel.PadWidth(maxNum) if len(issues) == 0 { log.Println(" No issues to process") @@ -79,14 +65,14 @@ func Issues(c *github.Client, owner, repo, outDir, since string, issueProjects m log.Printf(" %d issues/PRs to process", len(issues)) if since == "" { - return syncIssuesFull(c, owner, repo, issueDir, pad, issues, issueProjects) + return syncIssuesFull(c, s, owner, repo, pad, issues, issueProjects) } - return syncIssuesIncremental(c, owner, repo, issueDir, pad, issues, since, issueProjects) + return syncIssuesIncremental(c, s, owner, repo, pad, issues, since, issueProjects) } -// detectEvents compares the current API data with the previous on-disk state -// and returns any hook events that should fire for this issue/PR. -func detectEvents(path string, issue map[string]any, isPR bool, pr map[string]any, owner, repo string) []hooks.Event { +// detectEvents compares current API data with the prior stored state and +// returns hook events for this issue/PR. +func detectEvents(relPath string, issue map[string]any, isPR bool, pr map[string]any, owner, repo string, prev prevIssue) []hooks.Event { number := jsonutil.Int(issue, "number") state := jsonutil.Str(issue, "state") repoSlug := owner + "/" + repo @@ -97,16 +83,13 @@ func detectEvents(path string, issue map[string]any, isPR bool, pr map[string]an Author: jsonutil.UserLogin(issue, "user"), State: state, Labels: jsonutil.LabelNames(issue, "labels"), - File: path, + File: relPath, Repo: repoSlug, } - prev := readPrevState(path) - var events []hooks.Event - if prev == nil { - // File doesn't exist — new issue or PR + if !prev.exists { ev := base ev.Body = jsonutil.Str(issue, "body") if isPR { @@ -114,33 +97,31 @@ func detectEvents(path string, issue map[string]any, isPR bool, pr map[string]an } else { ev.Type = hooks.IssueCreated } - events = append(events, ev) - return events + return append(events, ev) } - // State transitions if isPR { merged := pr != nil && jsonutil.Bool(pr, "merged") - if merged && !prev.Merge.Merged { + if merged && !prev.merged { ev := base ev.Type = hooks.PRMerged events = append(events, ev) - } else if state == "closed" && prev.State == "open" && !merged { + } else if state == "closed" && prev.state == "open" && !merged { ev := base ev.Type = hooks.PRClosed events = append(events, ev) - } else if state == "open" && prev.State == "closed" { + } else if state == "open" && prev.state == "closed" { ev := base ev.Type = hooks.PRReopened events = append(events, ev) } } else { - if state == "closed" && prev.State == "open" { + if state == "closed" && prev.state == "open" { ev := base ev.Type = hooks.IssueClosed events = append(events, ev) } - if state == "open" && prev.State == "closed" { + if state == "open" && prev.state == "closed" { ev := base ev.Type = hooks.IssueReopened events = append(events, ev) @@ -152,7 +133,7 @@ func detectEvents(path string, issue map[string]any, isPR bool, pr map[string]an // detectTimelineEvents walks the timeline and emits one hook event per entry // that landed after the last sync. Only fires on incremental syncs (since != ""). -func detectTimelineEvents(path string, issue map[string]any, isPR bool, timeline []map[string]any, since, owner, repo string) []hooks.Event { +func detectTimelineEvents(relPath string, issue map[string]any, isPR bool, timeline []map[string]any, since, owner, repo string) []hooks.Event { if since == "" { return nil } @@ -170,7 +151,7 @@ func detectTimelineEvents(path string, issue map[string]any, isPR bool, timeline Author: actor, State: state, Labels: labels, - File: path, + File: relPath, Repo: repoSlug, } } @@ -268,7 +249,6 @@ func detectTimelineEvents(path string, issue map[string]any, isPR bool, timeline } si := jsonutil.Map(source, "issue") if si == nil || si["pull_request"] == nil { - // Only fire when the referencing source is a PR continue } e := base(actor) @@ -301,58 +281,55 @@ func detectTimelineEvents(path string, issue map[string]any, isPR bool, timeline return events } +// readPrevIssue reads the prior stored state for change detection. +func readPrevIssue(s *store.Store, number int64) prevIssue { + exists, state, _, merged, err := s.IssueState(number) + if err != nil { + log.Printf(" Warning: reading issue #%d state: %v", number, err) + return prevIssue{} + } + return prevIssue{exists: exists, state: state, merged: merged} +} + // syncIssuesFull uses bulk endpoints to minimize API calls for full syncs. -// -// API calls: ~5 paginated fetches + 1 per PR (reviews only). -// Compared to the per-issue approach (3 calls per issue), this is dramatically -// fewer calls for repos with many issues. -func syncIssuesFull(c *github.Client, owner, repo, issueDir string, pad int, issues []map[string]any, issueProjects map[int64][]string) ([]hooks.Event, error) { - // Bulk fetch all supporting data +func syncIssuesFull(c *github.Client, s *store.Store, owner, repo string, pad int, issues []map[string]any, issueProjects map[int64][]string) ([]hooks.Event, error) { comments, err := fetchAllIssueComments(c, owner, repo, "") if err != nil { log.Printf(" Warning: %v", err) comments = make(map[int64][]map[string]any) } - ghEvents, err := fetchAllIssueEvents(c, owner, repo) if err != nil { log.Printf(" Warning: %v", err) ghEvents = make(map[int64][]map[string]any) } - prDetails, err := fetchAllPRs(c, owner, repo, "") if err != nil { log.Printf(" Warning: %v", err) prDetails = make(map[int64]map[string]any) } - reviewComments, err := fetchAllReviewComments(c, owner, repo, "") if err != nil { log.Printf(" Warning: %v", err) reviewComments = make(map[int64][]map[string]any) } - - // Bulk-fetch reviews for all PRs via GraphQL — one paginated query per 100 - // PRs instead of one REST call per PR. allReviews, err := fetchAllReviewsGraphQL(c, owner, repo) if err != nil { log.Printf(" Warning: %v", err) allReviews = make(map[int64][]map[string]any) } - // Write files and detect events var hookEvents []hooks.Event for i, issue := range issues { number := jsonutil.Int(issue, "number") isPR := issue["pull_request"] != nil if (i+1)%100 == 0 || i == len(issues)-1 { - log.Printf(" Writing %d/%d (#%d)", i+1, len(issues), number) + log.Printf(" Storing %d/%d (#%d)", i+1, len(issues), number) } var pr map[string]any - var rc []map[string]any - var reviews []map[string]any + var rc, reviews []map[string]any if isPR { pr = prDetails[number] rc = reviewComments[number] @@ -360,14 +337,13 @@ func syncIssuesFull(c *github.Client, owner, repo, issueDir string, pad int, iss } timeline := buildTimeline(comments[number], ghEvents[number], rc, reviews) + relPath := issueRelPath(number, pad) - path := filepath.Join(issueDir, issueFilename(number, pad)) - - // Detect events before writing (so we can compare with previous state) - hookEvents = append(hookEvents, detectEvents(path, issue, isPR, pr, owner, repo)...) + prev := readPrevIssue(s, number) + hookEvents = append(hookEvents, detectEvents(relPath, issue, isPR, pr, owner, repo, prev)...) - if err := writeIssueFile(path, issue, isPR, pr, timeline, issueProjects[number]); err != nil { - log.Printf(" Warning: writing #%d: %v", number, err) + if err := s.UpsertIssue(number, isPR, issue, pr, timeline, issueProjects[number]); err != nil { + log.Printf(" Warning: storing #%d: %v", number, err) } } @@ -375,12 +351,8 @@ func syncIssuesFull(c *github.Client, owner, repo, issueDir string, pad int, iss } // syncIssuesIncremental uses the per-issue timeline endpoint for changed issues -// (gives complete comment+event history in one call) combined with bulk PR -// details (saves N per-PR detail calls). -// -// API calls: N timeline + ~few pages bulk PRs (no separate PR detail or review calls). -func syncIssuesIncremental(c *github.Client, owner, repo, issueDir string, pad int, issues []map[string]any, since string, issueProjects map[int64][]string) ([]hooks.Event, error) { - // Bulk fetch PR details (replaces N per-PR detail calls with ~few paginated pages) +// combined with bulk PR details. +func syncIssuesIncremental(c *github.Client, s *store.Store, owner, repo string, pad int, issues []map[string]any, since string, issueProjects map[int64][]string) ([]hooks.Event, error) { prDetails, err := fetchAllPRs(c, owner, repo, since) if err != nil { log.Printf(" Warning: %v", err) @@ -396,7 +368,6 @@ func syncIssuesIncremental(c *github.Client, owner, repo, issueDir string, pad i log.Printf(" Processing %d/%d (#%d)", i+1, len(issues), number) } - // Fetch per-issue timeline (gives complete comment+event+review history) timelineURL := fmt.Sprintf("%s/repos/%s/%s/issues/%d/timeline?per_page=%d", github.API, owner, repo, number, github.PerPage) timelineRaw, err := c.GetPaginated(timelineURL, nil) @@ -411,319 +382,21 @@ func syncIssuesIncremental(c *github.Client, owner, repo, issueDir string, pad i timeline = append(timeline, m) } - // PR details from bulk fetch var pr map[string]any if isPR { pr = prDetails[number] } - path := filepath.Join(issueDir, issueFilename(number, pad)) + relPath := issueRelPath(number, pad) - // Detect events before writing (so we can compare with previous state) - hookEvents = append(hookEvents, detectEvents(path, issue, isPR, pr, owner, repo)...) - hookEvents = append(hookEvents, detectTimelineEvents(path, issue, isPR, timeline, since, owner, repo)...) + prev := readPrevIssue(s, number) + hookEvents = append(hookEvents, detectEvents(relPath, issue, isPR, pr, owner, repo, prev)...) + hookEvents = append(hookEvents, detectTimelineEvents(relPath, issue, isPR, timeline, since, owner, repo)...) - if err := writeIssueFile(path, issue, isPR, pr, timeline, issueProjects[number]); err != nil { - log.Printf(" Warning: writing #%d: %v", number, err) + if err := s.UpsertIssue(number, isPR, issue, pr, timeline, issueProjects[number]); err != nil { + log.Printf(" Warning: storing #%d: %v", number, err) } } return hookEvents, nil } - -func writeIssueFile(path string, issue map[string]any, isPR bool, pr map[string]any, timeline []map[string]any, projects []string) error { - f, err := os.Create(path) - if err != nil { - return err - } - defer f.Close() - - d := &document.Writer{} - d.KV("number", jsonutil.Int(issue, "number")) - d.KV("title", jsonutil.Str(issue, "title")) - if isPR { - d.KV("type", "pull_request") - } - d.KV("state", jsonutil.Str(issue, "state")) - if reason := jsonutil.Str(issue, "state_reason"); reason != "" { - d.KV("state_reason", reason) - } - if isPR && pr != nil && jsonutil.Bool(pr, "draft") { - d.KV("draft", true) - } - if jsonutil.Bool(issue, "locked") { - d.KV("locked", true) - } - d.KV("created_at", jsonutil.Str(issue, "created_at")) - d.KV("updated_at", jsonutil.Str(issue, "updated_at")) - d.KV("closed_at", jsonutil.Str(issue, "closed_at")) - d.KV("author", jsonutil.UserLogin(issue, "user")) - d.List("assignees", jsonutil.Logins(issue, "assignees")) - d.List("labels", jsonutil.LabelNames(issue, "labels")) - if ms := jsonutil.Map(issue, "milestone"); ms != nil { - d.KV("milestone", jsonutil.Str(ms, "title")) - } - if len(projects) > 0 { - d.List("projects", projects) - } - - // PR-specific fields - if isPR && pr != nil { - if head := jsonutil.Map(pr, "head"); head != nil { - d.KV("source_branch", jsonutil.Str(head, "ref")) - } - if base := jsonutil.Map(pr, "base"); base != nil { - d.KV("target_branch", jsonutil.Str(base, "ref")) - } - // Cross-repo? - head := jsonutil.Map(pr, "head") - base := jsonutil.Map(pr, "base") - if head != nil && base != nil { - headRepo := jsonutil.Map(head, "repo") - baseRepo := jsonutil.Map(base, "repo") - if headRepo != nil && baseRepo != nil { - hf := jsonutil.Str(headRepo, "full_name") - bf := jsonutil.Str(baseRepo, "full_name") - if hf != "" && bf != "" && hf != bf { - d.KV("source_repo", hf) - } - } - } - - // Merge info - merged := jsonutil.Bool(pr, "merged") - fmt.Fprint(d.Buf(), "merge:\n") - d.KVIndent(" ", "merged", merged) - if merged { - d.KVIndent(" ", "merged_at", jsonutil.Str(pr, "merged_at")) - mergedBy := jsonutil.UserLogin(pr, "merged_by") - if mergedBy == "" { - // Bulk PR list endpoint omits merged_by; extract from timeline - for _, ev := range timeline { - if jsonutil.Str(ev, "event") == "merged" { - mergedBy = jsonutil.UserLogin(ev, "actor") - break - } - } - } - if mergedBy != "" { - d.KVIndent(" ", "merged_by", mergedBy) - } - d.KVIndent(" ", "commit_sha", jsonutil.Str(pr, "merge_commit_sha")) - } - - // Reviewers (deduplicated from reviewed events in timeline) - reviewerSet := map[string]bool{} - for _, ev := range timeline { - if jsonutil.Str(ev, "event") == "reviewed" { - if login := jsonutil.UserLogin(ev, "user"); login != "" { - reviewerSet[login] = true - } - } - } - if len(reviewerSet) > 0 { - var reviewers []string - for login := range reviewerSet { - reviewers = append(reviewers, login) - } - sort.Strings(reviewers) - d.List("reviewers", reviewers) - } - d.List("requested_reviewers", jsonutil.Logins(pr, "requested_reviewers")) - } - - // Reactions - if reactions := buildReactions(issue); len(reactions) > 0 { - fmt.Fprint(d.Buf(), "reactions:\n") - for _, r := range reactions { - fmt.Fprintf(d.Buf(), " %s: %d\n", document.YamlScalar(r.key), r.count) - } - } - - document.WriteFirstDoc(f, d.String(), jsonutil.Str(issue, "body")) - - // Timeline documents - for _, event := range timeline { - writeTimelineDoc(f, event) - } - - return nil -} - -type reaction struct { - key string - count int -} - -func buildReactions(issue map[string]any) []reaction { - r := jsonutil.Map(issue, "reactions") - if r == nil { - return nil - } - keys := []string{"+1", "-1", "laugh", "hooray", "confused", "heart", "rocket", "eyes"} - var out []reaction - for _, k := range keys { - if count := int(jsonutil.Int(r, k)); count > 0 { - out = append(out, reaction{k, count}) - } - } - return out -} - -func writeTimelineDoc(w io.Writer, event map[string]any) { - eventType := jsonutil.Str(event, "event") - - switch eventType { - case "commented": - d := &document.Writer{} - d.KV("document", "comment") - d.KV("id", jsonutil.Int(event, "id")) - d.KV("author", jsonutil.UserLogin(event, "user")) - d.KV("created_at", jsonutil.Str(event, "created_at")) - document.WriteSubDoc(w, d.String(), jsonutil.Str(event, "body")) - - case "reviewed": - d := &document.Writer{} - d.KV("document", "review") - d.KV("id", jsonutil.Int(event, "id")) - d.KV("author", jsonutil.UserLogin(event, "user")) - d.KV("state", strings.ToLower(jsonutil.Str(event, "state"))) - d.KV("commit_sha", jsonutil.Str(event, "commit_id")) - d.KV("submitted_at", jsonutil.Str(event, "submitted_at")) - document.WriteSubDoc(w, d.String(), jsonutil.Str(event, "body")) - - case "line-commented": - comments := jsonutil.List(event, "comments") - for _, c := range comments { - cm, ok := c.(map[string]any) - if !ok { - continue - } - d := &document.Writer{} - d.KV("document", "review_comment") - d.KV("id", jsonutil.Int(cm, "id")) - if rid := jsonutil.Int(cm, "pull_request_review_id"); rid > 0 { - d.KV("review_id", rid) - } - d.KV("author", jsonutil.UserLogin(cm, "user")) - d.KV("created_at", jsonutil.Str(cm, "created_at")) - d.KV("path", jsonutil.Str(cm, "path")) - if line := jsonutil.Int(cm, "line"); line > 0 { - d.KV("line", line) - } else if origLine := jsonutil.Int(cm, "original_line"); origLine > 0 { - d.KV("line", origLine) - } - if side := jsonutil.Str(cm, "side"); side != "" { - d.KV("side", side) - } - d.KV("commit_sha", jsonutil.Str(cm, "commit_id")) - document.WriteSubDoc(w, d.String(), jsonutil.Str(cm, "body")) - } - - case "labeled", "unlabeled": - label := jsonutil.Map(event, "label") - if label == nil { - return - } - d := &document.Writer{} - d.KV("document", "event") - d.KV("event", eventType) - d.KV("actor", jsonutil.UserLogin(event, "actor")) - d.KV("created_at", jsonutil.Str(event, "created_at")) - d.KV("label", jsonutil.Str(label, "name")) - document.WriteSubDoc(w, d.String(), "") - - case "assigned", "unassigned": - d := &document.Writer{} - d.KV("document", "event") - d.KV("event", eventType) - d.KV("actor", jsonutil.UserLogin(event, "actor")) - d.KV("created_at", jsonutil.Str(event, "created_at")) - d.KV("assignee", jsonutil.UserLogin(event, "assignee")) - document.WriteSubDoc(w, d.String(), "") - - case "milestoned", "demilestoned": - d := &document.Writer{} - d.KV("document", "event") - d.KV("event", eventType) - d.KV("actor", jsonutil.UserLogin(event, "actor")) - d.KV("created_at", jsonutil.Str(event, "created_at")) - if ms := jsonutil.Map(event, "milestone"); ms != nil { - d.KV("milestone", jsonutil.Str(ms, "title")) - } - document.WriteSubDoc(w, d.String(), "") - - case "renamed": - d := &document.Writer{} - d.KV("document", "event") - d.KV("event", eventType) - d.KV("actor", jsonutil.UserLogin(event, "actor")) - d.KV("created_at", jsonutil.Str(event, "created_at")) - if rename := jsonutil.Map(event, "rename"); rename != nil { - d.KV("from", jsonutil.Str(rename, "from")) - d.KV("to", jsonutil.Str(rename, "to")) - } - document.WriteSubDoc(w, d.String(), "") - - case "cross-referenced": - d := &document.Writer{} - d.KV("document", "event") - d.KV("event", eventType) - d.KV("actor", jsonutil.UserLogin(event, "actor")) - d.KV("created_at", jsonutil.Str(event, "created_at")) - if source := jsonutil.Map(event, "source"); source != nil { - if si := jsonutil.Map(source, "issue"); si != nil { - if n := jsonutil.Int(si, "number"); n > 0 { - d.KV("source_number", n) - } - if sr := jsonutil.Map(si, "repository"); sr != nil { - d.KV("source_repo", jsonutil.Str(sr, "full_name")) - } - } - } - document.WriteSubDoc(w, d.String(), "") - - case "committed": - // Skip individual commit entries — commit info is in PR details - return - - case "closed", "reopened", "merged", "referenced", - "locked", "unlocked", - "head_ref_force_pushed", "head_ref_deleted", "base_ref_changed", - "converted_to_draft", "ready_for_review", - "review_requested", "review_request_removed", - "review_dismissed", - "pinned", "unpinned", "transferred", - "connected", "disconnected", - "marked_as_duplicate", "unmarked_as_duplicate": - d := &document.Writer{} - d.KV("document", "event") - d.KV("event", eventType) - d.KV("actor", jsonutil.UserLogin(event, "actor")) - d.KV("created_at", jsonutil.Str(event, "created_at")) - if sha := jsonutil.Str(event, "commit_id"); sha != "" { - d.KV("commit_sha", sha) - } - if reviewer := jsonutil.UserLogin(event, "requested_reviewer"); reviewer != "" { - d.KV("reviewer", reviewer) - } - if reason := jsonutil.Str(event, "lock_reason"); reason != "" { - d.KV("lock_reason", reason) - } - if dismissal := jsonutil.Str(event, "dismissal_message"); dismissal != "" { - d.KV("dismissal_message", dismissal) - } - document.WriteSubDoc(w, d.String(), "") - - default: - // Unknown event — still record if it has an event type - if eventType != "" { - d := &document.Writer{} - d.KV("document", "event") - d.KV("event", eventType) - d.KV("actor", jsonutil.UserLogin(event, "actor")) - d.KV("created_at", jsonutil.Str(event, "created_at")) - document.WriteSubDoc(w, d.String(), "") - } - } -} diff --git a/internal/sync/projects.go b/internal/sync/projects.go index 017682c..2c15cb2 100644 --- a/internal/sync/projects.go +++ b/internal/sync/projects.go @@ -1,21 +1,16 @@ package sync import ( - "bufio" "encoding/json" "fmt" "log" - "os" - "path/filepath" - "sort" "strings" - "github.com/mevdschee/github-export/internal/document" + "github.com/mevdschee/github-export/internal/ghmodel" "github.com/mevdschee/github-export/internal/github" "github.com/mevdschee/github-export/internal/hooks" "github.com/mevdschee/github-export/internal/jsonutil" - - "gopkg.in/yaml.v3" + "github.com/mevdschee/github-export/internal/store" ) // listProjectsQuery fetches all projectsV2 linked to a repository, with their @@ -91,20 +86,18 @@ query($projectId: ID!, $cursor: String) { } }` -// Projects exports GitHub Projects v2 linked to the repository. +// Projects syncs GitHub Projects v2 linked to the repository into the store. // // Behavior: // - Lists repository.projectsV2 via GraphQL (open + closed are both fetched -// so we can detect open→closed transitions; only open ones are written). -// - If since is non-empty, projects whose updatedAt is older than since are -// skipped (no rewrite, no diff). -// - Closed projects whose file exists are deleted; a project_closed event is +// so we can detect open→closed transitions; only open ones are stored). +// - If since is non-empty, open projects older than since with an existing +// stored row are skipped (no rewrite, no diff). +// - Closed projects whose row exists are deleted; a project_closed event is // emitted. -// - Each kept project produces github-data/projects/.md with the -// project header + one document per item. // - Returns a map of issue/PR number → project titles for the issues sync to // cross-link, plus any hook events. -func Projects(c *github.Client, owner, repo, outDir, since string) (map[int64][]string, []hooks.Event, error) { +func Projects(c *github.Client, s *store.Store, owner, repo, since string) (map[int64][]string, []hooks.Event, error) { log.Println("Syncing projects...") projects, err := fetchAllProjects(c, owner, repo) @@ -114,11 +107,6 @@ func Projects(c *github.Client, owner, repo, outDir, since string) (map[int64][] log.Printf(" %d projects on repo", len(projects)) repoSlug := owner + "/" + repo - projectsDir := filepath.Join(outDir, "projects") - if err := os.MkdirAll(projectsDir, 0755); err != nil { - return nil, nil, fmt.Errorf("creating projects dir: %w", err) - } - issueProjects := make(map[int64][]string) var events []hooks.Event @@ -127,33 +115,35 @@ func Projects(c *github.Client, owner, repo, outDir, since string) (map[int64][] title := jsonutil.Str(p, "title") closed := jsonutil.Bool(p, "closed") updatedAt := jsonutil.Str(p, "updatedAt") - path := filepath.Join(projectsDir, fmt.Sprintf("%04d.md", number)) - fileExists := fileExistsOrFalse(path) + relPath := fmt.Sprintf("projects/%04d.md", number) + + prevItemMaps, existed, err := s.ProjectItems(number) + if err != nil { + log.Printf(" Warning: reading project #%d: %v", number, err) + } - // Closed: delete file if it existed, emit project_closed if closed { - if fileExists { - log.Printf(" Project #%d %q closed — removing file", number, title) - os.Remove(path) - events = append(events, projectEvent(hooks.ProjectClosed, p, path, repoSlug)) + if existed { + log.Printf(" Project #%d %q closed — removing", number, title) + if err := s.DeleteProject(number); err != nil { + log.Printf(" Warning: deleting project #%d: %v", number, err) + } + events = append(events, projectEvent(hooks.ProjectClosed, p, relPath, repoSlug)) } continue } - // Incremental: skip unchanged open projects - if since != "" && updatedAt < since && fileExists { + if since != "" && updatedAt < since && existed { log.Printf(" Project #%d %q skipped (unchanged)", number, title) continue } - // Fetch items for this project items, err := fetchProjectItems(c, jsonutil.Str(p, "id")) if err != nil { log.Printf(" Warning: items for project #%d: %v", number, err) items = nil } - // Track which issues/PRs this project contains (for the cross-link) for _, item := range items { content := jsonutil.Map(item, "content") if content == nil { @@ -171,98 +161,95 @@ func Projects(c *github.Client, owner, repo, outDir, since string) (map[int64][] } } - // Detect events: project_created (new file) or per-item diffs - if !fileExists { - events = append(events, projectEvent(hooks.ProjectCreated, p, path, repoSlug)) + if !existed { + events = append(events, projectEvent(hooks.ProjectCreated, p, relPath, repoSlug)) } else { - prevItems := readPrevProjectItems(path) - currentNumbers := map[int64]bool{} - for _, item := range items { - content := jsonutil.Map(item, "content") - if content == nil { - continue - } - n := jsonutil.Int(content, "number") - if n == 0 { - continue - } - currentNumbers[n] = true - currFields := extractItemFields(item) - extra := itemExtra(n, content, item) - - prev, existed := prevItems[n] - if !existed { - ev := projectEvent(hooks.ItemAdded, p, path, repoSlug) - ev.Extra = extra - events = append(events, ev) - continue - } - // Field-by-field diff - keys := map[string]bool{} - for k := range prev.Fields { - keys[k] = true - } - for k := range currFields { - keys[k] = true - } - for k := range keys { - if prev.Fields[k] == currFields[k] { - continue - } - evType := hooks.ItemFieldChanged - if k == "Status" { - evType = hooks.ItemStatusChanged - } - ev := projectEvent(evType, p, path, repoSlug) - ee := map[string]string{} - for ek, ev2 := range extra { - ee[ek] = ev2 - } - ee["field_name"] = k - if v := prev.Fields[k]; v != "" { - ee["from_value"] = v - } - if v := currFields[k]; v != "" { - ee["to_value"] = v - } - ev.Extra = ee - events = append(events, ev) - } - } - // Items present before but missing now - for n, prev := range prevItems { - if currentNumbers[n] { - continue - } - ev := projectEvent(hooks.ItemRemoved, p, path, repoSlug) - ee := map[string]string{ - "item_number": fmt.Sprintf("%d", n), - } - if prev.Title != "" { - ee["item_title"] = prev.Title - } - if prev.Type != "" { - ee["item_type"] = prev.Type - } - if prev.Repo != "" { - ee["item_repo"] = prev.Repo - } - ev.Extra = ee - events = append(events, ev) - } + events = append(events, diffProjectItems(p, relPath, repoSlug, buildPrevItems(prevItemMaps), items)...) } - if err := writeProjectFile(path, p, items); err != nil { - log.Printf(" Warning: writing project #%d: %v", number, err) + if err := s.UpsertProject(number, p, items); err != nil { + log.Printf(" Warning: storing project #%d: %v", number, err) } } return issueProjects, events, nil } -func fileExistsOrFalse(path string) bool { - _, err := os.Stat(path) - return err == nil +// diffProjectItems emits item_added / item_removed / item_status_changed / +// item_field_changed events by comparing stored items against fresh ones. +func diffProjectItems(p map[string]any, relPath, repoSlug string, prevItems map[int64]*prevProjectItem, items []map[string]any) []hooks.Event { + var events []hooks.Event + currentNumbers := map[int64]bool{} + for _, item := range items { + content := jsonutil.Map(item, "content") + if content == nil { + continue + } + n := jsonutil.Int(content, "number") + if n == 0 { + continue + } + currentNumbers[n] = true + currFields := ghmodel.ExtractItemFields(item) + extra := itemExtra(n, content, item) + + prev, existed := prevItems[n] + if !existed { + ev := projectEvent(hooks.ItemAdded, p, relPath, repoSlug) + ev.Extra = extra + events = append(events, ev) + continue + } + keys := map[string]bool{} + for k := range prev.Fields { + keys[k] = true + } + for k := range currFields { + keys[k] = true + } + for k := range keys { + if prev.Fields[k] == currFields[k] { + continue + } + evType := hooks.ItemFieldChanged + if k == "Status" { + evType = hooks.ItemStatusChanged + } + ev := projectEvent(evType, p, relPath, repoSlug) + ee := map[string]string{} + for ek, ev2 := range extra { + ee[ek] = ev2 + } + ee["field_name"] = k + if v := prev.Fields[k]; v != "" { + ee["from_value"] = v + } + if v := currFields[k]; v != "" { + ee["to_value"] = v + } + ev.Extra = ee + events = append(events, ev) + } + } + for n, prev := range prevItems { + if currentNumbers[n] { + continue + } + ev := projectEvent(hooks.ItemRemoved, p, relPath, repoSlug) + ee := map[string]string{"item_number": fmt.Sprintf("%d", n)} + if prev.Title != "" { + ee["item_title"] = prev.Title + } + if prev.Type != "" { + ee["item_type"] = prev.Type + } + if prev.Repo != "" { + ee["item_repo"] = prev.Repo + } + ev.Extra = ee + events = append(events, ev) + } + return events } func fetchAllProjects(c *github.Client, owner, repo string) ([]map[string]any, error) { @@ -335,7 +322,7 @@ func fetchProjectItems(c *github.Client, projectID string) ([]map[string]any, er return all, nil } -func projectEvent(eventType string, p map[string]any, path, repoSlug string) hooks.Event { +func projectEvent(eventType string, p map[string]any, relPath, repoSlug string) hooks.Event { number := jsonutil.Int(p, "number") state := "open" if jsonutil.Bool(p, "closed") { @@ -345,163 +332,16 @@ func projectEvent(eventType string, p map[string]any, path, repoSlug string) hoo Type: eventType, Number: number, Title: jsonutil.Str(p, "title"), - Author: ownerLogin(p), + Author: ghmodel.OwnerLogin(p), State: state, - File: path, + File: relPath, Repo: repoSlug, URL: jsonutil.Str(p, "url"), Body: jsonutil.Str(p, "shortDescription"), } } -func ownerLogin(p map[string]any) string { - o := jsonutil.Map(p, "owner") - if o == nil { - return "" - } - return jsonutil.Str(o, "login") -} - -func writeProjectFile(path string, p map[string]any, items []map[string]any) error { - f, err := os.Create(path) - if err != nil { - return err - } - defer f.Close() - - d := &document.Writer{} - d.KV("number", jsonutil.Int(p, "number")) - d.KV("title", jsonutil.Str(p, "title")) - d.KV("state", "open") - if jsonutil.Bool(p, "public") { - d.KV("public", true) - } - d.KV("url", jsonutil.Str(p, "url")) - d.KV("owner", ownerLogin(p)) - if desc := jsonutil.Str(p, "shortDescription"); desc != "" { - d.KV("description", desc) - } - d.KV("created_at", jsonutil.Str(p, "createdAt")) - d.KV("updated_at", jsonutil.Str(p, "updatedAt")) - - // Field definitions - fieldsMap := jsonutil.Map(p, "fields") - fieldNodes := jsonutil.List(fieldsMap, "nodes") - if len(fieldNodes) > 0 { - fmt.Fprint(d.Buf(), "fields:\n") - for _, fn := range fieldNodes { - fm, _ := fn.(map[string]any) - if fm == nil { - continue - } - name := jsonutil.Str(fm, "name") - if name == "" { - continue - } - fmt.Fprintf(d.Buf(), " - name: %s\n", document.YamlScalar(name)) - if dt := jsonutil.Str(fm, "dataType"); dt != "" { - fmt.Fprintf(d.Buf(), " type: %s\n", document.YamlScalar(dt)) - } - if opts := jsonutil.List(fm, "options"); len(opts) > 0 { - fmt.Fprint(d.Buf(), " options:\n") - for _, o := range opts { - om, _ := o.(map[string]any) - if om == nil { - continue - } - if optName := jsonutil.Str(om, "name"); optName != "" { - fmt.Fprintf(d.Buf(), " - %s\n", document.YamlScalar(optName)) - } - } - } - } - } - - document.WriteFirstDoc(f, d.String(), jsonutil.Str(p, "readme")) - - for _, item := range items { - writeProjectItem(f, item) - } - return nil -} - -func writeProjectItem(f *os.File, item map[string]any) { - content := jsonutil.Map(item, "content") - if content == nil { - return - } - typename := jsonutil.Str(content, "__typename") - var itemType string - switch typename { - case "Issue": - itemType = "issue" - case "PullRequest": - itemType = "pull_request" - default: - // Skip drafts and anything else we don't model - return - } - - d := &document.Writer{} - d.KV("document", "item") - d.KV("type", itemType) - d.KV("number", jsonutil.Int(content, "number")) - d.KV("title", jsonutil.Str(content, "title")) - if r := jsonutil.Map(content, "repository"); r != nil { - d.KV("repo", jsonutil.Str(r, "nameWithOwner")) - } - - // Field values - fvMap := jsonutil.Map(item, "fieldValues") - fvNodes := jsonutil.List(fvMap, "nodes") - var pairs [][2]string - for _, fv := range fvNodes { - fvm, _ := fv.(map[string]any) - if fvm == nil { - continue - } - fieldName := "" - if fld := jsonutil.Map(fvm, "field"); fld != nil { - fieldName = jsonutil.Str(fld, "name") - } - if fieldName == "" { - continue - } - value := fieldValueString(fvm) - if value == "" { - continue - } - pairs = append(pairs, [2]string{fieldName, value}) - } - if len(pairs) > 0 { - sort.Slice(pairs, func(i, j int) bool { return pairs[i][0] < pairs[j][0] }) - fmt.Fprint(d.Buf(), "fields:\n") - for _, kv := range pairs { - fmt.Fprintf(d.Buf(), " %s: %s\n", document.YamlScalar(kv[0]), document.YamlScalar(kv[1])) - } - } - - document.WriteSubDoc(f, d.String(), "") -} - -func fieldValueString(fv map[string]any) string { - switch jsonutil.Str(fv, "__typename") { - case "ProjectV2ItemFieldTextValue": - return jsonutil.Str(fv, "text") - case "ProjectV2ItemFieldNumberValue": - return jsonutil.Str(fv, "number") - case "ProjectV2ItemFieldDateValue": - return jsonutil.Str(fv, "date") - case "ProjectV2ItemFieldSingleSelectValue": - return jsonutil.Str(fv, "name") - case "ProjectV2ItemFieldIterationValue": - return jsonutil.Str(fv, "title") - } - return "" -} - -// prevProjectItem captures the state of a project item from a previously -// written project file so we can diff against the freshly-fetched data. +// prevProjectItem captures the prior state of a project item for diffing. type prevProjectItem struct { Title string Type string @@ -522,98 +362,28 @@ func itemExtra(n int64, content, item map[string]any) map[string]string { return out } -// extractItemFields flattens a project item's fieldValues into name → string-value pairs. -func extractItemFields(item map[string]any) map[string]string { - out := map[string]string{} - fvMap := jsonutil.Map(item, "fieldValues") - for _, fv := range jsonutil.List(fvMap, "nodes") { - fvm, _ := fv.(map[string]any) - if fvm == nil { - continue - } - fld := jsonutil.Map(fvm, "field") - if fld == nil { +// buildPrevItems converts stored item maps into the diff-friendly prev map keyed +// by issue/PR number. +func buildPrevItems(items []map[string]any) map[int64]*prevProjectItem { + out := map[int64]*prevProjectItem{} + for _, item := range items { + content := jsonutil.Map(item, "content") + if content == nil { continue } - name := jsonutil.Str(fld, "name") - if name == "" { + n := jsonutil.Int(content, "number") + if n == 0 { continue } - if v := fieldValueString(fvm); v != "" { - out[name] = v - } - } - return out -} - -// readPrevProjectItems parses item sub-documents out of a previously written -// project file and returns them keyed by issue/PR number. -func readPrevProjectItems(path string) map[int64]*prevProjectItem { - out := map[int64]*prevProjectItem{} - f, err := os.Open(path) - if err != nil { - return out - } - defer f.Close() - - // A project file looks like: - // --- - // - // --- - // - // --- - // document: item - // ... - // number: 42 - // fields: - // Status: In Progress - // --- - scanner := bufio.NewScanner(f) - scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) - - var current []string - inFrontmatter := false - flush := func() { - if len(current) == 0 { - return - } - var doc struct { - Document string `yaml:"document"` - Number int64 `yaml:"number"` - Title string `yaml:"title"` - Type string `yaml:"type"` - Repo string `yaml:"repo"` - Fields map[string]string `yaml:"fields"` - } - if err := yaml.Unmarshal([]byte(strings.Join(current, "\n")), &doc); err == nil { - if doc.Document == "item" && doc.Number > 0 { - if doc.Fields == nil { - doc.Fields = map[string]string{} - } - out[doc.Number] = &prevProjectItem{ - Title: doc.Title, - Type: doc.Type, - Repo: doc.Repo, - Fields: doc.Fields, - } - } - } - current = nil - } - - for scanner.Scan() { - line := scanner.Text() - if strings.TrimSpace(line) == "---" { - if inFrontmatter { - flush() - inFrontmatter = false - } else { - inFrontmatter = true - } - continue + repoName := "" + if r := jsonutil.Map(content, "repository"); r != nil { + repoName = jsonutil.Str(r, "nameWithOwner") } - if inFrontmatter { - current = append(current, line) + out[n] = &prevProjectItem{ + Title: jsonutil.Str(content, "title"), + Type: strings.ToLower(jsonutil.Str(item, "type")), + Repo: repoName, + Fields: ghmodel.ExtractItemFields(item), } } return out diff --git a/internal/sync/resync.go b/internal/sync/resync.go new file mode 100644 index 0000000..0d5c942 --- /dev/null +++ b/internal/sync/resync.go @@ -0,0 +1,50 @@ +package sync + +import ( + "encoding/json" + "fmt" + + "github.com/mevdschee/github-export/internal/github" + "github.com/mevdschee/github-export/internal/store" +) + +// ResyncIssue re-pulls a single issue/PR (and its timeline) into the store. It +// is used for read-after-write consistency by the write proxy: after a write to +// issue #n succeeds upstream, the local row is refreshed before the call +// returns. The issue's project cross-links are preserved from the existing row. +func ResyncIssue(c *github.Client, s *store.Store, owner, repo string, number int64) error { + issueURL := fmt.Sprintf("%s/repos/%s/%s/issues/%d", github.API, owner, repo, number) + rawIssue, err := c.GetJSON(issueURL, nil) + if err != nil { + return fmt.Errorf("fetching issue #%d: %w", number, err) + } + var issue map[string]any + if err := json.Unmarshal(rawIssue, &issue); err != nil { + return fmt.Errorf("parsing issue #%d: %w", number, err) + } + isPR := issue["pull_request"] != nil + + var pr map[string]any + if isPR { + prURL := fmt.Sprintf("%s/repos/%s/%s/pulls/%d", github.API, owner, repo, number) + if rawPR, err := c.GetJSON(prURL, nil); err == nil { + json.Unmarshal(rawPR, &pr) + } + } + + timelineURL := fmt.Sprintf("%s/repos/%s/%s/issues/%d/timeline?per_page=%d", + github.API, owner, repo, number, github.PerPage) + var timeline []map[string]any + if timelineRaw, err := c.GetPaginated(timelineURL, nil); err == nil { + for _, r := range timelineRaw { + var m map[string]any + json.Unmarshal(r, &m) + timeline = append(timeline, m) + } + } + + // Preserve existing project cross-links (not re-fetched here). + projects, _ := s.ProjectsForIssue(number) + + return s.UpsertIssue(number, isPR, issue, pr, timeline, projects) +} diff --git a/internal/sync/run.go b/internal/sync/run.go new file mode 100644 index 0000000..2cef825 --- /dev/null +++ b/internal/sync/run.go @@ -0,0 +1,65 @@ +package sync + +import ( + "log" + + "github.com/mevdschee/github-export/internal/github" + "github.com/mevdschee/github-export/internal/hooks" + "github.com/mevdschee/github-export/internal/store" +) + +// Run performs a full sync of all entities into the store and records the +// detected change events (stamped with syncStart) in the events table. It +// returns the detected events for summary logging. +// +// A failure to fetch or store an individual entity group is logged as a warning +// and the sync continues; only a failure to persist repo metadata (and the sync +// timestamp) is treated as fatal, since that is what drives incremental runs. +func Run(c *github.Client, s *store.Store, owner, repo, since, syncStart string) ([]hooks.Event, error) { + var events []hooks.Event + + skip := checkScopes(c) + + if err := Labels(c, s, owner, repo); err != nil { + log.Printf("Warning: %v", err) + } + if err := Milestones(c, s, owner, repo); err != nil { + log.Printf("Warning: %v", err) + } + var issueProjects map[int64][]string + if !skip["projects"] { + ip, projectEvents, err := Projects(c, s, owner, repo, since) + if err != nil { + log.Printf("Warning: %v", err) + } + issueProjects = ip + events = append(events, projectEvents...) + } + issueEvents, err := Issues(c, s, owner, repo, since, issueProjects) + if err != nil { + log.Printf("Warning: %v", err) + } + events = append(events, issueEvents...) + + releaseEvents, err := Releases(c, s, owner, repo) + if err != nil { + log.Printf("Warning: %v", err) + } + events = append(events, releaseEvents...) + + discussionEvents, err := Discussions(c, s, owner, repo, since) + if err != nil { + log.Printf("Warning: %v", err) + } + events = append(events, discussionEvents...) + + if err := Repo(c, s, owner, repo, syncStart); err != nil { + return events, err + } + + if err := s.InsertEvents(syncStart, events); err != nil { + return events, err + } + + return events, nil +} diff --git a/internal/sync/scopes.go b/internal/sync/scopes.go new file mode 100644 index 0000000..0386396 --- /dev/null +++ b/internal/sync/scopes.go @@ -0,0 +1,57 @@ +package sync + +import ( + "log" + "strings" + + "github.com/mevdschee/github-export/internal/github" +) + +// featureScopes maps each optional feature to the OAuth scopes that unlock it +// (holding any one is enough). Features absent from this map need only `repo`, +// without which nothing syncs at all. Projects v2 is the odd one out: GitHub +// rejects the query field-by-field ("the 'createdAt' field requires one of the +// following scopes: ['read:project']") when the scope is missing, so we skip +// the feature up front instead of letting it spew one error per field. +var featureScopes = map[string][]string{ + "projects": {"read:project", "project"}, +} + +// checkScopes inspects the token's granted scopes, logs them, and returns the +// set of features to skip because none of their accepted scopes is present. +// It returns nil (skip nothing) for tokens that do not report scopes — +// fine-grained PATs and GitHub App tokens — since their permissions are not +// expressed as a scope list and must not be second-guessed. +func checkScopes(c *github.Client) map[string]bool { + granted, known := c.Scopes() + if !known { + return nil + } + if len(granted) == 0 { + log.Println("Token scopes: (none)") + } else { + log.Printf("Token scopes: %s", strings.Join(granted, ", ")) + } + + have := make(map[string]bool, len(granted)) + for _, s := range granted { + have[s] = true + } + + skip := map[string]bool{} + for feature, accepted := range featureScopes { + ok := false + for _, s := range accepted { + if have[s] { + ok = true + break + } + } + if !ok { + skip[feature] = true + log.Printf("Skipping %s: token lacks the %q scope (add it with: gh auth refresh -s %s)", + feature, accepted[0], accepted[0]) + } + } + return skip +} diff --git a/internal/sync/state.go b/internal/sync/state.go deleted file mode 100644 index 0ab73a4..0000000 --- a/internal/sync/state.go +++ /dev/null @@ -1,50 +0,0 @@ -package sync - -import ( - "bufio" - "os" - "strings" - - "gopkg.in/yaml.v3" -) - -// prevState holds the fields we need from an existing issue file to detect state transitions. -type prevState struct { - State string `yaml:"state"` - Type string `yaml:"type"` - Merge struct { - Merged bool `yaml:"merged"` - } `yaml:"merge"` -} - -// readPrevState reads the YAML frontmatter from an existing issue markdown file. -// Returns nil if the file does not exist. -func readPrevState(path string) *prevState { - f, err := os.Open(path) - if err != nil { - return nil - } - defer f.Close() - - scanner := bufio.NewScanner(f) - - // Skip opening --- - if !scanner.Scan() || strings.TrimSpace(scanner.Text()) != "---" { - return nil - } - - var lines []string - for scanner.Scan() { - line := scanner.Text() - if strings.TrimSpace(line) == "---" { - break - } - lines = append(lines, line) - } - - var ps prevState - if err := yaml.Unmarshal([]byte(strings.Join(lines, "\n")), &ps); err != nil { - return nil - } - return &ps -} diff --git a/internal/sync/sync.go b/internal/sync/sync.go index a98bb0e..8cc849d 100644 --- a/internal/sync/sync.go +++ b/internal/sync/sync.go @@ -1,27 +1,23 @@ package sync import ( - "bufio" "encoding/json" "fmt" "log" - "os" "path/filepath" - "strings" - "github.com/mevdschee/github-export/internal/config" - "github.com/mevdschee/github-export/internal/document" + "github.com/mevdschee/github-export/internal/ghmodel" "github.com/mevdschee/github-export/internal/github" "github.com/mevdschee/github-export/internal/hooks" "github.com/mevdschee/github-export/internal/jsonutil" - - "gopkg.in/yaml.v3" + "github.com/mevdschee/github-export/internal/store" ) -// Repo fetches repository-level metadata and writes it to repo.yml. The -// syncedAt timestamp should be captured before any other sync calls so the -// next incremental run picks up anything updated during this one. -func Repo(c *github.Client, owner, repo, outDir, syncedAt string) error { +// Repo fetches repository-level metadata, stores it, and records the sync +// timestamp in the store's meta table. syncedAt should be captured before any +// other sync call so the next incremental run picks up anything updated during +// this one. +func Repo(c *github.Client, s *store.Store, owner, repo, syncedAt string) error { log.Println("Syncing repo metadata...") url := fmt.Sprintf("%s/repos/%s/%s", github.API, owner, repo) raw, err := c.GetJSON(url, nil) @@ -32,187 +28,79 @@ func Repo(c *github.Client, owner, repo, outDir, syncedAt string) error { if err := json.Unmarshal(raw, &data); err != nil { return fmt.Errorf("parsing repo metadata: %w", err) } - - cfg := &config.RepoConfig{ - Owner: owner, - Repo: repo, - DefaultBranch: jsonutil.Str(data, "default_branch"), - Description: jsonutil.Str(data, "description"), - Homepage: jsonutil.Str(data, "homepage"), - Visibility: jsonutil.Str(data, "visibility"), - Language: jsonutil.Str(data, "language"), - License: jsonutil.Str(jsonutil.Map(data, "license"), "name"), - Topics: stringList(jsonutil.List(data, "topics")), - Archived: jsonutil.Bool(data, "archived"), - HasIssues: jsonutil.Bool(data, "has_issues"), - HasProjects: jsonutil.Bool(data, "has_projects"), - HasWiki: jsonutil.Bool(data, "has_wiki"), - HasPages: jsonutil.Bool(data, "has_pages"), - HasDiscussions: jsonutil.Bool(data, "has_discussions"), - CreatedAt: jsonutil.Str(data, "created_at"), - UpdatedAt: jsonutil.Str(data, "updated_at"), - PushedAt: jsonutil.Str(data, "pushed_at"), - SyncedAt: syncedAt, + if err := s.UpsertRepo(owner, repo, data); err != nil { + return fmt.Errorf("storing repo metadata: %w", err) } - return config.WriteRepoConfig(filepath.Join(outDir, "repo.yml"), cfg) + return s.SetMeta("synced_at", syncedAt) } -func stringList(xs []any) []string { - out := make([]string, 0, len(xs)) - for _, x := range xs { - if s, ok := x.(string); ok { - out = append(out, s) - } - } - return out -} - -func Labels(c *github.Client, owner, repo, outDir string) error { +func Labels(c *github.Client, s *store.Store, owner, repo string) error { log.Println("Syncing labels...") url := fmt.Sprintf("%s/repos/%s/%s/labels?per_page=%d", github.API, owner, repo, github.PerPage) items, err := c.GetPaginated(url, nil) if err != nil { return fmt.Errorf("fetching labels: %w", err) } - - type Label struct { - Name string `yaml:"name"` - Color string `yaml:"color"` - Description string `yaml:"description,omitempty"` + maps := unmarshalMaps(items) + if err := s.ReplaceLabels(maps); err != nil { + return fmt.Errorf("storing labels: %w", err) } - - var labels []Label - for _, raw := range items { - var m map[string]any - json.Unmarshal(raw, &m) - labels = append(labels, Label{ - Name: jsonutil.Str(m, "name"), - Color: jsonutil.Str(m, "color"), - Description: jsonutil.Str(m, "description"), - }) - } - - data, err := yaml.Marshal(labels) - if err != nil { - return err - } - log.Printf(" %d labels", len(labels)) - return os.WriteFile(filepath.Join(outDir, "labels.yml"), data, 0644) + log.Printf(" %d labels", len(maps)) + return nil } -func Milestones(c *github.Client, owner, repo, outDir string) error { +func Milestones(c *github.Client, s *store.Store, owner, repo string) error { log.Println("Syncing milestones...") url := fmt.Sprintf("%s/repos/%s/%s/milestones?state=all&per_page=%d", github.API, owner, repo, github.PerPage) items, err := c.GetPaginated(url, nil) if err != nil { return fmt.Errorf("fetching milestones: %w", err) } - - type Milestone struct { - Title string `yaml:"title"` - State string `yaml:"state"` - Description string `yaml:"description,omitempty"` - DueOn string `yaml:"due_on,omitempty"` - ClosedAt string `yaml:"closed_at,omitempty"` - } - - var milestones []Milestone - for _, raw := range items { - var m map[string]any - json.Unmarshal(raw, &m) - milestones = append(milestones, Milestone{ - Title: jsonutil.Str(m, "title"), - State: jsonutil.Str(m, "state"), - Description: jsonutil.Str(m, "description"), - DueOn: jsonutil.Str(m, "due_on"), - ClosedAt: jsonutil.Str(m, "closed_at"), - }) - } - - data, err := yaml.Marshal(milestones) - if err != nil { - return err + maps := unmarshalMaps(items) + if err := s.ReplaceMilestones(maps); err != nil { + return fmt.Errorf("storing milestones: %w", err) } - log.Printf(" %d milestones", len(milestones)) - return os.WriteFile(filepath.Join(outDir, "milestones.yml"), data, 0644) + log.Printf(" %d milestones", len(maps)) + return nil } -func Releases(c *github.Client, owner, repo, outDir string) ([]hooks.Event, error) { +func Releases(c *github.Client, s *store.Store, owner, repo string) ([]hooks.Event, error) { log.Println("Syncing releases...") url := fmt.Sprintf("%s/repos/%s/%s/releases?per_page=%d", github.API, owner, repo, github.PerPage) items, err := c.GetPaginated(url, nil) if err != nil { return nil, fmt.Errorf("fetching releases: %w", err) } - - relDir := filepath.Join(outDir, "releases") - os.MkdirAll(relDir, 0755) repoSlug := owner + "/" + repo var events []hooks.Event - for _, raw := range items { - var m map[string]any - json.Unmarshal(raw, &m) - + for _, m := range unmarshalMaps(items) { tag := jsonutil.Str(m, "tag_name") if tag == "" { continue } - d := &document.Writer{} - d.KV("tag", tag) - d.KV("name", jsonutil.Str(m, "name")) - d.KV("draft", jsonutil.Bool(m, "draft")) - d.KV("prerelease", jsonutil.Bool(m, "prerelease")) - d.KV("author", jsonutil.UserLogin(m, "author")) - d.KV("created_at", jsonutil.Str(m, "created_at")) - d.KV("published_at", jsonutil.Str(m, "published_at")) - d.KV("target_commitish", jsonutil.Str(m, "target_commitish")) - - assets := jsonutil.List(m, "assets") - if len(assets) > 0 { - fmt.Fprint(d.Buf(), "assets:\n") - for _, a := range assets { - asset, _ := a.(map[string]any) - if asset == nil { - continue - } - fmt.Fprintf(d.Buf(), " - name: %s\n", document.YamlScalar(jsonutil.Str(asset, "name"))) - if ct := jsonutil.Str(asset, "content_type"); ct != "" { - fmt.Fprintf(d.Buf(), " content_type: %s\n", document.YamlScalar(ct)) - } - if sz := jsonutil.Int(asset, "size"); sz > 0 { - fmt.Fprintf(d.Buf(), " size_bytes: %d\n", sz) - } - if dc := jsonutil.Int(asset, "download_count"); dc > 0 { - fmt.Fprintf(d.Buf(), " download_count: %d\n", dc) - } - } - } - - safeTag := strings.ReplaceAll(tag, "/", "-") - path := filepath.Join(relDir, safeTag+".md") - prev := readPrevRelease(path) - - f, err := os.Create(path) + existed, prevPrerelease, err := s.ReleaseState(tag) if err != nil { - log.Printf(" Warning: cannot create %s: %v", path, err) + return events, fmt.Errorf("reading release state %q: %w", tag, err) + } + if err := s.UpsertRelease(m); err != nil { + log.Printf(" Warning: storing release %q: %v", tag, err) continue } - document.WriteFirstDoc(f, d.String(), jsonutil.Str(m, "body")) - f.Close() if jsonutil.Bool(m, "draft") { continue } prerelease := jsonutil.Bool(m, "prerelease") + relPath := filepath.Join("releases", ghmodel.SafeTag(tag)+".md") base := hooks.Event{ Number: jsonutil.Int(m, "id"), Title: jsonutil.Str(m, "name"), Author: jsonutil.UserLogin(m, "author"), State: "published", - File: path, + File: relPath, Repo: repoSlug, URL: jsonutil.Str(m, "html_url"), } @@ -221,13 +109,14 @@ func Releases(c *github.Client, owner, repo, outDir string) ([]hooks.Event, erro } extra := map[string]string{"tag": tag} - if !prev.exists { + switch { + case !existed: ev := base ev.Type = hooks.ReleasePublished ev.Body = jsonutil.Str(m, "body") ev.Extra = extra events = append(events, ev) - } else if prev.prerelease && !prerelease { + case prevPrerelease && !prerelease: ev := base ev.Type = hooks.PrereleasePromoted ev.Extra = extra @@ -239,37 +128,15 @@ func Releases(c *github.Client, owner, repo, outDir string) ([]hooks.Event, erro return events, nil } -type prevReleaseState struct { - exists bool - prerelease bool -} - -func readPrevRelease(path string) prevReleaseState { - f, err := os.Open(path) - if err != nil { - return prevReleaseState{} - } - defer f.Close() - out := prevReleaseState{exists: true} - - scanner := bufio.NewScanner(f) - scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) - if !scanner.Scan() || strings.TrimSpace(scanner.Text()) != "---" { - return out - } - var lines []string - for scanner.Scan() { - line := scanner.Text() - if strings.TrimSpace(line) == "---" { - break +// unmarshalMaps decodes a slice of raw JSON objects into maps, skipping any that +// fail to parse. +func unmarshalMaps(items []json.RawMessage) []map[string]any { + out := make([]map[string]any, 0, len(items)) + for _, raw := range items { + var m map[string]any + if err := json.Unmarshal(raw, &m); err == nil { + out = append(out, m) } - lines = append(lines, line) - } - var fm struct { - Prerelease bool `yaml:"prerelease"` - } - if err := yaml.Unmarshal([]byte(strings.Join(lines, "\n")), &fm); err == nil { - out.prerelease = fm.Prerelease } return out } diff --git a/internal/writeproxy/writeproxy.go b/internal/writeproxy/writeproxy.go new file mode 100644 index 0000000..a0dec6b --- /dev/null +++ b/internal/writeproxy/writeproxy.go @@ -0,0 +1,215 @@ +// Package writeproxy forwards requests we cannot (or must not) answer locally to +// api.github.com: all writes, and any read the local mirror does not implement. +// The upstream response is streamed back verbatim. After a successful write to a +// known entity it triggers a synchronous targeted re-sync so the local store is +// consistent the moment the call returns. +package writeproxy + +import ( + "context" + "fmt" + "io" + "log" + "net/http" + "regexp" + "strconv" + "strings" + "time" +) + +const upstream = "https://api.github.com" + +// ReSyncFunc re-pulls a single entity into the store after a write. number is 0 +// for whole-collection writes (e.g. creating a label). It runs synchronously on +// the write path; errors are logged, not fatal. +type ReSyncFunc func(kind string, number int64) error + +// Proxy forwards to GitHub using the server's token. +type Proxy struct { + Token string + Client *http.Client + ReSync ReSyncFunc + Disabled bool // --proxy=off: refuse to forward (offline-only mode) + BaseURL string // upstream base; defaults to api.github.com (overridable in tests) +} + +// New builds a Proxy. +func New(token string, resync ReSyncFunc) *Proxy { + return &Proxy{ + Token: token, + Client: &http.Client{Timeout: 60 * time.Second}, + ReSync: resync, + } +} + +// base returns the upstream base URL (the GitHub API by default). +func (p *Proxy) base() string { + if p.BaseURL != "" { + return p.BaseURL + } + return upstream +} + +// hopHeaders are managed by the transport and must not be copied. +var hopHeaders = map[string]bool{ + "Connection": true, "Keep-Alive": true, "Proxy-Authenticate": true, + "Proxy-Authorization": true, "Te": true, "Trailer": true, + "Transfer-Encoding": true, "Upgrade": true, +} + +// Forward proxies r to api.github.com and writes the response to w. It returns +// the upstream status code (or 0 on a transport error already written to w). +func (p *Proxy) Forward(w http.ResponseWriter, r *http.Request) int { + if p.Disabled { + http.Error(w, `{"message":"proxy disabled (--proxy=off); endpoint not mirrored locally"}`, http.StatusNotImplemented) + return http.StatusNotImplemented + } + + target := p.base() + r.URL.Path + if r.URL.RawQuery != "" { + target += "?" + r.URL.RawQuery + } + + var body io.Reader + if r.Body != nil { + body = r.Body + } + req, err := http.NewRequestWithContext(r.Context(), r.Method, target, body) + if err != nil { + http.Error(w, `{"message":"building upstream request: `+err.Error()+`"}`, http.StatusBadGateway) + return 0 + } + for k, vs := range r.Header { + if hopHeaders[http.CanonicalHeaderKey(k)] || strings.EqualFold(k, "Authorization") { + continue + } + for _, v := range vs { + req.Header.Add(k, v) + } + } + if req.Header.Get("Accept") == "" { + req.Header.Set("Accept", "application/vnd.github+json") + } + req.Header.Set("Authorization", "Bearer "+p.Token) + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + + resp, err := p.Client.Do(req) + if err != nil { + http.Error(w, `{"message":"upstream error: `+err.Error()+`"}`, http.StatusBadGateway) + return 0 + } + defer resp.Body.Close() + // Buffer the upstream body so the targeted re-sync can run BEFORE we return + // to the caller — read-after-write consistency means the local store is + // already updated by the time the write response is delivered. + respBody, _ := io.ReadAll(resp.Body) + + resyncFailed := false + if r.Method != http.MethodGet && r.Method != http.MethodHead && + resp.StatusCode >= 200 && resp.StatusCode < 300 && p.ReSync != nil { + if kind, number, ok := classifyWrite(r.Method, r.URL.Path); ok { + if err := p.ReSync(kind, number); err != nil { + log.Printf("write-proxy: re-sync %s #%d failed: %v", kind, number, err) + resyncFailed = true + } + } + } + + for k, vs := range resp.Header { + if hopHeaders[http.CanonicalHeaderKey(k)] { + continue + } + for _, v := range vs { + w.Header().Add(k, v) + } + } + w.Header().Set("X-GitHub-Export-Proxied", "true") + if resyncFailed { + w.Header().Set("X-GitHub-Export-Resync", "failed") + } + // Content-Length from upstream may not match after buffering; let the writer + // recompute it. + w.Header().Del("Content-Length") + w.WriteHeader(resp.StatusCode) + w.Write(respBody) + return resp.StatusCode +} + +// Request performs a programmatic call to api.github.com (used by the MCP write +// tools, which have no http.ResponseWriter to stream into). On a successful +// write to a known entity it triggers the same synchronous re-sync as Forward. +func (p *Proxy) Request(ctx context.Context, method, path string, body io.Reader) (status int, respBody []byte, err error) { + if p.Disabled { + return http.StatusNotImplemented, nil, fmt.Errorf("proxy disabled (--proxy=off)") + } + req, err := http.NewRequestWithContext(ctx, method, p.base()+path, body) + if err != nil { + return 0, nil, err + } + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("Authorization", "Bearer "+p.Token) + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := p.Client.Do(req) + if err != nil { + return 0, nil, err + } + defer resp.Body.Close() + respBody, err = io.ReadAll(resp.Body) + if err != nil { + return resp.StatusCode, nil, err + } + if method != http.MethodGet && resp.StatusCode >= 200 && resp.StatusCode < 300 && p.ReSync != nil { + if kind, number, ok := classifyWrite(method, path); ok { + if rerr := p.ReSync(kind, number); rerr != nil { + log.Printf("write-proxy: re-sync %s #%d failed: %v", kind, number, rerr) + } + } + } + return resp.StatusCode, respBody, nil +} + +var ( + reIssue = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/(\d+)$`) + reIssueCreate = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues$`) + reIssueComment = regexp.MustCompile(`^/repos/[^/]+/[^/]+/issues/(\d+)/comments$`) + rePull = regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls/(\d+)$`) + rePullCreate = regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls$`) + rePullMerge = regexp.MustCompile(`^/repos/[^/]+/[^/]+/pulls/(\d+)/merge$`) + reLabels = regexp.MustCompile(`^/repos/[^/]+/[^/]+/labels`) +) + +// classifyWrite maps a write path to (entity-kind, number) for targeted re-sync. +// number is 0 when the write creates a new entity (its number is unknown until +// the response body is parsed; the caller re-syncs the whole collection or the +// next sync picks it up). +func classifyWrite(method, path string) (kind string, number int64, ok bool) { + switch { + case reIssueComment.MatchString(path): + m := reIssueComment.FindStringSubmatch(path) + return "issue", atoi(m[1]), true + case reIssue.MatchString(path): + m := reIssue.FindStringSubmatch(path) + return "issue", atoi(m[1]), true + case rePullMerge.MatchString(path): + m := rePullMerge.FindStringSubmatch(path) + return "issue", atoi(m[1]), true + case rePull.MatchString(path): + m := rePull.FindStringSubmatch(path) + return "issue", atoi(m[1]), true + case reIssueCreate.MatchString(path) && method == http.MethodPost: + return "issues", 0, true + case rePullCreate.MatchString(path) && method == http.MethodPost: + return "issues", 0, true + case reLabels.MatchString(path): + return "labels", 0, true + } + return "", 0, false +} + +func atoi(s string) int64 { + n, _ := strconv.ParseInt(s, 10, 64) + return n +} diff --git a/internal/writeproxy/writeproxy_test.go b/internal/writeproxy/writeproxy_test.go new file mode 100644 index 0000000..807132b --- /dev/null +++ b/internal/writeproxy/writeproxy_test.go @@ -0,0 +1,190 @@ +package writeproxy + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// fakeUpstream records the requests it receives and returns a canned response. +type fakeUpstream struct { + srv *httptest.Server + lastAuth string + lastBody string + lastPath string +} + +func newFakeUpstream(t *testing.T, status int, respBody string) *fakeUpstream { + t.Helper() + f := &fakeUpstream{} + f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.lastAuth = r.Header.Get("Authorization") + f.lastPath = r.URL.RequestURI() + b, _ := io.ReadAll(r.Body) + f.lastBody = string(b) + w.Header().Set("X-RateLimit-Remaining", "4999") + w.WriteHeader(status) + w.Write([]byte(respBody)) + })) + t.Cleanup(f.srv.Close) + return f +} + +func newProxy(t *testing.T, f *fakeUpstream, resync ReSyncFunc) *Proxy { + p := New("secret-token", resync) + p.BaseURL = f.srv.URL + return p +} + +func TestForwardStreamsResponseAndAuth(t *testing.T) { + f := newFakeUpstream(t, 200, `{"ok":true}`) + p := newProxy(t, f, nil) + + req := httptest.NewRequest("GET", "/repos/o/r/actions/runs?per_page=5", nil) + rec := httptest.NewRecorder() + status := p.Forward(rec, req) + + if status != 200 { + t.Errorf("status=%d, want 200", status) + } + res := rec.Result() + body, _ := io.ReadAll(res.Body) + if string(body) != `{"ok":true}` { + t.Errorf("body=%s", body) + } + if res.Header.Get("X-GitHub-Export-Proxied") != "true" { + t.Errorf("missing proxied header") + } + if f.lastAuth != "Bearer secret-token" { + t.Errorf("upstream auth=%q", f.lastAuth) + } + if f.lastPath != "/repos/o/r/actions/runs?per_page=5" { + t.Errorf("upstream path=%q (query not preserved?)", f.lastPath) + } +} + +func TestForwardDisabledReturns501(t *testing.T) { + p := New("t", nil) + p.Disabled = true + req := httptest.NewRequest("GET", "/x", nil) + rec := httptest.NewRecorder() + if status := p.Forward(rec, req); status != http.StatusNotImplemented { + t.Errorf("status=%d, want 501", status) + } +} + +func TestForwardWriteTriggersResync(t *testing.T) { + f := newFakeUpstream(t, 201, `{"number":42}`) + var gotKind string + var gotNum int64 + called := 0 + p := newProxy(t, f, func(kind string, number int64) error { + called++ + gotKind, gotNum = kind, number + return nil + }) + + req := httptest.NewRequest("PATCH", "/repos/o/r/issues/42", strings.NewReader(`{"state":"closed"}`)) + rec := httptest.NewRecorder() + p.Forward(rec, req) + + if called != 1 || gotKind != "issue" || gotNum != 42 { + t.Errorf("resync called=%d kind=%q num=%d, want 1/issue/42", called, gotKind, gotNum) + } + if f.lastBody != `{"state":"closed"}` { + t.Errorf("upstream body=%q", f.lastBody) + } +} + +func TestForwardGetDoesNotResync(t *testing.T) { + f := newFakeUpstream(t, 200, `{}`) + called := 0 + p := newProxy(t, f, func(string, int64) error { called++; return nil }) + req := httptest.NewRequest("GET", "/repos/o/r/issues/42", nil) + p.Forward(httptest.NewRecorder(), req) + if called != 0 { + t.Errorf("resync called %d times on GET, want 0", called) + } +} + +func TestForwardFailedWriteNoResync(t *testing.T) { + f := newFakeUpstream(t, 422, `{"message":"validation failed"}`) + called := 0 + p := newProxy(t, f, func(string, int64) error { called++; return nil }) + req := httptest.NewRequest("POST", "/repos/o/r/issues", strings.NewReader(`{}`)) + p.Forward(httptest.NewRecorder(), req) + if called != 0 { + t.Errorf("resync ran on a 422, want 0") + } +} + +func TestForwardResyncFailureSetsHeader(t *testing.T) { + f := newFakeUpstream(t, 200, `{}`) + p := newProxy(t, f, func(string, int64) error { return io.EOF }) + req := httptest.NewRequest("POST", "/repos/o/r/issues/42/comments", strings.NewReader(`{"body":"hi"}`)) + rec := httptest.NewRecorder() + p.Forward(rec, req) + if rec.Result().Header.Get("X-GitHub-Export-Resync") != "failed" { + t.Errorf("expected resync-failed header") + } +} + +func TestRequestProgrammatic(t *testing.T) { + f := newFakeUpstream(t, 201, `{"id":7}`) + called := 0 + p := newProxy(t, f, func(kind string, number int64) error { + called++ + if kind != "issues" { + t.Errorf("kind=%q, want issues", kind) + } + return nil + }) + status, body, err := p.Request(context.Background(), "POST", "/repos/o/r/issues", strings.NewReader(`{"title":"x"}`)) + if err != nil || status != 201 || string(body) != `{"id":7}` { + t.Fatalf("Request → status=%d body=%s err=%v", status, body, err) + } + if called != 1 { + t.Errorf("resync called=%d, want 1", called) + } + if f.lastBody != `{"title":"x"}` { + t.Errorf("upstream body=%q", f.lastBody) + } +} + +func TestRequestDisabled(t *testing.T) { + p := New("t", nil) + p.Disabled = true + status, _, err := p.Request(context.Background(), "POST", "/x", nil) + if status != http.StatusNotImplemented || err == nil { + t.Errorf("status=%d err=%v, want 501/err", status, err) + } +} + +func TestClassifyWrite(t *testing.T) { + cases := []struct { + method, path string + wantKind string + wantNum int64 + wantOK bool + }{ + {"PATCH", "/repos/o/r/issues/42", "issue", 42, true}, + {"POST", "/repos/o/r/issues/42/comments", "issue", 42, true}, + {"PUT", "/repos/o/r/pulls/9/merge", "issue", 9, true}, + {"PATCH", "/repos/o/r/pulls/9", "issue", 9, true}, + {"POST", "/repos/o/r/issues", "issues", 0, true}, + {"POST", "/repos/o/r/pulls", "issues", 0, true}, + {"POST", "/repos/o/r/labels", "labels", 0, true}, + {"GET", "/repos/o/r/issues/42", "issue", 42, true}, // classify is method-agnostic for issues + {"POST", "/repos/o/r/actions/runs", "", 0, false}, + } + for _, c := range cases { + kind, num, ok := classifyWrite(c.method, c.path) + if kind != c.wantKind || num != c.wantNum || ok != c.wantOK { + t.Errorf("classifyWrite(%s %s) = %q,%d,%v want %q,%d,%v", + c.method, c.path, kind, num, ok, c.wantKind, c.wantNum, c.wantOK) + } + } +} diff --git a/main.go b/main.go index 3eabfd3..78cb001 100644 --- a/main.go +++ b/main.go @@ -5,36 +5,74 @@ import ( "fmt" "log" "os" - "path/filepath" + "os/exec" "regexp" "sort" "strconv" "strings" "time" - "github.com/mevdschee/github-export/internal/config" + "github.com/mevdschee/github-export/internal/exporter" "github.com/mevdschee/github-export/internal/github" "github.com/mevdschee/github-export/internal/hooks" + "github.com/mevdschee/github-export/internal/store" "github.com/mevdschee/github-export/internal/sync" ) +const defaultDB = "github.sqlite" + func main() { - maxAge := flag.String("max-age", "", + log.SetFlags(0) + + args := os.Args[1:] + sub := "sync" + switch { + case len(args) > 0 && isSubcommand(args[0]): + sub = args[0] + args = args[1:] + } + + switch sub { + case "export": + runExport(args) + case "serve": + runServe(args) + case "api": + runAPI(args) + case "mcp": + runMCP(args) + case "issue", "pr", "search", "release": + runNative(sub, args) + default: + runSync(args) + } +} + +func isSubcommand(s string) bool { + switch s { + case "sync", "export", "serve", "api", "mcp", "issue", "pr", "search", "release": + return true + } + return false +} + +func runSync(args []string) { + fs := flag.NewFlagSet("sync", flag.ExitOnError) + maxAge := fs.String("max-age", "", "only fetch issues/PRs/projects updated within this window (e.g. 2y, 6mo, 4w, 30d, 12h); useful for very large repos on first sync") - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage: %s [flags] [owner/repo] [output-dir]\n\n", os.Args[0]) - fmt.Fprintf(os.Stderr, "Exports GitHub issues, PRs, releases, labels, and milestones\n") - fmt.Fprintf(os.Stderr, "to a local directory in markdown format.\n\n") - fmt.Fprintf(os.Stderr, "Runs incrementally on subsequent invocations.\n\n") - fmt.Fprintf(os.Stderr, "If invoked with no arguments, the current directory is treated\n") - fmt.Fprintf(os.Stderr, "as an existing export and owner/repo are read from ./repo.yml.\n\n") - fmt.Fprintf(os.Stderr, "Requires GITHUB_TOKEN environment variable.\n") - fmt.Fprintf(os.Stderr, " export GITHUB_TOKEN=$(gh auth token)\n\n") + dbPath := fs.String("db", defaultDB, "path to the SQLite database (the primary store)") + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s sync [flags] [owner/repo]\n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Syncs GitHub issues, PRs, discussions, projects, releases, labels, and\n") + fmt.Fprintf(os.Stderr, "milestones into a local SQLite database. Runs incrementally on later runs.\n\n") + fmt.Fprintf(os.Stderr, "owner/repo is resolved from (1) the argument, (2) the existing database,\n") + fmt.Fprintf(os.Stderr, "or (3) the origin git remote of the current directory.\n\n") + fmt.Fprintf(os.Stderr, "Run 'github-export export' afterwards to render markdown files.\n\n") + fmt.Fprintf(os.Stderr, "Requires GITHUB_TOKEN:\n export GITHUB_TOKEN=$(gh auth token)\n\n") fmt.Fprintf(os.Stderr, "Flags:\n") - flag.PrintDefaults() + fs.PrintDefaults() } - flag.Parse() - args := flag.Args() + fs.Parse(args) token := os.Getenv("GITHUB_TOKEN") if token == "" { @@ -45,35 +83,6 @@ func main() { os.Exit(1) } - var owner, repo, outDir string - switch { - case len(args) == 0: - // Deduce mode: current directory must already hold a repo.yml from - // a previous sync. owner/repo are filled in from cfg below. If the - // current directory has no repo.yml but a ./github-data folder does, - // treat that as the existing export. - outDir = "." - if _, err := os.Stat(filepath.Join(outDir, "repo.yml")); os.IsNotExist(err) { - if _, err := os.Stat(filepath.Join("github-data", "repo.yml")); err == nil { - outDir = "github-data" - } - } - case len(args) == 1: - if strings.Contains(args[0], "/") { - parts := strings.SplitN(args[0], "/", 2) - owner = parts[0] - repo = parts[1] - } - outDir = "github-data" - default: - if strings.Contains(args[0], "/") { - parts := strings.SplitN(args[0], "/", 2) - owner = parts[0] - repo = parts[1] - } - outDir = args[1] - } - cutoff := "" if *maxAge != "" { t, err := parseMaxAge(*maxAge) @@ -83,33 +92,23 @@ func main() { cutoff = t.UTC().Format(time.RFC3339) } - // Create output directories - os.MkdirAll(filepath.Join(outDir, "issues"), 0755) - os.MkdirAll(filepath.Join(outDir, "releases"), 0755) - - // Read existing config for incremental sync - configPath := filepath.Join(outDir, "repo.yml") - cfg, err := config.ReadRepoConfig(configPath) + s, err := store.Open(*dbPath) if err != nil { - log.Fatalf("Reading repo.yml: %v", err) - } - if cfg != nil && owner == "" { - owner = cfg.Owner - repo = cfg.Repo + log.Fatalf("opening database %s: %v", *dbPath, err) } + defer s.Close() + + owner, repo := resolveOwnerRepo(fs.Arg(0), s) if owner == "" || repo == "" { - if len(args) == 0 { - log.Fatalf("no owner/repo given and no repo.yml found in %s; pass owner/repo (e.g., octocat/Hello-World) or run from a directory that already has an export", outDir) - } - log.Fatal("owner/repo must be specified as first argument (e.g., octocat/Hello-World)") + log.Fatalf("could not determine owner/repo; pass it (e.g. octocat/Hello-World), run from an existing database, or from a directory whose origin remote points at GitHub") } - since := "" - if cfg != nil { - since = cfg.SyncedAt + since, err := s.SyncedAt() + if err != nil { + log.Fatalf("reading sync state: %v", err) } - // Treat --max-age as a floor: if it's more recent than synced_at (or - // synced_at is empty), the cutoff drives the next fetch. + // --max-age is a floor: if it is more recent than synced_at (or there is no + // synced_at yet), the cutoff drives the next fetch. if cutoff != "" && cutoff > since { since = cutoff } @@ -125,46 +124,78 @@ func main() { syncStart := time.Now().UTC().Format(time.RFC3339) client := github.NewClient(token) - // Sync all entities - if err := sync.Labels(client, owner, repo, outDir); err != nil { - log.Printf("Warning: %v", err) + events, err := sync.Run(client, s, owner, repo, since, syncStart) + if err != nil { + log.Fatalf("sync: %v", err) } - if err := sync.Milestones(client, owner, repo, outDir); err != nil { - log.Printf("Warning: %v", err) + + log.Printf("Done. synced_at=%s, db=%s", syncStart, *dbPath) + if len(events) > 0 { + log.Printf("Recorded %d events — %s", len(events), summarizeEvents(events)) + log.Printf("Run 'github-export export' to write them to the events/ folder.") } - issueProjects, projectEvents, err := sync.Projects(client, owner, repo, outDir, since) - if err != nil { - log.Printf("Warning: %v", err) +} + +func runExport(args []string) { + fs := flag.NewFlagSet("export", flag.ExitOnError) + dbPath := fs.String("db", defaultDB, "path to the SQLite database to export from") + out := fs.String("out", "github-data", "output directory for the markdown export") + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s export [flags]\n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Renders the SQLite store to markdown files. One-way: reads the database\n") + fmt.Fprintf(os.Stderr, "and writes files; it never reads the markdown back.\n\n") + fmt.Fprintf(os.Stderr, "Flags:\n") + fs.PrintDefaults() } - events, err := sync.Issues(client, owner, repo, outDir, since, issueProjects) - if err != nil { - log.Printf("Warning: %v", err) + fs.Parse(args) + + if _, err := os.Stat(*dbPath); err != nil { + log.Fatalf("database %s not found; run 'github-export sync' first", *dbPath) } - events = append(events, projectEvents...) - releaseEvents, err := sync.Releases(client, owner, repo, outDir) + + s, err := store.Open(*dbPath) if err != nil { - log.Printf("Warning: %v", err) + log.Fatalf("opening database %s: %v", *dbPath, err) } - events = append(events, releaseEvents...) - discussionEvents, err := sync.Discussions(client, owner, repo, outDir, since) - if err != nil { - log.Printf("Warning: %v", err) + defer s.Close() + + if err := exporter.Export(s, *out); err != nil { + log.Fatalf("export: %v", err) + } + log.Printf("Exported %s to %s", *dbPath, *out) +} + +// resolveOwnerRepo determines the target repo from, in order: the positional +// argument, the existing database, and the origin git remote. +func resolveOwnerRepo(arg string, s *store.Store) (owner, repo string) { + if strings.Contains(arg, "/") { + parts := strings.SplitN(arg, "/", 2) + return parts[0], parts[1] + } + if o, r, err := s.OwnerRepo(); err == nil && o != "" && r != "" { + return o, r } - events = append(events, discussionEvents...) - if err := sync.Repo(client, owner, repo, outDir, syncStart); err != nil { - log.Fatalf("Writing repo.yml: %v", err) + if o, r, ok := detectRepoFromGit(); ok { + log.Printf("Detected %s/%s from git remote", o, r) + return o, r } + return "", "" +} - log.Printf("Done. synced_at=%s", syncStart) +var gitRemoteRe = regexp.MustCompile(`github\.com[:/]([^/]+)/(.+?)(?:\.git)?$`) - // Export events as markdown files for agents to pick up - if len(events) > 0 { - eventsDir := filepath.Join(outDir, "events") - log.Printf("Exporting %d events to %s — %s", len(events), eventsDir, summarizeEvents(events)) - if err := hooks.Export(eventsDir, events); err != nil { - log.Printf("Warning: exporting events: %v", err) - } +// detectRepoFromGit parses owner/repo out of the origin remote URL. +func detectRepoFromGit() (owner, repo string, ok bool) { + cmd := exec.Command("git", "remote", "get-url", "origin") + out, err := cmd.Output() + if err != nil { + return "", "", false + } + m := gitRemoteRe.FindStringSubmatch(strings.TrimSpace(string(out))) + if m == nil { + return "", "", false } + return m[1], m[2], true } var maxAgeRe = regexp.MustCompile(`^(\d+)(h|d|w|mo|y)$`) @@ -172,10 +203,10 @@ var maxAgeRe = regexp.MustCompile(`^(\d+)(h|d|w|mo|y)$`) // parseMaxAge turns a human age like "2y", "6mo", "4w", "30d", "12h" into the // UTC timestamp that age ago. Months are taken as 30 days and years as 365 // days — the cutoff is an approximate floor, not a calendar boundary. -func parseMaxAge(s string) (time.Time, error) { - m := maxAgeRe.FindStringSubmatch(s) +func parseMaxAge(str string) (time.Time, error) { + m := maxAgeRe.FindStringSubmatch(str) if m == nil { - return time.Time{}, fmt.Errorf("must be Nh, Nd, Nw, Nmo or Ny (got %q)", s) + return time.Time{}, fmt.Errorf("must be Nh, Nd, Nw, Nmo or Ny (got %q)", str) } n, err := strconv.Atoi(m[1]) if err != nil || n <= 0 { @@ -197,8 +228,8 @@ func parseMaxAge(s string) (time.Time, error) { return time.Now().Add(-d), nil } -// summarizeEvents returns a stable "type=N, type=N, ..." breakdown of an event -// slice, sorted by descending count then ascending type name. +// summarizeEvents returns a stable "type=N, ..." breakdown sorted by descending +// count then ascending type name. func summarizeEvents(events []hooks.Event) string { counts := map[string]int{} for _, ev := range events { diff --git a/mcp.go b/mcp.go new file mode 100644 index 0000000..be8cccd --- /dev/null +++ b/mcp.go @@ -0,0 +1,90 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net/http" + "os" + + mcpsdk "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/mevdschee/github-export/internal/gitbackend" + "github.com/mevdschee/github-export/internal/github" + "github.com/mevdschee/github-export/internal/mcp" + "github.com/mevdschee/github-export/internal/query" + "github.com/mevdschee/github-export/internal/store" + "github.com/mevdschee/github-export/internal/sync" + "github.com/mevdschee/github-export/internal/writeproxy" +) + +func runMCP(args []string) { + fs := flag.NewFlagSet("mcp", flag.ExitOnError) + dbPath := fs.String("db", defaultDB, "path to the SQLite database") + readOnly := fs.Bool("read-only", false, "register read tools only (no write/proxy tools)") + proxyMode := fs.String("proxy", "on", "proxy fallback for writes: on|off") + repoPath := fs.String("repo-path", ".", "local git clone for content/code tools") + httpAddr := fs.String("http", "", "serve the streamable-HTTP transport on this address instead of stdio (e.g. localhost:8081)") + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s mcp [flags]\n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Runs a Model Context Protocol server over stdio (default) whose tool names\n") + fmt.Fprintf(os.Stderr, "match GitHub's official MCP server. Reads come from the local store/git;\n") + fmt.Fprintf(os.Stderr, "writes proxy to GitHub and re-sync.\n\n") + fs.PrintDefaults() + } + fs.Parse(args) + + s, err := store.Open(*dbPath) + if err != nil { + log.Fatalf("opening database %s: %v", *dbPath, err) + } + defer s.Close() + owner, repo, err := s.OwnerRepo() + if err != nil || owner == "" || repo == "" { + log.Fatalf("database %s has no synced repo; run 'github-export sync' first", *dbPath) + } + syncedAt, _ := s.SyncedAt() + + token := os.Getenv("GITHUB_TOKEN") + var client *github.Client + if token != "" { + client = github.NewClient(token) + } + proxy := writeproxy.New(token, func(kind string, number int64) error { + if client != nil && kind == "issue" && number > 0 { + return sync.ResyncIssue(client, s, owner, repo, number) + } + return nil + }) + if *proxyMode == "off" { + proxy.Disabled = true + } + + var git *gitbackend.Backend + if *repoPath != "" { + git = gitbackend.New(*repoPath, owner, repo, false) + if !git.Available() { + git = nil + } + } + + srv := mcp.NewServer(mcp.Deps{ + Query: query.New(s), Git: git, Proxy: proxy, + Owner: owner, Repo: repo, SyncedAt: syncedAt, ReadOnly: *readOnly, + }) + + ctx := context.Background() + if *httpAddr != "" { + handler := mcpsdk.NewStreamableHTTPHandler(func(*http.Request) *mcpsdk.Server { return srv }, nil) + log.Printf("MCP streamable-HTTP server on http://%s", *httpAddr) + if err := http.ListenAndServe(*httpAddr, handler); err != nil { + log.Fatalf("mcp http: %v", err) + } + return + } + + if err := srv.Run(ctx, &mcpsdk.StdioTransport{}); err != nil { + log.Fatalf("mcp: %v", err) + } +} diff --git a/native.go b/native.go new file mode 100644 index 0000000..d4bc584 --- /dev/null +++ b/native.go @@ -0,0 +1,203 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "log" + "os" + "strconv" + "text/tabwriter" + + "github.com/mevdschee/github-export/internal/query" + "github.com/mevdschee/github-export/internal/store" +) + +// runNative implements the offline read subcommands (issue/pr/search/release) +// over internal/query. They mirror common `gh` reads for use without `gh` or a +// running server. Use --json for raw GitHub-shaped JSON. +func runNative(sub string, args []string) { + fs := flag.NewFlagSet(sub, flag.ExitOnError) + dbPath := fs.String("db", defaultDB, "path to the SQLite database") + asJSON := fs.Bool("json", false, "emit raw GitHub-shaped JSON") + state := fs.String("state", "open", "filter by state: open|closed|all") + limit := fs.Int("limit", 30, "maximum number of results") + // Allow flags and positionals to be interspersed (e.g. `issue list --state all`, + // `issue view 1 --json`), which Go's flag package does not do on its own. + positional := parseInterspersed(fs, args) + + s, err := store.Open(*dbPath) + if err != nil { + log.Fatalf("opening database %s: %v", *dbPath, err) + } + defer s.Close() + q := query.New(s) + + rest := positional + action := "list" + if len(rest) > 0 { + action = rest[0] + rest = rest[1:] + } + + switch sub { + case "issue": + nativeIssueOrPR(q, false, action, rest, *state, *limit, *asJSON) + case "pr": + nativeIssueOrPR(q, true, action, rest, *state, *limit, *asJSON) + case "release": + nativeReleases(q, *asJSON) + case "search": + nativeSearch(q, positional, *limit, *asJSON) + } +} + +// parseInterspersed parses flags from args while collecting positional +// arguments, allowing the two to be mixed in any order (unlike flag.Parse, +// which stops at the first positional). +func parseInterspersed(fs *flag.FlagSet, args []string) []string { + var positional []string + for { + if err := fs.Parse(args); err != nil { + return positional + } + rest := fs.Args() + if len(rest) == 0 { + return positional + } + positional = append(positional, rest[0]) + args = rest[1:] + } +} + +func nativeIssueOrPR(q *query.Query, isPR bool, action string, rest []string, state string, limit int, asJSON bool) { + switch action { + case "view": + if len(rest) == 0 { + log.Fatalf("usage: view ") + } + n, _ := strconv.ParseInt(rest[0], 10, 64) + var doc json.RawMessage + var ok bool + var err error + if isPR { + doc, ok, err = q.GetPull(n) + } else { + doc, ok, err = q.GetIssue(n) + } + if err != nil { + log.Fatal(err) + } + if !ok { + log.Fatalf("#%d not found in local store", n) + } + if asJSON { + os.Stdout.Write(doc) + fmt.Println() + return + } + printIssueDetail(doc) + default: // list + o := query.ListIssuesOpts{State: state, PerPage: limit, Page: 1, OnlyPulls: isPR, OnlyIssues: !isPR} + items, _, err := q.ListIssues(o) + if err != nil { + log.Fatal(err) + } + if asJSON { + emitJSONArray(items) + return + } + printIssueTable(items) + } +} + +func nativeReleases(q *query.Query, asJSON bool) { + items, err := q.ListReleases() + if err != nil { + log.Fatal(err) + } + if asJSON { + emitJSONArray(items) + return + } + tw := tabwriter.NewWriter(os.Stdout, 0, 2, 2, ' ', 0) + fmt.Fprintln(tw, "TAG\tNAME\tPUBLISHED") + for _, it := range items { + var m map[string]any + json.Unmarshal(it, &m) + fmt.Fprintf(tw, "%s\t%s\t%s\n", s(m, "tag_name"), s(m, "name"), s(m, "published_at")) + } + tw.Flush() +} + +func nativeSearch(q *query.Query, args []string, limit int, asJSON bool) { + queryStr := "" + for _, a := range args { + queryStr += a + " " + } + items, total, err := q.SearchIssues(queryStr, limit, 1) + if err != nil { + log.Fatal(err) + } + if asJSON { + emitJSONArray(items) + return + } + fmt.Printf("Found %d result(s):\n", total) + printIssueTable(items) +} + +func printIssueTable(items []json.RawMessage) { + tw := tabwriter.NewWriter(os.Stdout, 0, 2, 2, ' ', 0) + fmt.Fprintln(tw, "NUMBER\tSTATE\tTITLE\tAUTHOR") + for _, it := range items { + var m map[string]any + json.Unmarshal(it, &m) + author := "" + if u, ok := m["user"].(map[string]any); ok { + author, _ = u["login"].(string) + } + fmt.Fprintf(tw, "#%s\t%s\t%s\t%s\n", numStr(m), s(m, "state"), truncate(s(m, "title"), 60), author) + } + tw.Flush() +} + +func printIssueDetail(doc json.RawMessage) { + var m map[string]any + json.Unmarshal(doc, &m) + author := "" + if u, ok := m["user"].(map[string]any); ok { + author, _ = u["login"].(string) + } + fmt.Printf("#%s %s\n", numStr(m), s(m, "title")) + fmt.Printf("State: %s Author: %s Created: %s\n\n", s(m, "state"), author, s(m, "created_at")) + fmt.Println(s(m, "body")) +} + +func emitJSONArray(items []json.RawMessage) { + if items == nil { + items = []json.RawMessage{} + } + b, _ := json.MarshalIndent(items, "", " ") + os.Stdout.Write(b) + fmt.Println() +} + +func s(m map[string]any, k string) string { + v, _ := m[k].(string) + return v +} + +func numStr(m map[string]any) string { + if f, ok := m["number"].(float64); ok { + return strconv.FormatInt(int64(f), 10) + } + return "" +} + +func truncate(str string, n int) string { + if len(str) <= n { + return str + } + return str[:n-1] + "…" +} diff --git a/serve.go b/serve.go new file mode 100644 index 0000000..b469548 --- /dev/null +++ b/serve.go @@ -0,0 +1,167 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/mevdschee/github-export/internal/gitbackend" + "github.com/mevdschee/github-export/internal/github" + "github.com/mevdschee/github-export/internal/graphqlmirror" + "github.com/mevdschee/github-export/internal/httpapi" + "github.com/mevdschee/github-export/internal/query" + "github.com/mevdschee/github-export/internal/shadow" + "github.com/mevdschee/github-export/internal/store" + "github.com/mevdschee/github-export/internal/sync" + "github.com/mevdschee/github-export/internal/writeproxy" +) + +// projectRepoSlug is where shadow-compare files parity-gap issues. +const projectRepoSlug = "mevdschee/github-export" + +func runServe(args []string) { + fs := flag.NewFlagSet("serve", flag.ExitOnError) + addr := fs.String("addr", "localhost:8080", "address to bind (localhost only; no client auth)") + dbPath := fs.String("db", defaultDB, "path to the SQLite database to serve") + proxyMode := fs.String("proxy", "on", "proxy fallback to api.github.com for unmirrored reads and writes: on|off") + repoPath := fs.String("repo-path", ".", "local git clone for content endpoints (files/commits/branches); empty to disable") + gitFetch := fs.Bool("git-fetch", false, "git fetch before serving content endpoints") + autoSync := fs.Duration("auto-sync", 0, "if set, run an incremental sync on this interval (e.g. 15m)") + debugCompare := fs.Bool("debug-compare", false, "answer reads locally AND remotely, diff, and log divergences (doubles request cost)") + compareFile := fs.Bool("debug-compare-file", false, "with --debug-compare, file a deduplicated issue on "+projectRepoSlug+" per divergence") + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s serve [flags]\n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Serves the SQLite store over a GitHub-compatible HTTP API. Mirrored reads\n") + fmt.Fprintf(os.Stderr, "are answered locally; unmirrored reads and all writes proxy to GitHub.\n") + fmt.Fprintf(os.Stderr, "Binds localhost and trusts local callers (no client token needed).\n\n") + fmt.Fprintf(os.Stderr, "Flags:\n") + fs.PrintDefaults() + } + fs.Parse(args) + + s, err := store.Open(*dbPath) + if err != nil { + log.Fatalf("opening database %s: %v", *dbPath, err) + } + defer s.Close() + + owner, repo, err := s.OwnerRepo() + if err != nil || owner == "" || repo == "" { + log.Fatalf("database %s has no synced repo; run 'github-export sync' first", *dbPath) + } + syncedAt, _ := s.SyncedAt() + + token := os.Getenv("GITHUB_TOKEN") + var client *github.Client + if token != "" { + client = github.NewClient(token) + } + + proxy := writeproxy.New(token, func(kind string, number int64) error { + if client == nil { + return fmt.Errorf("no GITHUB_TOKEN: cannot re-sync after write") + } + if kind == "issue" && number > 0 { + return sync.ResyncIssue(client, s, owner, repo, number) + } + return nil // collection writes are picked up by the next full sync + }) + if *proxyMode == "off" { + proxy.Disabled = true + } else if token == "" { + log.Println("Warning: GITHUB_TOKEN not set; proxy fallback and writes will fail (mirrored reads still work)") + } + + var git *gitbackend.Backend + if *repoPath != "" { + git = gitbackend.New(*repoPath, owner, repo, *gitFetch) + if !git.Available() { + log.Printf("Note: %s is not a git work tree; content endpoints disabled", *repoPath) + git = nil + } else if *gitFetch { + if err := git.Fetch(); err != nil { + log.Printf("Warning: git fetch: %v", err) + } + } + } + + var comparator *shadow.Comparator + if *debugCompare { + if client == nil { + log.Println("Warning: --debug-compare needs GITHUB_TOKEN for the remote leg; disabling") + } else { + fetch := func(ctx context.Context, path string) (int, []byte, error) { + return proxy.Request(ctx, "GET", path, nil) + } + var fileIssue func(title, body string) error + if *compareFile { + fileIssue = func(title, body string) error { + payload := fmt.Sprintf(`{"title":%q,"body":%q,"labels":["shadow-compare"]}`, title, body) + status, resp, err := proxy.Request(context.Background(), "POST", + "/repos/"+projectRepoSlug+"/issues", strings.NewReader(payload)) + if err != nil { + return err + } + if status >= 400 { + return fmt.Errorf("filing issue: %d %s", status, string(resp)) + } + return nil + } + } + comparator = shadow.New(fetch, fileIssue, projectRepoSlug) + log.Printf("Shadow-compare enabled (file-issues=%v)", *compareFile) + } + } + + q := query.New(s) + srv := httpapi.New(httpapi.Config{ + Query: q, Proxy: proxy, Git: git, Compare: comparator, + GraphQL: graphqlmirror.New(q, owner, repo), + Owner: owner, Repo: repo, SyncedAt: syncedAt, + }) + + if *autoSync > 0 && client != nil { + go autoSyncLoop(client, s, owner, repo, *autoSync) + } + + httpServer := &http.Server{Addr: *addr, Handler: srv.Handler()} + go func() { + log.Printf("Serving %s/%s from %s on http://%s", owner, repo, *dbPath, *addr) + log.Printf(" API docs: http://%s/docs", *addr) + log.Printf(" Status: http://%s/status", *addr) + log.Printf(" Point gh: GH_HOST=%s gh issue list", *addr) + if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("serve: %v", err) + } + }() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + <-stop + log.Println("Shutting down...") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + httpServer.Shutdown(ctx) +} + +// autoSyncLoop runs an incremental sync on a fixed interval. +func autoSyncLoop(client *github.Client, s *store.Store, owner, repo string, every time.Duration) { + ticker := time.NewTicker(every) + defer ticker.Stop() + for range ticker.C { + since, _ := s.SyncedAt() + start := time.Now().UTC().Format(time.RFC3339) + if _, err := sync.Run(client, s, owner, repo, since, start); err != nil { + log.Printf("auto-sync: %v", err) + } else { + log.Printf("auto-sync complete at %s", start) + } + } +}