From 8706a491ee804c86f59429c563d8955f00e04acb Mon Sep 17 00:00:00 2001 From: DamnPatrick Date: Thu, 22 Oct 2026 11:51:10 +0800 Subject: [PATCH 1/2] feat: add --json and --flatten to spark repo scan - --json writes a formatted registry_.json alongside the YAML - --flatten moves repositories nested deeper than two levels (category/repo) up into the two-level layout before scanning; conflicts are skipped with a warning and never overwritten, empty intermediate dirs are removed Refs: #26 --- AGENTS.md | 11 ++- CHANGELOG.md | 7 ++ cmd/repo/scan.go | 34 ++++++++++ docs/en/Agents.md | 2 + docs/en/usage/repo.md | 24 ++++++- docs/zh/Agents.md | 2 + docs/zh/usage/repo.md | 24 ++++++- internal/registry/flatten.go | 105 +++++++++++++++++++++++++++++ internal/registry/registry.go | 19 ++++-- internal/registry/registry_test.go | 89 ++++++++++++++++++++++++ 10 files changed, 309 insertions(+), 8 deletions(-) create mode 100644 internal/registry/flatten.go diff --git a/AGENTS.md b/AGENTS.md index b331e8b..ca1f72a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -559,6 +559,8 @@ spark git scan . --db ~/.innate/feeds.db ```bash spark repo scan [folder-name] # 扫描目录并写入 registry_.yaml +spark repo scan [folder-name] --json # 同时写出格式化 registry_.json +spark repo scan [folder-name] --flatten # 先把超过两层的目录拍平为 category/repo 再扫描 spark repo clone -f registry_.yaml # 克隆 registry 中全部仓库 spark repo clone -r -f registry_.yaml # 仅克隆指定仓库 spark repo list -f registry_.yaml # 列出 registry 中的仓库 @@ -579,7 +581,14 @@ projects: #### `spark repo scan [folder-name]` -递归扫描目录，发现带 `origin` 远程的 Git 仓库，写入当前目录下的 `registry_.yaml`。扫描时跳过隐藏目录与 `node_modules`/`venv`/`.venv`/`__pycache__`/`dist`/`build`；SSH URL 归一化为 HTTPS 形式存储；重复扫描与已有 registry 合并（按 `path` 优先、其次按 `repo` URL 匹配，保留旧 `name`/`desc`，删除目录已不存在的旧条目，追加新发现仓库）。 +递归扫描目录，发现带 `origin` 远程的 Git 仓库，写入当前目录下的 `registry_.yaml`。目录约定为两层：第一层是 category，第二层是正式仓库。扫描时跳过隐藏目录与 `node_modules`/`venv`/`.venv`/`__pycache__`/`dist`/`build`；SSH URL 归一化为 HTTPS 形式存储；重复扫描与已有 registry 合并（按 `path` 优先、其次按 `repo` URL 匹配，保留旧 `name`/`desc`，删除目录已不存在的旧条目，追加新发现仓库）。 + +| 选项 | 说明 | +|------|------| +| `--json` | 在 YAML 之外额外写出格式化（2 空格缩进）的 `registry_.json`，内容一致 | +| `--flatten` | 扫描前把超过两层（`category/repo`）的仓库上移到两层布局（破坏性操作，显式开启） | + +`--flatten` 由 `internal/registry.Flatten` 实现：深层仓库移动到 `/<仓库名>`，目标已存在或本次运行中被占用时跳过并打印 `Skipped ...` 告警（不覆盖任何文件），移动后清理空的中间目录；按深度从深到浅处理，避免父目录先移动导致子路径失效。 #### `spark repo clone` diff --git a/CHANGELOG.md b/CHANGELOG.md index 96d9120..cb7857a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- `spark repo scan --json` — additionally writes a formatted (2-space indent) `registry_.json` with the same content as the YAML registry +- `spark repo scan --flatten` — moves repositories nested deeper than two levels (`category/repo` layout) up into the two-level layout before scanning; conflicts are skipped with a warning and never overwritten, empty intermediate directories are removed (fixes #26) + ## [0.3.3] - 2026-10-20 ### Added diff --git a/cmd/repo/scan.go b/cmd/repo/scan.go index 3c8be9e..b723a9a 100644 --- a/cmd/repo/scan.go +++ b/cmd/repo/scan.go @@ -9,12 +9,23 @@ import ( "github.com/spf13/cobra" ) +var ( + scanJSON bool + scanFlatten bool +) + var scanCmd = &cobra.Command{ Use: "scan [folder-name]", Short: "Scan a folder for git repositories and save to registry_.yaml", Long: `Recursively scan a folder for git repositories that have an origin remote and write them to a registry file named registry_.yaml in the current directory. +The folder is expected to hold a two-level layout: the first level is the category +and the second level holds the actual repositories. With --flatten, repositories +nested deeper than two levels are moved up into the two-level layout (no files are +overwritten; conflicts are reported and left in place) before scanning. With --json, +an additional registry_.json file with the same content is written. + The registry records each repository's name, remote URL and relative path. Re-scanning merges results and preserves existing descriptions.`, Args: cobra.MaximumNArgs(1), @@ -29,6 +40,19 @@ merges results and preserves existing descriptions.`, return fmt.Errorf("resolve folder: %w", err) } + if scanFlatten { + result, err := registry.Flatten(absFolder) + if err != nil { + return err + } + for _, m := range result.Moves { + fmt.Printf("Flattened %s -> %s\n", m.From, m.To) + } + for _, c := range result.Conflicts { + fmt.Printf("Skipped %s -> %s: %s\n", c.From, c.To, c.Reason) + } + } + projects, err := registry.Scan(absFolder) if err != nil { return err @@ -52,10 +76,20 @@ merges results and preserves existing descriptions.`, fmt.Printf(" %s -> %s\n", p.Path, p.Repo) } fmt.Printf("Wrote %d projects to %s\n", len(final), registryFile) + + if scanJSON { + jsonFile := fmt.Sprintf("registry_%s.json", folderName) + if err := registry.WriteJSON(jsonFile, ®istry.Registry{Projects: final}); err != nil { + return err + } + fmt.Printf("Wrote %d projects to %s\n", len(final), jsonFile) + } return nil }, } func init() { + scanCmd.Flags().BoolVar(&scanJSON, "json", false, "additionally write registry_.json") + scanCmd.Flags().BoolVar(&scanFlatten, "flatten", false, "move repositories deeper than two levels up into the category/repo layout before scanning") RepoCmd.AddCommand(scanCmd) } diff --git a/docs/en/Agents.md b/docs/en/Agents.md index 79a1a6b..19246b5 100644 --- a/docs/en/Agents.md +++ b/docs/en/Agents.md @@ -144,6 +144,8 @@ spark repo list -f registry_.yaml # List the repos in the reg | Option | Command | Description | |------|------|------| +| `--json` | `scan` | Additionally write a formatted (2-space indent) `registry_.json` alongside the YAML | +| `--flatten` | `scan` | Move repositories deeper than two levels (`category/repo`) into the two-level layout before scanning; conflicts are skipped, never overwritten | | `-r, --repo` | `clone` | Clone only the named repo (omit to clone all) | | `-f, --file` | `clone` / `list` | Registry file (required) | diff --git a/docs/en/usage/repo.md b/docs/en/usage/repo.md index 38684b5..dc06e05 100644 --- a/docs/en/usage/repo.md +++ b/docs/en/usage/repo.md @@ -6,6 +6,8 @@ Manage multiple GitHub repositories under a directory via a registry file, as an ```bash spark repo scan [folder-name] # Scan a directory and write registry_.yaml +spark repo scan [folder-name] --json # Also write a formatted registry_.json +spark repo scan [folder-name] --flatten # Flatten directories deeper than two levels to category/repo before scanning spark repo list -f registry_.yaml # List repositories in the registry spark repo clone -f registry_.yaml # Clone every repository in the registry spark repo clone -r -f registry_.yaml # Clone only the named repository @@ -20,12 +22,32 @@ Recursively scan a directory, find Git repositories with an `origin` remote, and ```bash spark repo scan # Scan the current directory spark repo scan ~/workspace/innate-apps # Scan a specific directory +spark repo scan ~/workspace/innate-apps --json # Also write registry_innate-apps.json +spark repo scan ~/workspace/innate-apps --flatten # Flatten deep directories before scanning ``` -**Flow**: recursively locate `.git` (including `.git` files) → parse the `origin` remote URL → normalize → merge with the existing registry → write `registry_.yaml`. +| Flag | Default | Description | +|------|---------|-------------| +| `--json` | `false` | Additionally write a formatted (2-space indent) `registry_.json` with the same content as the YAML | +| `--flatten` | `false` | Move repositories nested deeper than two levels (`category/repo`) up into the two-level layout before scanning | + +**Flow**: (optional) `--flatten` deep directories → recursively locate `.git` (including `.git` files) → parse the `origin` remote URL → normalize → merge with the existing registry → write `registry_.yaml` (plus `registry_.json` when `--json` is set). Scanning skips hidden directories and `node_modules`/`venv`/`.venv`/`__pycache__`/`dist`/`build`. SSH URLs are normalized to HTTPS form. Re-scanning preserves existing `name`/`desc` and removes entries whose directory no longer exists. +### Two-level layout and --flatten + +The directory layout follows a two-level convention: the first level is the category, the second level holds the actual repositories (`tooling/spark-cli`, `docs/quick-start`). When three or deeper levels exist, `--flatten` moves deep repositories up to `/`, for example: + +``` +apps/fe/innate/web/innate-fe-cli → apps/innate-fe-cli +tooling/spark/spark-cli → tooling/spark-cli +``` + +- Nothing is overwritten: when the target already exists (or was claimed by another repository in the same run), the repository stays in place and a `Skipped ...` warning is printed. +- Empty intermediate directories are removed after a move. +- `--flatten` modifies the directory structure on disk and is destructive; confirm before using it. + --- ## spark repo clone diff --git a/docs/zh/Agents.md b/docs/zh/Agents.md index a193792..6812974 100644 --- a/docs/zh/Agents.md +++ b/docs/zh/Agents.md @@ -143,6 +143,8 @@ spark repo list -f registry_.yaml # 列出 registry 中的仓 | 选项 | 命令 | 说明 | |------|------|------| +| `--json` | `scan` | 在 YAML 之外额外写出格式化（2 空格缩进）的 `registry_.json` | +| `--flatten` | `scan` | 扫描前把超过两层（`category/repo`）的仓库上移到两层布局，冲突跳过不覆盖 | | `-r, --repo` | `clone` | 仅克隆指定名称的仓库（省略则全部） | | `-f, --file` | `clone` / `list` | registry 文件（必填） | diff --git a/docs/zh/usage/repo.md b/docs/zh/usage/repo.md index 13cb948..f17c705 100644 --- a/docs/zh/usage/repo.md +++ b/docs/zh/usage/repo.md @@ -6,6 +6,8 @@ ```bash spark repo scan [folder-name] # 扫描目录并写入 registry_.yaml +spark repo scan [folder-name] --json # 同时写出格式化 registry_.json +spark repo scan [folder-name] --flatten # 先把超过两层的目录拍平为 category/repo 再扫描 spark repo list -f registry_.yaml # 列出 registry 中的仓库 spark repo clone -f registry_.yaml # 克隆 registry 中全部仓库 spark repo clone -r -f registry_.yaml # 仅克隆指定仓库 @@ -20,12 +22,32 @@ spark repo clone -r -f registry_.yaml # 仅克隆指定仓库 ```bash spark repo scan # 扫描当前目录 spark repo scan ~/workspace/innate-apps # 扫描指定目录 +spark repo scan ~/workspace/innate-apps --json # 同时写出 registry_innate-apps.json +spark repo scan ~/workspace/innate-apps --flatten # 先拍平深层目录再扫描 ``` -**流程**：递归查找 `.git`（含 `.git` 文件）→ 解析 `origin` 远程 URL → 归一化 → 与已有 registry 合并 → 写入 `registry_.yaml` +| 标志 | 默认值 | 说明 | +|------|--------|------| +| `--json` | `false` | 在 YAML 之外额外写出格式化（2 空格缩进）的 `registry_.json`，内容与 YAML 一致 | +| `--flatten` | `false` | 扫描前把超过两层（`category/repo`）的仓库上移到两层布局 | + +**流程**：（可选）`--flatten` 拍平深层目录 → 递归查找 `.git`（含 `.git` 文件）→ 解析 `origin` 远程 URL → 归一化 → 与已有 registry 合并 → 写入 `registry_.yaml`（及 `--json` 时的 `registry_.json`） 扫描时会跳过隐藏目录以及 `node_modules`/`venv`/`.venv`/`__pycache__`/`dist`/`build`。SSH URL 会归一化为 HTTPS 形式存储。重复扫描会保留已有的 `name`/`desc`，并清理目录已不存在的旧条目。 +### 两层目录布局与 --flatten + +目录约定为两层：第一层是 category，第二层是正式仓库（`tooling/spark-cli`、`docs/quick-start`）。当存在三层或更深目录时，`--flatten` 会把深层仓库移动到 `/<仓库名>`，例如： + +``` +apps/fe/innate/web/innate-fe-cli → apps/innate-fe-cli +tooling/spark/spark-cli → tooling/spark-cli +``` + +- 不覆盖任何文件：目标位置已存在（或本次运行中已被其他仓库占用）时，该仓库保持原位并在输出中打印 `Skipped ...` 告警。 +- 移动后自动清理空的中间目录。 +- `--flatten` 会直接修改磁盘上的目录结构，属于破坏性操作，请确认后再使用。 + --- ## spark repo clone diff --git a/internal/registry/flatten.go b/internal/registry/flatten.go new file mode 100644 index 0000000..56f90c4 --- /dev/null +++ b/internal/registry/flatten.go @@ -0,0 +1,105 @@ +package registry + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +type FlattenMove struct { + From string + To string +} + +type FlattenConflict struct { + From string + To string + Reason string +} + +type FlattenResult struct { + Moves []FlattenMove + Conflicts []FlattenConflict +} + +// Flatten moves repositories nested deeper than two levels (category/repo) +// up into the two-level layout. Depth-1 and depth-2 repositories are left +// untouched. When the target path already exists, the repository stays in +// place and the conflict is reported instead of overwriting anything. +func Flatten(root string) (*FlattenResult, error) { + absRoot, err := filepath.Abs(root) + if err != nil { + return nil, err + } + + projects, err := Scan(absRoot) + if err != nil { + return nil, err + } + + sort.SliceStable(projects, func(i, j int) bool { + return depth(projects[i].Path) > depth(projects[j].Path) + }) + + result := &FlattenResult{} + claimed := map[string]bool{} + + for _, p := range projects { + if depth(p.Path) <= 2 { + continue + } + + rel := filepath.FromSlash(p.Path) + targetRel := filepath.Join(strings.Split(p.Path, "/")[0], filepath.Base(rel)) + target := filepath.Join(absRoot, targetRel) + + if _, err := os.Stat(target); err == nil { + result.Conflicts = append(result.Conflicts, FlattenConflict{ + From: p.Path, + To: filepath.ToSlash(targetRel), + Reason: "target already exists", + }) + continue + } + if claimed[filepath.ToSlash(targetRel)] { + result.Conflicts = append(result.Conflicts, FlattenConflict{ + From: p.Path, + To: filepath.ToSlash(targetRel), + Reason: "target claimed by another repository in this run", + }) + continue + } + + src := filepath.Join(absRoot, rel) + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return nil, fmt.Errorf("flatten %s: %w", p.Path, err) + } + if err := os.Rename(src, target); err != nil { + return nil, fmt.Errorf("flatten %s: %w", p.Path, err) + } + claimed[filepath.ToSlash(targetRel)] = true + result.Moves = append(result.Moves, FlattenMove{ + From: p.Path, + To: filepath.ToSlash(targetRel), + }) + + removeEmptyParents(absRoot, filepath.Dir(src)) + } + + return result, nil +} + +func depth(rel string) int { + return strings.Count(rel, "/") + 1 +} + +func removeEmptyParents(root, dir string) { + for dir != root && strings.HasPrefix(dir, root) { + if err := os.Remove(dir); err != nil { + return + } + dir = filepath.Dir(dir) + } +} diff --git a/internal/registry/registry.go b/internal/registry/registry.go index 155dac2..5167208 100644 --- a/internal/registry/registry.go +++ b/internal/registry/registry.go @@ -1,6 +1,7 @@ package registry import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -10,14 +11,14 @@ import ( ) type Project struct { - Name string `yaml:"name"` - Repo string `yaml:"repo"` - Path string `yaml:"path,omitempty"` - Desc string `yaml:"desc,omitempty"` + Name string `yaml:"name" json:"name"` + Repo string `yaml:"repo" json:"repo"` + Path string `yaml:"path,omitempty" json:"path,omitempty"` + Desc string `yaml:"desc,omitempty" json:"desc,omitempty"` } type Registry struct { - Projects []Project `yaml:"projects"` + Projects []Project `yaml:"projects" json:"projects"` } func Read(path string) (*Registry, error) { @@ -46,6 +47,14 @@ func Write(path string, reg *Registry) error { return os.WriteFile(path, append([]byte(header), data...), 0644) } +func WriteJSON(path string, reg *Registry) error { + data, err := json.MarshalIndent(reg, "", " ") + if err != nil { + return fmt.Errorf("encode registry: %w", err) + } + return os.WriteFile(path, append(data, '\n'), 0644) +} + func FindByName(reg *Registry, name string) (*Project, error) { for i := range reg.Projects { if reg.Projects[i].Name == name { diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go index 968e04e..4d79b8a 100644 --- a/internal/registry/registry_test.go +++ b/internal/registry/registry_test.go @@ -1,6 +1,7 @@ package registry_test import ( + "encoding/json" "os" "path/filepath" @@ -150,6 +151,94 @@ var _ = Describe("Registry", func() { }) }) + Describe("WriteJSON", func() { + It("writes an indented json file that round-trips", func() { + path := filepath.Join(tempDir, "registry_demo.json") + reg := ®istry.Registry{Projects: []registry.Project{ + {Name: "spark-cli", Repo: "https://github.com/variableway/spark-cli.git", Path: "tooling/spark-cli", Desc: "Spark CLI"}, + }} + + Expect(registry.WriteJSON(path, reg)).To(Succeed()) + + data, err := os.ReadFile(path) + Expect(err).ToNot(HaveOccurred()) + Expect(string(data)).To(ContainSubstring("\n \"projects\"")) + Expect(string(data)).To(ContainSubstring("spark-cli")) + + var got registry.Registry + Expect(json.Unmarshal(data, &got)).To(Succeed()) + Expect(got.Projects).To(Equal(reg.Projects)) + }) + }) + + Describe("Flatten", func() { + It("moves a level-3 repository up into the two-level layout", func() { + writeGitRepo(filepath.Join(tempDir, "tooling", "spark"), "spark-cli", "https://github.com/variableway/spark-cli.git") + + result, err := registry.Flatten(tempDir) + Expect(err).ToNot(HaveOccurred()) + Expect(result.Moves).To(Equal([]registry.FlattenMove{{From: "tooling/spark/spark-cli", To: "tooling/spark-cli"}})) + Expect(result.Conflicts).To(BeEmpty()) + + _, err = os.Stat(filepath.Join(tempDir, "tooling", "spark-cli", ".git")) + Expect(err).ToNot(HaveOccurred()) + _, err = os.Stat(filepath.Join(tempDir, "tooling", "spark")) + Expect(os.IsNotExist(err)).To(BeTrue()) + + projects, err := registry.Scan(tempDir) + Expect(err).ToNot(HaveOccurred()) + Expect(projects).To(HaveLen(1)) + Expect(projects[0].Path).To(Equal("tooling/spark-cli")) + }) + + It("flattens deeper nesting to two levels", func() { + writeGitRepo(filepath.Join(tempDir, "apps", "fe", "innate", "web"), "innate-fe-cli", "https://github.com/variableway/innate-fe-cli.git") + + result, err := registry.Flatten(tempDir) + Expect(err).ToNot(HaveOccurred()) + Expect(result.Moves).To(Equal([]registry.FlattenMove{{From: "apps/fe/innate/web/innate-fe-cli", To: "apps/innate-fe-cli"}})) + + projects, err := registry.Scan(tempDir) + Expect(err).ToNot(HaveOccurred()) + Expect(projects).To(HaveLen(1)) + Expect(projects[0].Path).To(Equal("apps/innate-fe-cli")) + }) + + It("leaves two-level repositories untouched", func() { + writeGitRepo(filepath.Join(tempDir, "tooling"), "spark-cli", "https://github.com/variableway/spark-cli.git") + + result, err := registry.Flatten(tempDir) + Expect(err).ToNot(HaveOccurred()) + Expect(result.Moves).To(BeEmpty()) + Expect(result.Conflicts).To(BeEmpty()) + }) + + It("reports a conflict instead of overwriting an existing target", func() { + writeGitRepo(filepath.Join(tempDir, "tooling", "old"), "spark-cli", "https://github.com/variableway/spark-cli.git") + writeGitRepo(filepath.Join(tempDir, "tooling"), "spark-cli", "https://github.com/other/spark-cli.git") + + result, err := registry.Flatten(tempDir) + Expect(err).ToNot(HaveOccurred()) + Expect(result.Moves).To(BeEmpty()) + Expect(result.Conflicts).To(HaveLen(1)) + Expect(result.Conflicts[0].From).To(Equal("tooling/old/spark-cli")) + Expect(result.Conflicts[0].To).To(Equal("tooling/spark-cli")) + + _, err = os.Stat(filepath.Join(tempDir, "tooling", "old", "spark-cli", ".git")) + Expect(err).ToNot(HaveOccurred()) + }) + + It("reports a conflict when two repositories flatten to the same target", func() { + writeGitRepo(filepath.Join(tempDir, "tooling", "a"), "spark-cli", "https://github.com/a/spark-cli.git") + writeGitRepo(filepath.Join(tempDir, "tooling", "b"), "spark-cli", "https://github.com/b/spark-cli.git") + + result, err := registry.Flatten(tempDir) + Expect(err).ToNot(HaveOccurred()) + Expect(result.Moves).To(HaveLen(1)) + Expect(result.Conflicts).To(HaveLen(1)) + }) + }) + Describe("FindByName", func() { It("finds a project by name", func() { reg := ®istry.Registry{Projects: []registry.Project{ From d8f22947cd80724bf4aa26940e49c158850f22c5 Mon Sep 17 00:00:00 2001 From: DamnPatrick Date: Thu, 12 Nov 2026 14:23:20 +0800 Subject: [PATCH 2/2] =?UTF-8?q?feat(repo):=20=E5=88=86=E7=B1=BB=20registry?= =?UTF-8?q?=20(version=202)=E2=80=94=E2=80=94scan/list/clone=20=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20category/subcategory/others?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit spark repo scan 扫描三分层目录 <分类>/<子分类>/<项目> 时输出分类格式： - internal/registry: 新增 Subcategory / Category / Registry.Version(=2) 与 Categorize()——分类取一级子目录（git 仓目录不会被误当分类/子分类）； 无子分类的项目归 others 且每个分类恒有 others 桶；根级仓库跳过并告警； 无一级分类目录的目录回退旧扁平格式。Read 为旧调用者派生扁平视图， Write / WriteJSON 按格式分别序列化，FindByName 走 AllProjects - cmd/repo/scan: 合并后归组入库，按 分类/子分类 分组打印；全部项目都没有 分类时写旧扁平 registry 并打印提示；--json 同步输出 v2 - cmd/repo/list: 分类格式打印 `# cat / sub (n)` 分组头（行格式 name/repo/path 不变） - cmd/repo/clone: 按 path 深度从浅到深克隆（先父项目，再其内部嵌套仓） - 测试: internal/registry 新增 Categorize / v2 往返 / JSON / 旧格式兼容用例， 该包全过；docs zh+en/usage/repo、AGENTS.md、CHANGELOG 同步 - --flatten 保持旧两层语义，help 与 docs 明确标注禁止用于三分层分类树 验证：go vet ./... 通过；go test ./... 中 internal/registry、cmd/repo 全过 （整仓 328 passed / 2 failed 均为本机环境相关既有失败：flushDNS 需要提权； skill.Discover 扫到本机 builtin_skills 得到 58 个技能，与用例 len=1 的环境假设 冲突）。新二进制已安装 ~/.local/bin，并在 skills 仓真实扫出 37 仓的 v2 registry。 --- AGENTS.md | 43 ++++- CHANGELOG.md | 1 + cmd/repo/clone.go | 14 +- cmd/repo/list.go | 15 ++ cmd/repo/repo.go | 5 +- cmd/repo/scan.go | 71 ++++++-- docs/en/Agents.md | 12 +- docs/en/usage/repo.md | 79 +++++++-- docs/zh/Agents.md | 12 +- docs/zh/usage/repo.md | 79 +++++++-- internal/registry/category.go | 143 ++++++++++++++++ internal/registry/category_test.go | 255 +++++++++++++++++++++++++++++ internal/registry/registry.go | 99 ++++++++++- 13 files changed, 764 insertions(+), 64 deletions(-) create mode 100644 internal/registry/category.go create mode 100644 internal/registry/category_test.go diff --git a/AGENTS.md b/AGENTS.md index e130173..428a5ca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,7 @@ 11. **系统工具** — `spark magic` 提供 DNS 缓存刷新、`node_modules`/`.venv` 清理、pip/npm/go 镜像源切换、Neovim/Ghostty 模板部署 12. **文档管理** — `spark docs init`/`spark docs site`（docmd 站点初始化） 13. **进程诊断** — `spark witr`（Why Is This Running），检查进程或端口为何在运行 -14. **仓库管理** — `spark repo` 通过 registry 文件管理目录下的多个 GitHub 仓库（`scan`/`clone`/`list`），替代 submodule +14. **仓库管理** — `spark repo` 通过 registry 文件管理目录下的多个 GitHub 仓库（`scan`/`clone`/`list`），替代 submodule；`scan` 输出分类格式（version 2：`分类/子分类/others`），无分类目录的目录自动回退旧扁平格式 15. **Skill 管理** — `spark skill` 发现并同步 AI agent skills（`SKILL.md` 目录），支持 `list`/`sync` 16. **项目 Hub** — `spark hub scan` 扫描本地 Git 仓库并生成 innate-apps registry（`registry/apps.json`）注册信息 @@ -148,7 +148,7 @@ spark │ ├── push-all (-p) │ └── scan ([folder-path], -d, --db, --skip-api) ├── repo -│ ├── scan ([folder-name]) +│ ├── scan ([folder-name], --json, --flatten) │ ├── clone (-r, --repo, -f, --file) │ └── list (-f, --file) ├── script @@ -574,15 +574,42 @@ spark git scan . --db ~/.innate/feeds.db 父命令 `RepoCmd` 在 `cmd/repo/repo.go` 中定义，通过 registry 文件管理目录下的多个 GitHub 仓库（替代 submodule）。 ```bash -spark repo scan [folder-name] # 扫描目录并写入 registry_.yaml +spark repo scan [folder-name] # 扫描分类目录并写入 registry_.yaml spark repo scan [folder-name] --json # 同时写出格式化 registry_.json -spark repo scan [folder-name] --flatten # 先把超过两层的目录拍平为 category/repo 再扫描 -spark repo clone -f registry_.yaml # 克隆 registry 中全部仓库 +spark repo scan [folder-name] --flatten # （仅旧两层布局）拍平为 category/repo 再扫描 +spark repo clone -f registry_.yaml # 克隆 registry 中全部仓库（按 path 还原） spark repo clone -r -f registry_.yaml # 仅克隆指定仓库 -spark repo list -f registry_.yaml # 列出 registry 中的仓库 +spark repo list -f registry_.yaml # 列出 registry 中的仓库（分类格式按分组输出） ``` -registry 文件格式： +registry 文件格式（分类版，三分层目录扫描的默认输出；无一级分类目录时仍是旧扁平 `projects:` 格式）： + +```yaml +# Project registry +# Synced by spark repo scan + +version: 2 +categories: + - name: skills + subcategories: + - name: agent + projects: + - name: taste-skill + repo: https://github.com/Leonxlnx/taste-skill.git + path: skills/agent/taste-skill + - name: others + projects: [] + - name: products + subcategories: + - name: others + projects: [] + - name: plugins + subcategories: + - name: others + projects: [] +``` + +旧扁平格式： ```yaml # Project registry @@ -597,7 +624,7 @@ projects: #### `spark repo scan [folder-name]` -递归扫描目录，发现带 `origin` 远程的 Git 仓库，写入当前目录下的 `registry_.yaml`。目录约定为两层：第一层是 category，第二层是正式仓库。扫描时跳过隐藏目录与 `node_modules`/`venv`/`.venv`/`__pycache__`/`dist`/`build`；SSH URL 归一化为 HTTPS 形式存储；重复扫描与已有 registry 合并（按 `path` 优先、其次按 `repo` URL 匹配，保留旧 `name`/`desc`，删除目录已不存在的旧条目，追加新发现仓库）。 +递归扫描目录，发现带 `origin` 远程的 Git 仓库，写入当前目录下的 `registry_.yaml`。目录约定为**三层分类**：`<分类>/<子分类>/<项目>`——分类取自一级子目录（即使为空也入册），每个分类恒有 `others` 子分类，直接放在分类下的项目（无子分类）归入 `others`，项目内部更深的嵌套仓保全完整 path 但仍归入其前两段的 `分类/子分类`，直接放在扫描根目录的仓库打印 `WARNING` 且不入册（需先移入分类）；没有任何一级分类目录时回退旧扁平格式。扫描时跳过隐藏目录与 `node_modules`/`venv`/`.venv`/`__pycache__`/`dist`/`build`；SSH URL 归一化为 HTTPS 形式存储；重复扫描与已有 registry 合并（按 `path` 优先、其次按 `repo` URL 匹配，保留旧 `name`/`desc`，删除目录已不存在的旧条目，追加新发现仓库）。`--flatten` 只服务于旧两层布局——**禁止**在三分层分类树上使用（会把项目移出子分类）。 | 选项 | 说明 | |------|------| diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d89005..6fc0de5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `spark repo scan` — categorized registry format (version 2): groups repositories as `//` from the folder's own directory layout, always emits an `others` subcategory per category (projects directly under a category land there), warns and skips repositories that sit directly in the folder root, and falls back to the legacy flat `projects:` format when the folder has no category directories; `list` prints `# category / subcategory (n)` group headers for categorized files, `clone` restores each project to its recorded `path` shallowest-path-first - `spark repo scan --json` — additionally writes a formatted (2-space indent) `registry_.json` with the same content as the YAML registry - `spark repo scan --flatten` — moves repositories nested deeper than two levels (`category/repo` layout) up into the two-level layout before scanning; conflicts are skipped with a warning and never overwritten, empty intermediate directories are removed (fixes #26) - `spark skill list [--json]` — Discover agent skills (`SKILL.md` directories) across project (`.mimocode/`, `.agents/`), global (`~/.claude/`, `~/.agents/`, `~/.mimocode/`), and builtin skill-pack locations, deduplicated by name with version-aware builtin preference (`internal/skill`) diff --git a/cmd/repo/clone.go b/cmd/repo/clone.go index 25f9597..c08fc0d 100644 --- a/cmd/repo/clone.go +++ b/cmd/repo/clone.go @@ -5,6 +5,7 @@ import ( "os" "os/exec" "path/filepath" + "sort" "strings" "spark/internal/registry" @@ -37,7 +38,7 @@ otherwise all repositories in the registry are cloned.`, return err } - projects := reg.Projects + projects := reg.AllProjects() if cloneRepoName != "" { p, err := registry.FindByName(reg, cloneRepoName) if err != nil { @@ -51,6 +52,17 @@ otherwise all repositories in the registry are cloned.`, return nil } + // Clone shallow paths first so a parent project (and any nested clone + // recorded under its path) is created in the right order. + sort.SliceStable(projects, func(i, j int) bool { + di := strings.Count(projects[i].Path, "/") + dj := strings.Count(projects[j].Path, "/") + if di != dj { + return di < dj + } + return projects[i].Path < projects[j].Path + }) + root := folderFromRegistryFile(cloneFile) cloned, skipped, failed := 0, 0, 0 for _, p := range projects { diff --git a/cmd/repo/list.go b/cmd/repo/list.go index 7b0c1a4..0fdace0 100644 --- a/cmd/repo/list.go +++ b/cmd/repo/list.go @@ -25,6 +25,21 @@ var listCmd = &cobra.Command{ return err } + if reg.IsCategorized() { + // Categorized (v2) registry: print one "# category / subcategory (n)" + // header per bucket, then the same namerepopath rows as the + // legacy format (rows stay machine-readable, headers start with '#'). + for _, c := range reg.Categories { + for _, s := range c.Subcategories { + fmt.Printf("# %s / %s (%d)\n", c.Name, s.Name, len(s.Projects)) + for _, p := range s.Projects { + fmt.Printf("%s\t%s\t%s\n", p.Name, p.Repo, p.Path) + } + } + } + return nil + } + if len(reg.Projects) == 0 { fmt.Println("No repositories in registry.") return nil diff --git a/cmd/repo/repo.go b/cmd/repo/repo.go index b006b42..90577f9 100644 --- a/cmd/repo/repo.go +++ b/cmd/repo/repo.go @@ -12,8 +12,11 @@ git submodules. Commands: - scan: scan a folder and write its repositories to registry_.yaml + (categorized format: //, "others" + as the default subcategory; folders without categories stay flat) - clone: clone repositories from a registry file into the folder directory -- list: list repositories declared in a registry file`, + (categorized registries restore each project to its recorded path) +- list: list repositories declared in a registry file (grouped when categorized)`, } func init() { diff --git a/cmd/repo/scan.go b/cmd/repo/scan.go index b723a9a..244de58 100644 --- a/cmd/repo/scan.go +++ b/cmd/repo/scan.go @@ -20,11 +20,22 @@ var scanCmd = &cobra.Command{ Long: `Recursively scan a folder for git repositories that have an origin remote and write them to a registry file named registry_.yaml in the current directory. -The folder is expected to hold a two-level layout: the first level is the category -and the second level holds the actual repositories. With --flatten, repositories -nested deeper than two levels are moved up into the two-level layout (no files are -overwritten; conflicts are reported and left in place) before scanning. With --json, -an additional registry_.json file with the same content is written. +The folder is expected to hold the categorized layout: + + // e.g. skills/agent/taste-skill + +The registry is written in the categorized format (version: 2): categories are the +folder's immediate subdirectories (so empty categories still appear), every category +always contains an "others" subcategory, and a project directly under a category +(/, i.e. without a subcategory) is recorded under "others". +Folders without category directories keep the legacy flat "projects" format. + +With --json, an additional registry_.json file with the same content is +written. With --flatten, repositories nested deeper than two levels are moved up +into the LEGACY two-level / layout first (no files are overwritten; +conflicts are reported and left in place) — do NOT pass --flatten on a categorized +// tree, it would move projects out of their +subcategory. The registry records each repository's name, remote URL and relative path. Re-scanning merges results and preserves existing descriptions.`, @@ -66,23 +77,59 @@ merges results and preserves existing descriptions.`, return err } - final := registry.Merge(existing.Projects, projects) - if err := registry.Write(registryFile, ®istry.Registry{Projects: final}); err != nil { + final := registry.Merge(existing.AllProjects(), projects) + + groups, skipped, err := registry.Categorize(absFolder, final) + if err != nil { + return err + } + // Categorized output only once at least one project actually has a + // category (or the categorized tree holds no projects at all). A folder + // whose projects all sit in the root keeps the legacy flat format so + // nothing silently disappears from an existing registry. + classified := len(final) - len(skipped) + useCategories := len(groups) > 0 && (len(final) == 0 || classified > 0) + reg := ®istry.Registry{Projects: final} + if useCategories { + reg = ®istry.Registry{Categories: groups} + } + if err := registry.Write(registryFile, reg); err != nil { return err } fmt.Printf("Scanned %s: found %d repositories\n", absFolder, len(projects)) - for _, p := range projects { - fmt.Printf(" %s -> %s\n", p.Path, p.Repo) + if reg.IsCategorized() { + for _, c := range reg.Categories { + for _, s := range c.Subcategories { + if len(s.Projects) == 0 { + fmt.Printf("%s / %s (0)\n", c.Name, s.Name) + continue + } + fmt.Printf("%s / %s\n", c.Name, s.Name) + for _, p := range s.Projects { + fmt.Printf(" %s -> %s\n", p.Path, p.Repo) + } + } + } + for _, p := range skipped { + fmt.Printf("WARNING: %s sits directly in the folder root and has no category — move it into // (not written to the registry)\n", p.Path) + } + } else { + if len(groups) > 0 && len(final) > 0 { + fmt.Println("NOTE: no project has a category — wrote the legacy flat registry; move projects into // to get the categorized format") + } + for _, p := range projects { + fmt.Printf(" %s -> %s\n", p.Path, p.Repo) + } } - fmt.Printf("Wrote %d projects to %s\n", len(final), registryFile) + fmt.Printf("Wrote %d projects to %s\n", len(reg.AllProjects()), registryFile) if scanJSON { jsonFile := fmt.Sprintf("registry_%s.json", folderName) - if err := registry.WriteJSON(jsonFile, ®istry.Registry{Projects: final}); err != nil { + if err := registry.WriteJSON(jsonFile, reg); err != nil { return err } - fmt.Printf("Wrote %d projects to %s\n", len(final), jsonFile) + fmt.Printf("Wrote %d projects to %s\n", len(reg.AllProjects()), jsonFile) } return nil }, diff --git a/docs/en/Agents.md b/docs/en/Agents.md index c60272d..f663073 100644 --- a/docs/en/Agents.md +++ b/docs/en/Agents.md @@ -22,7 +22,7 @@ Core capabilities: 11. **System utilities** — `spark magic` provides DNS cache flushing, `node_modules`/`.venv` cleanup, pip/npm/go mirror switching, and Neovim/Ghostty template deployment 12. **Docs management** — `spark docs init`/`spark docs site` (docmd site initialization) 13. **Process diagnostics** — `spark witr` (Why Is This Running), inspecting why a process or port is running -14. **Repo management** — `spark repo` manages multiple GitHub repositories in a directory via a registry file (`scan`/`clone`/`list`), replacing submodules +14. **Repo management** — `spark repo` manages multiple GitHub repositories in a directory via a registry file (`scan`/`clone`/`list`), replacing submodules; `scan` emits the categorized format (version 2: `category/subcategory/others`) and falls back to the legacy flat format for directories without category subdirectories 15. **Skill management** — `spark skill` discovers and syncs AI agent skills (`SKILL.md` directories), with `list`/`sync` 16. **Project hub** — `spark hub scan` scans local git repositories and produces innate-apps registry (`registry/apps.json`) registration info @@ -139,19 +139,19 @@ GitLab implementation notes: ### `spark repo` — Repository management -Manages multiple GitHub repositories in a directory via a registry file (replacing submodules): +Manages multiple GitHub repositories in a directory via a registry file (replacing submodules). Scanning a three-level tree (`//`) emits the categorized format (version 2, always with an `others` subcategory); directories without category subdirectories keep the legacy flat `projects:` format: ```bash -spark repo scan [folder-name] # Scan the directory and write registry_.yaml -spark repo clone -f registry_.yaml # Clone every repo in the registry +spark repo scan [folder-name] # Scan a categorized directory and write registry_.yaml +spark repo clone -f registry_.yaml # Clone every repo in the registry (to its recorded path) spark repo clone -r -f registry_.yaml # Clone only the specified repo -spark repo list -f registry_.yaml # List the repos in the registry +spark repo list -f registry_.yaml # List the repos in the registry (grouped when categorized) ``` | Option | Command | Description | |------|------|------| | `--json` | `scan` | Additionally write a formatted (2-space indent) `registry_.json` alongside the YAML | -| `--flatten` | `scan` | Move repositories deeper than two levels (`category/repo`) into the two-level layout before scanning; conflicts are skipped, never overwritten | +| `--flatten` | `scan` | Legacy two-level layout only: move repositories deeper than two levels (`category/repo`) up before scanning; conflicts are skipped, never overwritten. Do not use it on a `category/subcategory/project` tree (it would move projects out of their subcategory) | | `-r, --repo` | `clone` | Clone only the named repo (omit to clone all) | | `-f, --file` | `clone` / `list` | Registry file (required) | diff --git a/docs/en/usage/repo.md b/docs/en/usage/repo.md index dc06e05..213580e 100644 --- a/docs/en/usage/repo.md +++ b/docs/en/usage/repo.md @@ -5,11 +5,11 @@ Manage multiple GitHub repositories under a directory via a registry file, as an ## Quick Reference ```bash -spark repo scan [folder-name] # Scan a directory and write registry_.yaml +spark repo scan [folder-name] # Scan a categorized directory and write registry_.yaml spark repo scan [folder-name] --json # Also write a formatted registry_.json -spark repo scan [folder-name] --flatten # Flatten directories deeper than two levels to category/repo before scanning -spark repo list -f registry_.yaml # List repositories in the registry -spark repo clone -f registry_.yaml # Clone every repository in the registry +spark repo scan [folder-name] --flatten # (legacy two-level layout only) flatten to category/repo before scanning +spark repo list -f registry_.yaml # List repositories (grouped output for categorized registries) +spark repo clone -f registry_.yaml # Clone every repository to its recorded path spark repo clone -r -f registry_.yaml # Clone only the named repository ``` @@ -17,33 +17,56 @@ spark repo clone -r -f registry_.yaml # Clone only the named rep ## spark repo scan -Recursively scan a directory, find Git repositories with an `origin` remote, and write them to `registry_.yaml` in the current directory. +Recursively scan a directory, find Git repositories with an `origin` remote, and write them to `registry_.yaml` in the current directory (categorized format). ```bash spark repo scan # Scan the current directory -spark repo scan ~/workspace/innate-apps # Scan a specific directory +spark repo scan ~/workspace/innate-references # Scan a specific directory (three-level layout) spark repo scan ~/workspace/innate-apps --json # Also write registry_innate-apps.json -spark repo scan ~/workspace/innate-apps --flatten # Flatten deep directories before scanning +spark repo scan ~/workspace/legacy-refs --flatten # Legacy two-level layout: flatten first ``` | Flag | Default | Description | |------|---------|-------------| | `--json` | `false` | Additionally write a formatted (2-space indent) `registry_.json` with the same content as the YAML | -| `--flatten` | `false` | Move repositories nested deeper than two levels (`category/repo`) up into the two-level layout before scanning | +| `--flatten` | `false` | Move repositories nested deeper than two levels (`category/repo`) up into the **legacy** two-level layout before scanning; do not use it on a `category/subcategory/project` tree | -**Flow**: (optional) `--flatten` deep directories → recursively locate `.git` (including `.git` files) → parse the `origin` remote URL → normalize → merge with the existing registry → write `registry_.yaml` (plus `registry_.json` when `--json` is set). +**Flow**: (optional) `--flatten` deep directories → recursively locate `.git` (including `.git` files) → parse the `origin` remote URL → normalize → merge with the existing registry → group by category (`Categorize`) → write `registry_.yaml` (plus `registry_.json` when `--json` is set). Scanning skips hidden directories and `node_modules`/`venv`/`.venv`/`__pycache__`/`dist`/`build`. SSH URLs are normalized to HTTPS form. Re-scanning preserves existing `name`/`desc` and removes entries whose directory no longer exists. -### Two-level layout and --flatten +### Three-level categorized layout (registry v2) -The directory layout follows a two-level convention: the first level is the category, the second level holds the actual repositories (`tooling/spark-cli`, `docs/quick-start`). When three or deeper levels exist, `--flatten` moves deep repositories up to `/`, for example: +The directory layout follows a three-level convention: `//`. + +``` +references/ +├── skills/ # level 1 = category (directory name becomes the registry category) +│ ├── agent/ # level 2 = subcategory +│ │ └── taste-skill/ # level 3 = project (a git repo with an origin remote) +│ └── others/ # projects without a subcategory go to others (always present) +├── products/ +│ └── others/ +└── plugins/ + └── others/ +``` + +- **Categories** are the folder's immediate subdirectories (they appear even when they hold no projects); **every category always contains an `others` subcategory**. +- A project directly under a category (`skills/some-project`) has no subcategory and is recorded under that category's `others` (move it on disk to `skills/others/some-project`). +- Repositories nested deeper inside a project (e.g. `fire-skills/downloads/*`) keep their full path but are grouped by their first two segments. +- A repository directly under the scan root has no category: a `WARNING` is printed and it is not written to the registry (move it into a category first). +- A directory **without any category subdirectories** automatically falls back to the legacy flat `projects:` format, exactly as before. + +### `--flatten` is for the legacy two-level layout only + +The legacy convention is two levels: the first is the category, the second holds the repositories (`tooling/spark-cli`). When three or deeper levels exist, `--flatten` moves deep repositories up to `/`, for example: ``` apps/fe/innate/web/innate-fe-cli → apps/innate-fe-cli tooling/spark/spark-cli → tooling/spark-cli ``` +- **Do not run `--flatten` on a three-level categorized tree**: it would move `skills/agent/taste-skill` up to `skills/taste-skill`, destroying the subcategory. The categorized layout is grouped by `Categorize` and moves nothing. - Nothing is overwritten: when the target already exists (or was claimed by another repository in the same run), the repository stays in place and a `Skipped ...` warning is printed. - Empty intermediate directories are removed after a move. - `--flatten` modifies the directory structure on disk and is destructive; confirm before using it. @@ -69,6 +92,8 @@ spark repo clone -r spark-cli -f registry_innate-apps.yaml Repositories that already exist (`.git` present or directory exists) are skipped; a `cloned / skipped / failed` summary is printed at the end. +Destinations restore the `path` recorded in the registry (for a categorized registry that is `//`), cloned **shallowest path first** so a parent project exists before any nested clone recorded under it. + --- ## spark repo list @@ -83,12 +108,42 @@ List the repositories in a registry file. spark repo list -f registry_innate-apps.yaml ``` -Each repository is printed as `namerepopath`. +Each repository is printed as `namerepopath`. For categorized (v2) registries a group header `# / ()` is printed before each bucket (rows stay machine-readable; filter lines starting with `#`). --- ## Registry file +**Categorized format (v2 — the default output when scanning a three-level tree)**: + +```yaml +# Project registry +# Synced by spark repo scan + +version: 2 +categories: + - name: skills + subcategories: + - name: agent + projects: + - name: taste-skill + repo: https://github.com/Leonxlnx/taste-skill.git + path: skills/agent/taste-skill + desc: Frontend taste skill + - name: others + projects: [] + - name: products + subcategories: + - name: others + projects: [] + - name: plugins + subcategories: + - name: others + projects: [] +``` + +**Legacy flat format (still written and read for directories without category subdirectories)**: + ```yaml # Project registry # Synced by spark repo scan diff --git a/docs/zh/Agents.md b/docs/zh/Agents.md index 0b1b0db..dc63d88 100644 --- a/docs/zh/Agents.md +++ b/docs/zh/Agents.md @@ -22,7 +22,7 @@ 11. **系统工具** — `spark magic` 提供 DNS 缓存刷新、`node_modules`/`.venv` 清理、pip/npm/go 镜像源切换、Neovim/Ghostty 模板部署 12. **文档管理** — `spark docs init`/`spark docs site`（docmd 站点初始化） 13. **进程诊断** — `spark witr`（Why Is This Running），检查进程或端口为何在运行 -14. **仓库管理** — `spark repo` 通过 registry 文件管理目录下的多个 GitHub 仓库（`scan`/`clone`/`list`），替代 submodule +14. **仓库管理** — `spark repo` 通过 registry 文件管理目录下的多个 GitHub 仓库（`scan`/`clone`/`list`），替代 submodule；`scan` 输出分类格式（version 2：`分类/子分类/others`），无分类目录的目录自动回退旧扁平格式 15. **Skill 管理** — `spark skill` 发现并同步 AI agent skills（`SKILL.md` 目录），支持 `list`/`sync` 16. **项目 Hub** — `spark hub scan` 扫描本地 Git 仓库并生成 innate-apps registry（`registry/apps.json`）注册信息 @@ -138,19 +138,19 @@ GitLab 实现要点： ### `spark repo` — 仓库管理 -通过 registry 文件管理目录下的多个 GitHub 仓库（替代 submodule）： +通过 registry 文件管理目录下的多个 GitHub 仓库（替代 submodule）。扫描三分层目录（`<分类>/<子分类>/<项目>`）时输出分类格式（version 2，恒含 `others` 子分类）；无一级分类目录时回退旧扁平 `projects:` 格式： ```bash -spark repo scan [folder-name] # 扫描目录写入 registry_.yaml -spark repo clone -f registry_.yaml # 克隆 registry 中全部仓库 +spark repo scan [folder-name] # 扫描分类目录写入 registry_.yaml +spark repo clone -f registry_.yaml # 克隆 registry 中全部仓库（按 path 还原） spark repo clone -r -f registry_.yaml # 仅克隆指定仓库 -spark repo list -f registry_.yaml # 列出 registry 中的仓库 +spark repo list -f registry_.yaml # 列出 registry 中的仓库（分类格式按分组输出） ``` | 选项 | 命令 | 说明 | |------|------|------| | `--json` | `scan` | 在 YAML 之外额外写出格式化（2 空格缩进）的 `registry_.json` | -| `--flatten` | `scan` | 扫描前把超过两层（`category/repo`）的仓库上移到两层布局，冲突跳过不覆盖 | +| `--flatten` | `scan` | 仅旧两层布局：扫描前把超过两层（`category/repo`）的仓库上移到两层布局，冲突跳过不覆盖；三分层分类树禁用（会把项目移出子分类） | | `-r, --repo` | `clone` | 仅克隆指定名称的仓库（省略则全部） | | `-f, --file` | `clone` / `list` | registry 文件（必填） | diff --git a/docs/zh/usage/repo.md b/docs/zh/usage/repo.md index f17c705..3dc35b2 100644 --- a/docs/zh/usage/repo.md +++ b/docs/zh/usage/repo.md @@ -5,11 +5,11 @@ ## 命令速查 ```bash -spark repo scan [folder-name] # 扫描目录并写入 registry_.yaml +spark repo scan [folder-name] # 扫描分类目录并写入分类 registry_.yaml spark repo scan [folder-name] --json # 同时写出格式化 registry_.json -spark repo scan [folder-name] --flatten # 先把超过两层的目录拍平为 category/repo 再扫描 -spark repo list -f registry_.yaml # 列出 registry 中的仓库 -spark repo clone -f registry_.yaml # 克隆 registry 中全部仓库 +spark repo scan [folder-name] --flatten # （仅旧两层布局）拍平为 category/repo 再扫描 +spark repo list -f registry_.yaml # 列出 registry 中的仓库（分类格式按分组输出） +spark repo clone -f registry_.yaml # 克隆 registry 中全部仓库（按记录的 path 还原） spark repo clone -r -f registry_.yaml # 仅克隆指定仓库 ``` @@ -17,33 +17,56 @@ spark repo clone -r -f registry_.yaml # 仅克隆指定仓库 ## spark repo scan -递归扫描目录，发现带 `origin` 远程的 Git 仓库，写入当前目录下的 `registry_.yaml`。 +递归扫描目录，发现带 `origin` 远程的 Git 仓库，写入当前目录下的 `registry_.yaml`（分类格式）。 ```bash spark repo scan # 扫描当前目录 -spark repo scan ~/workspace/innate-apps # 扫描指定目录 +spark repo scan ~/workspace/innate-references # 扫描指定目录（三分层布局） spark repo scan ~/workspace/innate-apps --json # 同时写出 registry_innate-apps.json -spark repo scan ~/workspace/innate-apps --flatten # 先拍平深层目录再扫描 +spark repo scan ~/workspace/legacy-refs --flatten # 旧两层布局：先拍平再扫描 ``` | 标志 | 默认值 | 说明 | |------|--------|------| | `--json` | `false` | 在 YAML 之外额外写出格式化（2 空格缩进）的 `registry_.json`，内容与 YAML 一致 | -| `--flatten` | `false` | 扫描前把超过两层（`category/repo`）的仓库上移到两层布局 | +| `--flatten` | `false` | 扫描前把超过两层（`category/repo`）的仓库上移到**旧两层布局**；三分层（`category/subcategory/project`）目录禁止使用 | -**流程**：（可选）`--flatten` 拍平深层目录 → 递归查找 `.git`（含 `.git` 文件）→ 解析 `origin` 远程 URL → 归一化 → 与已有 registry 合并 → 写入 `registry_.yaml`（及 `--json` 时的 `registry_.json`） +**流程**：（可选）`--flatten` 拍平深层目录 → 递归查找 `.git`（含 `.git` 文件）→ 解析 `origin` 远程 URL → 归一化 → 与已有 registry 合并 → 按分类目录归组（`Categorize`）→ 写入 `registry_.yaml`（及 `--json` 时的 `registry_.json`） 扫描时会跳过隐藏目录以及 `node_modules`/`venv`/`.venv`/`__pycache__`/`dist`/`build`。SSH URL 会归一化为 HTTPS 形式存储。重复扫描会保留已有的 `name`/`desc`，并清理目录已不存在的旧条目。 -### 两层目录布局与 --flatten +### 三层分类布局（registry v2） -目录约定为两层：第一层是 category，第二层是正式仓库（`tooling/spark-cli`、`docs/quick-start`）。当存在三层或更深目录时，`--flatten` 会把深层仓库移动到 `/<仓库名>`，例如： +目录约定为**三层**：`<分类>/<子分类>/<项目>`。 + +``` +references/ +├── skills/ # 一级 = 分类（目录名即 registry 的 category） +│ ├── agent/ # 二级 = 子分类 +│ │ └── taste-skill/ # 三级 = 开源项目（带 origin 的 git 仓） +│ └── others/ # 没有子分类的项目归入 others（每个分类恒有 others） +├── products/ +│ └── others/ +└── plugins/ + └── others/ +``` + +- **分类**取自一级子目录（即使一个项目都没有也会出现在 registry 里）；**每个分类恒有 `others` 子分类**。 +- 直接放在分类下的项目（`skills/some-project`）没有子分类，registry 中归入该分类的 `others`（磁盘上应移动到 `skills/others/some-project`）。 +- 项目内部更深的嵌套仓（如 `fire-skills/downloads/*`）保全完整 path，仍归入其前两段的 `分类/子分类`。 +- 直接放在扫描根目录的仓库没有分类，打印 `WARNING` 且不写入 registry（需先移入某个分类）。 +- **没有一级分类目录**的普通目录自动回退为旧的扁平 `projects:` 格式，行为与旧版一致。 + +### `--flatten` 仅适用于旧两层布局 + +旧布局约定为两层：第一层 category，第二层仓库（`tooling/spark-cli`）。当存在三层或更深目录时，`--flatten` 会把深层仓库移动到 `/<仓库名>`，例如： ``` apps/fe/innate/web/innate-fe-cli → apps/innate-fe-cli tooling/spark/spark-cli → tooling/spark-cli ``` +- **不要对三分层分类目录使用 `--flatten`**：它会把 `skills/agent/taste-skill` 上移到 `skills/taste-skill`，破坏子分类结构。分类布局靠 `Categorize` 归组，不需要移动任何文件。 - 不覆盖任何文件：目标位置已存在（或本次运行中已被其他仓库占用）时，该仓库保持原位并在输出中打印 `Skipped ...` 告警。 - 移动后自动清理空的中间目录。 - `--flatten` 会直接修改磁盘上的目录结构，属于破坏性操作，请确认后再使用。 @@ -69,6 +92,8 @@ spark repo clone -r spark-cli -f registry_innate-apps.yaml 已存在的仓库（`.git` 或目录已存在）会被跳过，结束时打印 `cloned / skipped / failed` 统计。 +克隆目标按 registry 记录的 `path` 还原（分类格式即 `<分类>/<子分类>/<项目>`），并按 path 深度**从浅到深**执行——先克隆父项目，再克隆记录在其内部的嵌套仓。 + --- ## spark repo list @@ -83,12 +108,42 @@ spark repo clone -r spark-cli -f registry_innate-apps.yaml spark repo list -f registry_innate-apps.yaml ``` -每行输出 `namerepopath`。 +每行输出 `namerepopath`。分类格式（v2）会先打印一行分组头 `# <分类> / <子分类> (<数量>)`，行格式不变（机器解析时跳过 `#` 开头的行即可）。 --- ## registry 文件 +**分类格式（v2，三分层目录扫描的默认输出）**： + +```yaml +# Project registry +# Synced by spark repo scan + +version: 2 +categories: + - name: skills + subcategories: + - name: agent + projects: + - name: taste-skill + repo: https://github.com/Leonxlnx/taste-skill.git + path: skills/agent/taste-skill + desc: 前端审美 skill + - name: others + projects: [] + - name: products + subcategories: + - name: others + projects: [] + - name: plugins + subcategories: + - name: others + projects: [] +``` + +**旧扁平格式（无一级分类目录的目录仍保持不变，可继续读取）**： + ```yaml # Project registry # Synced by spark repo scan diff --git a/internal/registry/category.go b/internal/registry/category.go new file mode 100644 index 0000000..aa69307 --- /dev/null +++ b/internal/registry/category.go @@ -0,0 +1,143 @@ +package registry + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// Categorize groups flat scan results into the canonical three-level layout: +// +// // (e.g. skills/agent/taste-skill) +// +// Rules: +// - categories are the folder's immediate subdirectories (sorted), so empty +// categories such as products/plugins still appear in the registry; +// - every category always gets an "others" subcategory; +// - a project directly under a category (/) has no +// subcategory and lands in that category's "others"; +// - deeper projects (e.g. a project vendoring nested clones) keep their full +// path but are grouped by their first two segments; +// - a project directly under the folder root has no category at all and is +// returned as skipped for the caller to warn about (it must be moved into +// a category first — the registry only ever has the three real categories). +// +// It returns (nil, projects) when the folder has no category directories, i.e. +// the layout is not categorized — the caller then falls back to the legacy +// flat registry format. +func Categorize(folder string, projects []Project) (groups []Category, skipped []Project, err error) { + cats, err := immediateSubdirs(folder) + if err != nil { + return nil, nil, fmt.Errorf("list categories in %s: %w", folder, err) + } + if len(cats) == 0 { + // Not a categorized layout (legacy "flat" folder): report every project + // as unclassified so the caller keeps the old flat format. + return nil, projects, nil + } + catSet := make(map[string]bool, len(cats)) + for _, c := range cats { + catSet[c] = true + } + + type key struct{ cat, sub string } + var order []key + buckets := map[key][]Project{} + + for _, p := range projects { + segs := strings.Split(strings.Trim(p.Path, "/"), "/") + var cat, sub string + switch { + case len(segs) >= 3: + cat, sub = segs[0], segs[1] + case len(segs) == 2: + cat, sub = segs[0], OthersSubcategory + default: + skipped = append(skipped, p) + continue + } + if !catSet[cat] { + skipped = append(skipped, p) + continue + } + k := key{cat: cat, sub: sub} + if _, ok := buckets[k]; !ok { + order = append(order, k) + } + buckets[k] = append(buckets[k], p) + } + + groups = make([]Category, 0, len(cats)) + for _, cat := range cats { + diskSubs, err := immediateSubdirs(filepath.Join(folder, cat)) + if err != nil { + return nil, nil, fmt.Errorf("list subcategories of %s: %w", cat, err) + } + subSet := map[string]bool{} + var subNames []string + for _, s := range diskSubs { + subSet[s] = true + subNames = append(subNames, s) + } + for _, k := range order { + if k.cat == cat && !subSet[k.sub] { + subSet[k.sub] = true + subNames = append(subNames, k.sub) + } + } + if !subSet[OthersSubcategory] { + // The default bucket always exists, even before any project needs it. + subNames = append(subNames, OthersSubcategory) + } + sortSubcategories(subNames) + + cat := Category{Name: cat, Subcategories: make([]Subcategory, 0, len(subNames))} + for _, sub := range subNames { + projs := buckets[key{cat: cat.Name, sub: sub}] + sorted := make([]Project, len(projs)) + copy(sorted, projs) + sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Path < sorted[j].Path }) + cat.Subcategories = append(cat.Subcategories, Subcategory{Name: sub, Projects: sorted}) + } + groups = append(groups, cat) + } + return groups, skipped, nil +} + +// sortSubcategories orders alphabetically but always keeps "others" last. +func sortSubcategories(names []string) { + sort.SliceStable(names, func(i, j int) bool { + if names[i] == OthersSubcategory { + return false + } + if names[j] == OthersSubcategory { + return true + } + return names[i] < names[j] + }) +} + +// immediateSubdirs returns the sorted names of the immediate subdirectories +// that are NOT git repositories. A directory holding a `.git` is a project, not +// a category/subcategory: root-level repos must not become categories and a +// repo directly under a category belongs in that category's "others" bucket. +func immediateSubdirs(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + var out []string + for _, e := range entries { + if !e.IsDir() || strings.HasPrefix(e.Name(), ".") { + continue + } + if isGitRepo(filepath.Join(dir, e.Name())) { + continue + } + out = append(out, e.Name()) + } + sort.Strings(out) + return out, nil +} diff --git a/internal/registry/category_test.go b/internal/registry/category_test.go new file mode 100644 index 0000000..e25d415 --- /dev/null +++ b/internal/registry/category_test.go @@ -0,0 +1,255 @@ +package registry_test + +import ( + "os" + "path/filepath" + "strings" + + "spark/internal/registry" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func mkdirAll(parts ...string) { + Expect(os.MkdirAll(filepath.Join(parts...), 0755)).To(Succeed()) +} + +var _ = Describe("Categorize", func() { + var tempDir string + + BeforeEach(func() { + var err error + tempDir, err = os.MkdirTemp("", "registry-categorize") + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + os.RemoveAll(tempDir) + }) + + It("groups projects by category/subcategory and defaults to others", func() { + // three real categories on disk; only skills has subcategories for now + mkdirAll(tempDir, "skills", "agent") + mkdirAll(tempDir, "skills", "ui-ux") + mkdirAll(tempDir, "products") + mkdirAll(tempDir, "plugins") + + projects := []registry.Project{ + {Name: "taste-skill", Repo: "https://github.com/a/taste.git", Path: "skills/ui-ux/taste-skill"}, + {Name: "agency", Repo: "https://github.com/b/agency.git", Path: "skills/agent/agency"}, + {Name: "legacy", Repo: "https://github.com/c/legacy.git", Path: "skills/legacy"}, // no subcategory + {Name: "stray", Repo: "https://github.com/d/stray.git", Path: "stray"}, // no category + } + + groups, skipped, err := registry.Categorize(tempDir, projects) + Expect(err).NotTo(HaveOccurred()) + + Expect(groups).To(HaveLen(3)) + Expect([]string{groups[0].Name, groups[1].Name, groups[2].Name}). + To(Equal([]string{"plugins", "products", "skills"})) + + find := func(cat string, sub string) []string { + for _, c := range groups { + if c.Name != cat { + continue + } + for _, s := range c.Subcategories { + if s.Name == sub { + var names []string + for _, p := range s.Projects { + names = append(names, p.Name) + } + return names + } + } + } + return nil + } + subNames := func(cat string) []string { + for _, c := range groups { + if c.Name == cat { + var out []string + for _, s := range c.Subcategories { + out = append(out, s.Name) + } + return out + } + } + return nil + } + + Expect(find("skills", "agent")).To(Equal([]string{"agency"})) + Expect(find("skills", "ui-ux")).To(Equal([]string{"taste-skill"})) + // project without a subcategory lands in others, path unchanged + Expect(find("skills", "others")).To(Equal([]string{"legacy"})) + // every category gets an others bucket, even with no project in it, + // and it is always sorted last inside its category + Expect(subNames("skills")).To(Equal([]string{"agent", "ui-ux", "others"})) + Expect(subNames("products")).To(Equal([]string{"others"})) + Expect(subNames("plugins")).To(Equal([]string{"others"})) + // root-level project has no category: skipped for the caller to warn + Expect(skipped).To(HaveLen(1)) + Expect(skipped[0].Name).To(Equal("stray")) + }) + + It("keeps nested projects under their first two segments with full path", func() { + mkdirAll(tempDir, "skills", "dev-tool") + projects := []registry.Project{ + {Name: "fire-skills", Repo: "https://github.com/x/fire.git", Path: "skills/dev-tool/fire-skills"}, + {Name: "01-superpowers", Repo: "https://github.com/y/super.git", Path: "skills/dev-tool/fire-skills/downloads/01-superpowers"}, + } + + groups, skipped, err := registry.Categorize(tempDir, projects) + Expect(err).NotTo(HaveOccurred()) + Expect(skipped).To(BeEmpty()) + Expect(groups).To(HaveLen(1)) + Expect(groups[0].Name).To(Equal("skills")) + Expect(groups[0].Subcategories).To(HaveLen(2)) // dev-tool + others + + var dev *registry.Subcategory + for i := range groups[0].Subcategories { + if groups[0].Subcategories[i].Name == "dev-tool" { + dev = &groups[0].Subcategories[i] + } + } + Expect(dev).NotTo(BeNil()) + Expect(dev.Projects).To(HaveLen(2)) + // parent before its nested clone (path order) + Expect(dev.Projects[0].Path).To(Equal("skills/dev-tool/fire-skills")) + Expect(dev.Projects[1].Path).To(Equal("skills/dev-tool/fire-skills/downloads/01-superpowers")) + }) + + It("returns no groups for a folder without category directories", func() { + projects := []registry.Project{ + {Name: "flat", Repo: "https://github.com/e/flat.git", Path: "flat"}, + } + groups, skipped, err := registry.Categorize(tempDir, projects) + Expect(err).NotTo(HaveOccurred()) + Expect(groups).To(BeNil()) + // every project is reported unclassified so the caller keeps the legacy format + Expect(skipped).To(HaveLen(1)) + }) + + It("does not treat git repositories as category or subcategory directories", func() { + mkdirAll(tempDir, "skills", "agent") + // a repo in the folder root must NOT become a category + writeGitRepo(tempDir, "rootproj", "https://github.com/f/rootproj.git") + // a repo directly under a category must NOT become a subcategory + writeGitRepo(filepath.Join(tempDir, "skills"), "legacyproj", "https://github.com/g/legacyproj.git") + + projects := []registry.Project{ + {Name: "rootproj", Repo: "https://github.com/f/rootproj.git", Path: "rootproj"}, + {Name: "legacyproj", Repo: "https://github.com/g/legacyproj.git", Path: "skills/legacyproj"}, + {Name: "agent-repo", Repo: "https://github.com/h/agent.git", Path: "skills/agent/agent-repo"}, + } + + groups, skipped, err := registry.Categorize(tempDir, projects) + Expect(err).NotTo(HaveOccurred()) + // only skills (the non-repo dir) is a category + Expect(groups).To(HaveLen(1)) + Expect(groups[0].Name).To(Equal("skills")) + Expect(groups[0].Subcategories).To(HaveLen(2)) // agent + others (legacyproj is not a subcategory) + var subNames []string + for _, s := range groups[0].Subcategories { + subNames = append(subNames, s.Name) + } + Expect(subNames).To(Equal([]string{"agent", "others"})) + + // the root-level repo has no category: skipped for a warning + Expect(skipped).To(HaveLen(1)) + Expect(skipped[0].Name).To(Equal("rootproj")) + }) +}) + +var _ = Describe("Categorized registry format", func() { + var tempDir string + + BeforeEach(func() { + var err error + tempDir, err = os.MkdirTemp("", "registry-v2") + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + os.RemoveAll(tempDir) + }) + + categorized := func() *registry.Registry { + return ®istry.Registry{Categories: []registry.Category{ + {Name: "skills", Subcategories: []registry.Subcategory{ + {Name: "agent", Projects: []registry.Project{ + {Name: "agency", Repo: "https://github.com/b/agency.git", Path: "skills/agent/agency", Desc: "keep me"}, + }}, + {Name: "others", Projects: []registry.Project{}}, + }}, + {Name: "products", Subcategories: []registry.Subcategory{ + {Name: "others", Projects: []registry.Project{}}, + }}, + }} + } + + It("writes version 2 with categories and no top-level projects", func() { + path := filepath.Join(tempDir, "registry_refs.yaml") + Expect(registry.Write(path, categorized())).To(Succeed()) + + data, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + text := string(data) + Expect(text).To(ContainSubstring("version: 2")) + Expect(text).To(ContainSubstring("categories:")) + Expect(text).To(ContainSubstring("name: products")) + // no legacy top-level projects key (nested subcategory keys are fine) + Expect(text).ToNot(MatchRegexp("(?m)^projects:")) + // subcategory order keeps others last inside its category + Expect(strings.Index(text, "name: agent")).To(BeNumerically("<", strings.Index(text, "name: others"))) + }) + + It("reads back flattened projects for legacy callers", func() { + path := filepath.Join(tempDir, "registry_refs2.yaml") + Expect(registry.Write(path, categorized())).To(Succeed()) + + got, err := registry.Read(path) + Expect(err).NotTo(HaveOccurred()) + Expect(got.IsCategorized()).To(BeTrue()) + Expect(got.Version).To(Equal(registry.RegistryVersion)) + Expect(got.AllProjects()).To(HaveLen(1)) + Expect(got.Projects).To(HaveLen(1)) // derived flat view + Expect(got.Projects[0].Path).To(Equal("skills/agent/agency")) + + p, err := registry.FindByName(got, "agency") + Expect(err).NotTo(HaveOccurred()) + Expect(p.Desc).To(Equal("keep me")) + }) + + It("writes the JSON mirror in the categorized shape", func() { + path := filepath.Join(tempDir, "registry_refs.json") + Expect(registry.WriteJSON(path, categorized())).To(Succeed()) + + data, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + text := string(data) + Expect(text).To(ContainSubstring(`"version": 2`)) + Expect(text).To(ContainSubstring(`"categories"`)) + // no legacy top-level "projects" key (nested subcategory keys are fine) + Expect(text).ToNot(MatchRegexp(`(?m)^ "projects":`)) + }) + + It("keeps the legacy flat shape for flat registries", func() { + path := filepath.Join(tempDir, "registry_legacy.yaml") + flat := ®istry.Registry{Projects: []registry.Project{ + {Name: "spark-cli", Repo: "https://github.com/variableway/spark-cli.git", Path: "spark-cli"}, + }} + Expect(registry.Write(path, flat)).To(Succeed()) + + data, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(ContainSubstring("projects:")) + Expect(string(data)).ToNot(ContainSubstring("version:")) + + got, err := registry.Read(path) + Expect(err).NotTo(HaveOccurred()) + Expect(got.IsCategorized()).To(BeFalse()) + Expect(got.Projects).To(HaveLen(1)) + }) +}) diff --git a/internal/registry/registry.go b/internal/registry/registry.go index 5167208..230c604 100644 --- a/internal/registry/registry.go +++ b/internal/registry/registry.go @@ -17,10 +17,57 @@ type Project struct { Desc string `yaml:"desc,omitempty" json:"desc,omitempty"` } -type Registry struct { +// RegistryVersion is written whenever the registry uses the categorized layout. +const RegistryVersion = 2 + +// OthersSubcategory is the default bucket for a project that sits directly +// under a category and therefore has no subcategory of its own. +const OthersSubcategory = "others" + +// Subcategory groups projects under one category. "others" is always present +// so unclassified projects have a home. +type Subcategory struct { + Name string `yaml:"name" json:"name"` Projects []Project `yaml:"projects" json:"projects"` } +// Category is one top-level bucket (e.g. skills / products / plugins). +type Category struct { + Name string `yaml:"name" json:"name"` + Subcategories []Subcategory `yaml:"subcategories" json:"subcategories"` +} + +// Registry holds either the categorized layout (Version=2, Categories set — +// the canonical format) or the legacy flat layout (Projects only). +// Read() always derives a flattened Projects slice, so callers written against +// the legacy shape keep working. +type Registry struct { + Version int `yaml:"version,omitempty" json:"version,omitempty"` + Projects []Project `yaml:"projects,omitempty" json:"projects,omitempty"` + Categories []Category `yaml:"categories,omitempty" json:"categories,omitempty"` +} + +// IsCategorized reports whether this registry uses the v2 category layout. +func (r *Registry) IsCategorized() bool { + return r != nil && len(r.Categories) > 0 +} + +// AllProjects flattens the registry into a single list, preserving order +// (categories -> subcategories -> projects). Legacy flat registries return +// their Projects unchanged. +func (r *Registry) AllProjects() []Project { + if !r.IsCategorized() { + return r.Projects + } + flat := make([]Project, 0, len(r.Projects)) + for _, c := range r.Categories { + for _, s := range c.Subcategories { + flat = append(flat, s.Projects...) + } + } + return flat +} + func Read(path string) (*Registry, error) { data, err := os.ReadFile(path) if err != nil { @@ -34,11 +81,42 @@ func Read(path string) (*Registry, error) { if err := yaml.Unmarshal(data, ®); err != nil { return nil, fmt.Errorf("parse registry %s: %w", path, err) } + if reg.IsCategorized() { + // v2 file: derive the flattened view so legacy callers (clone/list/find) + // keep working against a categorized registry. + reg.Version = RegistryVersion + reg.Projects = reg.AllProjects() + } return ®, nil } +// wire shapes keep the on-disk formats exactly distinguishable: +// legacy = {projects: [...]}, v2 = {version: 2, categories: [...]}. +type legacyYAML struct { + Projects []Project `yaml:"projects"` +} +type categorizedYAML struct { + Version int `yaml:"version"` + Categories []Category `yaml:"categories"` +} +type legacyJSON struct { + Projects []Project `json:"projects"` +} +type categorizedJSON struct { + Version int `json:"version"` + Categories []Category `json:"categories"` +} + func Write(path string, reg *Registry) error { - data, err := yaml.Marshal(reg) + var ( + data []byte + err error + ) + if reg.IsCategorized() { + data, err = yaml.Marshal(categorizedYAML{Version: RegistryVersion, Categories: reg.Categories}) + } else { + data, err = yaml.Marshal(legacyYAML{Projects: reg.Projects}) + } if err != nil { return fmt.Errorf("encode registry: %w", err) } @@ -48,7 +126,15 @@ func Write(path string, reg *Registry) error { } func WriteJSON(path string, reg *Registry) error { - data, err := json.MarshalIndent(reg, "", " ") + var ( + data []byte + err error + ) + if reg.IsCategorized() { + data, err = json.MarshalIndent(categorizedJSON{Version: RegistryVersion, Categories: reg.Categories}, "", " ") + } else { + data, err = json.MarshalIndent(legacyJSON{Projects: reg.Projects}, "", " ") + } if err != nil { return fmt.Errorf("encode registry: %w", err) } @@ -56,9 +142,10 @@ func WriteJSON(path string, reg *Registry) error { } func FindByName(reg *Registry, name string) (*Project, error) { - for i := range reg.Projects { - if reg.Projects[i].Name == name { - return ®.Projects[i], nil + projects := reg.AllProjects() + for i := range projects { + if projects[i].Name == name { + return &projects[i], nil } } return nil, fmt.Errorf("repository %q not found in registry", name)