Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -559,6 +559,8 @@ spark git scan . --db ~/.innate/feeds.db

```bash
spark repo scan [folder-name] # 扫描目录并写入 registry_<folder>.yaml
spark repo scan [folder-name] --json # 同时写出格式化 registry_<folder>.json
spark repo scan [folder-name] --flatten # 先把超过两层的目录拍平为 category/repo 再扫描
spark repo clone -f registry_<folder>.yaml # 克隆 registry 中全部仓库
spark repo clone -r <name> -f registry_<folder>.yaml # 仅克隆指定仓库
spark repo list -f registry_<folder>.yaml # 列出 registry 中的仓库
Expand All @@ -579,7 +581,14 @@ projects:

#### `spark repo scan [folder-name]`

递归扫描目录,发现带 `origin` 远程的 Git 仓库,写入当前目录下的 `registry_<folder>.yaml`。扫描时跳过隐藏目录与 `node_modules`/`venv`/`.venv`/`__pycache__`/`dist`/`build`;SSH URL 归一化为 HTTPS 形式存储;重复扫描与已有 registry 合并(按 `path` 优先、其次按 `repo` URL 匹配,保留旧 `name`/`desc`,删除目录已不存在的旧条目,追加新发现仓库)。
递归扫描目录,发现带 `origin` 远程的 Git 仓库,写入当前目录下的 `registry_<folder>.yaml`。目录约定为两层:第一层是 category,第二层是正式仓库。扫描时跳过隐藏目录与 `node_modules`/`venv`/`.venv`/`__pycache__`/`dist`/`build`;SSH URL 归一化为 HTTPS 形式存储;重复扫描与已有 registry 合并(按 `path` 优先、其次按 `repo` URL 匹配,保留旧 `name`/`desc`,删除目录已不存在的旧条目,追加新发现仓库)。

| 选项 | 说明 |
|------|------|
| `--json` | 在 YAML 之外额外写出格式化(2 空格缩进)的 `registry_<folder>.json`,内容一致 |
| `--flatten` | 扫描前把超过两层(`category/repo`)的仓库上移到两层布局(破坏性操作,显式开启) |

`--flatten` 由 `internal/registry.Flatten` 实现:深层仓库移动到 `<category>/<仓库名>`,目标已存在或本次运行中被占用时跳过并打印 `Skipped ...` 告警(不覆盖任何文件),移动后清理空的中间目录;按深度从深到浅处理,避免父目录先移动导致子路径失效。

#### `spark repo clone`

Expand Down
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]

### Added

- `spark repo scan --json` — additionally writes a formatted (2-space indent) `registry_<folder>.json` with the same content as the YAML registry
- `spark repo scan --flatten` — moves repositories nested deeper than two levels (`category/repo` layout) up into the two-level layout before scanning; conflicts are skipped with a warning and never overwritten, empty intermediate directories are removed (fixes #26)

## [0.3.3] - 2026-10-20

### Added
Expand Down
34 changes: 34 additions & 0 deletions cmd/repo/scan.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,23 @@ import (
"github.com/spf13/cobra"
)

var (
scanJSON bool
scanFlatten bool
)

var scanCmd = &cobra.Command{
Use: "scan [folder-name]",
Short: "Scan a folder for git repositories and save to registry_<folder>.yaml",
Long: `Recursively scan a folder for git repositories that have an origin remote and
write them to a registry file named registry_<folder>.yaml in the current directory.

The folder is expected to hold a two-level layout: the first level is the category
and the second level holds the actual repositories. With --flatten, repositories
nested deeper than two levels are moved up into the two-level layout (no files are
overwritten; conflicts are reported and left in place) before scanning. With --json,
an additional registry_<folder>.json file with the same content is written.

The registry records each repository's name, remote URL and relative path. Re-scanning
merges results and preserves existing descriptions.`,
Args: cobra.MaximumNArgs(1),
Expand All @@ -29,6 +40,19 @@ merges results and preserves existing descriptions.`,
return fmt.Errorf("resolve folder: %w", err)
}

if scanFlatten {
result, err := registry.Flatten(absFolder)
if err != nil {
return err
}
for _, m := range result.Moves {
fmt.Printf("Flattened %s -> %s\n", m.From, m.To)
}
for _, c := range result.Conflicts {
fmt.Printf("Skipped %s -> %s: %s\n", c.From, c.To, c.Reason)
}
}

projects, err := registry.Scan(absFolder)
if err != nil {
return err
Expand All @@ -52,10 +76,20 @@ merges results and preserves existing descriptions.`,
fmt.Printf(" %s -> %s\n", p.Path, p.Repo)
}
fmt.Printf("Wrote %d projects to %s\n", len(final), registryFile)

if scanJSON {
jsonFile := fmt.Sprintf("registry_%s.json", folderName)
if err := registry.WriteJSON(jsonFile, &registry.Registry{Projects: final}); err != nil {
return err
}
fmt.Printf("Wrote %d projects to %s\n", len(final), jsonFile)
}
return nil
},
}

func init() {
scanCmd.Flags().BoolVar(&scanJSON, "json", false, "additionally write registry_<folder>.json")
scanCmd.Flags().BoolVar(&scanFlatten, "flatten", false, "move repositories deeper than two levels up into the category/repo layout before scanning")
RepoCmd.AddCommand(scanCmd)
}
2 changes: 2 additions & 0 deletions docs/en/Agents.md
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,8 @@ spark repo list -f registry_<folder>.yaml # List the repos in the reg

| Option | Command | Description |
|------|------|------|
| `--json` | `scan` | Additionally write a formatted (2-space indent) `registry_<folder>.json` alongside the YAML |
| `--flatten` | `scan` | Move repositories deeper than two levels (`category/repo`) into the two-level layout before scanning; conflicts are skipped, never overwritten |
| `-r, --repo` | `clone` | Clone only the named repo (omit to clone all) |
| `-f, --file` | `clone` / `list` | Registry file (required) |

Expand Down
24 changes: 23 additions & 1 deletion docs/en/usage/repo.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ Manage multiple GitHub repositories under a directory via a registry file, as an

```bash
spark repo scan [folder-name] # Scan a directory and write registry_<folder>.yaml
spark repo scan [folder-name] --json # Also write a formatted registry_<folder>.json
spark repo scan [folder-name] --flatten # Flatten directories deeper than two levels to category/repo before scanning
spark repo list -f registry_<folder>.yaml # List repositories in the registry
spark repo clone -f registry_<folder>.yaml # Clone every repository in the registry
spark repo clone -r <name> -f registry_<folder>.yaml # Clone only the named repository
Expand All @@ -20,12 +22,32 @@ Recursively scan a directory, find Git repositories with an `origin` remote, and
```bash
spark repo scan # Scan the current directory
spark repo scan ~/workspace/innate-apps # Scan a specific directory
spark repo scan ~/workspace/innate-apps --json # Also write registry_innate-apps.json
spark repo scan ~/workspace/innate-apps --flatten # Flatten deep directories before scanning
```

**Flow**: recursively locate `.git` (including `.git` files) → parse the `origin` remote URL → normalize → merge with the existing registry → write `registry_<folder>.yaml`.
| Flag | Default | Description |
|------|---------|-------------|
| `--json` | `false` | Additionally write a formatted (2-space indent) `registry_<folder>.json` with the same content as the YAML |
| `--flatten` | `false` | Move repositories nested deeper than two levels (`category/repo`) up into the two-level layout before scanning |

**Flow**: (optional) `--flatten` deep directories → recursively locate `.git` (including `.git` files) → parse the `origin` remote URL → normalize → merge with the existing registry → write `registry_<folder>.yaml` (plus `registry_<folder>.json` when `--json` is set).

Scanning skips hidden directories and `node_modules`/`venv`/`.venv`/`__pycache__`/`dist`/`build`. SSH URLs are normalized to HTTPS form. Re-scanning preserves existing `name`/`desc` and removes entries whose directory no longer exists.

### Two-level layout and --flatten

The directory layout follows a two-level convention: the first level is the category, the second level holds the actual repositories (`tooling/spark-cli`, `docs/quick-start`). When three or deeper levels exist, `--flatten` moves deep repositories up to `<category>/<repo-name>`, for example:

```
apps/fe/innate/web/innate-fe-cli → apps/innate-fe-cli
tooling/spark/spark-cli → tooling/spark-cli
```

- Nothing is overwritten: when the target already exists (or was claimed by another repository in the same run), the repository stays in place and a `Skipped ...` warning is printed.
- Empty intermediate directories are removed after a move.
- `--flatten` modifies the directory structure on disk and is destructive; confirm before using it.

---

## spark repo clone
Expand Down
2 changes: 2 additions & 0 deletions docs/zh/Agents.md
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,8 @@ spark repo list -f registry_<folder>.yaml # 列出 registry 中的仓

| 选项 | 命令 | 说明 |
|------|------|------|
| `--json` | `scan` | 在 YAML 之外额外写出格式化(2 空格缩进)的 `registry_<folder>.json` |
| `--flatten` | `scan` | 扫描前把超过两层(`category/repo`)的仓库上移到两层布局,冲突跳过不覆盖 |
| `-r, --repo` | `clone` | 仅克隆指定名称的仓库(省略则全部) |
| `-f, --file` | `clone` / `list` | registry 文件(必填) |

Expand Down
24 changes: 23 additions & 1 deletion docs/zh/usage/repo.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@

```bash
spark repo scan [folder-name] # 扫描目录并写入 registry_<folder>.yaml
spark repo scan [folder-name] --json # 同时写出格式化 registry_<folder>.json
spark repo scan [folder-name] --flatten # 先把超过两层的目录拍平为 category/repo 再扫描
spark repo list -f registry_<folder>.yaml # 列出 registry 中的仓库
spark repo clone -f registry_<folder>.yaml # 克隆 registry 中全部仓库
spark repo clone -r <name> -f registry_<folder>.yaml # 仅克隆指定仓库
Expand All @@ -20,12 +22,32 @@ spark repo clone -r <name> -f registry_<folder>.yaml # 仅克隆指定仓库
```bash
spark repo scan # 扫描当前目录
spark repo scan ~/workspace/innate-apps # 扫描指定目录
spark repo scan ~/workspace/innate-apps --json # 同时写出 registry_innate-apps.json
spark repo scan ~/workspace/innate-apps --flatten # 先拍平深层目录再扫描
```

**流程**:递归查找 `.git`(含 `.git` 文件)→ 解析 `origin` 远程 URL → 归一化 → 与已有 registry 合并 → 写入 `registry_<folder>.yaml`
| 标志 | 默认值 | 说明 |
|------|--------|------|
| `--json` | `false` | 在 YAML 之外额外写出格式化(2 空格缩进)的 `registry_<folder>.json`,内容与 YAML 一致 |
| `--flatten` | `false` | 扫描前把超过两层(`category/repo`)的仓库上移到两层布局 |

**流程**:(可选)`--flatten` 拍平深层目录 → 递归查找 `.git`(含 `.git` 文件)→ 解析 `origin` 远程 URL → 归一化 → 与已有 registry 合并 → 写入 `registry_<folder>.yaml`(及 `--json` 时的 `registry_<folder>.json`)

扫描时会跳过隐藏目录以及 `node_modules`/`venv`/`.venv`/`__pycache__`/`dist`/`build`。SSH URL 会归一化为 HTTPS 形式存储。重复扫描会保留已有的 `name`/`desc`,并清理目录已不存在的旧条目。

### 两层目录布局与 --flatten

目录约定为两层:第一层是 category,第二层是正式仓库(`tooling/spark-cli`、`docs/quick-start`)。当存在三层或更深目录时,`--flatten` 会把深层仓库移动到 `<category>/<仓库名>`,例如:

```
apps/fe/innate/web/innate-fe-cli → apps/innate-fe-cli
tooling/spark/spark-cli → tooling/spark-cli
```

- 不覆盖任何文件:目标位置已存在(或本次运行中已被其他仓库占用)时,该仓库保持原位并在输出中打印 `Skipped ...` 告警。
- 移动后自动清理空的中间目录。
- `--flatten` 会直接修改磁盘上的目录结构,属于破坏性操作,请确认后再使用。

---

## spark repo clone
Expand Down
105 changes: 105 additions & 0 deletions internal/registry/flatten.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package registry

import (
"fmt"
"os"
"path/filepath"
"sort"
"strings"
)

type FlattenMove struct {
From string
To string
}

type FlattenConflict struct {
From string
To string
Reason string
}

type FlattenResult struct {
Moves []FlattenMove
Conflicts []FlattenConflict
}

// Flatten moves repositories nested deeper than two levels (category/repo)
// up into the two-level layout. Depth-1 and depth-2 repositories are left
// untouched. When the target path already exists, the repository stays in
// place and the conflict is reported instead of overwriting anything.
func Flatten(root string) (*FlattenResult, error) {
absRoot, err := filepath.Abs(root)
if err != nil {
return nil, err
}

projects, err := Scan(absRoot)
if err != nil {
return nil, err
}

sort.SliceStable(projects, func(i, j int) bool {
return depth(projects[i].Path) > depth(projects[j].Path)
})

result := &FlattenResult{}
claimed := map[string]bool{}

for _, p := range projects {
if depth(p.Path) <= 2 {
continue
}

rel := filepath.FromSlash(p.Path)
targetRel := filepath.Join(strings.Split(p.Path, "/")[0], filepath.Base(rel))
target := filepath.Join(absRoot, targetRel)

if _, err := os.Stat(target); err == nil {
result.Conflicts = append(result.Conflicts, FlattenConflict{
From: p.Path,
To: filepath.ToSlash(targetRel),
Reason: "target already exists",
})
continue
}
if claimed[filepath.ToSlash(targetRel)] {
result.Conflicts = append(result.Conflicts, FlattenConflict{
From: p.Path,
To: filepath.ToSlash(targetRel),
Reason: "target claimed by another repository in this run",
})
continue
}

src := filepath.Join(absRoot, rel)
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
return nil, fmt.Errorf("flatten %s: %w", p.Path, err)
}
if err := os.Rename(src, target); err != nil {
return nil, fmt.Errorf("flatten %s: %w", p.Path, err)
}
claimed[filepath.ToSlash(targetRel)] = true
result.Moves = append(result.Moves, FlattenMove{
From: p.Path,
To: filepath.ToSlash(targetRel),
})

removeEmptyParents(absRoot, filepath.Dir(src))
}

return result, nil
}

func depth(rel string) int {
return strings.Count(rel, "/") + 1
}

func removeEmptyParents(root, dir string) {
for dir != root && strings.HasPrefix(dir, root) {
if err := os.Remove(dir); err != nil {
return
}
dir = filepath.Dir(dir)
}
}
19 changes: 14 additions & 5 deletions internal/registry/registry.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package registry

import (
"encoding/json"
"fmt"
"os"
"path/filepath"
Expand All @@ -10,14 +11,14 @@ import (
)

type Project struct {
Name string `yaml:"name"`
Repo string `yaml:"repo"`
Path string `yaml:"path,omitempty"`
Desc string `yaml:"desc,omitempty"`
Name string `yaml:"name" json:"name"`
Repo string `yaml:"repo" json:"repo"`
Path string `yaml:"path,omitempty" json:"path,omitempty"`
Desc string `yaml:"desc,omitempty" json:"desc,omitempty"`
}

type Registry struct {
Projects []Project `yaml:"projects"`
Projects []Project `yaml:"projects" json:"projects"`
}

func Read(path string) (*Registry, error) {
Expand Down Expand Up @@ -46,6 +47,14 @@ func Write(path string, reg *Registry) error {
return os.WriteFile(path, append([]byte(header), data...), 0644)
}

func WriteJSON(path string, reg *Registry) error {
data, err := json.MarshalIndent(reg, "", " ")
if err != nil {
return fmt.Errorf("encode registry: %w", err)
}
return os.WriteFile(path, append(data, '\n'), 0644)
}

func FindByName(reg *Registry, name string) (*Project, error) {
for i := range reg.Projects {
if reg.Projects[i].Name == name {
Expand Down
Loading
Loading