Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions SYNC.md
Original file line number Diff line number Diff line change
Expand Up @@ -337,3 +337,15 @@ Source-only fix for Mole CLI (macOS), `fc4316b0`: `get_latest_commit_from_github
| Mole CLI (macOS) | Implemented in source | `lib/manage/update.sh:get_latest_commit_from_github`, `tests/update.bats` (`nightly commit lookup keeps the SHA from a large GitHub response`). |
| Mole CLI (Linux) | Intentional difference | `install.sh:latest_commit` at `d68d06b0` requests `application/vnd.github.sha` and validates the raw SHA; it has no JSON file-list pipeline to repair. Source reviewed, not a Linux runtime test. |
| Mole CLI (Windows) | Not applicable | `bin/update.ps1` at `2c6e8e99` updates source with Git; its stable update metadata in `lib/core/update_check.ps1` uses `Invoke-RestMethod`, not this Nightly commit parser. Source reviewed, not a Windows runtime test. |

## 2026-11-15 Home overview hardlink accounting

Source: Mole CLI (macOS), `7c83bad6`. The Home measurement passes its included immediate children to one `du` so file identities are shared across them and top-level files. It still omits only the immediate `~/Library` and mounted children. The bounded Go fallback also shares the existing identity set; overview cache schema 14 rejects old totals. Real-hardlink regressions fail on the prior main and pass here. This is a source change, not a verified package release.

| Product | Status | Consumer and evidence |
| --- | --- | --- |
| Mole CLI (macOS) | Implemented in source | `cmd/analyze/scanner.go:getDirectorySizeFromDuSkippingImmediateChild`, `TestHomeOverviewDeduplicatesHardlinks` and the neighboring permission/symlink/mount tests. |
| Mole CLI (Linux) | Pending | At `4de579cc`, Home excludes the immediate `.cache` through independent per-child `du` calls. Unchanged source run on Ubuntu 25.10 aarch64 reports 2097152 B for one 1048576 B file through both the real CLI and the Home measurement; uutils 0.2.2 and GNU du 9.5 agree, while the directory scanner deduplicates. Tracked in [#2013](https://github.com/tw93/Mole/issues/2013); its current CONTRIBUTING accepts issues rather than PRs. |
| Mole CLI (Windows) | Not applicable to this du repair | At `59f3dc5a`, `measureOverviewSize` uses `walkDirSize`, excluding AppData. Its file-ID/hardlink work remains the separate draft [#2012](https://github.com/tw93/Mole/pull/2012); no Windows runtime claim is made here. |

The Mole Mac and Mole for Windows consumers remain unverified in this task.
3 changes: 2 additions & 1 deletion cmd/analyze/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ import (
// v11: the overview leaves CloudStorage out of the User Library total.
// v12: traversal stops at mount points, so v11 totals can include mounted volumes.
// v13: the Home and old-Downloads overview totals stop at mount points too.
const cacheSchemaVersion = 13
// v14: Home totals deduplicate hardlinks across immediate children.
const cacheSchemaVersion = 14

type overviewSizeSnapshot struct {
Size int64 `json:"size"`
Expand Down
13 changes: 5 additions & 8 deletions cmd/analyze/crossmount_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
Expand Down Expand Up @@ -173,21 +172,19 @@ func TestDuSkippingImmediateChildStopsAtMountPoints(t *testing.T) {
t.Fatalf("create Library: %v", err)
}

var mu sync.Mutex
var measured []string
_, err := getDirectorySizeFromDuSkippingImmediateChild(context.Background(), fx.root, library, func(path string) (int64, error) {
mu.Lock()
measured = append(measured, path)
mu.Unlock()
_, err := getDirectorySizeFromDuSkippingImmediateChild(context.Background(), fx.root, library, func(paths ...string) (int64, error) {
measured = append(measured, paths...)
return 1 << 20, nil
})
if err != nil {
t.Fatalf("getDirectorySizeFromDuSkippingImmediateChild: %v", err)
}

local := filepath.Join(fx.root, "local")
if len(measured) != 1 || measured[0] != local {
t.Fatalf("measured %#v, want only %s (the mount point must not be measured)", measured, local)
rootFile := filepath.Join(fx.root, "root.txt")
if len(measured) != 2 || measured[0] != local || measured[1] != rootFile {
t.Fatalf("measured %#v, want %s and %s (the mount point must not be measured)", measured, local, rootFile)
}
}

Expand Down
83 changes: 41 additions & 42 deletions cmd/analyze/scanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -958,7 +958,7 @@ func measureOverviewSizeWithPublication(ctx context.Context, path string, public
defer cancel()
ignoreNames := overviewIgnoreNamesForPath(path)
size, err := getDirectorySizeFromDuWithExcludeAndIgnores(ctx, path, excludePath, ignoreNames, overviewDuTimeout)
if err != nil && size == 0 && ctx.Err() == nil {
if err != nil && (size == 0 || errors.Is(err, syscall.E2BIG)) && ctx.Err() == nil {
size, err = getDirectoryLogicalSizeWithExclude(ctx, path, excludePath, ignoreNames)
}
if overviewMeasurementStorable(size, err) {
Expand Down Expand Up @@ -1042,19 +1042,27 @@ func getDirectorySizeFromDuWithExcludeAndIgnores(ctx context.Context, path strin
}
}

runDuSize := func(target string) (int64, error) {
if _, err := os.Stat(target); err != nil {
return 0, err
runDuSize := func(targets ...string) (int64, error) {
if len(targets) == 0 {
return 0, nil
}
if len(targets) == 1 {
if _, err := os.Lstat(targets[0]); err != nil {
return 0, err
}
}

ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()

args := []string{"-skPx"}
if len(targets) > 1 {
args = append(args, "-c")
}
for _, ignoreName := range ignoreNames {
args = append(args, "-I", ignoreName)
}
args = append(args, target)
args = append(args, targets...)
cmd := exec.CommandContext(ctx, duBinary, args...)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
Expand All @@ -1063,7 +1071,14 @@ func getDirectorySizeFromDuWithExcludeAndIgnores(ctx context.Context, path strin
activeDu.Add(1)
runErr := cmd.Run()
activeDu.Add(-1)
fields := strings.Fields(stdout.String())
output := stdout.String()
if len(targets) > 1 {
// The final -c total shares du's hardlink set across operands.
// Parse only that line: file names can contain tabs and newlines.
output = strings.TrimSuffix(output, "\n")
output = output[strings.LastIndexByte(output, '\n')+1:]
}
fields := strings.Fields(output)
if ctx.Err() != nil {
runErr = ctx.Err()
} else if runErr != nil && duReportedOnlyPermissionDenials(stderr.Bytes()) {
Expand All @@ -1076,9 +1091,9 @@ func getDirectorySizeFromDuWithExcludeAndIgnores(ctx context.Context, path strin
}
if len(fields) == 0 {
if runErr != nil {
return 0, fmt.Errorf("du failed for %s: %w", target, runErr)
return 0, fmt.Errorf("du failed for %s: %w", path, runErr)
}
return 0, fmt.Errorf("du output empty for %s", target)
return 0, fmt.Errorf("du output empty for %s", path)
}
kb, parseErr := strconv.ParseInt(fields[0], 10, 64)
if parseErr != nil {
Expand All @@ -1090,7 +1105,7 @@ func getDirectorySizeFromDuWithExcludeAndIgnores(ctx context.Context, path strin
// Nonzero exits may still carry useful bytes for readable descendants.
// Keep both the bytes and the failure; do not trigger another full walk.
if runErr != nil {
return kb * 1024, fmt.Errorf("du incomplete for %s: %w", target, runErr)
return kb * 1024, fmt.Errorf("du incomplete for %s: %w", path, runErr)
}
return kb * 1024, nil
}
Expand Down Expand Up @@ -1169,7 +1184,7 @@ func overviewIgnoreNamesForPath(path string) []string {
return ignoreNames
}

func getDirectorySizeFromDuSkippingImmediateChild(ctx context.Context, path string, excludePath string, runDuSize func(string) (int64, error)) (int64, error) {
func getDirectorySizeFromDuSkippingImmediateChild(ctx context.Context, path string, excludePath string, runDuSize func(...string) (int64, error)) (int64, error) {
path = filepath.Clean(path)
excludePath = filepath.Clean(excludePath)

Expand All @@ -1192,7 +1207,7 @@ func getDirectorySizeFromDuSkippingImmediateChild(ctx context.Context, path stri
var total int64
var failures scanFailures
if info, err := os.Lstat(path); err == nil {
atomic.AddInt64(&total, getActualFileSize(path, info))
total = getActualFileSize(path, info)
} else {
failures.record(err)
}
Expand All @@ -1202,55 +1217,38 @@ func getDirectorySizeFromDuSkippingImmediateChild(ctx context.Context, path stri
// du over path would never enter it; skip it here to match.
rootDevice := scanDeviceOf(path)

var wg sync.WaitGroup
workerCount := min(max(runtime.NumCPU()*2, 2), 8)
sem := make(chan struct{}, workerCount)

operands := make([]string, 0, len(entries))
for _, entry := range entries {
if ctx.Err() != nil {
failures.record(ctx.Err())
break
return total, ctx.Err()
}
fullPath := filepath.Join(path, entry.Name())
if filepath.Clean(fullPath) == excludePath {
continue
}

if entry.Type()&fs.ModeSymlink != 0 || !entry.IsDir() {
info, err := entry.Info()
if err != nil {
failures.record(err)
continue
}
atomic.AddInt64(&total, getActualFileSize(fullPath, info))
if entry.IsDir() && rootDevice.isMountPoint(fullPath, entry) {
continue
}
if rootDevice.isMountPoint(fullPath, entry) {
continue
}

if err := acquireScanPermit(ctx, sem); err != nil {
failures.record(err)
break
}
wg.Go(func() {
defer func() { <-sem }()

size, err := runDuSize(fullPath)
failures.record(err)
atomic.AddInt64(&total, size)
})
operands = append(operands, fullPath)
}

wg.Wait()
// Independent du processes cannot deduplicate links between children,
// including a link directly in Home and another inside a directory.
if len(operands) == 0 {
return total, failures.err()
}
size, err := runDuSize(operands...)
failures.record(err)

return total, failures.err()
return total + size, failures.err()
}

func getDirectoryLogicalSizeWithExclude(ctx context.Context, path string, excludePath string, ignoreNames []string) (int64, error) {
var total int64
var failures scanFailures
rootDevice := scanDeviceOf(path)
var seen sync.Map
err := filepath.WalkDir(path, func(p string, d fs.DirEntry, err error) error {
if ctx.Err() != nil {
return ctx.Err()
Expand All @@ -1277,7 +1275,8 @@ func getDirectoryLogicalSizeWithExclude(ctx context.Context, path string, exclud
failures.record(err)
return nil
}
total += getActualFileSize(p, info)
size, _ := countableFileSize(p, info, &seen)
total += size
return nil
})
failures.record(err)
Expand Down
Loading
Loading