feat(demo): Add an ANALYZER=tango option to the local demo

Author: manjari25

Makefile

@@ -64,7 +64,13 @@ export SQ_PROVIDER_CONFIG_DIR ?= $(REPO_ROOT)/service/submitqueue/demo/provider/
 PROVIDER_COMPOSE_FILE_fake = $(abspath service/submitqueue/docker-compose.fake.yml)
 PROVIDER_COMPOSE_FILE_git = $(abspath service/submitqueue/docker-compose.git.yml)
 PROVIDER_COMPOSE_FILE_github = $(abspath service/submitqueue/docker-compose.provider.yml)
-PROVIDER_COMPOSE_FILE = $(PROVIDER_COMPOSE_FILE_$(PROVIDER))
+# What answers conflict analysis, chosen independently of the provider. Empty
+# keeps the provider's own analyzer; tango adds a fake Tango server and needs
+# PROVIDER=git, whose repository it reads targets from.
+ANALYZER ?=
+ANALYZER_COMPOSE_FILE_tango = $(abspath service/submitqueue/docker-compose.tango.yml)
+ANALYZER_COMPOSE_FILE = $(ANALYZER_COMPOSE_FILE_$(ANALYZER))
+PROVIDER_COMPOSE_FILE = $(PROVIDER_COMPOSE_FILE_$(PROVIDER))$(if $(ANALYZER_COMPOSE_FILE), -f $(ANALYZER_COMPOSE_FILE))
 
 # Where PROVIDER=git keeps the bare repository it lands into. Outside the
 # repository, so a demo leaves nothing in a checkout, and bind-mounted rather
@@ -165,7 +171,7 @@ build: ## Build all services and examples
 # Build Linux binaries required for the local docker-compose flows (local-*).
 # Bazel-run integration/e2e tests do NOT need these: they build the service
 # binaries hermetically as data dependencies.
-build-all-linux: build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-linux build-runway-linux ## Build all Linux binaries for Docker
+build-all-linux: build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-linux build-runway-linux build-tangofake-linux ## Build all Linux binaries for Docker
 	@echo "All Linux binaries ready for Docker"
 
 build-runway-linux: ## Build Runway Linux binary for Docker
@@ -389,7 +395,7 @@ local-submitqueue-clean: ## Stop the stack and remove its volumes, images, and P
 	@echo "Cleaning all services and data..."
 	@# The overlay is named so that volumes it declares — Runway's checkouts —
 	@# are removed too, rather than surviving as an orphan.
-	@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down -v --rmi local
+	@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down --remove-orphans -v --rmi local
 	@rm -rf "$(SQ_GIT_SANDBOX_DIR)"
 	@echo "All services, volumes, images, and $(SQ_GIT_SANDBOX_DIR) removed."
 
@@ -534,14 +540,18 @@ local-submitqueue-restart: build-all-linux ## Restart all services (rebuild and
 	@echo "Services restarted!"
 	@make local-submitqueue-ps
 
-local-submitqueue-start: build-all-linux web-image-load ## Start full stack (PROVIDER=fake|git|github; github needs GITHUB_TOKEN)
+local-submitqueue-start: build-all-linux web-image-load ## Start full stack (PROVIDER=fake|git|github; github needs GITHUB_TOKEN; ANALYZER=tango with PROVIDER=git adds a fake Tango server)
+	@test -z "$(ANALYZER)" || test -n "$(ANALYZER_COMPOSE_FILE)" \
+		|| { echo "Unknown ANALYZER '$(ANALYZER)'; the only choice is tango"; exit 2; }
+	@test -z "$(ANALYZER)" || test "$(PROVIDER)" = "git" \
+		|| { echo "ANALYZER=$(ANALYZER) needs PROVIDER=git, got '$(PROVIDER)'"; exit 2; }
 	@echo "Starting full stack against provider '$(PROVIDER)' ($(SQ_PROVIDER_CONFIG_DIR))..."
 	@test -f "$(SQ_PROVIDER_CONFIG_DIR)/merge.yaml" \
 		|| { echo "No such provider '$(PROVIDER)': $(SQ_PROVIDER_CONFIG_DIR)/merge.yaml not found"; exit 2; }
 	@test -n "$(PROVIDER_COMPOSE_FILE)" \
 		|| { echo "Provider '$(PROVIDER)' has no compose overlay; add PROVIDER_COMPOSE_FILE_$(PROVIDER) to the Makefile"; exit 2; }
 	@if [ "$(PROVIDER)" = "git" ]; then \
-		$(BAZEL) run //tool/gitsandbox -- -sandbox-dir "$(SQ_GIT_SANDBOX_DIR)" || exit 1; \
+		$(BAZEL) run //tool/gitsandbox -- -sandbox-dir "$(SQ_GIT_SANDBOX_DIR)" $(if $(ANALYZER),-targets) || exit 1; \
 	fi
 	@# Rootless Docker maps container root to the host user; rootful Docker needs
 	@# the host UID:GID explicitly, or the services write files into the sandbox
@@ -572,7 +582,7 @@ local-submitqueue-start: build-all-linux web-image-load ## Start full stack (PRO
 
 local-submitqueue-stop: ## Stop the SubmitQueue stack (keeps PROVIDER=git's sandbox; the databases do not survive)
 	@echo "Stopping SubmitQueue services..."
-	@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down
+	@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down --remove-orphans
 	@# Both MySQL services mount anonymous volumes, so `down` detaches them and
 	@# the next `up` creates fresh ones. Saying "data preserved" here would be
 	@# read as "your requests are still there", which they are not.
@@ -584,7 +594,7 @@ local-submitqueue-stop: ## Stop the SubmitQueue stack (keeps PROVIDER=git's sand
 
 local-stop: ## Stop every local stack — SubmitQueue, Stovepipe, and Runway
 	@echo "Stopping all services..."
-	@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down
+	@$(COMPOSE) -f $(COMPOSE_FILE) -f $(PROVIDER_COMPOSE_FILE) -f $(WEB_COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down --remove-orphans
 	@$(COMPOSE) -f $(STOVEPIPE_COMPOSE_FILE) -f $(STOVEPIPE_WEB_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) down
 	@$(COMPOSE) -f $(RUNWAY_COMPOSE_FILE) -p $(RUNWAY_LOCAL_PROJECT) down
 	@echo "Services stopped. Anonymous database volumes are not reused on the next start."

doc/howto/QUICKSTART.md

@@ -259,6 +259,30 @@ Three changes, one more ref update than before.
 
 The sandbox survives `make local-submitqueue-stop`, so what landed is still there to look at. `make local-submitqueue-clean` removes it.
 
+## Decide conflicts with Tango
+
+`PROVIDER=git` decides that two changes conflict by comparing the directories they touch. An analyzer built on a build graph can see more: a change to a library conflicts with a change to anything built on it, though their paths never overlap. `ANALYZER=tango` swaps the analyzer for one that asks a Tango server which build targets each batch affects, and runs a fake Tango server beside the stack so it needs no build system.
+
+```bash
+make local-submitqueue-stop
+make local-submitqueue-start PROVIDER=git ANALYZER=tango
+make demo-requests
+make local-submitqueue-stop PROVIDER=git ANALYZER=tango
+```
+
+Starting with `ANALYZER=tango` also commits a `TARGET` file into each `demo/NN` folder of the sandbox, on top of whatever is already there. `demo/00` and `demo/01` are libraries, and every other folder depends on one of them: even folders on `demo/00`, odd folders on `demo/01`. Pass the same `ANALYZER` when stopping, so the fake server is stopped with the rest. Only the folders `demo/00` to `demo/09` have targets, so `make demo-requests` refuses a `FOLDERS` above 10 against a sandbox seeded this way, because changes in a folder without a target would never conflict.
+
+Look at what the analyzer recorded, the same way as in the speculation section above:
+
+```bash
+docker exec submitqueue-mysql-app-1 mysql -uroot -proot submitqueue \
+  -N -e "select batch_id, dependents from batch_dependent where dependents != '[]'"
+docker exec submitqueue-mysql-app-1 mysql -uroot -proot submitqueue \
+  -N -e "select count(*) from target_signature"
+```
+
+The second count has a row for every batch analyzed, which shows the orchestrator really asked the fake server. Dependencies between batches can now join different folders, where under path overlap they only join batches sharing one.
+
 ## Land against GitHub
 
 The last rung is the only one that needs credentials, and the only one where a change is a pull request that a person can review.

service/submitqueue/demo/provider/README.md

@@ -19,6 +19,8 @@ Pick one with `make local-submitqueue-start PROVIDER=<name>`, which bind-mounts
 
 The three are a ladder, and the rung is the only thing that changes: the same commands land against all of them. `git/` is worth reading first of the two real ones. It is proof that neither half needs a provider — change metadata comes from reading the repository and the merge is the same Runway code path, both against a bare repository addressed by path, with no credential and no API — and it is what the hermetic git E2E (`make e2e-git-test`) runs against.
 
+The conflict analyzer is a separate choice from the provider. `make local-submitqueue-start PROVIDER=git ANALYZER=tango` keeps everything about `git/` and swaps `demo-queue`'s analyzer for Tango, answered by the fake server in [`service/tangofake`](../../../tangofake). It reads [`git/profiles.tango.yaml`](git/profiles.tango.yaml) instead of `profiles.yaml`, and the sandbox gains a `TARGET` file per `demo/NN` folder. Only `PROVIDER=git` supports it, because the fake reads targets from that provider's repository.
+
 Note that the orchestrator and Runway each keep their **own** copy of a queue's repository and configure their own remote for it: the change provider's copy is in `profiles.yaml`, the merger's checkout in `merge.yaml`, and they are independent even when they name the same remote. That is the same shape a deployment has when both point at a remote host.
 
 ## Adding a provider

service/submitqueue/demo/provider/git/BUILD.bazel

@@ -8,7 +8,15 @@ filegroup(
     name = "config",
     srcs = [
         "merge.yaml",
+        "profiles.tango.yaml",
         "profiles.yaml",
     ],
     visibility = ["//test:__subpackages__"],
 )
+
+# Read by the orchestrator's config test, so the profile `ANALYZER=tango`
+# mounts is checked against the loader that will read it.
+exports_files(
+    ["profiles.tango.yaml"],
+    visibility = ["//service/submitqueue/orchestrator/server:__pkg__"],
+)

service/submitqueue/demo/provider/git/profiles.tango.yaml

@@ -0,0 +1,43 @@
+# Extension profiles for the "git" example with the Tango conflict analyzer:
+# `make local-submitqueue-start PROVIDER=git ANALYZER=tango`.
+#
+# The same repository, change provider and build runner as profiles.yaml; the
+# only difference is how demo-queue decides that two batches conflict. Instead
+# of comparing paths, it asks a Tango server which build targets each batch
+# affects, so batches conflict when their targets overlap. The server here is the
+# fake in service/tangofake, which reads the TARGET files that tool/gitsandbox
+# commits into the sandbox.
+
+tango:
+  address: tangofake-service:8080
+  # Shared by every queue's analyzer.
+  signatureCacheBytes: 67108864
+
+defaults:
+  changeProvider: {type: fake}
+  buildRunner: {type: fake}
+  analyzer: {type: all}
+  # Same example factors as ../fake/profiles.yaml.
+  scorer:
+    type: evidence
+    factors:
+      pathPassed: 10
+      pathFailed: 0.3
+      landing: 12
+      cancelling: 0.1
+    base:
+      type: heuristic
+
+queues:
+  - name: demo-queue
+    changeProvider:
+      type: git
+      git:
+        remoteUrl: file:///srv/git/sandbox.git
+        target: main
+        repoPath: /var/submitqueue/changerepos/demo
+    analyzer:
+      type: tango
+      tango:
+        # The fake serves one repository and ignores the remote.
+        remote: sandbox

service/submitqueue/demo/requests/BUILD.bazel

@@ -47,6 +47,7 @@ go_test(
         "//platform/base/change/git:go_default_library",
         "//platform/git/exec:go_default_library",
         "//platform/git/exectest:go_default_library",
+        "//service/submitqueue/demo:go_default_library",
         "@com_github_stretchr_testify//assert:go_default_library",
         "@com_github_stretchr_testify//require:go_default_library",
     ],

service/submitqueue/demo/requests/main.go

@@ -221,7 +221,11 @@ func newChangeSource(ctx context.Context, cfg config) (changeSource, func(), err
 		if err != nil {
 			return nil, nil, err
 		}
-		src, err := newGitSource(ctx, git, filepath.Join(cfg.sandboxDir, sandboxRepo+".git"), sandboxRepo)
+		bare := filepath.Join(cfg.sandboxDir, sandboxRepo+".git")
+		if err := checkFoldersHaveTargets(ctx, git, bare, cfg.base, cfg.folders); err != nil {
+			return nil, nil, err
+		}
+		src, err := newGitSource(ctx, git, bare, sandboxRepo)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -235,6 +239,22 @@ func newChangeSource(ctx context.Context, cfg config) (changeSource, func(), err
 	return nil, nil, fmt.Errorf("unknown provider %q", cfg.provider)
 }
 
+// targetsMarker is the file tool/gitsandbox -targets commits first; it is on
+// the base branch only when the stack runs the Tango analyzer.
+const targetsMarker = "demo/00/TARGET"
+
+// checkFoldersHaveTargets rejects a folder count the sandbox's targets do not
+// cover, since changes in a folder without a target never conflict.
+func checkFoldersHaveTargets(ctx context.Context, git, bare, base string, folders int) error {
+	if folders <= demo.MaxShardDirs {
+		return nil
+	}
+	if err := gitexec.Run(ctx, git, bare, "cat-file", "-e", base+":"+targetsMarker); err != nil {
+		return nil
+	}
+	return fmt.Errorf("-folders %d is above %d, the folders with targets in this sandbox", folders, demo.MaxShardDirs)
+}
+
 // sandboxRepo is the repository the git provider's sandbox holds, matching what
 // tool/gitsandbox creates and what demo/provider/git/merge.yaml merges into.
 const sandboxRepo = "sandbox"

service/submitqueue/demo/requests/source_test.go

@@ -28,6 +28,7 @@ import (
 	gitchange "github.com/uber/submitqueue/platform/base/change/git"
 	gitexec "github.com/uber/submitqueue/platform/git/exec"
 	gitexectest "github.com/uber/submitqueue/platform/git/exectest"
+	"github.com/uber/submitqueue/service/submitqueue/demo"
 )
 
 // quietSpec is a changeSpec whose progress notes go nowhere, for tests that
@@ -245,3 +246,40 @@ func TestNewChangeSource_GitRejectsAMissingSandbox(t *testing.T) {
 	})
 	require.Error(t, err, "a missing sandbox must say so rather than fail later mid-run")
 }
+
+func TestCheckFoldersHaveTargets(t *testing.T) {
+	git := gitexectest.Git(t)
+	ctx := context.Background()
+	plain := sandbox(t, git)
+	withTargets := sandbox(t, git)
+	seed := t.TempDir()
+	require.NoError(t, gitexec.Run(ctx, git, "", "clone", withTargets, seed))
+	require.NoError(t, gitexec.Run(ctx, git, seed, "config", "user.name", "Test"))
+	require.NoError(t, gitexec.Run(ctx, git, seed, "config", "user.email", "test@example.invalid"))
+	require.NoError(t, os.MkdirAll(filepath.Join(seed, "demo", "00"), 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(seed, targetsMarker), nil, 0o644))
+	require.NoError(t, gitexec.Run(ctx, git, seed, "add", "."))
+	require.NoError(t, gitexec.Run(ctx, git, seed, "commit", "-m", "targets"))
+	require.NoError(t, gitexec.Run(ctx, git, seed, "push", "origin", "main"))
+
+	tests := []struct {
+		name    string
+		bare    string
+		folders int
+		wantErr bool
+	}{
+		{name: "targets cover the folders", bare: withTargets, folders: demo.MaxShardDirs},
+		{name: "targets do not cover the folders", bare: withTargets, folders: demo.MaxShardDirs + 1, wantErr: true},
+		{name: "no targets, any folder count", bare: plain, folders: 50},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := checkFoldersHaveTargets(ctx, git, tt.bare, "main", tt.folders)
+			if tt.wantErr {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+		})
+	}
+}

service/submitqueue/demo/workload.go

@@ -23,15 +23,16 @@ import (
 // Five to ten shards produce both shared-directory and independent changes.
 const (
 	minShardDirs = 5
-	maxShardDirs = 10
+	// MaxShardDirs is also how many shard folders have TARGET files in the Tango demo sandbox.
+	MaxShardDirs = 10
 )
 
 func resolveFolders(tag string, configured int) int {
 	if configured > 0 {
 		return configured
 	}
 	sum := sha256.Sum256([]byte("folders#" + tag))
-	return minShardDirs + int(sum[0])%(maxShardDirs-minShardDirs+1)
+	return minShardDirs + int(sum[0])%(MaxShardDirs-minShardDirs+1)
 }
 
 func changeShard(tag string, folders, change int) string {

service/submitqueue/demo/workload_test.go

@@ -86,7 +86,7 @@ func TestResolveFolders(t *testing.T) {
 		for minute := range 60 {
 			folders := resolveFolders(fmt.Sprintf("0810-12%02d", minute), 0)
 			assert.GreaterOrEqual(t, folders, minShardDirs)
-			assert.LessOrEqual(t, folders, maxShardDirs)
+			assert.LessOrEqual(t, folders, MaxShardDirs)
 			seen[folders] = struct{}{}
 		}
 		assert.Greater(t, len(seen), 1, "runs must not all pick the same number")