[Improvement] Code Editor Tasks

config/agent/default.yaml

@@ -54,6 +54,15 @@ save_som: False # Add a set of marks to the screenshot.
 # extract_visible_tag: False # Add a "visible" tag to visible elements in the AXTree.
 # extract_clickable_tag: False #  Add a "clickable" tag to clickable elements in the AXTree.
 extract_coords: False # Add the coordinates of the elements.
+
+# --- coordinate space ---
+# What an (x, y) in the model's output means. null = raw viewport pixels
+# (UI-TARS, GPT-4o); an integer N = a normalized [0, N) grid, converted to
+# pixels by action_parsers.coords.rescale_xy. Qwen-VL / GLM-VL use 1000; the
+# PaliGemma/Gemma lineage bins locations to 1024. Leave null unless you have
+# measured the model's convention -- a wrong scale clicks somewhere plausible
+# and scores 0 silently.
+coord_scale: null
 # filter_visible_elements_only: False  # filter elements that are not visible
 # use_focused_element: False  # use focused element
 

config/agent/gemma-4-computer-use.yaml → config/agent/gemma-4-2B-computer-use.yaml

@@ -4,12 +4,12 @@ defaults:
 
 # Must exactly match the model vLLM is serving (the --model / served_model_name).
 model_name: "google/gemma-4-E2B-it"
-model_pretty_name: "gemma-4-computer-use"
+model_pretty_name: "gemma-4-2B-computer-use"
 api_version: null
 client_type: "vllm"
 # For client_type=vllm the URL is built as http://${hostname}:${port}/v1
 # (base_url is ignored). The vLLM node changes every SLURM allocation, so pass it
-# at launch, e.g.: uv run launch_agent.py agent=gemma-4-e2b-it agent.hostname=h200-000-026
+# at launch, e.g.: uv run launch_agent.py agent=gemma-4-e2b-it agent.hostname=node-001
 hostname: null
 port: "8000"
 # vLLM does not check the key, but the OpenAI client requires a non-empty string.

config/agent/gemma-4-31B-computer-use.yaml

@@ -0,0 +1,137 @@
+# Gemma 4 31B, screenshot-only via set-of-marks.
+#
+# The model sees ONLY the screenshot (no AXTree, no HTML), but the screenshot is
+# annotated with numbered boxes (save_som: true -> ObsFlags.use_som), so it can
+# target elements by bid instead of by pixel coordinate. Gemma is not trained
+# for pixel-precise GUI grounding the way UI-TARS is, so reading a printed bid
+# off the image is the reliable path.
+#
+# The coordinate-mode counterpart is gemma-4-31B-coords.yaml, which declares a
+# 1000x1000 grid and sets coord_scale to convert it to viewport pixels. Run
+# both to compare; this one is the safer default.
+#
+# Pair with: browsergym_env_args=screenshot
+defaults:
+  - default
+  - _self_
+
+# Must exactly match the model vLLM is serving (the --model / served_model_name).
+model_name: "google/gemma-4-31B-it"
+model_pretty_name: "gemma-4-31B-computer-use"
+api_version: null
+client_type: "vllm"
+# For client_type=vllm the URL is built as http://${hostname}:${port}/v1
+# (base_url is ignored). The vLLM node changes every SLURM allocation, so pass it
+# at launch, e.g.: uv run launch_agent.py agent=gemma-4-31B-computer-use agent.hostname=h200-000-026
+hostname: null
+port: "8000"
+# vLLM does not check the key, but the OpenAI client requires a non-empty string.
+api_key: "EMPTY"
+# Grounding wants determinism, not diversity.
+temperature: 0
+max_tokens: 5000
+aws_access_key: null
+aws_secret_key: null
+aws_session_token: null
+aws_region: us-west-2
+
+# Bid-based actions only. Kept deliberately small: every action here is
+# described in action_prompt below, and a smaller space is easier for a model
+# that is not GUI-pretrained. All of these pass through flexible_parser
+# untouched (the UI-TARS remaps in utils.uitars_parser only fire on
+# click(point=/start_box=/x=, type(content=, scroll(direction=, right_single(,
+# and hotkey(key=).
+custom_actions:
+- click
+- fill
+- select_option
+- scroll
+- noop
+
+use_html: false
+use_axtree: false
+use_screenshot: true
+save_som: true # feeds ObsFlags.use_som -> the bid-annotated screenshot
+extract_visible_tag: false
+extract_clickable_tag: false
+extract_coords: false
+filter_visible_elements_only: false
+use_focused_element: false
+prompt_txt:
+  system_prompt: |-
+    You are a GUI agent operating a web browser. At each step you are given a
+    task, your action history, and a screenshot of the current page.
+
+    The screenshot is annotated with a set of marks: every interactive element
+    is outlined with a coloured box and labelled with a short id (its "bid"),
+    drawn at the corner of the box. Target elements by that id. Never guess
+    pixel coordinates -- there is no coordinate-based action available to you.
+
+    Read the bid off the image carefully; it is the label attached to the box
+    around the element you want, not the element's visible text.
+  output_format: '
+
+    <think>
+
+    </think>
+
+    <action>
+
+    </action>
+    '
+
+  think_prompt: null
+  # Overridden: agentlab's default abstract example tells the model to compute
+  # coordinates, which contradicts the bid-only action space above.
+  think_abstract_example: |
+    <think>
+    Think step by step. Describe what your previous action changed on the page,
+    name the element you need next, and read its bid off the annotated
+    screenshot.
+    </think>
+  think_concrete_example: null
+  action_prompt: |
+    ## Action Space
+
+    click("bid")
+      Click the element with this bid. Use for links, buttons, checkboxes,
+      tabs, files in a sidebar, and to focus a text field.
+
+    fill("bid", "text")
+      Type text into the input, textarea or editor with this bid. This
+      REPLACES the whole current value, so when you are editing existing
+      content include the existing text plus your change in one call.
+
+    select_option("bid", "option label")
+      Choose an option in a <select> dropdown.
+
+    scroll(delta_x, delta_y)
+      Scroll the page by a pixel amount. Positive delta_y scrolls down,
+      negative scrolls up. e.g. scroll(0, 400)
+
+    noop(wait_ms=5000)
+      Wait and re-screenshot. Use only when the page appears to still be
+      loading.
+
+    ## Note
+    - Exactly one action per step, inside a single <action> tag.
+    - Use English in the `think` part.
+    - Write a short plan, then state your next action and the element it
+      targets in one sentence.
+    - Check the history: if your last action did not change the page, pick a
+      different target rather than repeating it.
+    - Saving usually needs an explicit click on a Save/Send/Add button after
+      you fill a field.
+  action_abstract_example: |
+    <action>click("bid")</action>
+
+    or, to enter text:
+
+    <action>fill("bid", "the text to type")</action>
+  action_concrete_example: |
+    <action>click("92")</action>
+
+    then on the next step, to replace the contents of a code editor whose box
+    is labelled 47 with the original text plus a new line:
+
+    <action>fill("47", "import torch\nx = torch.randn(2, 3)\n# Reviewed by Bob")</action>

config/agent/gemma-4-31B-coords.yaml

@@ -0,0 +1,171 @@
+# Gemma 4 31B, screenshot-only via pixel coordinates.
+#
+# NOT THE RECOMMENDED CONFIG. Use gemma-4-31B-computer-use.yaml (set-of-marks
+# bids). This one is kept as a measured negative result, so the experiment is
+# not repeated from scratch.
+#
+# ## What was measured
+#
+# The theory was that Gemma's off-target clicks were a coordinate-space
+# problem -- that it emitted normalized coordinates which were being fed to
+# playwright as raw viewport pixels. Two cluster runs falsified that. Taking
+# the executed action and inverting the conversion:
+#
+#   model said (88, 238),  coord_scale=1024, 1920x1080 -> executed (165, 251)
+#   model said (611, 645), coord_scale=1000, 1920x1080 -> executed (1173, 697)
+#
+# Both exact to the pixel, at two different scales. The conversion put the
+# cursor precisely where the model asked; the model asked for the wrong place.
+# That is a grounding failure, not a scaling one, and no coord_scale value
+# fixes it -- see "Calibrating a new model" in src/open_apps/agent/README.md
+# for how to tell the two apart.
+#
+# A run with bids instead was mechanically sound by comparison -- clicks
+# executed and navigation worked -- but the model still misread which printed
+# label belonged to which row. So Gemma 4 31B struggles with pixel grounding
+# *and* with reading marks off an image. Feeding it the accessibility tree
+# (agent.use_axtree=true) turns "decode a label in an image" into "read a
+# name from text" and is the more promising direction.
+#
+# ## If you run this anyway
+#
+# The prompt declares a 1000x1000 grid and coord_scale matches it, so the
+# conversion is correct by construction as long as the model complies -- and
+# it does comply; that was never the failing part. Do not pass
+# agent.coord_scale=1024: it contradicts the declared grid and shrinks every
+# click ~2.3% toward the top-left.
+#
+# Pair with: browsergym_env_args=screenshot (also raises max_steps 10 -> 25;
+# on the default preset this task can run out of budget before it finishes).
+defaults:
+  - default
+  - _self_
+
+# Must exactly match the model vLLM is serving (the --model / served_model_name).
+model_name: "google/gemma-4-31B-it"
+model_pretty_name: "gemma-4-31B-coords"
+api_version: null
+client_type: "vllm"
+# For client_type=vllm the URL is built as http://${hostname}:${port}/v1
+# (base_url is ignored). The vLLM node changes every SLURM allocation, so pass it
+# at launch, e.g.: uv run launch_agent.py agent=gemma-4-31B-coords agent.hostname=h200-000-026
+hostname: null
+port: "8000"
+# vLLM does not check the key, but the OpenAI client requires a non-empty string.
+api_key: "EMPTY"
+# Grounding wants determinism, not diversity.
+temperature: 0
+max_tokens: 5000
+aws_access_key: null
+aws_secret_key: null
+aws_session_token: null
+aws_region: us-west-2
+
+# The model emits coordinates on the 0-1000 grid declared in system_prompt;
+# action_parsers.coords.rescale_xy converts them to viewport pixels.
+coord_scale: 1000
+
+# Coordinate-based actions only. Every action here is described in
+# action_prompt below and survives flexible_parser: mouse_click is emitted
+# directly, and keyboard_type / scroll / noop pass through untouched.
+custom_actions:
+- mouse_click
+- keyboard_type
+- keyboard_press
+- scroll
+- noop
+
+use_html: false
+use_axtree: false
+use_screenshot: true
+# No marks: this config is measuring raw pixel grounding, so the annotated
+# boxes would both give away the answer and clutter the image.
+save_som: false
+extract_visible_tag: false
+extract_clickable_tag: false
+extract_coords: false
+filter_visible_elements_only: false
+use_focused_element: false
+prompt_txt:
+  system_prompt: |-
+    You are a GUI agent operating a web browser. At each step you are given a
+    task, your action history, and a screenshot of the current page.
+
+    Coordinates use a fixed 1000x1000 grid, regardless of the real size of the
+    screenshot. x=0 is the left edge and x=1000 the right edge; y=0 is the top
+    edge and y=1000 the bottom edge. So the centre of the screen is (500, 500),
+    and a button in the bottom-right corner is near (900, 950).
+
+    Always give coordinates on that 1000x1000 grid. Never report coordinates in
+    the screenshot's own pixel size.
+
+    Aim for the centre of the element you want to click, not its edge or its
+    label's first character.
+  output_format: '
+
+    <think>
+
+    </think>
+
+    <action>
+
+    </action>
+    '
+
+  think_prompt: null
+  think_abstract_example: |
+    <think>
+    Think step by step. Describe what your previous action changed on the page,
+    name the element you need next, locate it in the screenshot, and convert
+    its centre to the 1000x1000 grid.
+    </think>
+  think_concrete_example: null
+  action_prompt: |
+    ## Action Space
+
+    mouse_click(x=X, y=Y)
+      Click at (X, Y) on the 1000x1000 grid. Use for links, buttons,
+      checkboxes, tabs, files in a sidebar, and to focus a text field.
+
+    keyboard_type(text="...")
+      Type text at the current keyboard focus. Click the field first to focus
+      it. This appends at the cursor, so to replace existing content select it
+      first with keyboard_press.
+
+    keyboard_press(key="...")
+      Press a key or chord, e.g. keyboard_press(key="Enter"),
+      keyboard_press(key="Control+a"), keyboard_press(key="Backspace").
+
+    scroll(delta_x, delta_y)
+      Scroll the page. Unlike mouse_click, these deltas are in screen pixels,
+      not grid units. Positive delta_y scrolls down, negative scrolls up.
+      e.g. scroll(0, 400)
+
+    noop(wait_ms=5000)
+      Wait and re-screenshot. Use only when the page appears to still be
+      loading.
+
+    ## Note
+    - Exactly one action per step, inside a single <action> tag.
+    - Use English in the `think` part.
+    - Write a short plan, then state your next action and the element it
+      targets in one sentence.
+    - Check the history: if your last action did not change the page, your
+      coordinate probably missed. Re-locate the element and try a different
+      point rather than repeating the same one.
+    - Saving usually needs an explicit click on a Save/Send/Add button after
+      you type into a field.
+  action_abstract_example: |
+    <action>mouse_click(x=X, y=Y)</action>
+
+    or, to enter text once a field is focused:
+
+    <action>keyboard_type(text="the text to type")</action>
+  action_concrete_example: |
+    To click a Send button in the bottom-right of the chat panel:
+
+    <action>mouse_click(x=870, y=940)</action>
+
+    then on the next step, to type into the field it focused and submit:
+
+    <action>keyboard_type(text="Hello world\n")</action>

config/apps/theme/bootstrap.yaml

@@ -0,0 +1,21 @@
+# @package apps.theme
+# Bootstrap 5 look approximated with design tokens (no Bootstrap CSS needed).
+name: bootstrap
+import_url: ""
+tokens:
+  color-bg: "#ffffff"
+  color-surface: "#f8f9fa"
+  color-fg: "#212529"
+  color-muted: "#6c757d"
+  color-border: "#dee2e6"
+  color-primary: "#0d6efd"
+  color-on-primary: "#ffffff"
+  color-accent: "#198754"
+  color-danger: "#dc3545"
+  color-neutral: "#6c757d"
+  color-btn-fg: "#ffffff"
+  font-family: "system-ui, -apple-system, 'Segoe UI', Roboto, sans-serif"
+  font-heading: "inherit"
+  font-size-base: "16px"
+  radius: "0.375rem"
+  space: "8px"

config/apps/theme/challenging_font.yaml

@@ -0,0 +1,22 @@
+# @package apps.theme
+# Faithful port of the former todo `challenging_font` appearance variant:
+# the default palette rendered in a hard-to-read script face.
+name: challenging_font
+import_url: ""
+tokens:
+  color-bg: "#ffffff"
+  color-surface: "#f4f5f7"
+  color-fg: "#1a1a1a"
+  color-muted: "#6b7280"
+  color-border: "#dfe1e6"
+  color-primary: "#2563eb"
+  color-on-primary: "#ffffff"
+  color-accent: "#2563eb"
+  color-danger: "#b22222"
+  color-neutral: "#36454f"
+  color-btn-fg: "#ffffff"
+  font-family: "'Brush Script MT', cursive"
+  font-heading: "inherit"
+  font-size-base: "16px"
+  radius: "8px"
+  space: "8px"