-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathindex.txt
More file actions
89 lines (89 loc) · 73.4 KB
/
Copy pathindex.txt
File metadata and controls
89 lines (89 loc) · 73.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
1:"$Sreact.fragment"
2:I[28422,["./_next/static/chunks/3d-g45tvvtwds.js"],"default"]
3:I[4703,["./_next/static/chunks/3d-g45tvvtwds.js"],"default"]
7:I[76400,["./_next/static/chunks/3d-g45tvvtwds.js"],"default",1]
:HL["./_next/static/chunks/0h5q2shr05yje.css","style"]
:HL["./_next/static/media/PlexMono_Regular-s.p.2x6m9-apme3b_.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
:HL["./_next/static/media/Silkscreen_Regular-s.p.3es8zrs5x_4vu.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
4:Tad0,{"@context":"https://schema.org","@type":"ScholarlyArticle","headline":"SceneAgent: 3D Capture-Derived Scenes with Predictive Physics for Policy Evaluation and Training Environments","name":"SceneAgent","description":"SceneAgent converts 3D captures of real-world scenes into simulation environments with predictive physics for robot policy evaluation, planning and training.","abstract":"SceneAgent converts 3d captures of real-world scenes into simulation environments that can be used for policy evaluation, online planning, and policy training or finetuning. The pipeline primarily focuses on converting environments captured with 3d Gaussian Splatting (3DGS) but also supports scenes captured with photogrammetry and LiDAR. It incorporates 3d semantic features, object segmentation, predictive per-gaussian physics properties, object decomposition, deformability, and articulation. To increase policy generalization, it is also able to generate “digital sisters” or similar versions of each object within an environment. We are still evaluating how well policies trained in this environment perform in the real world, and initial results show promising success rates similar to or improving on other recent papers such as SimFoundry and PolaRiS.","author":[{"@type":"Person","name":"Luke Hollis","affiliation":{"@type":"Organization","name":"Harvard Computational Robotics Group, School of Engineering and Applied Sciences, Harvard University","url":"https://computationalrobotics.seas.harvard.edu/"}},{"@type":"Person","name":"Tianxing Fan","affiliation":{"@type":"Organization","name":"Harvard Computational Robotics Group, School of Engineering and Applied Sciences, Harvard University","url":"https://computationalrobotics.seas.harvard.edu/"}},{"@type":"Person","name":"Heng Yang","affiliation":{"@type":"Organization","name":"Harvard Computational Robotics Group, School of Engineering and Applied Sciences, Harvard University","url":"https://computationalrobotics.seas.harvard.edu/"}}],"publisher":{"@type":"Organization","name":"Harvard Computational Robotics Group","url":"https://computationalrobotics.seas.harvard.edu/"},"datePublished":"2026","creativeWorkStatus":"Preprint","image":"https://computationalrobotics.seas.harvard.edu/SceneAgent/branding/sceneagent-social-share.jpg","url":"https://computationalrobotics.seas.harvard.edu/SceneAgent/","mainEntityOfPage":"https://computationalrobotics.seas.harvard.edu/SceneAgent/","inLanguage":"en","keywords":"SceneAgent, real-to-sim, sim-to-real, 3D Gaussian Splatting, simulation environments, predictive physics, robot policy evaluation, policy training, online planning, object segmentation, articulation, deformable objects, digital sisters, robot learning, Harvard Computational Robotics Group"}0:{"P":null,"c":["",""],"q":"","i":false,"f":[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",16],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"./_next/static/chunks/0h5q2shr05yje.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"./_next/static/chunks/3d-g45tvvtwds.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","className":"silkscreen_5c214b00-module__BDnqEG__variable plexmono_301f1454-module__3YJGwG__variable","children":["$","body",null,{"children":[["$","$L2",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L3",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}],["$","script",null,{"type":"application/ld+json","dangerouslySetInnerHTML":{"__html":"$4"}}]]}]}]]}],{"children":["$L5",{},null,false,null]},null,false,null],"$L6",false]],"m":"$undefined","G":["$7",["$L8"]],"S":true,"h":null,"s":"$undefined","l":"$undefined","p":"$undefined","d":"$undefined","b":"LUBGb-DDhf4v8HxDYyIX7"}
9:I[88409,["./_next/static/chunks/3d-g45tvvtwds.js","./_next/static/chunks/24ble59yb_9_o.js","./_next/static/chunks/1pndem1tl0drp.js","./_next/static/chunks/0p_6srangbw9h.js","./_next/static/chunks/0_sx6di6mxk5o.js"],"SideNav"]
a:I[90112,["./_next/static/chunks/3d-g45tvvtwds.js","./_next/static/chunks/24ble59yb_9_o.js","./_next/static/chunks/1pndem1tl0drp.js","./_next/static/chunks/0p_6srangbw9h.js","./_next/static/chunks/0_sx6di6mxk5o.js"],"VideoPlate"]
b:I[45828,["./_next/static/chunks/3d-g45tvvtwds.js","./_next/static/chunks/24ble59yb_9_o.js","./_next/static/chunks/1pndem1tl0drp.js","./_next/static/chunks/0p_6srangbw9h.js","./_next/static/chunks/0_sx6di6mxk5o.js"],"Button"]
c:I[7838,["./_next/static/chunks/3d-g45tvvtwds.js","./_next/static/chunks/24ble59yb_9_o.js","./_next/static/chunks/1pndem1tl0drp.js","./_next/static/chunks/0p_6srangbw9h.js","./_next/static/chunks/0_sx6di6mxk5o.js"],"PipelineChapter"]
1d:I[36215,["./_next/static/chunks/3d-g45tvvtwds.js"],"ViewportBoundary"]
1f:I[36215,["./_next/static/chunks/3d-g45tvvtwds.js"],"MetadataBoundary"]
20:"$Sreact.suspense"
:HL["./branding/computational-robotics.jpg","image"]
5:["$","$1","c",{"children":[["$","div",null,{"className":"shell","children":[["$","$L9",null,{}],["$","main",null,{"children":[["$","section",null,{"id":"top","style":{"background":"var(--paper)","padding":"40px var(--chapter-pad) 56px"},"children":["$","div",null,{"style":{"maxWidth":"calc(var(--page-w) - 2 * var(--chapter-pad))"},"children":[["$","div",null,{"className":"cover__brand","children":["$","a",null,{"className":"lab-logo","href":"https://computationalrobotics.seas.harvard.edu/","children":["$","img",null,{"src":"./branding/computational-robotics.jpg","alt":"Harvard Computational Robotics Group","width":2286,"height":299}]}]}],["$","div",null,{"style":{"borderTop":"3px solid var(--ink)","paddingTop":10,"display":"flex","justifyContent":"space-between","fontSize":13,"lineHeight":1.45,"color":"var(--ink)"},"children":[["$","div",null,{"children":["Harvard Computational Robotics Group",["$","br",null,{}],"School of Engineering and Applied Sciences, Harvard University"]}],["$","div",null,{"style":{"textAlign":"right"},"children":["Preprint",["$","br",null,{}],"2026"]}]]}],["$","$La",null,{"src":"./videos/cover_video.mp4","poster":"./videos/cover_video.jpg","ratio":"16 / 9","alt":"SceneAgent cover video","controls":false,"style":{"marginTop":40}}],["$","h1",null,{"className":"cover__title","children":"SceneAgent"}],["$","div",null,{"style":{"marginTop":12,"maxWidth":640,"fontSize":14,"color":"var(--text-muted)"},"children":"An agentic pipeline for converting 3D captures to simulation environments with predictive physics for policy evaluation, online planning, and policy training"}],["$","div",null,{"style":{"marginTop":24,"fontSize":15,"fontWeight":700,"color":"var(--ink)"},"children":[["$","span","Luke Hollis",{"children":[["$","a",null,{"href":"mailto:lhollis@g.harvard.edu","children":"Luke Hollis"}],["$","sup",null,{"children":[1,"*"]}],", "]}],["$","span","Tianxing Fan",{"children":[["$","a",null,{"href":"mailto:tianxing_fan@seas.harvard.edu","children":"Tianxing Fan"}],["$","sup",null,{"children":[1,"*"]}],", "]}],["$","span","Heng Yang",{"children":[["$","a",null,{"href":"mailto:hankyang@seas.harvard.edu","children":"Heng Yang"}],["$","sup",null,{"children":[1,""]}],""]}]]}],["$","div",null,{"style":{"marginTop":8,"fontSize":13,"lineHeight":1.5,"color":"var(--text-muted)"},"children":[[["$","span","1",{"children":[["$","sup",null,{"children":1}]," ",["$","a",null,{"href":"https://computationalrobotics.seas.harvard.edu/","target":"_blank","rel":"noopener","children":"Harvard Computational Robotics Group, School of Engineering and Applied Sciences, Harvard University"}]]}]],["$","br",null,{}],["$","sup",null,{"children":"*"}]," ","Equal contribution"]}],["$","div",null,{"style":{"marginTop":24,"display":"flex","alignItems":"center","gap":16,"flexWrap":"wrap"},"children":[["$","$Lb",null,{"size":"sm","disabled":true,"style":{"textTransform":"uppercase"},"children":"Paper"}],["$","$Lb",null,{"size":"sm","variant":"secondary","disabled":true,"style":{"textTransform":"uppercase"},"children":"Code"}],["$","span",null,{"style":{"fontSize":12,"textTransform":"uppercase","color":"var(--text-muted)"},"children":"[coming soon]"}]]}],["$","p",null,{"className":"chapter__text","children":"SceneAgent converts 3d captures of real-world scenes into simulation environments that can be used for policy evaluation, online planning, and policy training or finetuning. The pipeline primarily focuses on converting environments captured with 3d Gaussian Splatting (3DGS) but also supports scenes captured with photogrammetry and LiDAR. It incorporates 3d semantic features, object segmentation, predictive per-gaussian physics properties, object decomposition, deformability, and articulation. To increase policy generalization, it is also able to generate “digital sisters” or similar versions of each object within an environment. We are still evaluating how well policies trained in this environment perform in the real world, and initial results show promising success rates similar to or improving on other recent papers such as SimFoundry and PolaRiS."}]]}]}],["$","$Lc",null,{}],["$","section",null,{"className":"chapter","id":"cousins","children":[["$","div",null,{"className":"chapter__rule","aria-hidden":"true"}],["$","div",null,{"className":"chapter__head","children":[["$","div",null,{"className":"chapter__num","aria-hidden":"true","children":"2"}],"$Ld"]}],["$Le","$Lf","$L10","$L11"]]}],"$L12","$L13","$L14","$L15","$L16","$L17"]}]]}],["$L18","$L19","$L1a","$L1b"],"$L1c"]}]
6:["$","$1","h",{"children":[null,["$","$L1d",null,{"children":"$L1e"}],["$","div",null,{"hidden":true,"children":["$","$L1f",null,{"children":["$","$20",null,{"name":"Next.Metadata","children":"$L21"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}]
8:["$","link","0",{"rel":"stylesheet","href":"./_next/static/chunks/0h5q2shr05yje.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}]
22:I[79999,["./_next/static/chunks/3d-g45tvvtwds.js","./_next/static/chunks/24ble59yb_9_o.js","./_next/static/chunks/1pndem1tl0drp.js","./_next/static/chunks/0p_6srangbw9h.js","./_next/static/chunks/0_sx6di6mxk5o.js"],"CousinsGallery"]
44:I[36215,["./_next/static/chunks/3d-g45tvvtwds.js"],"OutletBoundary"]
d:["$","div",null,{"children":[["$","h2",null,{"className":"chapter__title","children":"Digital Twins and Sisters"}],["$","p",null,{"className":"chapter__lead","children":"In order to increase training generalizability from simulation back to the real world, we create slight variations of each object in the scenes converted by the pipeline."}]]}]
e:["$","div",null,{"className":"chapter__exhibit","data-bleed":"$undefined","children":[["$","$L22",null,{}],["$","div",null,{"className":"chapter__caption","children":["$undefined",["$","div",null,{"style":{"fontFamily":"var(--font-sans)","fontSize":10,"fontWeight":700,"lineHeight":1.5,"letterSpacing":"0.02em","color":"var(--accent)","marginLeft":"auto","flexShrink":0},"children":"2.2"}]]}]]}]
f:["$","div",null,{"className":"chapter__text","children":[["$","p",null,{"children":"Instead of digital twins, these are rather “digital sisters,” not exact replicas of reality but similar variations that reflect the range of diversity that a manipulator may encounter in real-world scenarios. We also randomize placement of objects and lighting to continue to improve policy performance in real-world rollouts."}],["$","p",null,{"children":"In our initial testing, this increases real-world task completion rates after training in our converted simulation digital sister scenes, but we are continuing to test and evaluate how different lighting, placement, and generative 3d models influence policy performance from simulation back to the real world."}],["$","p",null,{"children":"We chose to randomize each of these factors when composing full digital sister scenes, but future work could test each in isolation to verify its influence on policy generalizability during finetuning to study which is best to spend more iterations on."}]]}]
10:["$","div",null,{"className":"chapter__exhibit sub","data-bleed":"$undefined","children":[["$","div",null,{"className":"clips","style":{"gridTemplateColumns":"repeat(2, minmax(0, 1fr))"},"children":[["$","$La","warehouse_suitcase_suitcase",{"src":"./videos/warehouse_suitcase_suitcase.mp4","poster":"./videos/warehouse_suitcase_suitcase.jpg","caption":"Contact rich tasks with predictive physics, such as rolling a red rubber ball from one side of the suitcase to the other","ratio":"4 / 3","alt":"Warehouse · a different suitcase"}],["$","$La","studio_flower_noir_cinema",{"src":"./videos/studio_flower_noir_cinema.mp4","poster":"./videos/studio_flower_noir_cinema.jpg","caption":"Picking up delicate items such as the flower 3d capture from the previous greenhouse scene in hybrid digital art and 3d capture environments","ratio":"4 / 3","alt":"Studio · flower grasp, relit"}]]}],["$","div",null,{"className":"chapter__caption","children":["$undefined",["$","div",null,{"style":{"fontFamily":"var(--font-sans)","fontSize":10,"fontWeight":700,"lineHeight":1.5,"letterSpacing":"0.02em","color":"var(--accent)","marginLeft":"auto","flexShrink":0},"children":"2.3"}]]}]]}]
11:["$","div",null,{"className":"chapter__text","children":[["$","p",null,{"children":"Along with the typical tasks being trained in simulation policy, we are seeking to understand better how the predictive physics properties can help us achieve a series of contact-rich tasks, such as rolling a red rubber ball from one side of the suitcase to another with a gripper on the Franka arm or picking a columnar object from a tightly packed box."}],["$","p",null,{"children":"In future iterations, we also hope to study further how small perturbations of the physics properties may also influence training in simulation. For example, if an object’s mass, friction coefficient, or rigidity are varied slightly between versions, will this help our policy during finetuning more accurately grasp and manipulate deformable objects or objects with heterogeneous physics properties (such as shoes, hats, cardboard boxes, and similar)."}]]}]
12:["$","section",null,{"className":"chapter chapter--bleed","id":"hybrid","children":[["$","div",null,{"className":"chapter__rule","aria-hidden":"true"}],["$","div",null,{"className":"chapter__head","children":[["$","div",null,{"className":"chapter__num","aria-hidden":"true","children":"3"}],["$","div",null,{"children":[["$","h2",null,{"className":"chapter__title","children":"Hybrid Scenes with 3d Captures and Digital Art"}],["$","p",null,{"className":"chapter__lead","children":"We convert individual 3d captured objects into simulation assets with our SceneAgent pipeline and incorporate them into synthetic scenes prepared by 3d artists."}]]}]]}],[[["$","$1","scenes",{"children":[false,["$","div",null,{"className":"chapter__exhibit chapter__exhibit--bleed","data-bleed":"","children":[["$","div",null,{"className":"clips","style":{"gridTemplateColumns":"repeat(3, minmax(0, 1fr))"},"children":[["$","$La","kitchen_apple",{"src":"./videos/kitchen_apple.mp4","poster":"./videos/kitchen_apple.jpg","caption":"Hybrid scenes from available assets in Isaac Lab, a Kitchen scene picking fruit, similar to RoboCasa","ratio":"4 / 3","alt":"Kitchen · apple"}],["$","$La","warehouse_pack",{"src":"./videos/warehouse_pack.mp4","poster":"./videos/warehouse_pack.jpg","caption":"Contact rich tasks picking tightly packed vertical cylinders","ratio":"4 / 3","alt":"Warehouse · pack"}],["$","$La","chem_lab_sort",{"src":"./videos/chem_lab_sort.mp4","poster":"./videos/chem_lab_sort.jpg","caption":"Chemistry lab, sorting vials into beakers","ratio":"4 / 3","alt":"Chemistry lab · sort"}],["$","$La","datacenter_cable",{"src":"./videos/datacenter_cable.mp4","poster":"./videos/datacenter_cable.jpg","caption":"Picking and plugging cables into a data center server rack","ratio":"4 / 3","alt":"Data center · cable"}],["$","$La","studio_cardboard_fold",{"src":"./videos/studio_cardboard_fold.mp4","poster":"./videos/studio_cardboard_fold.jpg","caption":"Two arms folding a cardboard packaging in simulation, packing a deformable object","ratio":"4 / 3","alt":"Studio · cardboard fold"}],["$","$La","studio_succulent_watering",{"src":"./videos/studio_succulent_watering.mp4","poster":"./videos/studio_succulent_watering.jpg","caption":"Hybrid digital art and 3d capture scene with a Franka Emika Panda arm watering a succulent with an eyedropper","ratio":"4 / 3","alt":"Studio · succulent watering"}]]}],["$","div",null,{"className":"chapter__caption","children":["$undefined",["$","div",null,{"style":{"fontFamily":"var(--font-sans)","fontSize":10,"fontWeight":700,"lineHeight":1.5,"letterSpacing":"0.02em","color":"var(--accent)","marginLeft":"auto","flexShrink":0},"children":"3.2"}]]}]]}]]}],["$","$1","hands",{"children":[["$","div",null,{"className":"chapter__text","children":[["$","p",null,{"children":"Because many backgrounds and environments are redundant across common tasks, we combine specific 3d captured objects processed with our SceneAgent pipeline with other existing art assets created by 3d artists, such as in the NVIDIA Omniverse asset repository. This allows for rapid development of common scenes for policy evaluation along with scenes with specific objects for individual tasks."}],["$","p",null,{"children":"For example, when a manipulator may need to learn how to use a specific part, say for a new healthcare device or a new cooling fan in a server rack, that specific part individually could be 3d captured and processed in our pipeline and then incorporated into a pre-existing hospital or data center scene such as the default environments that accompany NVIDIA Isaac Lab."}]]}],["$","div",null,{"className":"chapter__exhibit chapter__exhibit--bleed sub","data-bleed":"","children":[["$","div",null,{"className":"clips","style":{"gridTemplateColumns":"repeat(3, minmax(0, 1fr))"},"children":[["$","$La","sharpa_hand_flower_scoop_realtime",{"src":"./videos/sharpa_hand_flower_scoop_realtime.mp4","poster":"./videos/sharpa_hand_flower_scoop_realtime.jpg","caption":"A Sharpa dexterous hand scooping to cup a delicate flower in its palm","ratio":"4 / 3","alt":"Studio · Sharpa hand, real time"}],"$L23","$L24","$L25","$L26","$L27"]}],"$L28"]}]]}]],"$L29"]]}]
13:["$","section",null,{"className":"chapter chapter--bleed","id":"agents","children":[["$","div",null,{"className":"chapter__rule","aria-hidden":"true"}],["$","div",null,{"className":"chapter__head","children":[["$","div",null,{"className":"chapter__num","aria-hidden":"true","children":"4"}],["$","div",null,{"children":[["$","h2",null,{"className":"chapter__title","children":"Agentic Pipeline for SceneAgent"}],["$","p",null,{"className":"chapter__lead","children":"We provide a team of agents to convert many types of 3d captures into fully usable simulation scenes ready for training in Isaac Lab, MuJoCo, or Unreal. The easiest way to convert a 3d capture is by an agent swarm that processes each step of the pipeline and visually inspects and reviews it for errors."}]]}]]}],[["$","div",null,{"className":"chapter__exhibit chapter__exhibit--bleed","data-bleed":"","children":[["$","div",null,{"className":"spread","role":"img","aria-label":"The agentic workflow: nine agents in three stages, Input, Conversion and Sisters, processing 3d capture data into usable simulation scenes.","children":["$","div",null,{"className":"spread__grid","style":{"--spread-fluid":"minmax(0, 480fr) minmax(0, 144fr) minmax(0, 480fr) minmax(0, 144fr) minmax(0, 480fr)","--spread-fixed":"calc(480px * var(--s)) minmax(0, 1fr) calc(480px * var(--s)) minmax(0, 1fr) calc(480px * var(--s))"},"children":[["$","$1","00",{"children":[["$","svg",null,{"className":"spread__col","viewBox":"0 0 480 932","style":{"fontFamily":"Helvetica, \"Helvetica Neue\", Arimo, Arial, sans-serif"},"children":[["$","g",null,{"children":[["$","text",null,{"x":0,"y":26,"fontSize":26,"fontWeight":700,"fill":"var(--accent)","children":"00"}],["$","text",null,{"x":46,"y":26,"fontSize":26,"fontWeight":700,"fill":"var(--ink)","children":"Input"}],["$","line",null,{"x1":0,"y1":38.5,"x2":480,"y2":38.5,"stroke":"var(--ink)"}],[["$","text","0",{"x":0,"y":64,"fontSize":24,"fill":"var(--text-muted)","children":"Process input 3d data from a diverse"}],["$","text","1",{"x":0,"y":92,"fontSize":24,"fill":"var(--text-muted)","children":"range of sources, including RGB images,"}],["$","text","2",{"x":0,"y":120,"fontSize":24,"fill":"var(--text-muted)","children":"pre-processed photogrammetry files,"}],["$","text","3",{"x":0,"y":148,"fontSize":24,"fill":"var(--text-muted)","children":"or LiDAR"}]]]}],[["$","g","01",{"children":[["$","g",null,{"className":"agent-node","role":"img","aria-label":"01 Orchestrator: Plans the full conversion pipeline based on the status of the input data","children":[["$","rect",null,{"x":0.5,"y":168.5,"width":479,"height":165,"fill":"var(--paper)","stroke":"var(--ink)"}],["$","g",null,{"transform":"translate(14 184) scale(7)","fill":"currentColor","shapeRendering":"crispEdges","aria-hidden":"true","children":["$","g",null,{"className":"agent-sprite","style":{"--i":0},"children":[["$","g","leg",{"className":"agent-sprite__leg","children":[["$","rect","6-4",{"x":4,"y":6,"width":1,"height":1}],["$","rect","6-6",{"x":6,"y":6,"width":1,"height":1}],["$","rect","6-8",{"x":8,"y":6,"width":1,"height":1}]]}],["$","g","body",{"className":"agent-sprite__body","children":[["$","rect","3-2",{"x":2,"y":3,"width":9,"height":1}],["$","rect","4-2",{"x":2,"y":4,"width":2,"height":1}],["$","rect","4-5",{"x":5,"y":4,"width":3,"height":1}],["$","rect","4-9",{"x":9,"y":4,"width":2,"height":1}],["$","rect","5-2",{"x":2,"y":5,"width":9,"height":1}]]}],["$","g","tool",{"className":"agent-sprite__tool","children":[["$","rect","0-6",{"x":6,"y":0,"width":1,"height":1}],["$","rect","1-6",{"x":6,"y":1,"width":1,"height":1}],["$","rect","2-6",{"x":6,"y":2,"width":1,"height":1}]]}]]}]}],["$","text",null,{"x":129,"y":210,"fontSize":26,"fontWeight":700,"fill":"var(--accent)","children":"01"}],["$","text",null,{"x":173,"y":210,"fontSize":26,"fontWeight":700,"fill":"var(--ink)","children":"Orchestrator"}],[["$","text","0",{"x":14,"y":265,"fontSize":21,"fill":"var(--text-body)","children":"Plans the full conversion pipeline based"}],["$","text","1",{"x":14,"y":291,"fontSize":21,"fill":"var(--text-body)","children":"on the status of the input data"}]],["$","text",null,{"x":14,"y":320,"fontSize":15,"fontFamily":"var(--font-pixel)","letterSpacing":"0.08em","fill":"var(--text-label)","children":"REAL2SIM-RUN-PIPELINE"}]]}],["$","line",null,{"x1":240,"y1":334,"x2":240,"y2":362,"stroke":"var(--text-label)"}]]}],["$","g","02",{"children":[["$","g",null,{"className":"agent-node","role":"img","aria-label":"02 Ingest: Process input RGB or other data sources to 3DGS","children":[["$","rect",null,{"x":0.5,"y":362.5,"width":479,"height":165,"fill":"var(--paper)","stroke":"var(--ink)"}],["$","g",null,{"transform":"translate(14 378) scale(7)","fill":"currentColor","shapeRendering":"crispEdges","aria-hidden":"true","children":"$L2a"}],"$L2b","$L2c",["$L2d","$L2e"],"$L2f"]}],"$L30"]}],"$L31"]]}],"$L32"]}],"$L33","$L34"]}]}],"$L35"]}],"$L36","$L37","$L38","$L39","$L3a","$L3b"]]}]
14:["$","section",null,{"className":"chapter","id":"policy-training","children":[["$","div",null,{"className":"chapter__rule","aria-hidden":"true"}],["$","div",null,{"className":"chapter__head","children":[["$","div",null,{"className":"chapter__num","aria-hidden":"true","children":"5"}],["$","div",null,{"children":[["$","h2",null,{"className":"chapter__title","children":"SceneAgent pipeline: Policy training"}],["$","p",null,{"className":"chapter__lead","children":"The policy training part of the SceneAgent pipeline starts from a reconstructed intractable simulation environment, which is a Gaussian-splat scene with segmented object meshes and predicted physics, of a robot's workspace. It then converts the simulation environment into manipulation policies that can be deployed on the real robot."}]]}]]}],["$","div",null,{"className":"chapter__text","children":[["$","p",null,{"children":"The SceneAgent pipeline generally consists of two major blocks: (i) a demonstration factory that generates thousands of expert episodes with photo-realistic observations inside the twin; (ii) policy training on these episodes alone, and zero-shot transfer to real world environment."}],["$","p",null,{"children":"The demonstration factory creates demonstrations for policy training with two scalable blocks: initial state randomization; and a physics-bounded scripted expert for generating motion control. Randomization creates possible distributions where the reality may be in at large scales: the objects are placed anywhere in the region on the table that the real task uses, the manipulated object's size varies, and the camera and robot base are perturbed around their calibrated poses. Sampled layouts are also physics-bounded: objects are dropped and settled under physics, so every initial state is one that the real table could actually present."}],["$","p",null,{"children":"The scripted expert supplies the actions: it plans from prescient ground-truth object poses inside the simulator and pairs a pick-and-place routine — locate, approach, descend, grasp, lift, carry, lower, release, retreat — to the sampled poses. Finally, a success gate checks whether the object truly ended up in the container and discards episodes that fail. In each recorded episode, observations are rendered by depth-compositing a mesh foreground (robot and objects) over the Gaussian-splat background of the deployment environment."}],["$","p",null,{"children":"The model-agnostic training uses the simulation-generated demonstration episodes for imitation-learning: camera streams are paired with the expert's actions in a standard format, and each episode carries a natural-language instruction sampled from many phrasings of the task. The pipeline then finetunes a pretrained vision language action model on the demonstration dataset via LoRA."}]]}]]}]
15:["$","section",null,{"className":"chapter","id":"sim2real","children":[["$","div",null,{"className":"chapter__rule","aria-hidden":"true"}],["$","div",null,{"className":"chapter__head","children":[["$","div",null,{"className":"chapter__num","aria-hidden":"true","children":"6"}],["$","div",null,{"children":[["$","h2",null,{"className":"chapter__title","children":"SceneAgent Evaluation in Progress"}],["$","p",null,{"className":"chapter__lead","children":"After fine-tuning a Vision-Language-Action (VLA) model policy in simulation only, we run the policy in the real world and see improvement in performance. We are still completing the evaluation of our pipeline and will update this with our results soon."}]]}]]}],[["$","div",null,{"className":"chapter__exhibit","data-bleed":"$undefined","children":[["$","$La","evaluation/real_sim_wrist_episodes_1_5_18_mosaic_real15s",{"src":"./videos/evaluation/real_sim_wrist_episodes_1_5_18_mosaic_real15s.mp4","poster":"./videos/evaluation/real_sim_wrist_episodes_1_5_18_mosaic_real15s.jpg","ratio":"16 / 9","alt":"Evaluation mosaic · episodes 1, 5, and 18 from left to right; real world, simulation, and wrist camera from top to bottom; real-world rollouts sped up to 15 seconds","style":{"border":0,"background":"var(--paper)"}}],["$","div",null,{"className":"chapter__caption","children":[["$","div",null,{"style":{"fontFamily":"var(--font-sans)","fontSize":10,"fontWeight":700,"lineHeight":1.5,"letterSpacing":"0.02em","color":"var(--ink)"},"children":"Policy evaluation in realworld on the Franka Emika Panda after finetuning in simulation."}],["$","div",null,{"style":{"fontFamily":"var(--font-sans)","fontSize":10,"fontWeight":700,"lineHeight":1.5,"letterSpacing":"0.02em","color":"var(--accent)","marginLeft":"auto","flexShrink":0},"children":"6.2"}]]}]]}],["$","div",null,{"className":"chapter__exhibit sub","data-bleed":"$undefined","children":[["$","div",null,{"className":"clips","style":{"gridTemplateColumns":"repeat(3, minmax(0, 1fr))"},"children":[["$","figure","evaluation_layout_1",{"style":{"margin":0,"minWidth":0,"fontFamily":"var(--font-sans)"},"children":[["$","div",null,{"style":{"position":"relative","border":"1px solid var(--hairline)","background":"var(--paper)","aspectRatio":"842 / 369","overflow":"hidden"},"children":[["$","img",null,{"src":"./figures/evaluation_layout_1.webp","alt":"Trial layout for episode 1: the egg and the basket in plan view about P0","loading":"lazy","style":{"position":"absolute","inset":0,"width":"100%","height":"100%","objectFit":"cover","objectPosition":"$undefined","display":"block","padding":0}}],false,false,"$undefined"]}],"$undefined"]}],["$","figure","evaluation_layout_5",{"style":{"margin":0,"minWidth":0,"fontFamily":"var(--font-sans)"},"children":[["$","div",null,{"style":{"position":"relative","border":"1px solid var(--hairline)","background":"var(--paper)","aspectRatio":"842 / 369","overflow":"hidden"},"children":[["$","img",null,{"src":"./figures/evaluation_layout_5.webp","alt":"Trial layout for episode 5: the egg and the basket in plan view about P0","loading":"lazy","style":{"position":"absolute","inset":0,"width":"100%","height":"100%","objectFit":"cover","objectPosition":"$undefined","display":"block","padding":0}}],false,false,"$undefined"]}],"$undefined"]}],["$","figure","evaluation_layout_18",{"style":{"margin":0,"minWidth":0,"fontFamily":"var(--font-sans)"},"children":[["$","div",null,{"style":{"position":"relative","border":"1px solid var(--hairline)","background":"var(--paper)","aspectRatio":"842 / 369","overflow":"hidden"},"children":[["$","img",null,{"src":"./figures/evaluation_layout_18.webp","alt":"Trial layout for episode 18: the egg and the basket in plan view about P0","loading":"lazy","style":{"position":"absolute","inset":0,"width":"100%","height":"100%","objectFit":"cover","objectPosition":"$undefined","display":"block","padding":0}}],false,false,"$undefined"]}],"$undefined"]}]]}],["$","div",null,{"className":"chapter__caption","children":["$undefined",["$","div",null,{"style":{"fontFamily":"var(--font-sans)","fontSize":10,"fontWeight":700,"lineHeight":1.5,"letterSpacing":"0.02em","color":"var(--accent)","marginLeft":"auto","flexShrink":0},"children":"6.3"}]]}]]}],["$","div",null,{"className":"chapter__text","children":[["$","p",null,{"children":"After finetuning generalist VLAs such as Physical Intelligence’s pi0.5 and NVIDIA’s GR00T 1.6 in simulation only on the digital twin and sisters, we see similar performance in the real world as in simulation. Initial results show a positive increase in task completion rates in the real world after training for 10,000 episodes in simulation of our lab space and example objects processed through our SceneAgent pipeline."}],"$L3c"]}]]]}]
16:["$","section",null,{"className":"chapter","id":"citation","children":[["$","div",null,{"className":"chapter__rule","aria-hidden":"true"}],["$","div",null,{"className":"chapter__head","children":[["$","div",null,{"className":"chapter__num","aria-hidden":"true","children":"7"}],["$","div",null,{"children":[["$","h2",null,{"className":"chapter__title","children":"Citation"}],"$undefined"]}]]}],[["$","div",null,{"className":"chapter__exhibit","data-bleed":"$undefined","children":["$","div",null,{"style":{"fontFamily":"var(--font-sans)","background":"var(--surface-card)","border":"1px solid var(--hairline)","padding":0,"boxShadow":"none","color":"var(--text-body)"},"children":["$","pre",null,{"style":{"fontSize":12,"lineHeight":1.6,"padding":20,"color":"var(--text-body)"},"children":"@misc{sceneagent2026,\n title = {SceneAgent: 3D Capture-Derived Scenes with Predictive\n Physics for Policy Evaluation and Training Environments},\n author = {Luke Hollis and Tianxing Fan and Heng Yang},\n year = {2026},\n note = {Preprint. Computational Robotics Group, Harvard University}\n}"}]}]}],["$","div",null,{"className":"chapter__text","children":["$","p",null,{"children":"The code, processed scenes, and example converted objects will be released with our paper soon."}]}],["$","div",null,{"className":"chapter__exhibit sub","data-bleed":"$undefined","children":[["$","h3",null,{"className":"chapter__subtitle","children":"Featured Art and 3D Captures"}],["$","ol",null,{"className":"credits","children":[["$","li","flower",{"children":[["$","b",null,{"children":["[",1,"]"]}],["$","span",null,{"children":["3dhdscan, ","“",["$","a",null,{"href":"https://sketchfab.com/3d-models/rudbeckia-flower-eee1a1ebdb614f38a722b52e26ebe038","target":"_blank","rel":"noopener noreferrer","children":"Rudbeckia flower"}],",” ","Sketchfab",", CC BY 4.0","."]}]]}],["$","li","radio",{"children":[["$","b",null,{"children":["[",2,"]"]}],["$","span",null,{"children":["","“",["$","a",null,{"href":"https://poly.cam/capture/71974300-d679-4e71-a7a9-af78c27b2f98","target":"_blank","rel":"noopener noreferrer","children":"Vintage Sony radio CFS-43"}],",” ","Polycam","","."]}]]}],["$","li","totoro",{"children":[["$","b",null,{"children":["[",3,"]"]}],["$","span",null,{"children":["TechnologicalFrostfang2470, ","“",["$","a",null,{"href":"https://poly.cam/capture/cae4a43a-87aa-4ed8-9e21-cff222662d0b","target":"_blank","rel":"noopener noreferrer","children":"Totoro"}],",” ","Polycam","","."]}]]}],["$","li","statue",{"children":[["$","b",null,{"children":["[",4,"]"]}],["$","span",null,{"children":["netgis, ","“",["$","a",null,{"href":"https://sketchfab.com/3d-models/chinese-statue-e0ccdb444522470f9e6567870e63c40c","target":"_blank","rel":"noopener noreferrer","children":"Chinese statue"}],",” ","Sketchfab",", CC BY-NC 4.0","."]}]]}],["$","li","motherboard",{"children":[["$","b",null,{"children":["[",5,"]"]}],["$","span",null,{"children":["Lassi Kaukonen, ","“",["$","a",null,{"href":"https://sketchfab.com/3d-models/asus-z170-p-motherboard-b998596cfc4945a0bc7b016005c39321","target":"_blank","rel":"noopener noreferrer","children":"Asus Z170-P motherboard"}],",” ","Sketchfab",", CC BY 4.0","."]}]]}],["$","li","gpu",{"children":[["$","b",null,{"children":["[",6,"]"]}],["$","span",null,{"children":["Igor.Jop, ","“",["$","a",null,{"href":"https://sketchfab.com/3d-models/rx-480-gpu-61e69f50bcd44c7284bef31a8f21c6a7","target":"_blank","rel":"noopener noreferrer","children":"RX 480 GPU"}],",” ","Sketchfab",", CC BY 4.0","."]}]]}],["$","li","stegosaurus",{"children":[["$","b",null,{"children":["[",7,"]"]}],["$","span",null,{"children":["","“",["$","a",null,{"href":"https://poly.cam/capture/df2dae99-a7ac-4d18-a308-3b7166aa4040","target":"_blank","rel":"noopener noreferrer","children":"Stegosaurus composite skeleton, American Museum of Natural History"}],",” ","Polycam","","."]}]]}],["$","li","bencini",{"children":[["$","b",null,{"children":["[",8,"]"]}],["$","span",null,{"children":["","“",["$","a",null,{"href":"https://poly.cam/capture/4e1f9431-425f-49be-844d-bc97a664f2e4","target":"_blank","rel":"noopener noreferrer","children":"Bencini Comet Universal 444 camera (1977)"}],",” ","Polycam","","."]}]]}],["$","li","cessna",{"children":[["$","b",null,{"children":["[",9,"]"]}],["$","span",null,{"children":["wlanfr3ak, ","“",["$","a",null,{"href":"https://poly.cam/capture/5DF483F7-5ECB-48EA-BDFC-EA4FD93A5A2C","target":"_blank","rel":"noopener noreferrer","children":"Cessna C172R cockpit"}],",” ","Polycam","","."]}]]}],"$L3d","$L3e","$L3f","$L40","$L41","$L42","$L43"]}]]}]]]}]
17:["$","footer",null,{"className":"colophon","children":["$","div",null,{"className":"colophon__field","children":["$","div",null,{"className":"colophon__content","children":[["$","a",null,{"className":"lab-logo","href":"https://computationalrobotics.seas.harvard.edu/","children":["$","img",null,{"src":"./branding/computational-robotics.jpg","alt":"Harvard Computational Robotics Group","width":2286,"height":299}]}],["$","div",null,{"className":"colophon__credit","children":["$","div",null,{"style":{"fontFamily":"var(--font-sans)","fontSize":13,"fontWeight":700,"lineHeight":1.5,"letterSpacing":"0.02em","color":"var(--gray-cool)","textAlign":"right"},"children":["Harvard Computational Robotics Group",["$","br",null,{}],"School of Engineering and Applied Sciences",["$","br",null,{}],"Harvard University"]}]}]]}]}]}]
18:["$","script","script-0",{"src":"./_next/static/chunks/24ble59yb_9_o.js","async":true,"nonce":"$undefined"}]
19:["$","script","script-1",{"src":"./_next/static/chunks/1pndem1tl0drp.js","async":true,"nonce":"$undefined"}]
1a:["$","script","script-2",{"src":"./_next/static/chunks/0p_6srangbw9h.js","async":true,"nonce":"$undefined"}]
1b:["$","script","script-3",{"src":"./_next/static/chunks/0_sx6di6mxk5o.js","async":true,"nonce":"$undefined"}]
1c:["$","$L44",null,{"children":["$","$20",null,{"name":"Next.MetadataOutlet","children":"$@45"}]}]
23:["$","$La","hands_clipping_in_gpu",{"src":"./videos/hands_clipping_in_gpu.mp4","poster":"./videos/hands_clipping_in_gpu.jpg","caption":"Two dextrous Sharpa hands clipping a GPU into place into a motherboard","ratio":"4 / 3","alt":"Dexterous hands · GPU installation"}]
24:["$","$La","hands_sunglasses_5x",{"src":"./videos/hands_sunglasses_5x.mp4","poster":"./videos/hands_sunglasses_5x.jpg","caption":"Sharpa hand picking up and unfolding sunglasses then putting them on a face","ratio":"4 / 3","alt":"Dexterous hands · sunglasses, 5× speed"}]
25:["$","$La","dexmate_sharpa_pizza_and_dishes",{"src":"./videos/dexmate_sharpa_pizza_and_dishes.mp4","poster":"./videos/dexmate_sharpa_pizza_and_dishes.jpg","caption":"A Dexmate mobile robot with dual Sharpa dexterous hands serving a pizza slice onto a plate and clearing the dishes to the sink","ratio":"4 / 3","alt":"Kitchen · Dexmate dual Sharpa"}]
26:["$","$La","studio_sunglasses_noir",{"src":"./videos/studio_sunglasses_noir.mp4","poster":"./videos/studio_sunglasses_noir.jpg","caption":"Two Sharpa dexterous hands lifting and unfolding a pair of sunglasses, then setting them on a bust","ratio":"4 / 3","alt":"Studio · Sharpa sunglasses, noir"}]
27:["$","$La","sharpa_hello_world_lift_then_uncap",{"src":"./videos/sharpa_hello_world_lift_then_uncap.mp4","poster":"./videos/sharpa_hello_world_lift_then_uncap.jpg","caption":"Dual robot arms lifting a marker into a Sharpa dexterous hand, uncapping it, and writing Hello World on a whiteboard","ratio":"4 / 3","alt":"Studio · Sharpa lift and uncap"}]
28:["$","div",null,{"className":"chapter__caption","children":["$undefined",["$","div",null,{"style":{"fontFamily":"var(--font-sans)","fontSize":10,"fontWeight":700,"lineHeight":1.5,"letterSpacing":"0.02em","color":"var(--accent)","marginLeft":"auto","flexShrink":0},"children":"3.3"}]]}]
29:["$","div",null,{"className":"chapter__text","children":[["$","p",null,{"children":"The generated environments can be adapted for policy evaluation and training for multiple embodiments, and we believe that they will be especially useful for training dextrous hands. Whereas manipulators like the UMI gripper on the Franka Emika Panda can benefit from more accurate physics properties on objects in simulation, dextrous hands must learn to accomplish much more complex tasks with deformable fabrics, plastics, sticky surfaces, and similar challenges. We hope in future simulations to have a pipeline with increased accuracy around specifically supporting packing in plastic bags and applying stickers to diverse surfaces."}],["$","p",null,{"children":"Alternative methods of course may use generative diffusion- or autogressive-based world models to generate the multi-view video used for policy evaluation and finetuning. Our method may be beneficial because it is much less computationally intensive than methods relying on generative video models and can be precise for specific environments or parts."}]]}]
2a:["$","g",null,{"className":"agent-sprite","style":{"--i":1},"children":[["$","g","leg",{"className":"agent-sprite__leg","children":[["$","rect","6-4",{"x":4,"y":6,"width":1,"height":1}],["$","rect","6-6",{"x":6,"y":6,"width":1,"height":1}],["$","rect","6-8",{"x":8,"y":6,"width":1,"height":1}]]}],["$","g","body",{"className":"agent-sprite__body","children":[["$","rect","3-2",{"x":2,"y":3,"width":9,"height":1}],["$","rect","4-2",{"x":2,"y":4,"width":2,"height":1}],["$","rect","4-5",{"x":5,"y":4,"width":3,"height":1}],["$","rect","4-9",{"x":9,"y":4,"width":2,"height":1}],["$","rect","5-2",{"x":2,"y":5,"width":9,"height":1}]]}],["$","g","tool",{"className":"agent-sprite__tool","children":[["$","rect","0-2",{"x":2,"y":0,"width":9,"height":1}],["$","rect","1-4",{"x":4,"y":1,"width":5,"height":1}]]}]]}]
2b:["$","text",null,{"x":129,"y":404,"fontSize":26,"fontWeight":700,"fill":"var(--accent)","children":"02"}]
2c:["$","text",null,{"x":173,"y":404,"fontSize":26,"fontWeight":700,"fill":"var(--ink)","children":"Ingest"}]
2d:["$","text","0",{"x":14,"y":459,"fontSize":21,"fill":"var(--text-body)","children":"Process input RGB or other data sources"}]
2e:["$","text","1",{"x":14,"y":485,"fontSize":21,"fill":"var(--text-body)","children":"to 3DGS"}]
2f:["$","text",null,{"x":14,"y":514,"fontSize":15,"fontFamily":"var(--font-pixel)","letterSpacing":"0.08em","fill":"var(--text-label)","children":"REAL2SIM-INGEST-DATA"}]
30:["$","line",null,{"x1":240,"y1":528,"x2":240,"y2":556,"stroke":"var(--text-label)"}]
31:["$","g","03",{"children":[["$","g",null,{"className":"agent-node","role":"img","aria-label":"03 Step runner: Process the tools for pipeline with visual review of the results, including object masks, semantic features, physics properties, background crop and infill, etc","children":[["$","rect",null,{"x":0.5,"y":556.5,"width":479,"height":217,"fill":"var(--paper)","stroke":"var(--ink)"}],["$","g",null,{"transform":"translate(14 572) scale(7)","fill":"currentColor","shapeRendering":"crispEdges","aria-hidden":"true","children":["$","g",null,{"className":"agent-sprite","style":{"--i":2},"children":[["$","g","leg",{"className":"agent-sprite__leg","children":[["$","rect","6-4",{"x":4,"y":6,"width":1,"height":1}],["$","rect","6-6",{"x":6,"y":6,"width":1,"height":1}],["$","rect","6-8",{"x":8,"y":6,"width":1,"height":1}]]}],["$","g","body",{"className":"agent-sprite__body","children":[["$","rect","3-2",{"x":2,"y":3,"width":9,"height":1}],["$","rect","4-2",{"x":2,"y":4,"width":2,"height":1}],["$","rect","4-5",{"x":5,"y":4,"width":3,"height":1}],["$","rect","4-9",{"x":9,"y":4,"width":2,"height":1}],["$","rect","5-2",{"x":2,"y":5,"width":9,"height":1}]]}],["$","g","tool",{"className":"agent-sprite__tool","children":[["$","rect","0-4",{"x":4,"y":0,"width":1,"height":1}],["$","rect","0-6",{"x":6,"y":0,"width":1,"height":1}],["$","rect","0-8",{"x":8,"y":0,"width":1,"height":1}],["$","rect","1-4",{"x":4,"y":1,"width":1,"height":1}],["$","rect","1-6",{"x":6,"y":1,"width":1,"height":1}],["$","rect","1-8",{"x":8,"y":1,"width":1,"height":1}]]}]]}]}],["$","text",null,{"x":129,"y":598,"fontSize":26,"fontWeight":700,"fill":"var(--accent)","children":"03"}],["$","text",null,{"x":173,"y":598,"fontSize":26,"fontWeight":700,"fill":"var(--ink)","children":"Step runner"}],[["$","text","0",{"x":14,"y":653,"fontSize":21,"fill":"var(--text-body)","children":"Process the tools for pipeline with visual"}],["$","text","1",{"x":14,"y":679,"fontSize":21,"fill":"var(--text-body)","children":"review of the results, including object"}],["$","text","2",{"x":14,"y":705,"fontSize":21,"fill":"var(--text-body)","children":"masks, semantic features, physics"}],["$","text","3",{"x":14,"y":731,"fontSize":21,"fill":"var(--text-body)","children":"properties, background crop and infill, etc"}]],["$","text",null,{"x":14,"y":760,"fontSize":15,"fontFamily":"var(--font-pixel)","letterSpacing":"0.08em","fill":"var(--text-label)","children":"REAL2SIM-RUN-STEP"}]]}],false]}]
32:["$","div",null,{"className":"spread__gutter","aria-hidden":"true","children":["$","svg",null,{"viewBox":"0 0 144 932","preserveAspectRatio":"none","children":["$","polyline",null,{"points":"0,665 72,665 72,251 144,251","fill":"none","stroke":"var(--text-label)","vectorEffect":"non-scaling-stroke"}]}]}]
33:["$","$1","01",{"children":[["$","svg",null,{"className":"spread__col","viewBox":"0 0 480 932","style":{"fontFamily":"Helvetica, \"Helvetica Neue\", Arimo, Arial, sans-serif"},"children":[["$","g",null,{"children":[["$","text",null,{"x":0,"y":26,"fontSize":26,"fontWeight":700,"fill":"var(--accent)","children":"01"}],["$","text",null,{"x":46,"y":26,"fontSize":26,"fontWeight":700,"fill":"var(--ink)","children":"Conversion"}],["$","line",null,{"x1":0,"y1":38.5,"x2":480,"y2":38.5,"stroke":"var(--ink)"}],[["$","text","0",{"x":0,"y":64,"fontSize":24,"fill":"var(--text-muted)","children":"Segment and predict properties for"}],["$","text","1",{"x":0,"y":92,"fontSize":24,"fill":"var(--text-muted)","children":"objects in scan, then package and inspect"}]]]}],[["$","g","04",{"children":[["$","g",null,{"className":"agent-node","role":"img","aria-label":"04 Alignment: Scale, place and rotate each asset by photometric, silhouette and edge residuals","children":[["$","rect",null,{"x":0.5,"y":168.5,"width":479,"height":165,"fill":"var(--paper)","stroke":"var(--ink)"}],["$","g",null,{"transform":"translate(14 184) scale(7)","fill":"currentColor","shapeRendering":"crispEdges","aria-hidden":"true","children":["$","g",null,{"className":"agent-sprite","style":{"--i":3},"children":[["$","g","leg",{"className":"agent-sprite__leg","children":[["$","rect","6-4",{"x":4,"y":6,"width":1,"height":1}],["$","rect","6-6",{"x":6,"y":6,"width":1,"height":1}],["$","rect","6-8",{"x":8,"y":6,"width":1,"height":1}]]}],["$","g","body",{"className":"agent-sprite__body","children":[["$","rect","3-2",{"x":2,"y":3,"width":9,"height":1}],["$","rect","4-2",{"x":2,"y":4,"width":2,"height":1}],["$","rect","4-5",{"x":5,"y":4,"width":3,"height":1}],["$","rect","4-9",{"x":9,"y":4,"width":2,"height":1}],["$","rect","5-2",{"x":2,"y":5,"width":9,"height":1}]]}],["$","g","tool",{"className":"agent-sprite__tool","children":[["$","rect","0-2",{"x":2,"y":0,"width":1,"height":1}],["$","rect","0-4",{"x":4,"y":0,"width":5,"height":1}],["$","rect","0-10",{"x":10,"y":0,"width":1,"height":1}]]}]]}]}],["$","text",null,{"x":129,"y":210,"fontSize":26,"fontWeight":700,"fill":"var(--accent)","children":"04"}],["$","text",null,{"x":173,"y":210,"fontSize":26,"fontWeight":700,"fill":"var(--ink)","children":"Alignment"}],[["$","text","0",{"x":14,"y":265,"fontSize":21,"fill":"var(--text-body)","children":"Scale, place and rotate each asset by"}],["$","text","1",{"x":14,"y":291,"fontSize":21,"fill":"var(--text-body)","children":"photometric, silhouette and edge residuals"}]],["$","text",null,{"x":14,"y":320,"fontSize":15,"fontFamily":"var(--font-pixel)","letterSpacing":"0.08em","fill":"var(--text-label)","children":"REAL2SIM-ALIGN-SCENE"}]]}],["$","line",null,{"x1":240,"y1":334,"x2":240,"y2":362,"stroke":"var(--text-label)"}]]}],["$","g","05",{"children":[["$","g",null,{"className":"agent-node","role":"img","aria-label":"05 Orientation: Isaac Lab sweeps that orient the completed background 3DGS to the exported scene","children":[["$","rect",null,{"x":0.5,"y":362.5,"width":479,"height":165,"fill":"var(--paper)","stroke":"var(--ink)"}],["$","g",null,{"transform":"translate(14 378) scale(7)","fill":"currentColor","shapeRendering":"crispEdges","aria-hidden":"true","children":["$","g",null,{"className":"agent-sprite","style":{"--i":4},"children":[["$","g","leg",{"className":"agent-sprite__leg","children":[["$","rect","6-4",{"x":4,"y":6,"width":1,"height":1}],["$","rect","6-6",{"x":6,"y":6,"width":1,"height":1}],["$","rect","6-8",{"x":8,"y":6,"width":1,"height":1}]]}],["$","g","body",{"className":"agent-sprite__body","children":[["$","rect","3-2",{"x":2,"y":3,"width":9,"height":1}],["$","rect","4-2",{"x":2,"y":4,"width":2,"height":1}],["$","rect","4-5",{"x":5,"y":4,"width":3,"height":1}],["$","rect","4-9",{"x":9,"y":4,"width":2,"height":1}],["$","rect","5-2",{"x":2,"y":5,"width":9,"height":1}]]}],["$","g","tool",{"className":"agent-sprite__tool","children":[["$","rect","0-5",{"x":5,"y":0,"width":3,"height":1}],["$","rect","1-6",{"x":6,"y":1,"width":1,"height":1}]]}]]}]}],["$","text",null,{"x":129,"y":404,"fontSize":26,"fontWeight":700,"fill":"var(--accent)","children":"05"}],["$","text",null,{"x":173,"y":404,"fontSize":26,"fontWeight":700,"fill":"var(--ink)","children":"Orientation"}],[["$","text","0",{"x":14,"y":459,"fontSize":21,"fill":"var(--text-body)","children":"Isaac Lab sweeps that orient the completed"}],["$","text","1",{"x":14,"y":485,"fontSize":21,"fill":"var(--text-body)","children":"background 3DGS to the exported scene"}]],["$","text",null,{"x":14,"y":514,"fontSize":15,"fontFamily":"var(--font-pixel)","letterSpacing":"0.08em","fill":"var(--text-label)","children":"REAL2SIM-ORIENT-BACKGROUND"}]]}],["$","line",null,{"x1":240,"y1":528,"x2":240,"y2":556,"stroke":"var(--text-label)"}]]}],["$","g","06",{"children":[["$","g",null,{"className":"agent-node","role":"img","aria-label":"06 Packager: Publishes the data_processed package for the editor as USDZ","children":[["$","rect",null,{"x":0.5,"y":556.5,"width":479,"height":165,"fill":"var(--paper)","stroke":"var(--ink)"}],["$","g",null,{"transform":"translate(14 572) scale(7)","fill":"currentColor","shapeRendering":"crispEdges","aria-hidden":"true","children":["$","g",null,{"className":"agent-sprite","style":{"--i":5},"children":["$L46","$L47","$L48"]}]}],"$L49","$L4a",["$L4b","$L4c"],"$L4d"]}],"$L4e"]}],"$L4f"]]}],"$L50"]}]
34:["$","$1","02",{"children":[["$","svg",null,{"className":"spread__col","viewBox":"0 0 480 932","style":{"fontFamily":"Helvetica, \"Helvetica Neue\", Arimo, Arial, sans-serif"},"children":[["$","g",null,{"children":[["$","text",null,{"x":0,"y":26,"fontSize":26,"fontWeight":700,"fill":"var(--accent)","children":"02"}],["$","text",null,{"x":46,"y":26,"fontSize":26,"fontWeight":700,"fill":"var(--ink)","children":"Sisters"}],["$","line",null,{"x1":0,"y1":38.5,"x2":480,"y2":38.5,"stroke":"var(--ink)"}],[["$","text","0",{"x":0,"y":64,"fontSize":24,"fill":"var(--text-muted)","children":"From the fully processed digital twin,"}],["$","text","1",{"x":0,"y":92,"fontSize":24,"fill":"var(--text-muted)","children":"generate a family of sisters"}]]]}],[["$","g","08",{"children":[["$","g",null,{"className":"agent-node","role":"img","aria-label":"08 Object sisters: Generate small variations of each object in the scene with generative 3d models, focusing on geometric and visual differences","children":[["$","rect",null,{"x":0.5,"y":168.5,"width":479,"height":191,"fill":"var(--paper)","stroke":"var(--ink)"}],["$","g",null,{"transform":"translate(14 184) scale(7)","fill":"currentColor","shapeRendering":"crispEdges","aria-hidden":"true","children":["$","g",null,{"className":"agent-sprite","style":{"--i":7},"children":[["$","g","leg",{"className":"agent-sprite__leg","children":[["$","rect","6-4",{"x":4,"y":6,"width":1,"height":1}],["$","rect","6-6",{"x":6,"y":6,"width":1,"height":1}],["$","rect","6-8",{"x":8,"y":6,"width":1,"height":1}]]}],["$","g","body",{"className":"agent-sprite__body","children":[["$","rect","3-2",{"x":2,"y":3,"width":9,"height":1}],["$","rect","4-2",{"x":2,"y":4,"width":2,"height":1}],["$","rect","4-5",{"x":5,"y":4,"width":3,"height":1}],["$","rect","4-9",{"x":9,"y":4,"width":2,"height":1}],["$","rect","5-2",{"x":2,"y":5,"width":9,"height":1}]]}],["$","g","tool",{"className":"agent-sprite__tool","children":[["$","rect","0-3",{"x":3,"y":0,"width":2,"height":1}],["$","rect","0-8",{"x":8,"y":0,"width":2,"height":1}],["$","rect","1-3",{"x":3,"y":1,"width":2,"height":1}],["$","rect","1-8",{"x":8,"y":1,"width":2,"height":1}]]}]]}]}],["$","text",null,{"x":129,"y":210,"fontSize":26,"fontWeight":700,"fill":"var(--accent)","children":"08"}],["$","text",null,{"x":173,"y":210,"fontSize":26,"fontWeight":700,"fill":"var(--ink)","children":"Object sisters"}],[["$","text","0",{"x":14,"y":265,"fontSize":21,"fill":"var(--text-body)","children":"Generate small variations of each object"}],["$","text","1",{"x":14,"y":291,"fontSize":21,"fill":"var(--text-body)","children":"in the scene with generative 3d models,"}],["$","text","2",{"x":14,"y":317,"fontSize":21,"fill":"var(--text-body)","children":"focusing on geometric and visual differences"}]],["$","text",null,{"x":14,"y":346,"fontSize":15,"fontFamily":"var(--font-pixel)","letterSpacing":"0.08em","fill":"var(--text-label)","children":"REAL2SIM-OBJECT-SISTERS"}]]}],["$","line",null,{"x1":240,"y1":360,"x2":240,"y2":388,"stroke":"var(--text-label)"}]]}],["$","g","09",{"children":[["$","g",null,{"className":"agent-node","role":"img","aria-label":"09 Scene sisters: Generate different layouts and arrangement of the sister objects and export them as separate USDZ scenes","children":[["$","rect",null,{"x":0.5,"y":388.5,"width":479,"height":191,"fill":"var(--paper)","stroke":"var(--ink)"}],["$","g",null,{"transform":"translate(14 404) scale(7)","fill":"currentColor","shapeRendering":"crispEdges","aria-hidden":"true","children":["$","g",null,{"className":"agent-sprite","style":{"--i":8},"children":[["$","g","leg",{"className":"agent-sprite__leg","children":[["$","rect","6-4",{"x":4,"y":6,"width":1,"height":1}],["$","rect","6-6",{"x":6,"y":6,"width":1,"height":1}],["$","rect","6-8",{"x":8,"y":6,"width":1,"height":1}]]}],["$","g","body",{"className":"agent-sprite__body","children":[["$","rect","3-2",{"x":2,"y":3,"width":9,"height":1}],["$","rect","4-2",{"x":2,"y":4,"width":2,"height":1}],["$","rect","4-5",{"x":5,"y":4,"width":3,"height":1}],["$","rect","4-9",{"x":9,"y":4,"width":2,"height":1}],["$","rect","5-2",{"x":2,"y":5,"width":9,"height":1}]]}],["$","g","tool",{"className":"agent-sprite__tool","children":[["$","rect","0-3",{"x":3,"y":0,"width":1,"height":1}],["$","rect","0-9",{"x":9,"y":0,"width":1,"height":1}],["$","rect","1-2",{"x":2,"y":1,"width":9,"height":1}]]}]]}]}],["$","text",null,{"x":129,"y":430,"fontSize":26,"fontWeight":700,"fill":"var(--accent)","children":"09"}],["$","text",null,{"x":173,"y":430,"fontSize":26,"fontWeight":700,"fill":"var(--ink)","children":"Scene sisters"}],[["$","text","0",{"x":14,"y":485,"fontSize":21,"fill":"var(--text-body)","children":"Generate different layouts and arrangement"}],["$","text","1",{"x":14,"y":511,"fontSize":21,"fill":"var(--text-body)","children":"of the sister objects and export them as"}],["$","text","2",{"x":14,"y":537,"fontSize":21,"fill":"var(--text-body)","children":"separate USDZ scenes"}]],["$","text",null,{"x":14,"y":566,"fontSize":15,"fontFamily":"var(--font-pixel)","letterSpacing":"0.08em","fill":"var(--text-label)","children":"REAL2SIM-SCENE-SISTERS"}]]}],false]}]]]}],"$undefined"]}]
35:["$","div",null,{"className":"chapter__caption","children":[["$","div",null,{"style":{"fontFamily":"var(--font-sans)","fontSize":10,"fontWeight":700,"lineHeight":1.5,"letterSpacing":"0.02em","color":"var(--ink)"},"children":"Figure 4 · An overview of the full agentic pipeline with various tools and skills for processing 3d capture data into usable simulation scenes"}],["$","div",null,{"style":{"fontFamily":"var(--font-sans)","fontSize":10,"fontWeight":700,"lineHeight":1.5,"letterSpacing":"0.02em","color":"var(--accent)","marginLeft":"auto","flexShrink":0},"children":"4.2"}]]}]
36:["$","div",null,{"className":"chapter__text","children":["$","p",null,{"children":"Our agentic workflow processes 3D capture data of diverse origins into fully usable simulation scenes in Isaac Lab, MuJoCo, or Unreal Engine. The agents are helpful in this case specifically because of the diversity of data formats of 3d data and error-prone nature of object conversion and decomposition models. For example, a common failure mode is that the 3d object segmentation will incorrectly transform the scale, rotation, or placement of the objects in the scene, and agents can take camera renders from within Isaac Lab to visually inspect the conversion results and rapidly correct such small errors."}]}]
37:["$","div",null,{"className":"chapter__exhibit chapter__exhibit--bleed","data-bleed":"","children":[["$","div",null,{"className":"clips","style":{"gridTemplateColumns":"repeat(3, minmax(0, 1fr))"},"children":[["$","$La","objects/sony_radio_clean",{"src":"./videos/objects/sony_radio_clean.mp4","poster":"./videos/objects/sony_radio_clean.jpg","caption":"Sony CFS-43, photogrammetry of a radio cassette player and articulations of its door, keys, knobs, and selector","ratio":"4 / 3","alt":"Sony radio · full control articulation"}],["$","$La","stegosaurus_walking",{"src":"./videos/stegosaurus_walking.mp4","poster":"./videos/stegosaurus_walking.jpg","caption":"A 3dgs of a stegosaurus skeleton from the AMNH, fully articulated and walking (without hand-created animations)","ratio":"4 / 3","alt":"Stegosaurus · walking"}],["$","$La","objects/ge90_engine_orbit_zoom",{"src":"./videos/objects/ge90_engine_orbit_zoom.mp4","poster":"./videos/objects/ge90_engine_orbit_zoom.jpg","caption":"A jet engine with processed turbofan, articulated and running based on engine schematics","ratio":"4 / 3","alt":"Jet engine · orbit and detail zoom"}],["$","$La","objects/sceneagent_bencini_camera",{"src":"./videos/objects/sceneagent_bencini_camera.mp4","poster":"./videos/objects/sceneagent_bencini_camera.jpg","caption":"A Bencini cine camera converted into a usable simulation asset, with usable frame-rate dial and battery compartment","ratio":"4 / 3","alt":"Bencini cine camera · orbit and detail zoom"}],["$","$La","objects/sceneagent_cessna_cockpit",{"src":"./videos/objects/sceneagent_cessna_cockpit.mp4","poster":"./videos/objects/sceneagent_cessna_cockpit.jpg","caption":"A Cessna cockpit, converted into simulation asset with extreme detail in its instrument panel, switches, and placards","ratio":"4 / 3","alt":"Cessna cockpit · orbit and detail zoom"}],["$","$La","objects/sceneagent_motorcycle",{"src":"./videos/objects/sceneagent_motorcycle.mp4","poster":"./videos/objects/sceneagent_motorcycle.jpg","caption":"A motorcycle converted from 3d capture into a usable simulation asset, including many details","ratio":"4 / 3","alt":"Motorcycle · orbit and detail zoom"}]]}],["$","div",null,{"className":"chapter__caption","children":["$undefined",["$","div",null,{"style":{"fontFamily":"var(--font-sans)","fontSize":10,"fontWeight":700,"lineHeight":1.5,"letterSpacing":"0.02em","color":"var(--accent)","marginLeft":"auto","flexShrink":0},"children":"4.3"}]]}]]}]
38:["$","div",null,{"className":"chapter__text","children":["$","p",null,{"children":["One of the most unique and beneficial aspects about our agentic workflow is that it can support converting multiple different data types from diverse sources with useful results. For example, it can similarly process input RGB images and align them with structure-from-motion with colmap and then train a 3DGS scene with NerfStudio’s ",["$","code",null,{"children":"splatfacto"}]," big and inspect, review, and correct any results, such as the per image color calibration of the input images. The same pipeline will equally process public internet 3d capture data from photogrammetry or pre-existing 3DGS .ply files."]}]}]
39:["$","div",null,{"className":"chapter__exhibit chapter__exhibit--bleed","data-bleed":"","children":[["$","div",null,{"className":"clips","style":{"gridTemplateColumns":"repeat(3, minmax(0, 1fr))"},"children":[["$","$La","deformables_textile_rack_orbit",{"src":"./videos/deformables_textile_rack_orbit.mp4","poster":"./videos/deformables_textile_rack_orbit.jpg","caption":"A patterned textile draped over a garment rack, orbited to read the folds from every side","ratio":"16 / 9","alt":"Textile rack · orbit"}],["$","$La","deformables_hoodie_lift_drape",{"src":"./videos/deformables_hoodie_lift_drape.mp4","poster":"./videos/deformables_hoodie_lift_drape.jpg","caption":"A hoodie lifted from a surface and left to drape under gravity","ratio":"16 / 9","alt":"Hoodie · lift and drape"}],["$","$La","deformables_towel_rack_orbit",{"src":"./videos/deformables_towel_rack_orbit.mp4","poster":"./videos/deformables_towel_rack_orbit.jpg","caption":"Three towels hung on a wall rack, orbited to read the folds from every side","ratio":"16 / 9","alt":"Towel rack · orbit"}],["$","$La","deformables_firefighter_hose_unwind",{"src":"./videos/deformables_firefighter_hose_unwind.mp4","poster":"./videos/deformables_firefighter_hose_unwind.jpg","caption":"A coiled firefighter hose unwinding under soft-body dynamics","ratio":"16 / 9","alt":"Firefighter hose · unwind"}],["$","$La","liquid_death_hat_physics",{"src":"./videos/liquid_death_hat_physics.mp4","poster":"./videos/liquid_death_hat_physics.jpg","caption":"A Liquid Death corduroy cap with soft-body physics, deforming under contact dynamics","ratio":"16 / 9","alt":"Liquid Death hat · physics"}],["$","$La","deformables_extension_cord_unwind",{"src":"./videos/deformables_extension_cord_unwind.mp4","poster":"./videos/deformables_extension_cord_unwind.jpg","caption":"An extension cord uncoiling from its wound state","ratio":"16 / 9","alt":"Extension cord · unwind"}]]}],["$","div",null,{"className":"chapter__caption","children":["$undefined",["$","div",null,{"style":{"fontFamily":"var(--font-sans)","fontSize":10,"fontWeight":700,"lineHeight":1.5,"letterSpacing":"0.02em","color":"var(--accent)","marginLeft":"auto","flexShrink":0},"children":"4.4"}]]}]]}]
3a:["$","div",null,{"className":"chapter__text","children":["$","p",null,{"children":"Likewise, it is able to turn synthetic datasets from generative 3DGS models (such as World Labs Marble, NVIDIA Lyra, or TenCent Hy-World). One can prompt the pipeline to convert the full scene into segmented 3d objects with predictive physics properties or instead focus only foreground objects. The first is much more time-consuming and error prone at this point, but we believe that future iterations of the pipeline will easily improve this."}]}]
3b:["$","div",null,{"className":"chapter__text","children":["$","p",null,{"children":"We imagine a future application of this agentic workflow running on individual in-home or in-factory robots that can 3d capture a space, process that space on-device only with an NVIDIA Jetson Orin or Thor or equivalent, and then train a finetune of its policy on the results with no data leaving device. We hope that this will continue the use of 3DGS in the future to be part of the solution to the data problem of generalist robotic policy development in the future."}]}]
3c:["$","p",null,{"children":"We are still evaluating the results and are optimistic about the results of real world performance improvements after training in converted 3DGS scenes to use in simulation with predictive physics. We will update our work with the full results soon."}]
3d:["$","li","motorcycle",{"children":[["$","b",null,{"children":["[",10,"]"]}],["$","span",null,{"children":["","“",["$","a",null,{"href":"https://poly.cam/capture/e762d944-86f8-405e-8cf7-a7872c4097ba","target":"_blank","rel":"noopener noreferrer","children":"Motorcycle"}],",” ","Polycam","","."]}]]}]
3e:["$","li","towel-rack",{"children":[["$","b",null,{"children":["[",11,"]"]}],["$","span",null,{"children":["","“",["$","a",null,{"href":"https://poly.cam/capture/0c8cabee-75cf-454e-864a-8c27ef91f41f","target":"_blank","rel":"noopener noreferrer","children":"Towel rack"}],",” ","Polycam","","."]}]]}]
3f:["$","li","red-towel",{"children":[["$","b",null,{"children":["[",12,"]"]}],["$","span",null,{"children":["","“",["$","a",null,{"href":"https://poly.cam/capture/c78e1538-2c3f-4486-ad01-f8eb97098a8a","target":"_blank","rel":"noopener noreferrer","children":"Red towel"}],",” ","Polycam","","."]}]]}]
40:["$","li","hoodie",{"children":[["$","b",null,{"children":["[",13,"]"]}],["$","span",null,{"children":["","“",["$","a",null,{"href":"https://poly.cam/capture/33170b4a-c8d2-43ef-82d6-9d5c442c7c6a","target":"_blank","rel":"noopener noreferrer","children":"Dept Artisté anorak"}],",” ","Polycam","","."]}]]}]
41:["$","li","hose",{"children":[["$","b",null,{"children":["[",14,"]"]}],["$","span",null,{"children":["","“",["$","a",null,{"href":"https://poly.cam/capture/099a56df-bdb6-429f-89cf-91f1a9bcbee8","target":"_blank","rel":"noopener noreferrer","children":"Firefighter hose"}],",” ","Polycam","","."]}]]}]
42:["$","li","cord",{"children":[["$","b",null,{"children":["[",15,"]"]}],["$","span",null,{"children":["","“",["$","a",null,{"href":"https://poly.cam/capture/72007c87-fa9d-41db-bff6-05e230f68cf7","target":"_blank","rel":"noopener noreferrer","children":"Electric extension lead"}],",” ","Polycam","","."]}]]}]
43:["$","li","hat",{"children":[["$","b",null,{"children":["[",16,"]"]}],["$","span",null,{"children":["","“",["$","a",null,{"href":"https://poly.cam/capture/ad4a76f3-2078-48df-81e2-1460cadae520","target":"_blank","rel":"noopener noreferrer","children":"Liquid Death hat"}],",” ","Polycam","","."]}]]}]
46:["$","g","leg",{"className":"agent-sprite__leg","children":[["$","rect","6-4",{"x":4,"y":6,"width":1,"height":1}],["$","rect","6-6",{"x":6,"y":6,"width":1,"height":1}],["$","rect","6-8",{"x":8,"y":6,"width":1,"height":1}]]}]
47:["$","g","body",{"className":"agent-sprite__body","children":[["$","rect","3-2",{"x":2,"y":3,"width":9,"height":1}],["$","rect","4-2",{"x":2,"y":4,"width":2,"height":1}],["$","rect","4-5",{"x":5,"y":4,"width":3,"height":1}],["$","rect","4-9",{"x":9,"y":4,"width":2,"height":1}],["$","rect","5-2",{"x":2,"y":5,"width":9,"height":1}]]}]
48:["$","g","tool",{"className":"agent-sprite__tool","children":[["$","rect","0-4",{"x":4,"y":0,"width":5,"height":1}],["$","rect","1-4",{"x":4,"y":1,"width":5,"height":1}]]}]
49:["$","text",null,{"x":129,"y":598,"fontSize":26,"fontWeight":700,"fill":"var(--accent)","children":"06"}]
4a:["$","text",null,{"x":173,"y":598,"fontSize":26,"fontWeight":700,"fill":"var(--ink)","children":"Packager"}]
4b:["$","text","0",{"x":14,"y":653,"fontSize":21,"fill":"var(--text-body)","children":"Publishes the data_processed package for"}]
4c:["$","text","1",{"x":14,"y":679,"fontSize":21,"fill":"var(--text-body)","children":"the editor as USDZ"}]
4d:["$","text",null,{"x":14,"y":708,"fontSize":15,"fontFamily":"var(--font-pixel)","letterSpacing":"0.08em","fill":"var(--text-label)","children":"REAL2SIM-PACKAGE-DATA-PROCESSED"}]
4e:["$","line",null,{"x1":240,"y1":722,"x2":240,"y2":750,"stroke":"var(--text-label)"}]
4f:["$","g","07",{"children":[["$","g",null,{"className":"agent-node","role":"img","aria-label":"07 Inspector: Visually reviews and fixes anything off in the final prepared USDZ in NVIDIA Isaac Lab","children":[["$","rect",null,{"x":0.5,"y":750.5,"width":479,"height":165,"fill":"var(--paper)","stroke":"var(--ink)"}],["$","g",null,{"transform":"translate(14 766) scale(7)","fill":"currentColor","shapeRendering":"crispEdges","aria-hidden":"true","children":["$","g",null,{"className":"agent-sprite","style":{"--i":6},"children":[["$","g","leg",{"className":"agent-sprite__leg","children":[["$","rect","6-4",{"x":4,"y":6,"width":1,"height":1}],["$","rect","6-6",{"x":6,"y":6,"width":1,"height":1}],["$","rect","6-8",{"x":8,"y":6,"width":1,"height":1}]]}],["$","g","body",{"className":"agent-sprite__body","children":[["$","rect","3-2",{"x":2,"y":3,"width":9,"height":1}],["$","rect","4-2",{"x":2,"y":4,"width":2,"height":1}],["$","rect","4-5",{"x":5,"y":4,"width":3,"height":1}],["$","rect","4-9",{"x":9,"y":4,"width":2,"height":1}],["$","rect","5-2",{"x":2,"y":5,"width":9,"height":1}]]}],["$","g","tool",{"className":"agent-sprite__tool","children":[["$","rect","0-4",{"x":4,"y":0,"width":4,"height":1}],["$","rect","1-4",{"x":4,"y":1,"width":2,"height":1}]]}]]}]}],["$","text",null,{"x":129,"y":792,"fontSize":26,"fontWeight":700,"fill":"var(--accent)","children":"07"}],["$","text",null,{"x":173,"y":792,"fontSize":26,"fontWeight":700,"fill":"var(--ink)","children":"Inspector"}],[["$","text","0",{"x":14,"y":847,"fontSize":21,"fill":"var(--text-body)","children":"Visually reviews and fixes anything off in"}],["$","text","1",{"x":14,"y":873,"fontSize":21,"fill":"var(--text-body)","children":"the final prepared USDZ in NVIDIA Isaac Lab"}]],["$","text",null,{"x":14,"y":902,"fontSize":15,"fontFamily":"var(--font-pixel)","letterSpacing":"0.08em","fill":"var(--text-label)","children":"REAL2SIM-INSPECT-ISAAC"}]]}],false]}]
50:["$","div",null,{"className":"spread__gutter","aria-hidden":"true","children":["$","svg",null,{"viewBox":"0 0 144 932","preserveAspectRatio":"none","children":["$","polyline",null,{"points":"0,833 72,833 72,264 144,264","fill":"none","stroke":"var(--text-label)","vectorEffect":"non-scaling-stroke"}]}]}]
1e:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]
21:[["$","title","0",{"children":"SceneAgent | Harvard Computational Robotics Group"}],["$","meta","1",{"name":"description","content":"SceneAgent converts 3D captures of real-world scenes into simulation environments with predictive physics for robot policy evaluation, planning and training."}],["$","link","2",{"rel":"author","href":"https://computationalrobotics.seas.harvard.edu/"}],["$","meta","3",{"name":"author","content":"Luke Hollis"}],["$","link","4",{"rel":"author","href":"https://computationalrobotics.seas.harvard.edu/"}],["$","meta","5",{"name":"author","content":"Tianxing Fan"}],["$","link","6",{"rel":"author","href":"https://computationalrobotics.seas.harvard.edu/"}],["$","meta","7",{"name":"author","content":"Heng Yang"}],["$","meta","8",{"name":"keywords","content":"SceneAgent,real-to-sim,sim-to-real,3D Gaussian Splatting,simulation environments,predictive physics,robot policy evaluation,policy training,online planning,object segmentation,articulation,deformable objects,digital sisters,robot learning,Harvard Computational Robotics Group"}],["$","meta","9",{"name":"creator","content":"Harvard Computational Robotics Group"}],["$","meta","10",{"name":"publisher","content":"Harvard Computational Robotics Group"}],["$","meta","11",{"name":"robots","content":"index, follow"}],["$","meta","12",{"name":"googlebot","content":"index, follow, max-video-preview:-1, max-image-preview:large, max-snippet:-1"}],["$","meta","13",{"name":"citation_title","content":"SceneAgent: 3D Capture-Derived Scenes with Predictive Physics for Policy Evaluation and Training Environments"}],["$","meta","14",{"name":"citation_author","content":"Luke Hollis"}],["$","meta","15",{"name":"citation_author","content":"Tianxing Fan"}],["$","meta","16",{"name":"citation_author","content":"Heng Yang"}],["$","meta","17",{"name":"citation_publication_date","content":"2026"}],["$","meta","18",{"name":"citation_publisher","content":"Harvard Computational Robotics Group"}],["$","meta","19",{"name":"citation_abstract_html_url","content":"https://computationalrobotics.seas.harvard.edu/SceneAgent/"}],["$","link","20",{"rel":"canonical","href":"https://computationalrobotics.seas.harvard.edu/SceneAgent/"}],["$","meta","21",{"property":"og:title","content":"SceneAgent: 3D captures to simulation environments with predictive physics"}],["$","meta","22",{"property":"og:description","content":"An agentic pipeline from 3D Gaussian Splatting, photogrammetry and LiDAR captures of real scenes to simulation environments with predictive physics. Harvard Computational Robotics Group, 2026 preprint."}],["$","meta","23",{"property":"og:url","content":"https://computationalrobotics.seas.harvard.edu/SceneAgent/"}],["$","meta","24",{"property":"og:site_name","content":"Harvard Computational Robotics Group"}],["$","meta","25",{"property":"og:locale","content":"en_US"}],["$","meta","26",{"property":"og:image","content":"https://computationalrobotics.seas.harvard.edu/SceneAgent/branding/sceneagent-social-share.jpg"}],["$","meta","27",{"property":"og:image:secure_url","content":"https://computationalrobotics.seas.harvard.edu/SceneAgent/branding/sceneagent-social-share.jpg"}],["$","meta","28",{"property":"og:image:type","content":"image/jpeg"}],["$","meta","29",{"property":"og:image:width","content":"1200"}],["$","meta","30",{"property":"og:image:height","content":"630"}],["$","meta","31",{"property":"og:image:alt","content":"SceneAgent: two Franka robot arms over a simulated tabletop scene, above the paper title and subtitle"}],["$","meta","32",{"property":"og:type","content":"website"}],["$","meta","33",{"name":"twitter:card","content":"summary_large_image"}],["$","meta","34",{"name":"twitter:title","content":"SceneAgent: 3D captures to simulation environments with predictive physics"}],["$","meta","35",{"name":"twitter:description","content":"An agentic pipeline from 3D Gaussian Splatting, photogrammetry and LiDAR captures of real scenes to simulation environments with predictive physics. Harvard Computational Robotics Group, 2026 preprint."}],"$L51","$L52","$L53","$L54"]
45:null
55:I[29447,["./_next/static/chunks/3d-g45tvvtwds.js"],"IconMark"]
51:["$","meta","36",{"name":"twitter:image","content":"https://computationalrobotics.seas.harvard.edu/SceneAgent/branding/sceneagent-social-share.jpg"}]
52:["$","meta","37",{"name":"twitter:image:alt","content":"SceneAgent: two Franka robot arms over a simulated tabletop scene, above the paper title and subtitle"}]
53:["$","link","38",{"rel":"icon","href":"./favicon.png","type":"image/png"}]
54:["$","$L55","39",{}]