From df7f4dd1277836ec20932d82d12c245b2a09fd79 Mon Sep 17 00:00:00 2001 From: Md Farhan Ishmam <45528856+farhanishmam@users.noreply.github.com> Date: Sun, 19 Jul 2026 19:49:04 -0600 Subject: [PATCH] Added Knows: a Google Workspace benchmark for web agents (Docs, Sheets, Slides) Knows evaluates browser agents on long-horizon document authoring in Google Workspace. An agent drives a real browser to build a Doc, Sheet, or Slide deck from a natural-language goal, and the result is graded checkpoint-by-checkpoint via the Workspace APIs, yielding a fractional reward rather than binary success. 110 tasks across 22 families and 3 Workspace apps (25 docs / 45 sheets / 40 slides), exposed as a single `knows` split. As with TimeWarp, the task code lives in an external package (`pip install browsergym-knows`); this change is config wiring only: - `knows` action subset in core, adding file upload and direct mouse control on top of the usual bid actions, both needed to edit Workspace canvases - `knows` benchmark backend, action set args, and metadata CSV - lazy `import browsergym.knows` in `prepare_backend` and `_get_env_name` - `knows` extra, dev requirement, and expected task count in the tests Knows requires a Google account and a Google Cloud service account; see the benchmark README for setup. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 17 +++ .../src/browsergym/core/action/highlevel.py | 29 +++++ browsergym/experiments/pyproject.toml | 4 + .../browsergym/experiments/benchmark/base.py | 1 + .../experiments/benchmark/configs.py | 23 ++++ .../experiments/benchmark/metadata/knows.csv | 111 ++++++++++++++++++ .../browsergym/experiments/benchmark/utils.py | 2 + .../src/browsergym/experiments/loop.py | 2 + dev/requirements.txt | 1 + tests/experiments/test_benchmark.py | 1 + 10 files changed, 191 insertions(+) create mode 100644 browsergym/experiments/src/browsergym/experiments/benchmark/metadata/knows.csv diff --git a/README.md b/README.md index fc5d188d..28343ec5 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ BrowserGym includes the following benchmarks by default: - [WebLINX](https://github.com/McGill-NLP/weblinx) (static benchmark) - [OpenApps](https://facebookresearch.github.io/OpenApps/) - [TimeWarp](https://timewarp-web.github.io) + - [Knows](https://github.com/alexgill321/Agent-Benchmark) Designing new web benchmarks with BrowserGym is easy, and simply requires to inherit the [`AbstractBrowserTask`](https://github.com/ServiceNow/BrowserGym/blob/main/browsergym/core/src/browsergym/core/task.py#L7C7-L7C26) class. @@ -64,6 +65,7 @@ pip install browsergym-workarena # core + workarena pip install browsergym-assistantbench # core + assistantbench pip install weblinx-browsergym # core + weblinx pip install browsergym-timewarp # core + timewarp +pip install browsergym-knows # core + knows ``` Then setup playwright by running @@ -80,6 +82,7 @@ Finally, each benchmark comes with its own specific setup that requires to follo - for AssistantBench, see [assistantbench/README.md](browsergym/assistantbench/README.md) - for OpenApps, see [OpenApps docs](https://facebookresearch.github.io/OpenApps/) - for TimeWarp, see [timewarp/README.md](https://github.com/sparklabutah/timewarp) + - for Knows, see [knows/README.md](https://github.com/alexgill321/Agent-Benchmark) (requires a Google account and a Google Cloud service account) ### 🏗️ Development setup @@ -211,6 +214,20 @@ env_ids = [id for id in gym.envs.registry.keys() if id.startswith("browsergym/ti print("\n".join(env_ids)) ``` +Knows +```python +import gymnasium as gym +import browsergym.knows # register knows tasks as gym environments + +# start a knows task +env = gym.make("browsergym/knows.docs_1_formal_letter.1") +... + +# list all the available knows tasks +env_ids = [id for id in gym.envs.registry.keys() if id.startswith("browsergym/knows")] +print("\n".join(env_ids)) +``` + ## 💻 Demo If you want to experiment with a demo agent in BrowserGym, follow these steps diff --git a/browsergym/core/src/browsergym/core/action/highlevel.py b/browsergym/core/src/browsergym/core/action/highlevel.py index 40358c28..c7374537 100644 --- a/browsergym/core/src/browsergym/core/action/highlevel.py +++ b/browsergym/core/src/browsergym/core/action/highlevel.py @@ -259,6 +259,34 @@ tab_close, # Closes a tab tab_focus, # Switches tabs ], + "knows": [ # Google Workspace tasks: on top of the usual bid actions, they need + # file upload and direct mouse control to edit Docs/Sheets/Slides canvases + click, + dblclick, + drag_and_drop, + fill, + go_back, + go_forward, + goto, + hover, + keyboard_insert_text, + keyboard_press, + keyboard_type, + mouse_click, + mouse_dblclick, + mouse_drag_and_drop, + mouse_move, + mouse_upload_file, + new_tab, + noop, + report_infeasible, + scroll, + select_option, + send_msg_to_user, + tab_close, + tab_focus, + upload_file, + ], } @@ -291,6 +319,7 @@ class HighLevelActionSet(AbstractActionSet): "weblinx", "assistantbench", "timewarp", + "knows", "custom", ] DemoMode = typing.Literal["off", "default", "all_blue", "only_visible_elements"] diff --git a/browsergym/experiments/pyproject.toml b/browsergym/experiments/pyproject.toml index 6d96c0b5..9f9e711e 100644 --- a/browsergym/experiments/pyproject.toml +++ b/browsergym/experiments/pyproject.toml @@ -46,6 +46,9 @@ weblinx = [ timewarp = [ "browsergym-timewarp" ] +knows = [ + "browsergym-knows" +] all = [ "browsergym-experiment[miniwob]", "browsergym-experiment[workarena]", @@ -54,6 +57,7 @@ all = [ "browsergym-experiment[assistantbench]", "browsergym-experiment[weblinx]", "browsergym-experiment[timewarp]", + "browsergym-experiment[knows]", ] [project.urls] diff --git a/browsergym/experiments/src/browsergym/experiments/benchmark/base.py b/browsergym/experiments/src/browsergym/experiments/benchmark/base.py index ab4b6db1..f51f7803 100644 --- a/browsergym/experiments/src/browsergym/experiments/benchmark/base.py +++ b/browsergym/experiments/src/browsergym/experiments/benchmark/base.py @@ -61,6 +61,7 @@ def make_action_set(self): "assistantbench", "weblinx", "timewarp", + "knows", ] diff --git a/browsergym/experiments/src/browsergym/experiments/benchmark/configs.py b/browsergym/experiments/src/browsergym/experiments/benchmark/configs.py index d10a3c18..0b4cfd0a 100644 --- a/browsergym/experiments/src/browsergym/experiments/benchmark/configs.py +++ b/browsergym/experiments/src/browsergym/experiments/benchmark/configs.py @@ -94,6 +94,13 @@ retry_with_force=True, demo_mode="off", ), + "knows": HighLevelActionSetArgs( + subsets=["knows"], + multiaction=False, + strict=False, + retry_with_force=True, + demo_mode="off", + ), } # all benchmarks are callables designed for lazy loading, i.e. `bench = DEFAULT_BENCHMARKS["miniwob_all"]()` @@ -312,4 +319,20 @@ ), task_metadata=task_metadata("timewarp"), ), + "knows": lambda n_repeats=1: Benchmark( + name="knows", + high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["knows"], + is_multi_tab=True, + supports_parallel_seeds=True, + backends=["knows"], + env_args_list=make_env_args_list_from_repeat_tasks( + task_list=task_list_from_metadata(metadata=task_metadata("knows")), + # Knows tasks are long-horizon document authoring; past this the env's + # TimeLimit truncates the episode and the document is graded as-is. + max_steps=120, + n_repeats=n_repeats, + seeds_rng=np.random.RandomState(42), + ), + task_metadata=task_metadata("knows"), + ), } diff --git a/browsergym/experiments/src/browsergym/experiments/benchmark/metadata/knows.csv b/browsergym/experiments/src/browsergym/experiments/benchmark/metadata/knows.csv new file mode 100644 index 00000000..13010e30 --- /dev/null +++ b/browsergym/experiments/src/browsergym/experiments/benchmark/metadata/knows.csv @@ -0,0 +1,111 @@ +task_name,sites,eval_types,task_id,browsergym_split +knows.docs_1_formal_letter.1,docs,checkpoint_rubric,1,test +knows.docs_1_formal_letter.2,docs,checkpoint_rubric,2,test +knows.docs_1_formal_letter.3,docs,checkpoint_rubric,3,test +knows.docs_1_formal_letter.4,docs,checkpoint_rubric,4,test +knows.docs_1_formal_letter.5,docs,checkpoint_rubric,5,test +knows.sheets_2_personal_recipe.1,sheets,checkpoint_rubric,6,test +knows.sheets_2_personal_recipe.2,sheets,checkpoint_rubric,7,test +knows.sheets_2_personal_recipe.3,sheets,checkpoint_rubric,8,test +knows.sheets_2_personal_recipe.4,sheets,checkpoint_rubric,9,test +knows.sheets_2_personal_recipe.5,sheets,checkpoint_rubric,10,test +knows.docs_5_influential_papers.1,docs,checkpoint_rubric,11,test +knows.docs_5_influential_papers.2,docs,checkpoint_rubric,12,test +knows.docs_5_influential_papers.3,docs,checkpoint_rubric,13,test +knows.docs_5_influential_papers.4,docs,checkpoint_rubric,14,test +knows.docs_5_influential_papers.5,docs,checkpoint_rubric,15,test +knows.sheets_6_stock_tracker.1,sheets,checkpoint_rubric,16,test +knows.sheets_6_stock_tracker.2,sheets,checkpoint_rubric,17,test +knows.sheets_6_stock_tracker.3,sheets,checkpoint_rubric,18,test +knows.sheets_6_stock_tracker.4,sheets,checkpoint_rubric,19,test +knows.sheets_6_stock_tracker.5,sheets,checkpoint_rubric,20,test +knows.sheets_7_running_analysis.1,sheets,checkpoint_rubric,21,test +knows.sheets_7_running_analysis.2,sheets,checkpoint_rubric,22,test +knows.sheets_7_running_analysis.3,sheets,checkpoint_rubric,23,test +knows.sheets_7_running_analysis.4,sheets,checkpoint_rubric,24,test +knows.sheets_7_running_analysis.5,sheets,checkpoint_rubric,25,test +knows.sheets_10_paper_sorting.1,sheets,checkpoint_rubric,26,test +knows.sheets_10_paper_sorting.2,sheets,checkpoint_rubric,27,test +knows.sheets_10_paper_sorting.3,sheets,checkpoint_rubric,28,test +knows.sheets_10_paper_sorting.4,sheets,checkpoint_rubric,29,test +knows.sheets_10_paper_sorting.5,sheets,checkpoint_rubric,30,test +knows.docs_11_personal_recipe_ocr.1,docs,checkpoint_rubric,31,test +knows.docs_11_personal_recipe_ocr.2,docs,checkpoint_rubric,32,test +knows.docs_11_personal_recipe_ocr.3,docs,checkpoint_rubric,33,test +knows.docs_11_personal_recipe_ocr.4,docs,checkpoint_rubric,34,test +knows.docs_11_personal_recipe_ocr.5,docs,checkpoint_rubric,35,test +knows.slides_17_remove_images_add_placeholders.1,slides,checkpoint_rubric,36,test +knows.slides_17_remove_images_add_placeholders.2,slides,checkpoint_rubric,37,test +knows.slides_17_remove_images_add_placeholders.3,slides,checkpoint_rubric,38,test +knows.slides_17_remove_images_add_placeholders.4,slides,checkpoint_rubric,39,test +knows.slides_17_remove_images_add_placeholders.5,slides,checkpoint_rubric,40,test +knows.slides_20_illustrated_book_report.1,slides,checkpoint_rubric,41,test +knows.slides_20_illustrated_book_report.2,slides,checkpoint_rubric,42,test +knows.slides_20_illustrated_book_report.3,slides,checkpoint_rubric,43,test +knows.slides_20_illustrated_book_report.4,slides,checkpoint_rubric,44,test +knows.slides_20_illustrated_book_report.5,slides,checkpoint_rubric,45,test +knows.sheets_25_skitourplan.1,sheets,checkpoint_rubric,46,test +knows.sheets_25_skitourplan.2,sheets,checkpoint_rubric,47,test +knows.sheets_25_skitourplan.3,sheets,checkpoint_rubric,48,test +knows.sheets_25_skitourplan.4,sheets,checkpoint_rubric,49,test +knows.sheets_25_skitourplan.5,sheets,checkpoint_rubric,50,test +knows.slides_26_basic_educational_slide_deck.1,slides,checkpoint_rubric,51,test +knows.slides_26_basic_educational_slide_deck.2,slides,checkpoint_rubric,52,test +knows.slides_26_basic_educational_slide_deck.3,slides,checkpoint_rubric,53,test +knows.slides_26_basic_educational_slide_deck.4,slides,checkpoint_rubric,54,test +knows.slides_26_basic_educational_slide_deck.5,slides,checkpoint_rubric,55,test +knows.sheets_28_personal_travel_planner.1,sheets,checkpoint_rubric,56,test +knows.sheets_28_personal_travel_planner.2,sheets,checkpoint_rubric,57,test +knows.sheets_28_personal_travel_planner.3,sheets,checkpoint_rubric,58,test +knows.sheets_28_personal_travel_planner.4,sheets,checkpoint_rubric,59,test +knows.sheets_28_personal_travel_planner.5,sheets,checkpoint_rubric,60,test +knows.slides_29_buy_car_pres.1,slides,checkpoint_rubric,61,test +knows.slides_29_buy_car_pres.2,slides,checkpoint_rubric,62,test +knows.slides_29_buy_car_pres.3,slides,checkpoint_rubric,63,test +knows.slides_29_buy_car_pres.4,slides,checkpoint_rubric,64,test +knows.slides_29_buy_car_pres.5,slides,checkpoint_rubric,65,test +knows.slides_30_wikipedia_photos.1,slides,checkpoint_rubric,66,test +knows.slides_30_wikipedia_photos.2,slides,checkpoint_rubric,67,test +knows.slides_30_wikipedia_photos.3,slides,checkpoint_rubric,68,test +knows.slides_30_wikipedia_photos.4,slides,checkpoint_rubric,69,test +knows.slides_30_wikipedia_photos.5,slides,checkpoint_rubric,70,test +knows.docs_31_education_lesson_plan.1,docs,checkpoint_rubric,71,test +knows.docs_31_education_lesson_plan.2,docs,checkpoint_rubric,72,test +knows.docs_31_education_lesson_plan.3,docs,checkpoint_rubric,73,test +knows.docs_31_education_lesson_plan.4,docs,checkpoint_rubric,74,test +knows.docs_31_education_lesson_plan.5,docs,checkpoint_rubric,75,test +knows.docs_37_reference_list.1,docs,checkpoint_rubric,76,test +knows.docs_37_reference_list.2,docs,checkpoint_rubric,77,test +knows.docs_37_reference_list.3,docs,checkpoint_rubric,78,test +knows.docs_37_reference_list.4,docs,checkpoint_rubric,79,test +knows.docs_37_reference_list.5,docs,checkpoint_rubric,80,test +knows.sheets_38_apartment_finder.1,sheets,checkpoint_rubric,81,test +knows.sheets_38_apartment_finder.2,sheets,checkpoint_rubric,82,test +knows.sheets_38_apartment_finder.3,sheets,checkpoint_rubric,83,test +knows.sheets_38_apartment_finder.4,sheets,checkpoint_rubric,84,test +knows.sheets_38_apartment_finder.5,sheets,checkpoint_rubric,85,test +knows.slides_39_personal_lookbook_paintcolors.1,slides,checkpoint_rubric,86,test +knows.slides_39_personal_lookbook_paintcolors.2,slides,checkpoint_rubric,87,test +knows.slides_39_personal_lookbook_paintcolors.3,slides,checkpoint_rubric,88,test +knows.slides_39_personal_lookbook_paintcolors.4,slides,checkpoint_rubric,89,test +knows.slides_39_personal_lookbook_paintcolors.5,slides,checkpoint_rubric,90,test +knows.slides_42_product_comparison.1,slides,checkpoint_rubric,91,test +knows.slides_42_product_comparison.2,slides,checkpoint_rubric,92,test +knows.slides_42_product_comparison.3,slides,checkpoint_rubric,93,test +knows.slides_42_product_comparison.4,slides,checkpoint_rubric,94,test +knows.slides_42_product_comparison.5,slides,checkpoint_rubric,95,test +knows.sheets_45_wedding_planner.1,sheets,checkpoint_rubric,96,test +knows.sheets_45_wedding_planner.2,sheets,checkpoint_rubric,97,test +knows.sheets_45_wedding_planner.3,sheets,checkpoint_rubric,98,test +knows.sheets_45_wedding_planner.4,sheets,checkpoint_rubric,99,test +knows.sheets_45_wedding_planner.5,sheets,checkpoint_rubric,100,test +knows.slides_51_event_announcement_poster.1,slides,checkpoint_rubric,101,test +knows.slides_51_event_announcement_poster.2,slides,checkpoint_rubric,102,test +knows.slides_51_event_announcement_poster.3,slides,checkpoint_rubric,103,test +knows.slides_51_event_announcement_poster.4,slides,checkpoint_rubric,104,test +knows.slides_51_event_announcement_poster.5,slides,checkpoint_rubric,105,test +knows.sheets_55_movie_recommendation.1,sheets,checkpoint_rubric,106,test +knows.sheets_55_movie_recommendation.2,sheets,checkpoint_rubric,107,test +knows.sheets_55_movie_recommendation.3,sheets,checkpoint_rubric,108,test +knows.sheets_55_movie_recommendation.4,sheets,checkpoint_rubric,109,test +knows.sheets_55_movie_recommendation.5,sheets,checkpoint_rubric,110,test diff --git a/browsergym/experiments/src/browsergym/experiments/benchmark/utils.py b/browsergym/experiments/src/browsergym/experiments/benchmark/utils.py index f2a7069b..8865717a 100644 --- a/browsergym/experiments/src/browsergym/experiments/benchmark/utils.py +++ b/browsergym/experiments/src/browsergym/experiments/benchmark/utils.py @@ -262,6 +262,8 @@ def prepare_backend(backend: str): weblinx_browsergym.download_and_unzip_demos(demo_ids=demo_ids, cache_dir=cache_dir) case "timewarp": import browsergym.timewarp + case "knows": + import browsergym.knows case _: raise NotImplementedError(f"Unknown benchmark backend {repr(backend)}") diff --git a/browsergym/experiments/src/browsergym/experiments/loop.py b/browsergym/experiments/src/browsergym/experiments/loop.py index 05d57dd7..cd960a2f 100644 --- a/browsergym/experiments/src/browsergym/experiments/loop.py +++ b/browsergym/experiments/src/browsergym/experiments/loop.py @@ -946,6 +946,8 @@ def _get_env_name(task_name: str): import weblinx_browsergym elif task_name.startswith("timewarp"): import browsergym.timewarp + elif task_name.startswith("knows"): + import browsergym.knows return f"browsergym/{task_name}" diff --git a/dev/requirements.txt b/dev/requirements.txt index 81e33ae2..d1cf5460 100644 --- a/dev/requirements.txt +++ b/dev/requirements.txt @@ -16,3 +16,4 @@ tenacity browsergym-workarena weblinx_browsergym browsergym-timewarp +browsergym-knows diff --git a/tests/experiments/test_benchmark.py b/tests/experiments/test_benchmark.py index 8420273a..c7f88544 100644 --- a/tests/experiments/test_benchmark.py +++ b/tests/experiments/test_benchmark.py @@ -60,6 +60,7 @@ def test_build_benchmarks(): "assistantbench": 214, "weblinx": 31586, "timewarp": 231, + "knows": 110, } for name, benchmark_builder in DEFAULT_BENCHMARKS.items(): benchmark = benchmark_builder()