Added Knows: a Google Workspace benchmark for web agents (Docs, Sheet, Slides)

README.md

@@ -46,6 +46,7 @@ BrowserGym includes the following benchmarks by default:
  - [WebLINX](https://github.com/McGill-NLP/weblinx) (static benchmark)
  - [OpenApps](https://facebookresearch.github.io/OpenApps/)
  - [TimeWarp](https://timewarp-web.github.io)
+ - [Knows](https://github.com/alexgill321/Agent-Benchmark)
 
 Designing new web benchmarks with BrowserGym is easy, and simply requires to inherit the [`AbstractBrowserTask`](https://github.com/ServiceNow/BrowserGym/blob/main/browsergym/core/src/browsergym/core/task.py#L7C7-L7C26) class.
 
@@ -64,6 +65,7 @@ pip install browsergym-workarena  # core + workarena
 pip install browsergym-assistantbench  # core + assistantbench
 pip install weblinx-browsergym  # core + weblinx
 pip install browsergym-timewarp  # core + timewarp 
+pip install browsergym-knows  # core + knows
 ```
 
 Then setup playwright by running
@@ -80,6 +82,7 @@ Finally, each benchmark comes with its own specific setup that requires to follo
  - for AssistantBench, see [assistantbench/README.md](browsergym/assistantbench/README.md)
  - for OpenApps, see [OpenApps docs](https://facebookresearch.github.io/OpenApps/)
   - for TimeWarp, see [timewarp/README.md](https://github.com/sparklabutah/timewarp)
+  - for Knows, see [knows/README.md](https://github.com/alexgill321/Agent-Benchmark) (requires a Google account and a Google Cloud service account)
 
 ### 🏗️ Development setup
 
@@ -211,6 +214,20 @@ env_ids = [id for id in gym.envs.registry.keys() if id.startswith("browsergym/ti
 print("\n".join(env_ids))
 ```
 
+Knows
+```python
+import gymnasium as gym
+import browsergym.knows  # register knows tasks as gym environments
+
+# start a knows task
+env = gym.make("browsergym/knows.docs_1_formal_letter.1")
+...
+
+# list all the available knows tasks
+env_ids = [id for id in gym.envs.registry.keys() if id.startswith("browsergym/knows")]
+print("\n".join(env_ids))
+```
+
 ## 💻 Demo
 
 If you want to experiment with a demo agent in BrowserGym, follow these steps

browsergym/core/src/browsergym/core/action/highlevel.py

@@ -259,6 +259,34 @@
         tab_close,  # Closes a tab
         tab_focus,  # Switches tabs
     ],
+    "knows": [  # Google Workspace tasks: on top of the usual bid actions, they need
+        # file upload and direct mouse control to edit Docs/Sheets/Slides canvases
+        click,
+        dblclick,
+        drag_and_drop,
+        fill,
+        go_back,
+        go_forward,
+        goto,
+        hover,
+        keyboard_insert_text,
+        keyboard_press,
+        keyboard_type,
+        mouse_click,
+        mouse_dblclick,
+        mouse_drag_and_drop,
+        mouse_move,
+        mouse_upload_file,
+        new_tab,
+        noop,
+        report_infeasible,
+        scroll,
+        select_option,
+        send_msg_to_user,
+        tab_close,
+        tab_focus,
+        upload_file,
+    ],
 }
 
 
@@ -291,6 +319,7 @@ class HighLevelActionSet(AbstractActionSet):
         "weblinx",
         "assistantbench",
         "timewarp",
+        "knows",
         "custom",
     ]
     DemoMode = typing.Literal["off", "default", "all_blue", "only_visible_elements"]

browsergym/experiments/pyproject.toml

@@ -46,6 +46,9 @@ weblinx = [
 timewarp = [
   "browsergym-timewarp"
 ]
+knows = [
+  "browsergym-knows"
+]
 all = [
   "browsergym-experiment[miniwob]",
   "browsergym-experiment[workarena]",
@@ -54,6 +57,7 @@ all = [
   "browsergym-experiment[assistantbench]",
   "browsergym-experiment[weblinx]",
   "browsergym-experiment[timewarp]",
+  "browsergym-experiment[knows]",
 ]
 
 [project.urls]

browsergym/experiments/src/browsergym/experiments/benchmark/base.py

@@ -61,6 +61,7 @@ def make_action_set(self):
     "assistantbench",
     "weblinx",
     "timewarp",
+    "knows",
 ]
 
 

browsergym/experiments/src/browsergym/experiments/benchmark/configs.py

@@ -94,6 +94,13 @@
         retry_with_force=True,
         demo_mode="off",
     ),
+    "knows": HighLevelActionSetArgs(
+        subsets=["knows"],
+        multiaction=False,
+        strict=False,
+        retry_with_force=True,
+        demo_mode="off",
+    ),
 }
 
 # all benchmarks are callables designed for lazy loading, i.e. `bench = DEFAULT_BENCHMARKS["miniwob_all"]()`
@@ -312,4 +319,20 @@
         ),
         task_metadata=task_metadata("timewarp"),
     ),
+    "knows": lambda n_repeats=1: Benchmark(
+        name="knows",
+        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["knows"],
+        is_multi_tab=True,
+        supports_parallel_seeds=True,
+        backends=["knows"],
+        env_args_list=make_env_args_list_from_repeat_tasks(
+            task_list=task_list_from_metadata(metadata=task_metadata("knows")),
+            # Knows tasks are long-horizon document authoring; past this the env's
+            # TimeLimit truncates the episode and the document is graded as-is.
+            max_steps=120,
+            n_repeats=n_repeats,
+            seeds_rng=np.random.RandomState(42),
+        ),
+        task_metadata=task_metadata("knows"),
+    ),
 }

browsergym/experiments/src/browsergym/experiments/benchmark/metadata/knows.csv

@@ -0,0 +1,111 @@
+task_name,sites,eval_types,task_id,browsergym_split
+knows.docs_1_formal_letter.1,docs,checkpoint_rubric,1,test
+knows.docs_1_formal_letter.2,docs,checkpoint_rubric,2,test
+knows.docs_1_formal_letter.3,docs,checkpoint_rubric,3,test
+knows.docs_1_formal_letter.4,docs,checkpoint_rubric,4,test
+knows.docs_1_formal_letter.5,docs,checkpoint_rubric,5,test
+knows.sheets_2_personal_recipe.1,sheets,checkpoint_rubric,6,test
+knows.sheets_2_personal_recipe.2,sheets,checkpoint_rubric,7,test
+knows.sheets_2_personal_recipe.3,sheets,checkpoint_rubric,8,test
+knows.sheets_2_personal_recipe.4,sheets,checkpoint_rubric,9,test
+knows.sheets_2_personal_recipe.5,sheets,checkpoint_rubric,10,test
+knows.docs_5_influential_papers.1,docs,checkpoint_rubric,11,test
+knows.docs_5_influential_papers.2,docs,checkpoint_rubric,12,test
+knows.docs_5_influential_papers.3,docs,checkpoint_rubric,13,test
+knows.docs_5_influential_papers.4,docs,checkpoint_rubric,14,test
+knows.docs_5_influential_papers.5,docs,checkpoint_rubric,15,test
+knows.sheets_6_stock_tracker.1,sheets,checkpoint_rubric,16,test
+knows.sheets_6_stock_tracker.2,sheets,checkpoint_rubric,17,test
+knows.sheets_6_stock_tracker.3,sheets,checkpoint_rubric,18,test
+knows.sheets_6_stock_tracker.4,sheets,checkpoint_rubric,19,test
+knows.sheets_6_stock_tracker.5,sheets,checkpoint_rubric,20,test
+knows.sheets_7_running_analysis.1,sheets,checkpoint_rubric,21,test
+knows.sheets_7_running_analysis.2,sheets,checkpoint_rubric,22,test
+knows.sheets_7_running_analysis.3,sheets,checkpoint_rubric,23,test
+knows.sheets_7_running_analysis.4,sheets,checkpoint_rubric,24,test
+knows.sheets_7_running_analysis.5,sheets,checkpoint_rubric,25,test
+knows.sheets_10_paper_sorting.1,sheets,checkpoint_rubric,26,test
+knows.sheets_10_paper_sorting.2,sheets,checkpoint_rubric,27,test
+knows.sheets_10_paper_sorting.3,sheets,checkpoint_rubric,28,test
+knows.sheets_10_paper_sorting.4,sheets,checkpoint_rubric,29,test
+knows.sheets_10_paper_sorting.5,sheets,checkpoint_rubric,30,test
+knows.docs_11_personal_recipe_ocr.1,docs,checkpoint_rubric,31,test
+knows.docs_11_personal_recipe_ocr.2,docs,checkpoint_rubric,32,test
+knows.docs_11_personal_recipe_ocr.3,docs,checkpoint_rubric,33,test
+knows.docs_11_personal_recipe_ocr.4,docs,checkpoint_rubric,34,test
+knows.docs_11_personal_recipe_ocr.5,docs,checkpoint_rubric,35,test
+knows.slides_17_remove_images_add_placeholders.1,slides,checkpoint_rubric,36,test
+knows.slides_17_remove_images_add_placeholders.2,slides,checkpoint_rubric,37,test
+knows.slides_17_remove_images_add_placeholders.3,slides,checkpoint_rubric,38,test
+knows.slides_17_remove_images_add_placeholders.4,slides,checkpoint_rubric,39,test
+knows.slides_17_remove_images_add_placeholders.5,slides,checkpoint_rubric,40,test
+knows.slides_20_illustrated_book_report.1,slides,checkpoint_rubric,41,test
+knows.slides_20_illustrated_book_report.2,slides,checkpoint_rubric,42,test
+knows.slides_20_illustrated_book_report.3,slides,checkpoint_rubric,43,test
+knows.slides_20_illustrated_book_report.4,slides,checkpoint_rubric,44,test
+knows.slides_20_illustrated_book_report.5,slides,checkpoint_rubric,45,test
+knows.sheets_25_skitourplan.1,sheets,checkpoint_rubric,46,test
+knows.sheets_25_skitourplan.2,sheets,checkpoint_rubric,47,test
+knows.sheets_25_skitourplan.3,sheets,checkpoint_rubric,48,test
+knows.sheets_25_skitourplan.4,sheets,checkpoint_rubric,49,test
+knows.sheets_25_skitourplan.5,sheets,checkpoint_rubric,50,test
+knows.slides_26_basic_educational_slide_deck.1,slides,checkpoint_rubric,51,test
+knows.slides_26_basic_educational_slide_deck.2,slides,checkpoint_rubric,52,test
+knows.slides_26_basic_educational_slide_deck.3,slides,checkpoint_rubric,53,test
+knows.slides_26_basic_educational_slide_deck.4,slides,checkpoint_rubric,54,test
+knows.slides_26_basic_educational_slide_deck.5,slides,checkpoint_rubric,55,test
+knows.sheets_28_personal_travel_planner.1,sheets,checkpoint_rubric,56,test
+knows.sheets_28_personal_travel_planner.2,sheets,checkpoint_rubric,57,test
+knows.sheets_28_personal_travel_planner.3,sheets,checkpoint_rubric,58,test
+knows.sheets_28_personal_travel_planner.4,sheets,checkpoint_rubric,59,test
+knows.sheets_28_personal_travel_planner.5,sheets,checkpoint_rubric,60,test
+knows.slides_29_buy_car_pres.1,slides,checkpoint_rubric,61,test
+knows.slides_29_buy_car_pres.2,slides,checkpoint_rubric,62,test
+knows.slides_29_buy_car_pres.3,slides,checkpoint_rubric,63,test
+knows.slides_29_buy_car_pres.4,slides,checkpoint_rubric,64,test
+knows.slides_29_buy_car_pres.5,slides,checkpoint_rubric,65,test
+knows.slides_30_wikipedia_photos.1,slides,checkpoint_rubric,66,test
+knows.slides_30_wikipedia_photos.2,slides,checkpoint_rubric,67,test
+knows.slides_30_wikipedia_photos.3,slides,checkpoint_rubric,68,test
+knows.slides_30_wikipedia_photos.4,slides,checkpoint_rubric,69,test
+knows.slides_30_wikipedia_photos.5,slides,checkpoint_rubric,70,test
+knows.docs_31_education_lesson_plan.1,docs,checkpoint_rubric,71,test
+knows.docs_31_education_lesson_plan.2,docs,checkpoint_rubric,72,test
+knows.docs_31_education_lesson_plan.3,docs,checkpoint_rubric,73,test
+knows.docs_31_education_lesson_plan.4,docs,checkpoint_rubric,74,test
+knows.docs_31_education_lesson_plan.5,docs,checkpoint_rubric,75,test
+knows.docs_37_reference_list.1,docs,checkpoint_rubric,76,test
+knows.docs_37_reference_list.2,docs,checkpoint_rubric,77,test
+knows.docs_37_reference_list.3,docs,checkpoint_rubric,78,test
+knows.docs_37_reference_list.4,docs,checkpoint_rubric,79,test
+knows.docs_37_reference_list.5,docs,checkpoint_rubric,80,test
+knows.sheets_38_apartment_finder.1,sheets,checkpoint_rubric,81,test
+knows.sheets_38_apartment_finder.2,sheets,checkpoint_rubric,82,test
+knows.sheets_38_apartment_finder.3,sheets,checkpoint_rubric,83,test
+knows.sheets_38_apartment_finder.4,sheets,checkpoint_rubric,84,test
+knows.sheets_38_apartment_finder.5,sheets,checkpoint_rubric,85,test
+knows.slides_39_personal_lookbook_paintcolors.1,slides,checkpoint_rubric,86,test
+knows.slides_39_personal_lookbook_paintcolors.2,slides,checkpoint_rubric,87,test
+knows.slides_39_personal_lookbook_paintcolors.3,slides,checkpoint_rubric,88,test
+knows.slides_39_personal_lookbook_paintcolors.4,slides,checkpoint_rubric,89,test
+knows.slides_39_personal_lookbook_paintcolors.5,slides,checkpoint_rubric,90,test
+knows.slides_42_product_comparison.1,slides,checkpoint_rubric,91,test
+knows.slides_42_product_comparison.2,slides,checkpoint_rubric,92,test
+knows.slides_42_product_comparison.3,slides,checkpoint_rubric,93,test
+knows.slides_42_product_comparison.4,slides,checkpoint_rubric,94,test
+knows.slides_42_product_comparison.5,slides,checkpoint_rubric,95,test
+knows.sheets_45_wedding_planner.1,sheets,checkpoint_rubric,96,test
+knows.sheets_45_wedding_planner.2,sheets,checkpoint_rubric,97,test
+knows.sheets_45_wedding_planner.3,sheets,checkpoint_rubric,98,test
+knows.sheets_45_wedding_planner.4,sheets,checkpoint_rubric,99,test
+knows.sheets_45_wedding_planner.5,sheets,checkpoint_rubric,100,test
+knows.slides_51_event_announcement_poster.1,slides,checkpoint_rubric,101,test
+knows.slides_51_event_announcement_poster.2,slides,checkpoint_rubric,102,test
+knows.slides_51_event_announcement_poster.3,slides,checkpoint_rubric,103,test
+knows.slides_51_event_announcement_poster.4,slides,checkpoint_rubric,104,test
+knows.slides_51_event_announcement_poster.5,slides,checkpoint_rubric,105,test
+knows.sheets_55_movie_recommendation.1,sheets,checkpoint_rubric,106,test
+knows.sheets_55_movie_recommendation.2,sheets,checkpoint_rubric,107,test
+knows.sheets_55_movie_recommendation.3,sheets,checkpoint_rubric,108,test
+knows.sheets_55_movie_recommendation.4,sheets,checkpoint_rubric,109,test
+knows.sheets_55_movie_recommendation.5,sheets,checkpoint_rubric,110,test

browsergym/experiments/src/browsergym/experiments/benchmark/utils.py

@@ -262,6 +262,8 @@ def prepare_backend(backend: str):
             weblinx_browsergym.download_and_unzip_demos(demo_ids=demo_ids, cache_dir=cache_dir)
         case "timewarp":
             import browsergym.timewarp
+        case "knows":
+            import browsergym.knows
         case _:
             raise NotImplementedError(f"Unknown benchmark backend {repr(backend)}")
 

browsergym/experiments/src/browsergym/experiments/loop.py

@@ -946,6 +946,8 @@ def _get_env_name(task_name: str):
         import weblinx_browsergym
     elif task_name.startswith("timewarp"):
         import browsergym.timewarp
+    elif task_name.startswith("knows"):
+        import browsergym.knows
 
     return f"browsergym/{task_name}"
 

dev/requirements.txt

@@ -16,3 +16,4 @@ tenacity
 browsergym-workarena
 weblinx_browsergym
 browsergym-timewarp
+browsergym-knows

tests/experiments/test_benchmark.py

@@ -60,6 +60,7 @@ def test_build_benchmarks():
         "assistantbench": 214,
         "weblinx": 31586,
         "timewarp": 231,
+        "knows": 110,
     }
     for name, benchmark_builder in DEFAULT_BENCHMARKS.items():
         benchmark = benchmark_builder()