[Pratt Parser] Add an option to enable Pratt Parser

PiperOrigin-RevId: 945370852

Author: dmitriplotnikov

common/expr_factory.h

@@ -38,6 +38,11 @@ namespace tools {
 class ProtoToPredicateBuilder;
 }
 
+namespace parser_internal {
+template <typename ExprNode>
+class AstFactoryInterface;
+}
+
 class ExprFactory {
  protected:
   // `IsExprLike` determines whether `T` is some `Expr`. Currently that means
@@ -385,6 +390,7 @@ class ExprFactory {
   friend class ParserMacroExprFactory;
   friend class OptimizerExprFactory;
   friend class tools::ProtoToPredicateBuilder;
+  friend class parser_internal::AstFactoryInterface<Expr>;
 
   ExprFactory() : accu_var_(kAccumulatorVariableName) {}
 

conformance/BUILD

@@ -247,7 +247,7 @@ _TESTS_TO_SKIP_LEGACY_DASHBOARD = [
 ]
 
 # Generates a bunch of `cc_test` whose names follow the pattern
-# `conformance_(...)_{arena|refcount}_{optimized|unoptimized}_{recursive|iterative}`.
+# `conformance_(...)_{pratt|antlr}_{optimized|unoptimized}_{recursive|iterative}`.
 gen_conformance_tests(
     name = "conformance_parse_only",
     data = _ALL_TESTS,
@@ -316,7 +316,7 @@ gen_conformance_tests(
 )
 
 # Generates a bunch of `cc_test` whose names follow the pattern
-# `conformance_dashboard_..._{arena|refcount}_{optimized|unoptimized}_{recursive|iterative}`.
+# `conformance_dashboard_..._{pratt|antlr}_{optimized|unoptimized}_{recursive|iterative}`.
 gen_conformance_tests(
     name = "conformance_dashboard_parse_only",
     dashboard = True,

conformance/run.bzl

@@ -47,16 +47,17 @@ def _expand_tests_to_skip(tests_to_skip):
             result.append(test_to_skip[0:slash] + part)
     return result
 
-def _conformance_test_name(name, optimize, recursive):
+def _conformance_test_name(name, pratt, optimize, recursive):
     return "_".join(
         [
             name,
+            "pratt" if pratt else "antlr",
             "optimized" if optimize else "unoptimized",
             "recursive" if recursive else "iterative",
         ],
     )
 
-def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard, enable_variadic_logical_operators):
+def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard, enable_variadic_logical_operators, pratt):
     args = []
     if modern:
         args.append("--modern")
@@ -74,12 +75,16 @@ def _conformance_test_args(modern, optimize, recursive, select_opt, skip_check,
         args.append("--dashboard")
     if enable_variadic_logical_operators:
         args.append("--enable_variadic_logical_operators")
+    if pratt:
+        args.append("--enable_pratt_parser")
+    else:
+        args.append("--noenable_pratt_parser")
     return args
 
-def _conformance_test(name, data, modern, optimize, recursive, select_opt, skip_check, skip_tests, tags, dashboard, enable_variadic_logical_operators):
+def _conformance_test(name, data, modern, optimize, recursive, select_opt, skip_check, skip_tests, tags, dashboard, enable_variadic_logical_operators, pratt):
     cc_test(
-        name = _conformance_test_name(name, optimize, recursive),
-        args = _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard, enable_variadic_logical_operators) + ["$(rlocationpath {})".format(test) for test in data],
+        name = _conformance_test_name(name, pratt, optimize, recursive),
+        args = _conformance_test_args(modern, optimize, recursive, select_opt, skip_check, dashboard, enable_variadic_logical_operators, pratt) + ["$(rlocationpath {})".format(test) for test in data],
         env = select(
             {
                 "@platforms//os:windows": {"CEL_SKIP_TESTS": ",".join(skip_tests + _TESTS_TO_SKIP_WINDOWS)},
@@ -108,23 +113,25 @@ def gen_conformance_tests(name, data, modern = False, checked = False, select_op
     """
     skip_check = not checked
     tests = []
-    for optimize in (True, False):
-        for recursive in (True, False):
-            test_name = _conformance_test_name(name, optimize, recursive)
-            tests.append(test_name)
-            _conformance_test(
-                name,
-                data,
-                modern = modern,
-                optimize = optimize,
-                recursive = recursive,
-                select_opt = select_opt,
-                skip_check = skip_check,
-                skip_tests = _expand_tests_to_skip(skip_tests),
-                tags = tags,
-                dashboard = dashboard,
-                enable_variadic_logical_operators = enable_variadic_logical_operators,
-            )
+    for pratt in (True, False):
+        for optimize in (True, False):
+            for recursive in (True, False):
+                test_name = _conformance_test_name(name, pratt, optimize, recursive)
+                tests.append(test_name)
+                _conformance_test(
+                    name,
+                    data,
+                    modern = modern,
+                    optimize = optimize,
+                    recursive = recursive,
+                    select_opt = select_opt,
+                    skip_check = skip_check,
+                    skip_tests = _expand_tests_to_skip(skip_tests),
+                    tags = tags,
+                    dashboard = dashboard,
+                    enable_variadic_logical_operators = enable_variadic_logical_operators,
+                    pratt = pratt,
+                )
     native.test_suite(
         name = name,
         tests = tests,

conformance/run.cc

@@ -69,6 +69,8 @@ ABSL_FLAG(bool, select_optimization, false, "Enable select optimization.");
 ABSL_FLAG(bool, enable_variadic_logical_operators, false,
           "Enable parsing logical AND & OR operators as a single flat variadic "
           "call.");
+ABSL_FLAG(bool, enable_pratt_parser, true,
+          "Enable manual (Pratt) parser instead of ANTLR parser.");
 
 namespace {
 
@@ -266,6 +268,7 @@ NewConformanceServiceFromFlags() {
           .select_optimization = absl::GetFlag(FLAGS_select_optimization),
           .enable_variadic_logical_operators =
               absl::GetFlag(FLAGS_enable_variadic_logical_operators),
+          .enable_pratt_parser = absl::GetFlag(FLAGS_enable_pratt_parser),
       });
   ABSL_CHECK_OK(status_or_service);
   return std::shared_ptr<cel_conformance::ConformanceServiceInterface>(

conformance/service.cc

@@ -129,14 +129,16 @@ cel::expr::Expr ExtractExpr(
 absl::Status LegacyParse(const conformance::v1alpha1::ParseRequest& request,
                          conformance::v1alpha1::ParseResponse& response,
                          bool enable_optional_syntax,
-                         bool enable_variadic_logical_operators) {
+                         bool enable_variadic_logical_operators,
+                         bool enable_pratt_parser) {
   if (request.cel_source().empty()) {
     return absl::InvalidArgumentError("no source code");
   }
   cel::ParserOptions options;
   options.enable_optional_syntax = enable_optional_syntax;
   options.enable_quoted_identifiers = true;
   options.enable_variadic_logical_operators = enable_variadic_logical_operators;
+  options.enable_pratt_parser = enable_pratt_parser;
   cel::MacroRegistry macros;
   CEL_RETURN_IF_ERROR(cel::RegisterStandardMacros(macros, options));
   CEL_RETURN_IF_ERROR(
@@ -239,7 +241,7 @@ class LegacyConformanceServiceImpl : public ConformanceServiceInterface {
  public:
   static absl::StatusOr<std::unique_ptr<LegacyConformanceServiceImpl>> Create(
       bool optimize, bool recursive, bool select_optimization,
-      bool enable_variadic_logical_operators) {
+      bool enable_variadic_logical_operators, bool enable_pratt_parser) {
     static auto* constant_arena = new Arena();
 
     google::protobuf::LinkMessageReflection<
@@ -317,14 +319,15 @@ class LegacyConformanceServiceImpl : public ConformanceServiceInterface {
         builder->GetRegistry(), options));
 
     return absl::WrapUnique(new LegacyConformanceServiceImpl(
-        std::move(builder), enable_variadic_logical_operators));
+        std::move(builder), enable_variadic_logical_operators,
+        enable_pratt_parser));
   }
 
   void Parse(const conformance::v1alpha1::ParseRequest& request,
              conformance::v1alpha1::ParseResponse& response) override {
     auto status =
         LegacyParse(request, response, /*enable_optional_syntax=*/false,
-                    enable_variadic_logical_operators_);
+                    enable_variadic_logical_operators_, enable_pratt_parser_);
     if (!status.ok()) {
       auto* issue = response.add_issues();
       issue->set_code(ToGrpcCode(status.code()));
@@ -423,19 +426,22 @@ class LegacyConformanceServiceImpl : public ConformanceServiceInterface {
 
  private:
   LegacyConformanceServiceImpl(std::unique_ptr<CelExpressionBuilder> builder,
-                               bool enable_variadic_logical_operators)
+                               bool enable_variadic_logical_operators,
+                               bool enable_pratt_parser)
       : builder_(std::move(builder)),
-        enable_variadic_logical_operators_(enable_variadic_logical_operators) {}
+        enable_variadic_logical_operators_(enable_variadic_logical_operators),
+        enable_pratt_parser_(enable_pratt_parser) {}
 
   std::unique_ptr<CelExpressionBuilder> builder_;
   bool enable_variadic_logical_operators_;
+  bool enable_pratt_parser_;
 };
 
 class ModernConformanceServiceImpl : public ConformanceServiceInterface {
  public:
   static absl::StatusOr<std::unique_ptr<ModernConformanceServiceImpl>> Create(
       bool optimize, bool recursive, bool select_optimization,
-      bool enable_variadic_logical_operators) {
+      bool enable_variadic_logical_operators, bool enable_pratt_parser) {
     google::protobuf::LinkMessageReflection<
         cel::expr::conformance::proto3::TestAllTypes>();
     google::protobuf::LinkMessageReflection<
@@ -477,9 +483,9 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface {
       options.max_recursion_depth = 48;
     }
 
-    return absl::WrapUnique(
-        new ModernConformanceServiceImpl(options, optimize, select_optimization,
-                                         enable_variadic_logical_operators));
+    return absl::WrapUnique(new ModernConformanceServiceImpl(
+        options, optimize, select_optimization,
+        enable_variadic_logical_operators, enable_pratt_parser));
   }
 
   absl::StatusOr<std::unique_ptr<const cel::Runtime>> Setup(
@@ -532,7 +538,7 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface {
              conformance::v1alpha1::ParseResponse& response) override {
     auto status =
         LegacyParse(request, response, /*enable_optional_syntax=*/true,
-                    enable_variadic_logical_operators_);
+                    enable_variadic_logical_operators_, enable_pratt_parser_);
     if (!status.ok()) {
       auto* issue = response.add_issues();
       issue->set_code(ToGrpcCode(status.code()));
@@ -624,11 +630,13 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface {
   ModernConformanceServiceImpl(const RuntimeOptions& options,
                                bool enable_optimizations,
                                bool enable_select_optimization,
-                               bool enable_variadic_logical_operators)
+                               bool enable_variadic_logical_operators,
+                               bool enable_pratt_parser)
       : options_(options),
         enable_optimizations_(enable_optimizations),
         enable_select_optimization_(enable_select_optimization),
-        enable_variadic_logical_operators_(enable_variadic_logical_operators) {}
+        enable_variadic_logical_operators_(enable_variadic_logical_operators),
+        enable_pratt_parser_(enable_pratt_parser) {}
 
   static absl::StatusOr<std::unique_ptr<cel::TraceableProgram>> Plan(
       const cel::Runtime& runtime,
@@ -660,6 +668,7 @@ class ModernConformanceServiceImpl : public ConformanceServiceInterface {
   bool enable_optimizations_;
   bool enable_select_optimization_;
   bool enable_variadic_logical_operators_;
+  bool enable_pratt_parser_;
 };
 
 }  // namespace
@@ -673,11 +682,11 @@ NewConformanceService(const ConformanceServiceOptions& options) {
   if (options.modern) {
     return google::api::expr::runtime::ModernConformanceServiceImpl::Create(
         options.optimize, options.recursive, options.select_optimization,
-        options.enable_variadic_logical_operators);
+        options.enable_variadic_logical_operators, options.enable_pratt_parser);
   } else {
     return google::api::expr::runtime::LegacyConformanceServiceImpl::Create(
         options.optimize, options.recursive, options.select_optimization,
-        options.enable_variadic_logical_operators);
+        options.enable_variadic_logical_operators, options.enable_pratt_parser);
   }
 }
 

conformance/service.h

@@ -47,6 +47,7 @@ struct ConformanceServiceOptions {
   bool recursive;
   bool select_optimization;
   bool enable_variadic_logical_operators = false;
+  bool enable_pratt_parser = true;
 };
 
 absl::StatusOr<std::unique_ptr<ConformanceServiceInterface>>

parser/BUILD

@@ -42,6 +42,7 @@ cc_library(
         ":source_factory",
         "//common:ast",
         "//common:constant",
+        "//common:expr",
         "//common:expr_factory",
         "//common:operators",
         "//common:source",
@@ -50,8 +51,8 @@ cc_library(
         "//internal:lexis",
         "//internal:status_macros",
         "//internal:strings",
-        "//internal:utf8",
         "//parser/internal:cel_cc_parser",
+        "//parser/internal:pratt_parser",
         "@antlr4-cpp-runtime",
         "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/cleanup",
@@ -60,6 +61,7 @@ cc_library(
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/functional:overload",
         "@com_google_absl//absl/log:absl_check",
+        "@com_google_absl//absl/log:check",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/status:statusor",
@@ -197,6 +199,31 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "pratt_parser_test",
+    srcs = ["parser_test.cc"],
+    defines = ["CEL_TEST_ENABLE_PRATT_PARSER=1"],
+    deps = [
+        ":macro",
+        ":options",
+        ":parser",
+        ":parser_interface",
+        ":source_factory",
+        "//common:constant",
+        "//common:expr",
+        "//common:source",
+        "//internal:testing",
+        "//testutil:expr_printer",
+        "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:status_matchers",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_cel_spec//proto/cel/expr:syntax_cc_proto",
+    ],
+)
+
 cc_test(
     name = "parser_benchmarks",
     srcs = ["parser_benchmarks.cc"],