Add experimental RoaringLookupResources to the Materialize API

authzed/api/materialize/v0/roaringlookupresources.proto

@@ -0,0 +1,107 @@
+syntax = "proto3";
+package authzed.api.materialize.v0;
+
+import "authzed/api/v1/core.proto";
+import "authzed/api/v1/permission_service.proto";
+import "buf/validate/validate.proto";
+import "validate/validate.proto";
+
+option go_package = "github.com/authzed/authzed-go/proto/authzed/api/materialize/v0";
+option java_multiple_files = true;
+option java_package = "com.authzed.api.materialize.v0";
+
+service RoaringLookupResourcesService {
+  // EXPERIMENTAL: RoaringLookupResources returns a roaring64 bitmap of the IDs
+  // of the resources of the given type on which the given subject has the
+  // given permission. This API is experimental and subject to change or
+  // removal.
+  //
+  // The bitmap is serialized in the RoaringFormatSpec 64-bit portable format
+  // (https://github.com/RoaringBitmap/RoaringFormatSpec#extention-for-64-bit-implementations),
+  // which OpenSearch consumes directly via a `"value_type": "bitmap"` terms
+  // query against a `long` field, once Base64-encoded.
+  //
+  // The IDs in the bitmap are the resource object IDs exactly as they appear
+  // in the relationships: no surrogate or internal ID is introduced, so the
+  // caller can use the bitmap directly against its own data (for example, a
+  // search index keyed by the same IDs).
+  //
+  // For this API to be usable, every resource object ID of the requested type
+  // must be a canonical decimal integer that fits in 44 bits -- at most
+  // 17592186044415 (2^44 - 1). Canonical means the string round-trips through
+  // uint64 formatting unchanged: `document:007` and `document:7` are distinct
+  // objects that would collide as the integer 7, so non-canonical IDs are
+  // rejected. If any resource object ID violates either rule, the call fails
+  // with FAILED_PRECONDITION rather than returning a partial bitmap, since a
+  // bitmap that is quietly too small is a wrong authorization answer. When the
+  // permission relates a type to itself (for example `group#member` looked up
+  // for a `group#member` subject), the subject is one of its own resources, so
+  // the subject's object ID is held to the same rules and can itself be the ID
+  // named in that error.
+  //
+  // Response size: the whole bitmap is returned in a single unary message, and
+  // most gRPC clients default to refusing messages larger than 4 MiB. Roaring
+  // is compact -- a million sequential IDs encode in a few hundred bytes -- but
+  // sparse IDs cost close to 10 bytes each, so a result of more than roughly
+  // 400,000 widely-spread IDs can exceed that default and fail on the CLIENT
+  // side with RESOURCE_EXHAUSTED ("received message larger than max"). This is
+  // a limit of the caller's own gRPC configuration, not of the service: raise
+  // it to match the largest result you expect (in Go,
+  // grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(n)); other languages
+  // have an equivalent channel option). The `cardinality` field is returned so
+  // callers can see how large a result is once received; no server-side result
+  // limit is applied.
+  rpc ExperimentalRoaringLookupResources(ExperimentalRoaringLookupResourcesRequest) returns (ExperimentalRoaringLookupResourcesResponse) {}
+}
+
+message ExperimentalRoaringLookupResourcesRequest {
+  // consistency selects the snapshot at which the lookup is performed. If
+  // unspecified, minimize_latency is used.
+  authzed.api.v1.Consistency consistency = 1;
+
+  // resource_object_type is the type of resource over which to look up access.
+  string resource_object_type = 2 [
+    (validate.rules).string = {
+      pattern: "^([a-z][a-z0-9_]{1,61}[a-z0-9]/)*[a-z][a-z0-9_]{1,62}[a-z0-9]$"
+      max_bytes: 128
+    },
+    (buf.validate.field).string = {
+      pattern: "^([a-z][a-z0-9_]{1,61}[a-z0-9]/)*[a-z][a-z0-9_]{1,62}[a-z0-9]$"
+      max_bytes: 128
+    }
+  ];
+
+  // permission is the name of the permission or relation to look up.
+  string permission = 3 [
+    (validate.rules).string = {
+      pattern: "^[a-z][a-z0-9_]{1,62}[a-z0-9]$"
+      max_bytes: 64
+    },
+    (buf.validate.field).string = {
+      pattern: "^[a-z][a-z0-9_]{1,62}[a-z0-9]$"
+      max_bytes: 64
+    }
+  ];
+
+  // subject is the subject for which access is being looked up.
+  authzed.api.v1.SubjectReference subject = 4 [
+    (validate.rules).message.required = true,
+    (buf.validate.field).required = true
+  ];
+}
+
+message ExperimentalRoaringLookupResourcesResponse {
+  // bitmap is the roaring64 bitmap, in RoaringFormatSpec 64-bit portable
+  // format, of the resource object IDs accessible to the subject. The IDs are
+  // the object IDs from the relationships themselves, as 44-bit integers.
+  bytes bitmap = 1;
+
+  // cardinality is the number of resource IDs in the bitmap.
+  uint64 cardinality = 2;
+
+  // at_revision is the ZedToken at which the lookup was performed.
+  authzed.api.v1.ZedToken at_revision = 3 [
+    (validate.rules).message.required = true,
+    (buf.validate.field).required = true
+  ];
+}

docs/apidocs.swagger.json

@@ -1022,11 +1022,11 @@
       "properties": {
         "@type": {
           "type": "string",
-          "description": "A URL/resource name that uniquely identifies the type of the serialized\nprotocol buffer message. This string must contain at least\none \"/\" character. The last segment of the URL's path must represent\nthe fully qualified name of the type (as in\n`path/google.protobuf.Duration`). The name should be in a canonical form\n(e.g., leading \".\" is not accepted).\n\nIn practice, teams usually precompile into the binary all types that they\nexpect it to use in the context of Any. However, for URLs which use the\nscheme `http`, `https`, or no scheme, one can optionally set up a type\nserver that maps type URLs to message definitions as follows:\n\n* If no scheme is provided, `https` is assumed.\n* An HTTP GET on the URL must yield a [google.protobuf.Type][]\n  value in binary format, or produce an error.\n* Applications are allowed to cache lookup results based on the\n  URL, or have them precompiled into a binary to avoid any\n  lookup. Therefore, binary compatibility needs to be preserved\n  on changes to types. (Use versioned type names to manage\n  breaking changes.)\n\nNote: this functionality is not currently available in the official\nprotobuf release, and it is not used for type URLs beginning with\ntype.googleapis.com. As of May 2023, there are no widely used type server\nimplementations and no plans to implement one.\n\nSchemes other than `http`, `https` (or the empty scheme) might be\nused with implementation specific semantics."
+          "description": "Identifies the type of the serialized Protobuf message with a URI reference\nconsisting of a prefix ending in a slash and the fully-qualified type name.\n\nExample: type.googleapis.com/google.protobuf.StringValue\n\nThis string must contain at least one `/` character, and the content after\nthe last `/` must be the fully-qualified name of the type in canonical\nform, without a leading dot. Do not write a scheme on these URI references\nso that clients do not attempt to contact them.\n\nThe prefix is arbitrary and Protobuf implementations are expected to\nsimply strip off everything up to and including the last `/` to identify\nthe type. `type.googleapis.com/` is a common default prefix that some\nlegacy implementations require. This prefix does not indicate the origin of\nthe type, and URIs containing it are not expected to respond to any\nrequests.\n\nAll type URL strings must be legal URI references with the additional\nrestriction (for the text format) that the content of the reference\nmust consist only of alphanumeric characters, percent-encoded escapes, and\ncharacters in the following set (not including the outer backticks):\n`/-.~_!$\u0026()*+,;=`. Despite our allowing percent encodings, implementations\nshould not unescape them to prevent confusion with existing parsers. For\nexample, `type.googleapis.com%2FFoo` should be rejected.\n\nIn the original design of `Any`, the possibility of launching a type\nresolution service at these type URLs was considered but Protobuf never\nimplemented one and considers contacting these URLs to be problematic and\na potential security issue. Do not attempt to contact type URLs."
         }
       },
       "additionalProperties": {},
-      "description": "`Any` contains an arbitrary serialized protocol buffer message along with a\nURL that describes the type of the serialized message.\n\nProtobuf library provides support to pack/unpack Any values in the form\nof utility functions or additional generated methods of the Any type.\n\nExample 1: Pack and unpack a message in C++.\n\n    Foo foo = ...;\n    Any any;\n    any.PackFrom(foo);\n    ...\n    if (any.UnpackTo(\u0026foo)) {\n      ...\n    }\n\nExample 2: Pack and unpack a message in Java.\n\n    Foo foo = ...;\n    Any any = Any.pack(foo);\n    ...\n    if (any.is(Foo.class)) {\n      foo = any.unpack(Foo.class);\n    }\n    // or ...\n    if (any.isSameTypeAs(Foo.getDefaultInstance())) {\n      foo = any.unpack(Foo.getDefaultInstance());\n    }\n\n Example 3: Pack and unpack a message in Python.\n\n    foo = Foo(...)\n    any = Any()\n    any.Pack(foo)\n    ...\n    if any.Is(Foo.DESCRIPTOR):\n      any.Unpack(foo)\n      ...\n\n Example 4: Pack and unpack a message in Go\n\n     foo := \u0026pb.Foo{...}\n     any, err := anypb.New(foo)\n     if err != nil {\n       ...\n     }\n     ...\n     foo := \u0026pb.Foo{}\n     if err := any.UnmarshalTo(foo); err != nil {\n       ...\n     }\n\nThe pack methods provided by protobuf library will by default use\n'type.googleapis.com/full.type.name' as the type URL and the unpack\nmethods only use the fully qualified type name after the last '/'\nin the type URL, for example \"foo.bar.com/x/y.z\" will yield type\nname \"y.z\".\n\nJSON\n====\nThe JSON representation of an `Any` value uses the regular\nrepresentation of the deserialized, embedded message, with an\nadditional field `@type` which contains the type URL. Example:\n\n    package google.profile;\n    message Person {\n      string first_name = 1;\n      string last_name = 2;\n    }\n\n    {\n      \"@type\": \"type.googleapis.com/google.profile.Person\",\n      \"firstName\": \u003cstring\u003e,\n      \"lastName\": \u003cstring\u003e\n    }\n\nIf the embedded message type is well-known and has a custom JSON\nrepresentation, that representation will be embedded adding a field\n`value` which holds the custom JSON in addition to the `@type`\nfield. Example (for message [google.protobuf.Duration][]):\n\n    {\n      \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n      \"value\": \"1.212s\"\n    }"
+      "description": "`Any` contains an arbitrary serialized protocol buffer message along with a\nURL that describes the type of the serialized message.\n\nIn its binary encoding, an `Any` is an ordinary message; but in other wire\nforms like JSON, it has a special encoding. The format of the type URL is\ndescribed on the `type_url` field.\n\nProtobuf APIs provide utilities to interact with `Any` values:\n\n- A 'pack' operation accepts a message and constructs a generic `Any` wrapper\n  around it.\n- An 'unpack' operation reads the content of an `Any` message, either into an\n  existing message or a new one. Unpack operations must check the type of the\n  value they unpack against the declared `type_url`.\n- An 'is' operation decides whether an `Any` contains a message of the given\n  type, i.e. whether it can 'unpack' that type.\n\nThe JSON format representation of an `Any` follows one of these cases:\n\n- For types without special-cased JSON encodings, the JSON format\n  representation of the `Any` is the same as that of the message, with an\n  additional `@type` field which contains the type URL.\n- For types with special-cased JSON encodings (typically called 'well-known'\n  types, listed in https://protobuf.dev/programming-guides/json/#any), the\n  JSON format representation has a key `@type` which contains the type URL\n  and a key `value` which contains the JSON-serialized value.\n\nThe text format representation of an `Any` is like a message with one field\nwhose name is the type URL in brackets. For example, an `Any` containing a\n`foo.Bar` message may be written `[type.googleapis.com/foo.Bar] { a: 2 }`."
     },
     "BreakingSchemaChange": {
       "type": "object",
@@ -2124,6 +2124,25 @@
     "ExperimentalRegisterRelationshipCounterResponse": {
       "type": "object"
     },
+    "ExperimentalRoaringLookupResourcesResponse": {
+      "type": "object",
+      "properties": {
+        "bitmap": {
+          "type": "string",
+          "format": "byte",
+          "description": "bitmap is the roaring64 bitmap, in RoaringFormatSpec 64-bit portable\nformat, of the resource object IDs accessible to the subject. The IDs are\nthe object IDs from the relationships themselves, as 44-bit integers."
+        },
+        "cardinality": {
+          "type": "string",
+          "format": "uint64",
+          "description": "cardinality is the number of resource IDs in the bitmap."
+        },
+        "atRevision": {
+          "$ref": "#/definitions/ZedToken",
+          "description": "at_revision is the ZedToken at which the lookup was performed."
+        }
+      }
+    },
     "ExperimentalUnregisterRelationshipCounterRequest": {
       "type": "object",
       "properties": {
@@ -2417,7 +2436,7 @@
         "NULL_VALUE"
       ],
       "default": "NULL_VALUE",
-      "description": "`NullValue` is a singleton enumeration to represent the null value for the\n`Value` type union.\n\nThe JSON representation for `NullValue` is JSON `null`.\n\n - NULL_VALUE: Null value."
+      "description": "Represents a JSON `null`.\n\n`NullValue` is a sentinel, using an enum with only one value to represent\nthe null value for the `Value` type union.\n\nA field of type `NullValue` with any value other than `0` is considered\ninvalid. Most ProtoJSON serializers will emit a Value with a `null_value` set\nas a JSON `null` regardless of the integer value, and so will round trip to\na `0` value.\n\n - NULL_VALUE: Null value."
     },
     "ObjectReference": {
       "type": "object",