diff --git a/.changeset/bedrock-embedding-calls.md b/.changeset/bedrock-embedding-calls.md new file mode 100644 index 000000000..20c75b53d --- /dev/null +++ b/.changeset/bedrock-embedding-calls.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat(bedrock): Capture embedding calls diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3-latest.cassette.json b/e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3-latest.cassette.json index c4cdf7a5c..1af1bd500 100644 --- a/e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3-latest.cassette.json +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3-latest.cassette.json @@ -4,7 +4,7 @@ "callIndex": 0, "id": "167baa531c51f0bf", "matchKey": "POST bedrock-runtime.aws-region.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse", - "recordedAt": "2026-09-03T04:29:52.164Z", + "recordedAt": "2026-09-11T12:57:57.001Z", "request": { "body": { "kind": "json", @@ -43,7 +43,7 @@ "kind": "json", "value": { "metrics": { - "latencyMs": 770 + "latencyMs": 551 }, "output": { "message": { @@ -57,10 +57,10 @@ }, "stopReason": "end_turn", "usage": { - "cacheReadInputTokenCount": 0, - "cacheReadInputTokens": 0, - "cacheWriteInputTokenCount": 14921, - "cacheWriteInputTokens": 14921, + "cacheReadInputTokenCount": 14921, + "cacheReadInputTokens": 14921, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, "inputTokens": 6, "outputTokens": 2, "serverToolUsage": {}, @@ -83,7 +83,7 @@ "callIndex": 0, "id": "5f6e11e6074e64db", "matchKey": "POST bedrock-runtime.aws-region.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse-stream", - "recordedAt": "2026-09-03T04:29:52.707Z", + "recordedAt": "2026-09-11T12:57:57.586Z", "request": { "body": { "kind": "json", @@ -121,7 +121,7 @@ "body": { "contentType": "application/vnd.amazon.eventstream", "kind": "base64", - "value": "AAAAmgAAAFJ78dAACzpldmVudC10eXBlBwAMbWVzc2FnZVN0YXJ0DTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDIiwicm9sZSI6ImFzc2lzdGFudCJ92EeA9QAAALMAAABXx0pp+gs6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiIifSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6In3rkVCrAAAA2wAAAFduCFV2CzpldmVudC10eXBlBwARY29udGVudEJsb2NrRGVsdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IlNUUkVBTSJ9LCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0NTY3In1cbHI9AAAAnAAAAFeE29EvCzpldmVudC10eXBlBwARY29udGVudEJsb2NrRGVsdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IiJ9LCJwIjoiYWJjIn0CsgalAAAArAAAAFZS/Vk/CzpldmVudC10eXBlBwAQY29udGVudEJsb2NrU3RvcA06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTiJ9MZDPXQAAAIwAAABRDVjjmAs6ZXZlbnQtdHlwZQcAC21lc3NhZ2VTdG9wDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprIiwic3RvcFJlYXNvbiI6ImVuZF90dXJuIn2GB4QBAAABVgAAAE6ISVowCzpldmVudC10eXBlBwAIbWV0YWRhdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJtZXRyaWNzIjp7ImxhdGVuY3lNcyI6NTA3fSwicCI6ImFiY2RlZmdoaWoiLCJ1c2FnZSI6eyJjYWNoZVJlYWRJbnB1dFRva2VuQ291bnQiOjE0OTIyLCJjYWNoZVJlYWRJbnB1dFRva2VucyI6MTQ5MjIsImNhY2hlV3JpdGVJbnB1dFRva2VuQ291bnQiOjAsImNhY2hlV3JpdGVJbnB1dFRva2VucyI6MCwiaW5wdXRUb2tlbnMiOjYsIm91dHB1dFRva2VucyI6Mywic2VydmVyVG9vbFVzYWdlIjp7fSwidG90YWxUb2tlbnMiOjE0OTMxfX3f3E6/" + "value": "AAAAmgAAAFJ78dAACzpldmVudC10eXBlBwAMbWVzc2FnZVN0YXJ0DTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDIiwicm9sZSI6ImFzc2lzdGFudCJ92EeA9QAAAMIAAABXA/ighQs6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiIifSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PIn2vzwFiAAAAvwAAAFcCuoT7CzpldmVudC10eXBlBwARY29udGVudEJsb2NrRGVsdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IlNUUkVBTSJ9LCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUYifSb/SVUAAADWAAAAV5aYkccLOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiIn0sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEyMzQ1Njc4In2oFQFAAAAAuAAAAFbHnWh9CzpldmVudC10eXBlBwAQY29udGVudEJsb2NrU3RvcA06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWiJ9i57JpAAAAIMAAABRjwh0SQs6ZXZlbnQtdHlwZQcAC21lc3NhZ2VTdG9wDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiIiwic3RvcFJlYXNvbiI6ImVuZF90dXJuIn2QVumkAAABUgAAAE59yfzwCzpldmVudC10eXBlBwAIbWV0YWRhdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJtZXRyaWNzIjp7ImxhdGVuY3lNcyI6NTQ1fSwicCI6ImFiY2RlZiIsInVzYWdlIjp7ImNhY2hlUmVhZElucHV0VG9rZW5Db3VudCI6MTQ5MjIsImNhY2hlUmVhZElucHV0VG9rZW5zIjoxNDkyMiwiY2FjaGVXcml0ZUlucHV0VG9rZW5Db3VudCI6MCwiY2FjaGVXcml0ZUlucHV0VG9rZW5zIjowLCJpbnB1dFRva2VucyI6Niwib3V0cHV0VG9rZW5zIjozLCJzZXJ2ZXJUb29sVXNhZ2UiOnt9LCJ0b3RhbFRva2VucyI6MTQ5MzF9fegnRzQ=" }, "headers": { "connection": "keep-alive", @@ -138,7 +138,7 @@ "callIndex": 0, "id": "3e9c682f68d2d756", "matchKey": "POST bedrock-runtime.aws-region.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/invoke", - "recordedAt": "2026-09-03T04:29:53.308Z", + "recordedAt": "2026-09-11T12:57:58.196Z", "request": { "body": { "kind": "json", @@ -203,7 +203,7 @@ "x-amzn-bedrock-cache-read-input-token-count": "14921", "x-amzn-bedrock-cache-write-input-token-count": "0", "x-amzn-bedrock-input-token-count": "6", - "x-amzn-bedrock-invocation-latency": "560", + "x-amzn-bedrock-invocation-latency": "567", "x-amzn-bedrock-output-token-count": "2", "x-amzn-requestid": "[REDACTED]" }, @@ -211,11 +211,252 @@ "statusText": "OK" } }, + { + "callIndex": 0, + "id": "0d28290b31e60524", + "matchKey": "POST bedrock-runtime.aws-region.amazonaws.com/model/amazon.titan-embed-text-v2%3A0/invoke", + "recordedAt": "2026-09-11T12:57:58.473Z", + "request": { + "body": { + "kind": "json", + "value": { + "dimensions": 256, + "inputText": "Embed this short Braintrust test sentence.", + "normalize": true + } + }, + "headers": {}, + "method": "POST", + "url": "https://bedrock-runtime.us-east-2.amazonaws.com/model/amazon.titan-embed-text-v2%3A0/invoke" + }, + "response": { + "body": { + "kind": "json", + "value": { + "embedding": [ + -0.12292258441448212, -0.07959577441215515, 0.08431042730808258, + -0.10088402777910233, 0.05975078046321869, 0.04403328150510788, + 0.024066895246505737, -0.04080509766936302, 0.02489643730223179, + -0.0795687884092331, 0.02351302281022072, 0.07398539036512375, + 0.04453573003411293, 0.08846784383058548, -0.10815097391605377, + 0.05521947890520096, -0.16448578238487244, 0.004347097594290972, + 0.07222197949886322, 0.0463339164853096, -0.07095752656459808, + 0.059447288513183594, 0.09525425732135773, 0.022096727043390274, + -0.017248446121811867, 0.048663217574357986, 0.023638635873794556, + -0.1004452258348465, 0.0075063626281917095, + -0.0059416936710476875, 0.027133848518133163, 0.06913647055625916, + -0.03559283912181854, -0.02739856019616127, 0.038251765072345734, + 0.06140438839793205, 0.1045832559466362, 0.014584042131900787, + -0.06437397003173828, -0.047034475952386856, 0.11060492694377899, + -0.038175467401742935, -0.0009408244513906538, 0.0793125107884407, + 0.007091590669006109, 0.018523111939430237, -0.06668557226657867, + 0.006105242297053337, 0.13666148483753204, -0.05786997452378273, + 0.014439462684094906, -0.044528983533382416, 0.024444574490189552, + -0.005831677932292223, 0.017307458445429802, 0.04847774654626846, + -0.02452550269663334, 0.06572577357292175, 0.0029202650766819715, + 0.007529967464506626, -0.1039273738861084, -0.03952137008309364, + -0.004398839082568884, 0.07740472257137299, 0.04621167480945587, + -0.17100746929645538, -0.04082975909113884, -0.0588807687163353, + -0.04599248990416527, -0.020512986928224564, -0.09188719093799591, + 0.00426911748945713, -0.000692129775416106, 0.004748803097754717, + 0.13025866448879242, 0.01573941856622696, 0.17338144779205322, + 0.003193407552316785, -0.029229626059532166, 0.024968940764665604, + -0.04834539070725441, -0.15418729186058044, -0.03690817579627037, + -0.06842221319675446, -0.10160229355096817, -0.09653398394584656, + 0.03875800594687462, 0.05118556693196297, 0.07510367035865784, + 0.03143162652850151, 0.02311595529317856, -0.015048659406602383, + 0.01710296981036663, -0.11315520852804184, 0.018481381237506866, + -0.0338292121887207, -0.058479487895965576, -0.036149658262729645, + -0.09464769810438156, 0.12142283469438553, 0.029261663556098938, + -0.004921624902635813, -0.04159269854426384, 0.08198323100805283, + 0.03196780011057854, 0.07600276172161102, -0.019774172455072403, + 0.02676628716289997, 0.007893314585089684, -0.08676154911518097, + 0.0862523540854454, 0.07928552478551865, -0.00991069059818983, + -0.14990806579589844, -0.023241829127073288, + -0.041857410222291946, 0.0588386170566082, -0.0015747849829494953, + -0.0013943759258836508, 0.057771336287260056, + -0.030339965596795082, -0.11415335536003113, 0.10198081284761429, + 0.0721922516822815, -0.05151677504181862, -0.023935591802001, + 0.08501140773296356, -0.19109611213207245, 0.018602358177304268, + 0.008469948545098305, 0.08628860116004944, -0.04712215065956116, + -0.12366276979446411, -0.056274112313985825, + -0.006444351747632027, -0.09390708804130554, 0.010054955258965492, + -0.025729354470968246, 0.032284777611494064, 0.020303597673773766, + 0.10665374249219894, 0.03838158771395683, 0.006953333038836718, + -0.0051728482358157635, 0.001733275013975799, + -0.09658793359994888, 0.10766538977622986, -0.022696124389767647, + -0.05594448745250702, 0.046537090092897415, -0.018275262787938118, + -0.018421949818730354, 0.08769562840461731, 0.052673522382974625, + -0.08586961776018143, -0.0235922671854496, -0.0772588774561882, + -0.014924732968211174, -0.05366745963692665, -0.05541253462433815, + 0.045496780425310135, 0.002971057780086994, -0.03147293999791145, + 0.002948084846138954, -0.021311864256858826, + -0.039724960923194885, 0.010902097448706627, 0.06243161857128143, + -0.026628030464053154, 0.11030576378107071, -0.1452612727880478, + 0.005462851375341415, 0.02862938679754734, -0.06087370216846466, + 0.010403022170066833, 0.10341817885637283, -0.033940497785806656, + 0.020730171352624893, 0.028514735400676727, 0.008983355015516281, + -0.10639071464538574, -0.00584490317851305, 0.0036663489881902933, + 0.020740289241075516, -0.044970735907554626, 0.04415973275899887, + 0.028332114219665527, 0.0067518483847379684, + -0.054336823523044586, 0.03544783592224121, 0.0672975555062294, + 0.01806703209877014, 0.04297527298331261, 0.05716772750020027, + -0.09931936860084534, 0.034363698214292526, 0.09905970841646194, + 0.02531627006828785, 0.0007899216143414378, -0.034105729311704636, + 0.11593195050954819, -0.04631452634930611, -0.015666916966438293, + 0.07577851414680481, 0.05583911016583443, -0.07117555290460587, + -0.01634724624454975, -0.02705981954932213, 0.10862980782985687, + -0.04369606822729111, -0.05634155496954918, -0.03384438529610634, + -0.02436027117073536, -0.01253589428961277, + -0.0010310289217159152, -0.03024463914334774, + -0.01715908572077751, 0.0645122304558754, 0.030614729970693588, + 0.08481582999229431, -0.0016961815999820828, 0.024990860372781754, + 0.03251070901751518, 0.08395930379629135, 0.10277410596609116, + -0.0586211159825325, 0.021863209083676338, -0.006060561165213585, + 0.08362884074449539, -0.0015376914525404572, 0.05520851910114288, + 0.007351244799792767, 0.009044053964316845, 0.05301115661859512, + -0.14014236629009247, -0.07494518160820007, -0.03454863652586937, + 0.005095289088785648, -0.011873271316289902, -0.06659957021474838, + -0.003867833875119686, -0.04873403161764145, -0.03669552505016327, + -0.10232392698526382, -0.027862651273608208, + -0.023979220539331436, -0.03040650114417076, 0.003351897466927767, + -0.08534693717956543, -0.044603168964385986, + -0.009662418626248837, -0.042438264936208725, 0.0830184817314148, + 0.013444683514535427, 0.06837331503629684, 0.0042699603363871574 + ], + "embeddingsByType": { + "float": [ + -0.12292258441448212, -0.07959577441215515, 0.08431042730808258, + -0.10088402777910233, 0.05975078046321869, 0.04403328150510788, + 0.024066895246505737, -0.04080509766936302, 0.02489643730223179, + -0.0795687884092331, 0.02351302281022072, 0.07398539036512375, + 0.04453573003411293, 0.08846784383058548, -0.10815097391605377, + 0.05521947890520096, -0.16448578238487244, 0.004347097594290972, + 0.07222197949886322, 0.0463339164853096, -0.07095752656459808, + 0.059447288513183594, 0.09525425732135773, 0.022096727043390274, + -0.017248446121811867, 0.048663217574357986, + 0.023638635873794556, -0.1004452258348465, + 0.0075063626281917095, -0.0059416936710476875, + 0.027133848518133163, 0.06913647055625916, -0.03559283912181854, + -0.02739856019616127, 0.038251765072345734, 0.06140438839793205, + 0.1045832559466362, 0.014584042131900787, -0.06437397003173828, + -0.047034475952386856, 0.11060492694377899, + -0.038175467401742935, -0.0009408244513906538, + 0.0793125107884407, 0.007091590669006109, 0.018523111939430237, + -0.06668557226657867, 0.006105242297053337, 0.13666148483753204, + -0.05786997452378273, 0.014439462684094906, + -0.044528983533382416, 0.024444574490189552, + -0.005831677932292223, 0.017307458445429802, + 0.04847774654626846, -0.02452550269663334, 0.06572577357292175, + 0.0029202650766819715, 0.007529967464506626, + -0.1039273738861084, -0.03952137008309364, + -0.004398839082568884, 0.07740472257137299, 0.04621167480945587, + -0.17100746929645538, -0.04082975909113884, -0.0588807687163353, + -0.04599248990416527, -0.020512986928224564, + -0.09188719093799591, 0.00426911748945713, + -0.000692129775416106, 0.004748803097754717, + 0.13025866448879242, 0.01573941856622696, 0.17338144779205322, + 0.003193407552316785, -0.029229626059532166, + 0.024968940764665604, -0.04834539070725441, + -0.15418729186058044, -0.03690817579627037, + -0.06842221319675446, -0.10160229355096817, + -0.09653398394584656, 0.03875800594687462, 0.05118556693196297, + 0.07510367035865784, 0.03143162652850151, 0.02311595529317856, + -0.015048659406602383, 0.01710296981036663, + -0.11315520852804184, 0.018481381237506866, -0.0338292121887207, + -0.058479487895965576, -0.036149658262729645, + -0.09464769810438156, 0.12142283469438553, 0.029261663556098938, + -0.004921624902635813, -0.04159269854426384, + 0.08198323100805283, 0.03196780011057854, 0.07600276172161102, + -0.019774172455072403, 0.02676628716289997, + 0.007893314585089684, -0.08676154911518097, 0.0862523540854454, + 0.07928552478551865, -0.00991069059818983, -0.14990806579589844, + -0.023241829127073288, -0.041857410222291946, + 0.0588386170566082, -0.0015747849829494953, + -0.0013943759258836508, 0.057771336287260056, + -0.030339965596795082, -0.11415335536003113, + 0.10198081284761429, 0.0721922516822815, -0.05151677504181862, + -0.023935591802001, 0.08501140773296356, -0.19109611213207245, + 0.018602358177304268, 0.008469948545098305, 0.08628860116004944, + -0.04712215065956116, -0.12366276979446411, + -0.056274112313985825, -0.006444351747632027, + -0.09390708804130554, 0.010054955258965492, + -0.025729354470968246, 0.032284777611494064, + 0.020303597673773766, 0.10665374249219894, 0.03838158771395683, + 0.006953333038836718, -0.0051728482358157635, + 0.001733275013975799, -0.09658793359994888, 0.10766538977622986, + -0.022696124389767647, -0.05594448745250702, + 0.046537090092897415, -0.018275262787938118, + -0.018421949818730354, 0.08769562840461731, + 0.052673522382974625, -0.08586961776018143, -0.0235922671854496, + -0.0772588774561882, -0.014924732968211174, + -0.05366745963692665, -0.05541253462433815, + 0.045496780425310135, 0.002971057780086994, + -0.03147293999791145, 0.002948084846138954, + -0.021311864256858826, -0.039724960923194885, + 0.010902097448706627, 0.06243161857128143, + -0.026628030464053154, 0.11030576378107071, -0.1452612727880478, + 0.005462851375341415, 0.02862938679754734, -0.06087370216846466, + 0.010403022170066833, 0.10341817885637283, + -0.033940497785806656, 0.020730171352624893, + 0.028514735400676727, 0.008983355015516281, + -0.10639071464538574, -0.00584490317851305, + 0.0036663489881902933, 0.020740289241075516, + -0.044970735907554626, 0.04415973275899887, + 0.028332114219665527, 0.0067518483847379684, + -0.054336823523044586, 0.03544783592224121, 0.0672975555062294, + 0.01806703209877014, 0.04297527298331261, 0.05716772750020027, + -0.09931936860084534, 0.034363698214292526, 0.09905970841646194, + 0.02531627006828785, 0.0007899216143414378, + -0.034105729311704636, 0.11593195050954819, + -0.04631452634930611, -0.015666916966438293, + 0.07577851414680481, 0.05583911016583443, -0.07117555290460587, + -0.01634724624454975, -0.02705981954932213, 0.10862980782985687, + -0.04369606822729111, -0.05634155496954918, + -0.03384438529610634, -0.02436027117073536, + -0.01253589428961277, -0.0010310289217159152, + -0.03024463914334774, -0.01715908572077751, 0.0645122304558754, + 0.030614729970693588, 0.08481582999229431, + -0.0016961815999820828, 0.024990860372781754, + 0.03251070901751518, 0.08395930379629135, 0.10277410596609116, + -0.0586211159825325, 0.021863209083676338, + -0.006060561165213585, 0.08362884074449539, + -0.0015376914525404572, 0.05520851910114288, + 0.007351244799792767, 0.009044053964316845, 0.05301115661859512, + -0.14014236629009247, -0.07494518160820007, + -0.03454863652586937, 0.005095289088785648, + -0.011873271316289902, -0.06659957021474838, + -0.003867833875119686, -0.04873403161764145, + -0.03669552505016327, -0.10232392698526382, + -0.027862651273608208, -0.023979220539331436, + -0.03040650114417076, 0.003351897466927767, + -0.08534693717956543, -0.044603168964385986, + -0.009662418626248837, -0.042438264936208725, + 0.0830184817314148, 0.013444683514535427, 0.06837331503629684, + 0.0042699603363871574 + ] + }, + "inputTextTokenCount": 9 + } + }, + "headers": { + "connection": "keep-alive", + "content-length": "10752", + "content-type": "application/json", + "date": "[REDACTED]", + "x-amzn-bedrock-input-token-count": "9", + "x-amzn-bedrock-invocation-latency": "252", + "x-amzn-requestid": "[REDACTED]" + }, + "status": 200, + "statusText": "OK" + } + }, { "callIndex": 0, "id": "6b97f66ee2a0ebd7", "matchKey": "POST bedrock-runtime.aws-region.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/invoke-with-response-stream", - "recordedAt": "2026-09-03T04:29:53.975Z", + "recordedAt": "2026-09-11T12:57:59.231Z", "request": { "body": { "kind": "json", @@ -252,7 +493,7 @@ "body": { "contentType": "application/vnd.amazon.eventstream", "kind": "base64", - "value": "AAAApwAAAEtGKwT3CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SnRaWE56WVdkbFUzUmhjblFpT25zaWNtOXNaU0k2SW1GemMybHpkR0Z1ZENKOWZRPT0iLCJwIjoiYWJjZGUifZ0S/N0AAAEAAAAAS09wlNQLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydEVaV3gwWVNJNmV5SmtaV3gwWVNJNmV5SjBaWGgwSWpvaVVrRlhJbjBzSW1OdmJuUmxiblJDYkc5amEwbHVaR1Y0SWpvd2ZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxIn3eemuTAAAA4gAAAEvWONPOCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRUZEc5d0lqcDdJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam93ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEyMyJ9cDMuGwAAAPoAAABLhqgPjQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0RVpXeDBZU0k2ZXlKa1pXeDBZU0k2ZXlKMFpYaDBJam9pVTFSU1JVRk5JbjBzSW1OdmJuUmxiblJDYkc5amEwbHVaR1Y0SWpveGZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVIifZZg/3kAAADFAAAAS6XZINoLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydFRkRzl3SWpwN0ltTnZiblJsYm5SQ2JHOWphMGx1WkdWNElqb3hmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QSJ9mhWMGwAAANoAAABLR2kgiQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0RVpXeDBZU0k2ZXlKa1pXeDBZU0k2ZXlKMFpYaDBJam9pSW4wc0ltTnZiblJsYm5SQ2JHOWphMGx1WkdWNElqb3lmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0In11bwjBAAAA4QAAAEuRmKkeCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRUZEc5d0lqcDdJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam95ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEyIn2C1bquAAAAqAAAAEvEe5MmCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SnRaWE56WVdkbFUzUnZjQ0k2ZXlKemRHOXdVbVZoYzI5dUlqb2laVzVrWDNSMWNtNGlmWDA9IiwicCI6ImFiIn0qaxypAAACQAAAAEuRF74zCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SnRaWFJoWkdGMFlTSTZleUoxYzJGblpTSTZleUpwYm5CMWRGUnZhMlZ1Y3lJNk5pd2liM1YwY0hWMFZHOXJaVzV6SWpvekxDSmpZV05vWlZKbFlXUkpibkIxZEZSdmEyVnVRMjkxYm5RaU9qRTBPVEl5TENKallXTm9aVmR5YVhSbFNXNXdkWFJVYjJ0bGJrTnZkVzUwSWpvd2ZTd2liV1YwY21samN5STZlMzBzSW5SeVlXTmxJanA3Zlgwc0ltRnRZWHB2YmkxaVpXUnliMk5yTFdsdWRtOWpZWFJwYjI1TlpYUnlhV056SWpwN0ltbHVjSFYwVkc5clpXNURiM1Z1ZENJNk5pd2liM1YwY0hWMFZHOXJaVzVEYjNWdWRDSTZNeXdpYVc1MmIyTmhkR2x2Ymt4aGRHVnVZM2tpT2pZek1pd2labWx5YzNSQ2VYUmxUR0YwWlc1amVTSTZOVGMzTENKallXTm9aVkpsWVdSSmJuQjFkRlJ2YTJWdVEyOTFiblFpT2pFME9USXlMQ0pqWVdOb1pWZHlhWFJsU1c1d2RYUlViMnRsYmtOdmRXNTBJam93ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2In0bVo2U" + "value": "AAAAvgAAAEsr2/EECzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SnRaWE56WVdkbFUzUmhjblFpT25zaWNtOXNaU0k2SW1GemMybHpkR0Z1ZENKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQiJ9NOrvXwAAAQIAAABLNbDHtAs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0RVpXeDBZU0k2ZXlKa1pXeDBZU0k2ZXlKMFpYaDBJam9pVWtGWEluMHNJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam93ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEyMyJ9Q9XCZwAAANAAAABLDdk4KAs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0VGRHOXdJanA3SW1OdmJuUmxiblJDYkc5amEwbHVaR1Y0SWpvd2ZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0wifWn+3P4AAADdAAAAS/VJ/JkLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydEVaV3gwWVNJNmV5SmtaV3gwWVNJNmV5SjBaWGgwSWpvaVUxUlNSVUZOSW4wc0ltTnZiblJsYm5SQ2JHOWphMGx1WkdWNElqb3hmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ubyJ9421YdgAAALEAAABLqYtm1Qs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0VGRHOXdJanA3SW1OdmJuUmxiblJDYkc5amEwbHVaR1Y0SWpveGZYMD0iLCJwIjoiYWJjZGVmZyJ9z/gnGgAAAOQAAABLWXgmbgs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0RVpXeDBZU0k2ZXlKa1pXeDBZU0k2ZXlKMFpYaDBJam9pSW4wc0ltTnZiblJsYm5SQ2JHOWphMGx1WkdWNElqb3lmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDRCJ9PCuwJwAAANMAAABLSnlC+As6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0VGRHOXdJanA3SW1OdmJuUmxiblJDYkc5amEwbHVaR1Y0SWpveWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk8ifXBEvIMAAADTAAAAS0p5QvgLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKdFpYTnpZV2RsVTNSdmNDSTZleUp6ZEc5d1VtVmhjMjl1SWpvaVpXNWtYM1IxY200aWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTIn0h7xHBAAACYAAAAEtQ1pE3CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SnRaWFJoWkdGMFlTSTZleUoxYzJGblpTSTZleUpwYm5CMWRGUnZhMlZ1Y3lJNk5pd2liM1YwY0hWMFZHOXJaVzV6SWpvekxDSmpZV05vWlZKbFlXUkpibkIxZEZSdmEyVnVRMjkxYm5RaU9qQXNJbU5oWTJobFYzSnBkR1ZKYm5CMWRGUnZhMlZ1UTI5MWJuUWlPakUwT1RJeWZTd2liV1YwY21samN5STZlMzBzSW5SeVlXTmxJanA3Zlgwc0ltRnRZWHB2YmkxaVpXUnliMk5yTFdsdWRtOWpZWFJwYjI1TlpYUnlhV056SWpwN0ltbHVjSFYwVkc5clpXNURiM1Z1ZENJNk5pd2liM1YwY0hWMFZHOXJaVzVEYjNWdWRDSTZNeXdpYVc1MmIyTmhkR2x2Ymt4aGRHVnVZM2tpT2pjeU15d2labWx5YzNSQ2VYUmxUR0YwWlc1amVTSTZOalUzTENKallXTm9aVkpsWVdSSmJuQjFkRlJ2YTJWdVEyOTFiblFpT2pBc0ltTmhZMmhsVjNKcGRHVkpibkIxZEZSdmEyVnVRMjkxYm5RaU9qRTBPVEl5ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEifW1I4EI=" }, "headers": { "connection": "keep-alive", diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3.cassette.json b/e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3.cassette.json index 0b53b4af4..338b36fe4 100644 --- a/e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3.cassette.json +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__cassettes__/bedrock-runtime-v3.cassette.json @@ -4,7 +4,7 @@ "callIndex": 0, "id": "167baa531c51f0bf", "matchKey": "POST bedrock-runtime.aws-region.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse", - "recordedAt": "2026-09-03T04:29:45.476Z", + "recordedAt": "2026-09-11T12:57:49.978Z", "request": { "body": { "kind": "json", @@ -43,7 +43,7 @@ "kind": "json", "value": { "metrics": { - "latencyMs": 504 + "latencyMs": 510 }, "output": { "message": { @@ -83,7 +83,7 @@ "callIndex": 0, "id": "5f6e11e6074e64db", "matchKey": "POST bedrock-runtime.aws-region.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse-stream", - "recordedAt": "2026-09-03T04:29:46.080Z", + "recordedAt": "2026-09-11T12:57:50.653Z", "request": { "body": { "kind": "json", @@ -121,7 +121,7 @@ "body": { "contentType": "application/vnd.amazon.eventstream", "kind": "base64", - "value": "AAAAugAAAFK6MP8ECzpldmVudC10eXBlBwAMbWVzc2FnZVN0YXJ0DTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIzNDU2NzgiLCJyb2xlIjoiYXNzaXN0YW50In00xKBOAAAAoQAAAFfdaq0YCzpldmVudC10eXBlBwARY29udGVudEJsb2NrRGVsdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IiJ9LCJwIjoiYWJjZGVmZ2gifeuV2vEAAACrAAAAV5fatbkLOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiU1RSRUFNIn0sInAiOiJhYmNkZWZnaGlqa2wifdlQ8xMAAADNAAAAV4GoN1QLOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiIn0sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaIn2Hkb5OAAAAmAAAAFYGXEd5CzpldmVudC10eXBlBwAQY29udGVudEJsb2NrU3RvcA06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3QifRIDkV4AAAClAAAAUcGJru0LOmV2ZW50LXR5cGUHAAttZXNzYWdlU3RvcA06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7InAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUoiLCJzdG9wUmVhc29uIjoiZW5kX3R1cm4ifUn7n6cAAAFoAAAATpZYXNcLOmV2ZW50LXR5cGUHAAhtZXRhZGF0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7Im1ldHJpY3MiOnsibGF0ZW5jeU1zIjo1Njd9LCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQiIsInVzYWdlIjp7ImNhY2hlUmVhZElucHV0VG9rZW5Db3VudCI6MTQ5MjIsImNhY2hlUmVhZElucHV0VG9rZW5zIjoxNDkyMiwiY2FjaGVXcml0ZUlucHV0VG9rZW5Db3VudCI6MCwiY2FjaGVXcml0ZUlucHV0VG9rZW5zIjowLCJpbnB1dFRva2VucyI6Niwib3V0cHV0VG9rZW5zIjozLCJzZXJ2ZXJUb29sVXNhZ2UiOnt9LCJ0b3RhbFRva2VucyI6MTQ5MzF9fdR5bz4=" + "value": "AAAAlwAAAFKDYRSxCzpldmVudC10eXBlBwAMbWVzc2FnZVN0YXJ0DTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6Iiwicm9sZSI6ImFzc2lzdGFudCJ9G/AFrwAAAKIAAABXmsrXyAs6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiIifSwicCI6ImFiY2RlZmdoaSJ9NTLrNAAAANgAAABXKagvpgs6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiJTVFJFQU0ifSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIzNCJ9FcansAAAAL8AAABXArqE+ws6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiIifSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMIn3CiVpEAAAAigAAAFYcfIObCzpldmVudC10eXBlBwAQY29udGVudEJsb2NrU3RvcA06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJwIjoiYWJjZGVmIn1263kMAAAAhQAAAFEASIHpCzpldmVudC10eXBlBwALbWVzc2FnZVN0b3ANOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJwIjoiYWJjZCIsInN0b3BSZWFzb24iOiJlbmRfdHVybiJ9HPrdBQAAAU4AAABO2NmGcws6ZXZlbnQtdHlwZQcACG1ldGFkYXRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsibWV0cmljcyI6eyJsYXRlbmN5TXMiOjYzN30sInAiOiJhYiIsInVzYWdlIjp7ImNhY2hlUmVhZElucHV0VG9rZW5Db3VudCI6MTQ5MjIsImNhY2hlUmVhZElucHV0VG9rZW5zIjoxNDkyMiwiY2FjaGVXcml0ZUlucHV0VG9rZW5Db3VudCI6MCwiY2FjaGVXcml0ZUlucHV0VG9rZW5zIjowLCJpbnB1dFRva2VucyI6Niwib3V0cHV0VG9rZW5zIjozLCJzZXJ2ZXJUb29sVXNhZ2UiOnt9LCJ0b3RhbFRva2VucyI6MTQ5MzF9ffEXrP0=" }, "headers": { "connection": "keep-alive", @@ -138,7 +138,7 @@ "callIndex": 0, "id": "3e9c682f68d2d756", "matchKey": "POST bedrock-runtime.aws-region.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/invoke", - "recordedAt": "2026-09-03T04:29:46.985Z", + "recordedAt": "2026-09-11T12:57:51.208Z", "request": { "body": { "kind": "json", @@ -203,7 +203,7 @@ "x-amzn-bedrock-cache-read-input-token-count": "14921", "x-amzn-bedrock-cache-write-input-token-count": "0", "x-amzn-bedrock-input-token-count": "6", - "x-amzn-bedrock-invocation-latency": "866", + "x-amzn-bedrock-invocation-latency": "514", "x-amzn-bedrock-output-token-count": "2", "x-amzn-requestid": "[REDACTED]" }, @@ -211,11 +211,252 @@ "statusText": "OK" } }, + { + "callIndex": 0, + "id": "0d28290b31e60524", + "matchKey": "POST bedrock-runtime.aws-region.amazonaws.com/model/amazon.titan-embed-text-v2%3A0/invoke", + "recordedAt": "2026-09-11T12:57:51.491Z", + "request": { + "body": { + "kind": "json", + "value": { + "dimensions": 256, + "inputText": "Embed this short Braintrust test sentence.", + "normalize": true + } + }, + "headers": {}, + "method": "POST", + "url": "https://bedrock-runtime.us-east-2.amazonaws.com/model/amazon.titan-embed-text-v2%3A0/invoke" + }, + "response": { + "body": { + "kind": "json", + "value": { + "embedding": [ + -0.12292258441448212, -0.07959577441215515, 0.08431042730808258, + -0.10088402777910233, 0.05975078046321869, 0.04403328150510788, + 0.024066895246505737, -0.04080509766936302, 0.02489643730223179, + -0.0795687884092331, 0.02351302281022072, 0.07398539036512375, + 0.04453573003411293, 0.08846784383058548, -0.10815097391605377, + 0.05521947890520096, -0.16448578238487244, 0.004347097594290972, + 0.07222197949886322, 0.0463339164853096, -0.07095752656459808, + 0.059447288513183594, 0.09525425732135773, 0.022096727043390274, + -0.017248446121811867, 0.048663217574357986, 0.023638635873794556, + -0.1004452258348465, 0.0075063626281917095, + -0.0059416936710476875, 0.027133848518133163, 0.06913647055625916, + -0.03559283912181854, -0.02739856019616127, 0.038251765072345734, + 0.06140438839793205, 0.1045832559466362, 0.014584042131900787, + -0.06437397003173828, -0.047034475952386856, 0.11060492694377899, + -0.038175467401742935, -0.0009408244513906538, 0.0793125107884407, + 0.007091590669006109, 0.018523111939430237, -0.06668557226657867, + 0.006105242297053337, 0.13666148483753204, -0.05786997452378273, + 0.014439462684094906, -0.044528983533382416, 0.024444574490189552, + -0.005831677932292223, 0.017307458445429802, 0.04847774654626846, + -0.02452550269663334, 0.06572577357292175, 0.0029202650766819715, + 0.007529967464506626, -0.1039273738861084, -0.03952137008309364, + -0.004398839082568884, 0.07740472257137299, 0.04621167480945587, + -0.17100746929645538, -0.04082975909113884, -0.0588807687163353, + -0.04599248990416527, -0.020512986928224564, -0.09188719093799591, + 0.00426911748945713, -0.000692129775416106, 0.004748803097754717, + 0.13025866448879242, 0.01573941856622696, 0.17338144779205322, + 0.003193407552316785, -0.029229626059532166, 0.024968940764665604, + -0.04834539070725441, -0.15418729186058044, -0.03690817579627037, + -0.06842221319675446, -0.10160229355096817, -0.09653398394584656, + 0.03875800594687462, 0.05118556693196297, 0.07510367035865784, + 0.03143162652850151, 0.02311595529317856, -0.015048659406602383, + 0.01710296981036663, -0.11315520852804184, 0.018481381237506866, + -0.0338292121887207, -0.058479487895965576, -0.036149658262729645, + -0.09464769810438156, 0.12142283469438553, 0.029261663556098938, + -0.004921624902635813, -0.04159269854426384, 0.08198323100805283, + 0.03196780011057854, 0.07600276172161102, -0.019774172455072403, + 0.02676628716289997, 0.007893314585089684, -0.08676154911518097, + 0.0862523540854454, 0.07928552478551865, -0.00991069059818983, + -0.14990806579589844, -0.023241829127073288, + -0.041857410222291946, 0.0588386170566082, -0.0015747849829494953, + -0.0013943759258836508, 0.057771336287260056, + -0.030339965596795082, -0.11415335536003113, 0.10198081284761429, + 0.0721922516822815, -0.05151677504181862, -0.023935591802001, + 0.08501140773296356, -0.19109611213207245, 0.018602358177304268, + 0.008469948545098305, 0.08628860116004944, -0.04712215065956116, + -0.12366276979446411, -0.056274112313985825, + -0.006444351747632027, -0.09390708804130554, 0.010054955258965492, + -0.025729354470968246, 0.032284777611494064, 0.020303597673773766, + 0.10665374249219894, 0.03838158771395683, 0.006953333038836718, + -0.0051728482358157635, 0.001733275013975799, + -0.09658793359994888, 0.10766538977622986, -0.022696124389767647, + -0.05594448745250702, 0.046537090092897415, -0.018275262787938118, + -0.018421949818730354, 0.08769562840461731, 0.052673522382974625, + -0.08586961776018143, -0.0235922671854496, -0.0772588774561882, + -0.014924732968211174, -0.05366745963692665, -0.05541253462433815, + 0.045496780425310135, 0.002971057780086994, -0.03147293999791145, + 0.002948084846138954, -0.021311864256858826, + -0.039724960923194885, 0.010902097448706627, 0.06243161857128143, + -0.026628030464053154, 0.11030576378107071, -0.1452612727880478, + 0.005462851375341415, 0.02862938679754734, -0.06087370216846466, + 0.010403022170066833, 0.10341817885637283, -0.033940497785806656, + 0.020730171352624893, 0.028514735400676727, 0.008983355015516281, + -0.10639071464538574, -0.00584490317851305, 0.0036663489881902933, + 0.020740289241075516, -0.044970735907554626, 0.04415973275899887, + 0.028332114219665527, 0.0067518483847379684, + -0.054336823523044586, 0.03544783592224121, 0.0672975555062294, + 0.01806703209877014, 0.04297527298331261, 0.05716772750020027, + -0.09931936860084534, 0.034363698214292526, 0.09905970841646194, + 0.02531627006828785, 0.0007899216143414378, -0.034105729311704636, + 0.11593195050954819, -0.04631452634930611, -0.015666916966438293, + 0.07577851414680481, 0.05583911016583443, -0.07117555290460587, + -0.01634724624454975, -0.02705981954932213, 0.10862980782985687, + -0.04369606822729111, -0.05634155496954918, -0.03384438529610634, + -0.02436027117073536, -0.01253589428961277, + -0.0010310289217159152, -0.03024463914334774, + -0.01715908572077751, 0.0645122304558754, 0.030614729970693588, + 0.08481582999229431, -0.0016961815999820828, 0.024990860372781754, + 0.03251070901751518, 0.08395930379629135, 0.10277410596609116, + -0.0586211159825325, 0.021863209083676338, -0.006060561165213585, + 0.08362884074449539, -0.0015376914525404572, 0.05520851910114288, + 0.007351244799792767, 0.009044053964316845, 0.05301115661859512, + -0.14014236629009247, -0.07494518160820007, -0.03454863652586937, + 0.005095289088785648, -0.011873271316289902, -0.06659957021474838, + -0.003867833875119686, -0.04873403161764145, -0.03669552505016327, + -0.10232392698526382, -0.027862651273608208, + -0.023979220539331436, -0.03040650114417076, 0.003351897466927767, + -0.08534693717956543, -0.044603168964385986, + -0.009662418626248837, -0.042438264936208725, 0.0830184817314148, + 0.013444683514535427, 0.06837331503629684, 0.0042699603363871574 + ], + "embeddingsByType": { + "float": [ + -0.12292258441448212, -0.07959577441215515, 0.08431042730808258, + -0.10088402777910233, 0.05975078046321869, 0.04403328150510788, + 0.024066895246505737, -0.04080509766936302, 0.02489643730223179, + -0.0795687884092331, 0.02351302281022072, 0.07398539036512375, + 0.04453573003411293, 0.08846784383058548, -0.10815097391605377, + 0.05521947890520096, -0.16448578238487244, 0.004347097594290972, + 0.07222197949886322, 0.0463339164853096, -0.07095752656459808, + 0.059447288513183594, 0.09525425732135773, 0.022096727043390274, + -0.017248446121811867, 0.048663217574357986, + 0.023638635873794556, -0.1004452258348465, + 0.0075063626281917095, -0.0059416936710476875, + 0.027133848518133163, 0.06913647055625916, -0.03559283912181854, + -0.02739856019616127, 0.038251765072345734, 0.06140438839793205, + 0.1045832559466362, 0.014584042131900787, -0.06437397003173828, + -0.047034475952386856, 0.11060492694377899, + -0.038175467401742935, -0.0009408244513906538, + 0.0793125107884407, 0.007091590669006109, 0.018523111939430237, + -0.06668557226657867, 0.006105242297053337, 0.13666148483753204, + -0.05786997452378273, 0.014439462684094906, + -0.044528983533382416, 0.024444574490189552, + -0.005831677932292223, 0.017307458445429802, + 0.04847774654626846, -0.02452550269663334, 0.06572577357292175, + 0.0029202650766819715, 0.007529967464506626, + -0.1039273738861084, -0.03952137008309364, + -0.004398839082568884, 0.07740472257137299, 0.04621167480945587, + -0.17100746929645538, -0.04082975909113884, -0.0588807687163353, + -0.04599248990416527, -0.020512986928224564, + -0.09188719093799591, 0.00426911748945713, + -0.000692129775416106, 0.004748803097754717, + 0.13025866448879242, 0.01573941856622696, 0.17338144779205322, + 0.003193407552316785, -0.029229626059532166, + 0.024968940764665604, -0.04834539070725441, + -0.15418729186058044, -0.03690817579627037, + -0.06842221319675446, -0.10160229355096817, + -0.09653398394584656, 0.03875800594687462, 0.05118556693196297, + 0.07510367035865784, 0.03143162652850151, 0.02311595529317856, + -0.015048659406602383, 0.01710296981036663, + -0.11315520852804184, 0.018481381237506866, -0.0338292121887207, + -0.058479487895965576, -0.036149658262729645, + -0.09464769810438156, 0.12142283469438553, 0.029261663556098938, + -0.004921624902635813, -0.04159269854426384, + 0.08198323100805283, 0.03196780011057854, 0.07600276172161102, + -0.019774172455072403, 0.02676628716289997, + 0.007893314585089684, -0.08676154911518097, 0.0862523540854454, + 0.07928552478551865, -0.00991069059818983, -0.14990806579589844, + -0.023241829127073288, -0.041857410222291946, + 0.0588386170566082, -0.0015747849829494953, + -0.0013943759258836508, 0.057771336287260056, + -0.030339965596795082, -0.11415335536003113, + 0.10198081284761429, 0.0721922516822815, -0.05151677504181862, + -0.023935591802001, 0.08501140773296356, -0.19109611213207245, + 0.018602358177304268, 0.008469948545098305, 0.08628860116004944, + -0.04712215065956116, -0.12366276979446411, + -0.056274112313985825, -0.006444351747632027, + -0.09390708804130554, 0.010054955258965492, + -0.025729354470968246, 0.032284777611494064, + 0.020303597673773766, 0.10665374249219894, 0.03838158771395683, + 0.006953333038836718, -0.0051728482358157635, + 0.001733275013975799, -0.09658793359994888, 0.10766538977622986, + -0.022696124389767647, -0.05594448745250702, + 0.046537090092897415, -0.018275262787938118, + -0.018421949818730354, 0.08769562840461731, + 0.052673522382974625, -0.08586961776018143, -0.0235922671854496, + -0.0772588774561882, -0.014924732968211174, + -0.05366745963692665, -0.05541253462433815, + 0.045496780425310135, 0.002971057780086994, + -0.03147293999791145, 0.002948084846138954, + -0.021311864256858826, -0.039724960923194885, + 0.010902097448706627, 0.06243161857128143, + -0.026628030464053154, 0.11030576378107071, -0.1452612727880478, + 0.005462851375341415, 0.02862938679754734, -0.06087370216846466, + 0.010403022170066833, 0.10341817885637283, + -0.033940497785806656, 0.020730171352624893, + 0.028514735400676727, 0.008983355015516281, + -0.10639071464538574, -0.00584490317851305, + 0.0036663489881902933, 0.020740289241075516, + -0.044970735907554626, 0.04415973275899887, + 0.028332114219665527, 0.0067518483847379684, + -0.054336823523044586, 0.03544783592224121, 0.0672975555062294, + 0.01806703209877014, 0.04297527298331261, 0.05716772750020027, + -0.09931936860084534, 0.034363698214292526, 0.09905970841646194, + 0.02531627006828785, 0.0007899216143414378, + -0.034105729311704636, 0.11593195050954819, + -0.04631452634930611, -0.015666916966438293, + 0.07577851414680481, 0.05583911016583443, -0.07117555290460587, + -0.01634724624454975, -0.02705981954932213, 0.10862980782985687, + -0.04369606822729111, -0.05634155496954918, + -0.03384438529610634, -0.02436027117073536, + -0.01253589428961277, -0.0010310289217159152, + -0.03024463914334774, -0.01715908572077751, 0.0645122304558754, + 0.030614729970693588, 0.08481582999229431, + -0.0016961815999820828, 0.024990860372781754, + 0.03251070901751518, 0.08395930379629135, 0.10277410596609116, + -0.0586211159825325, 0.021863209083676338, + -0.006060561165213585, 0.08362884074449539, + -0.0015376914525404572, 0.05520851910114288, + 0.007351244799792767, 0.009044053964316845, 0.05301115661859512, + -0.14014236629009247, -0.07494518160820007, + -0.03454863652586937, 0.005095289088785648, + -0.011873271316289902, -0.06659957021474838, + -0.003867833875119686, -0.04873403161764145, + -0.03669552505016327, -0.10232392698526382, + -0.027862651273608208, -0.023979220539331436, + -0.03040650114417076, 0.003351897466927767, + -0.08534693717956543, -0.044603168964385986, + -0.009662418626248837, -0.042438264936208725, + 0.0830184817314148, 0.013444683514535427, 0.06837331503629684, + 0.0042699603363871574 + ] + }, + "inputTextTokenCount": 9 + } + }, + "headers": { + "connection": "keep-alive", + "content-length": "10752", + "content-type": "application/json", + "date": "[REDACTED]", + "x-amzn-bedrock-input-token-count": "9", + "x-amzn-bedrock-invocation-latency": "257", + "x-amzn-requestid": "[REDACTED]" + }, + "status": 200, + "statusText": "OK" + } + }, { "callIndex": 0, "id": "6b97f66ee2a0ebd7", "matchKey": "POST bedrock-runtime.aws-region.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/invoke-with-response-stream", - "recordedAt": "2026-09-03T04:29:47.624Z", + "recordedAt": "2026-09-11T12:57:52.117Z", "request": { "body": { "kind": "json", @@ -252,7 +493,7 @@ "body": { "contentType": "application/vnd.amazon.eventstream", "kind": "base64", - "value": "AAAArQAAAEsMmxxWCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SnRaWE56WVdkbFUzUmhjblFpT25zaWNtOXNaU0k2SW1GemMybHpkR0Z1ZENKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamsifYa70P8AAAD5AAAAS8EIdV0LOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydEVaV3gwWVNJNmV5SmtaV3gwWVNJNmV5SjBaWGgwSWpvaVVrRlhJbjBzSW1OdmJuUmxiblJDYkc5amEwbHVaR1Y0SWpvd2ZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFUifaaaBRoAAAC0AAAAS2Fr6aULOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydFRkRzl3SWpwN0ltTnZiblJsYm5SQ2JHOWphMGx1WkdWNElqb3dmWDA9IiwicCI6ImFiY2RlZmdoaWoifaX9LCwAAADtAAAAS1RoRB8LOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydEVaV3gwWVNJNmV5SmtaV3gwWVNJNmV5SjBaWGgwSWpvaVUxUlNSVUZOSW4wc0ltTnZiblJsYm5SQ2JHOWphMGx1WkdWNElqb3hmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREUifZFmLQQAAADbAAAAS3oJCTkLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydFRkRzl3SWpwN0ltTnZiblJsYm5SQ2JHOWphMGx1WkdWNElqb3hmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVlcifTkXwsUAAAD3AAAAS344yzwLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydEVaV3gwWVNJNmV5SmtaV3gwWVNJNmV5SjBaWGgwSWpvaUluMHNJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam95ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXIn1byIHKAAAArgAAAEtLO2aGCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRUZEc5d0lqcDdJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam95ZlgwPSIsInAiOiJhYmNkIn0R0rymAAAA1gAAAEuCmc2ICzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SnRaWE56WVdkbFUzUnZjQ0k2ZXlKemRHOXdVbVZoYzI5dUlqb2laVzVrWDNSMWNtNGlmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVViJ9SAJgoQAAAl8AAABLc6e+YAs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUp0WlhSaFpHRjBZU0k2ZXlKMWMyRm5aU0k2ZXlKcGJuQjFkRlJ2YTJWdWN5STZOaXdpYjNWMGNIVjBWRzlyWlc1eklqb3pMQ0pqWVdOb1pWSmxZV1JKYm5CMWRGUnZhMlZ1UTI5MWJuUWlPakUwT1RJeUxDSmpZV05vWlZkeWFYUmxTVzV3ZFhSVWIydGxia052ZFc1MElqb3dmU3dpYldWMGNtbGpjeUk2ZTMwc0luUnlZV05sSWpwN2ZYMHNJbUZ0WVhwdmJpMWlaV1J5YjJOckxXbHVkbTlqWVhScGIyNU5aWFJ5YVdOeklqcDdJbWx1Y0hWMFZHOXJaVzVEYjNWdWRDSTZOaXdpYjNWMGNIVjBWRzlyWlc1RGIzVnVkQ0k2TXl3aWFXNTJiMk5oZEdsdmJreGhkR1Z1WTNraU9qWXdOU3dpWm1seWMzUkNlWFJsVEdGMFpXNWplU0k2TlRVekxDSmpZV05vWlZKbFlXUkpibkIxZEZSdmEyVnVRMjkxYm5RaU9qRTBPVEl5TENKallXTm9aVmR5YVhSbFNXNXdkWFJVYjJ0bGJrTnZkVzUwSWpvd2ZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWjAifUsLbco=" + "value": "AAAAvwAAAEsWu9i0CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SnRaWE56WVdkbFUzUmhjblFpT25zaWNtOXNaU0k2SW1GemMybHpkR0Z1ZENKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkMifcbWGmMAAADZAAAASwDJWlkLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydEVaV3gwWVNJNmV5SmtaV3gwWVNJNmV5SjBaWGgwSWpvaVVrRlhJbjBzSW1OdmJuUmxiblJDYkc5amEwbHVaR1Y0SWpvd2ZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vIn3GKXdiAAAAxAAAAEuYuQlqCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRUZEc5d0lqcDdJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam93ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5eiJ9ofEvuQAAAPMAAABLi7ht/As6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0RVpXeDBZU0k2ZXlKa1pXeDBZU0k2ZXlKMFpYaDBJam9pVTFSU1JVRk5JbjBzSW1OdmJuUmxiblJDYkc5amEwbHVaR1Y0SWpveGZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKSyJ9k2H1XAAAAK0AAABLDJscVgs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0VGRHOXdJanA3SW1OdmJuUmxiblJDYkc5amEwbHVaR1Y0SWpveGZYMD0iLCJwIjoiYWJjIn27IgluAAAAywAAAEsa6Z67CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRFWld4MFlTSTZleUprWld4MFlTSTZleUowWlhoMElqb2lJbjBzSW1OdmJuUmxiblJDYkc5amEwbHVaR1Y0SWpveWZYMD0iLCJwIjoiYWJjZGUifS/bTEwAAAC4AAAAS6SbBKQLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydFRkRzl3SWpwN0ltTnZiblJsYm5SQ2JHOWphMGx1WkdWNElqb3lmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1uIn2sEZFtAAAAqQAAAEv5G7qWCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SnRaWE56WVdkbFUzUnZjQ0k2ZXlKemRHOXdVbVZoYzI5dUlqb2laVzVrWDNSMWNtNGlmWDA9IiwicCI6ImFiYyJ9lp53VAAAAlwAAABLNAfEsAs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUp0WlhSaFpHRjBZU0k2ZXlKMWMyRm5aU0k2ZXlKcGJuQjFkRlJ2YTJWdWN5STZOaXdpYjNWMGNIVjBWRzlyWlc1eklqb3pMQ0pqWVdOb1pWSmxZV1JKYm5CMWRGUnZhMlZ1UTI5MWJuUWlPakUwT1RJeUxDSmpZV05vWlZkeWFYUmxTVzV3ZFhSVWIydGxia052ZFc1MElqb3dmU3dpYldWMGNtbGpjeUk2ZTMwc0luUnlZV05sSWpwN2ZYMHNJbUZ0WVhwdmJpMWlaV1J5YjJOckxXbHVkbTlqWVhScGIyNU5aWFJ5YVdOeklqcDdJbWx1Y0hWMFZHOXJaVzVEYjNWdWRDSTZOaXdpYjNWMGNIVjBWRzlyWlc1RGIzVnVkQ0k2TXl3aWFXNTJiMk5oZEdsdmJreGhkR1Z1WTNraU9qVTVNU3dpWm1seWMzUkNlWFJsVEdGMFpXNWplU0k2TlRJMExDSmpZV05vWlZKbFlXUkpibkIxZEZSdmEyVnVRMjkxYm5RaU9qRTBPVEl5TENKallXTm9aVmR5YVhSbFNXNXdkWFJVYjJ0bGJrTnZkVzUwSWpvd2ZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1gifS27Y54=" }, "headers": { "connection": "keep-alive", diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-auto.span-tree.json b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-auto.span-tree.json index 2559d106c..a98284ac8 100644 --- a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-auto.span-tree.json +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-auto.span-tree.json @@ -125,6 +125,43 @@ "testRunId": "" } }, + { + "name": "bedrock-embedding-operation", + "children": [ + { + "name": "bedrock.invokeModel", + "type": "llm", + "children": [], + "input": { + "inputs": [ + { + "content": "Embed this short Braintrust test sentence." + } + ], + "output_dimensions": 256 + }, + "output": { + "count": 1 + }, + "metadata": { + "accept": "application/json", + "command": "InvokeModelCommand", + "contentType": "application/json", + "model": "amazon.titan-embed-text-v2:0", + "operation": "invokeModel", + "provider": "aws-bedrock" + }, + "metrics": { + "prompt_tokens": 9, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "embedding", + "testRunId": "" + } + }, { "name": "bedrock-invoke-model-operation", "children": [ diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-auto.span-tree.txt b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-auto.span-tree.txt index 716d4fe5e..4cacfd581 100644 --- a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-auto.span-tree.txt +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-auto.span-tree.txt @@ -109,6 +109,35 @@ span_tree: │ "time_to_first_token": 0, │ "tokens": 14931 │ } + ├── bedrock-embedding-operation + │ metadata: { + │ "operation": "embedding", + │ "testRunId": "" + │ } + │ └── bedrock.invokeModel [llm] + │ input: { + │ "inputs": [ + │ { + │ "content": "Embed this short Braintrust test sentence." + │ } + │ ], + │ "output_dimensions": 256 + │ } + │ output: { + │ "count": 1 + │ } + │ metadata: { + │ "accept": "application/json", + │ "command": "InvokeModelCommand", + │ "contentType": "application/json", + │ "model": "amazon.titan-embed-text-v2:0", + │ "operation": "invokeModel", + │ "provider": "aws-bedrock" + │ } + │ metrics: { + │ "prompt_tokens": 9, + │ "tokens": 9 + │ } ├── bedrock-invoke-model-operation │ metadata: { │ "operation": "invoke-model", diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-auto.span-tree.json b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-auto.span-tree.json index 8d2e9ff57..a4dfcb321 100644 --- a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-auto.span-tree.json +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-auto.span-tree.json @@ -52,8 +52,8 @@ "metrics": { "completion_tokens": 2, "latency_ms": 0, - "prompt_cache_creation_tokens": 14921, - "prompt_cached_tokens": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 14921, "prompt_tokens": 6, "tokens": 14929 } @@ -125,6 +125,43 @@ "testRunId": "" } }, + { + "name": "bedrock-embedding-operation", + "children": [ + { + "name": "bedrock.invokeModel", + "type": "llm", + "children": [], + "input": { + "inputs": [ + { + "content": "Embed this short Braintrust test sentence." + } + ], + "output_dimensions": 256 + }, + "output": { + "count": 1 + }, + "metadata": { + "accept": "application/json", + "command": "InvokeModelCommand", + "contentType": "application/json", + "model": "amazon.titan-embed-text-v2:0", + "operation": "invokeModel", + "provider": "aws-bedrock" + }, + "metrics": { + "prompt_tokens": 9, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "embedding", + "testRunId": "" + } + }, { "name": "bedrock-invoke-model-operation", "children": [ @@ -242,16 +279,16 @@ "provider": "aws-bedrock", "trace": {}, "usage": { - "cacheReadInputTokenCount": 14922, - "cacheWriteInputTokenCount": 0, + "cacheReadInputTokenCount": 0, + "cacheWriteInputTokenCount": 14922, "inputTokens": 6, "outputTokens": 3 } }, "metrics": { "completion_tokens": 3, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 14922, + "prompt_cache_creation_tokens": 14922, + "prompt_cached_tokens": 0, "prompt_tokens": 6, "time_to_first_token": 0, "tokens": 14931 diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-auto.span-tree.txt b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-auto.span-tree.txt index c52aea7aa..d49b2d46e 100644 --- a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-auto.span-tree.txt +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-auto.span-tree.txt @@ -51,8 +51,8 @@ span_tree: │ metrics: { │ "completion_tokens": 2, │ "latency_ms": 0, - │ "prompt_cache_creation_tokens": 14921, - │ "prompt_cached_tokens": 0, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 14921, │ "prompt_tokens": 6, │ "tokens": 14929 │ } @@ -109,6 +109,35 @@ span_tree: │ "time_to_first_token": 0, │ "tokens": 14931 │ } + ├── bedrock-embedding-operation + │ metadata: { + │ "operation": "embedding", + │ "testRunId": "" + │ } + │ └── bedrock.invokeModel [llm] + │ input: { + │ "inputs": [ + │ { + │ "content": "Embed this short Braintrust test sentence." + │ } + │ ], + │ "output_dimensions": 256 + │ } + │ output: { + │ "count": 1 + │ } + │ metadata: { + │ "accept": "application/json", + │ "command": "InvokeModelCommand", + │ "contentType": "application/json", + │ "model": "amazon.titan-embed-text-v2:0", + │ "operation": "invokeModel", + │ "provider": "aws-bedrock" + │ } + │ metrics: { + │ "prompt_tokens": 9, + │ "tokens": 9 + │ } ├── bedrock-invoke-model-operation │ metadata: { │ "operation": "invoke-model", @@ -217,16 +246,16 @@ span_tree: "provider": "aws-bedrock", "trace": {}, "usage": { - "cacheReadInputTokenCount": 14922, - "cacheWriteInputTokenCount": 0, + "cacheReadInputTokenCount": 0, + "cacheWriteInputTokenCount": 14922, "inputTokens": 6, "outputTokens": 3 } } metrics: { "completion_tokens": 3, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 14922, + "prompt_cache_creation_tokens": 14922, + "prompt_cached_tokens": 0, "prompt_tokens": 6, "time_to_first_token": 0, "tokens": 14931 diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-wrapped.span-tree.json b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-wrapped.span-tree.json index 8d2e9ff57..a4dfcb321 100644 --- a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-wrapped.span-tree.json +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-wrapped.span-tree.json @@ -52,8 +52,8 @@ "metrics": { "completion_tokens": 2, "latency_ms": 0, - "prompt_cache_creation_tokens": 14921, - "prompt_cached_tokens": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 14921, "prompt_tokens": 6, "tokens": 14929 } @@ -125,6 +125,43 @@ "testRunId": "" } }, + { + "name": "bedrock-embedding-operation", + "children": [ + { + "name": "bedrock.invokeModel", + "type": "llm", + "children": [], + "input": { + "inputs": [ + { + "content": "Embed this short Braintrust test sentence." + } + ], + "output_dimensions": 256 + }, + "output": { + "count": 1 + }, + "metadata": { + "accept": "application/json", + "command": "InvokeModelCommand", + "contentType": "application/json", + "model": "amazon.titan-embed-text-v2:0", + "operation": "invokeModel", + "provider": "aws-bedrock" + }, + "metrics": { + "prompt_tokens": 9, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "embedding", + "testRunId": "" + } + }, { "name": "bedrock-invoke-model-operation", "children": [ @@ -242,16 +279,16 @@ "provider": "aws-bedrock", "trace": {}, "usage": { - "cacheReadInputTokenCount": 14922, - "cacheWriteInputTokenCount": 0, + "cacheReadInputTokenCount": 0, + "cacheWriteInputTokenCount": 14922, "inputTokens": 6, "outputTokens": 3 } }, "metrics": { "completion_tokens": 3, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 14922, + "prompt_cache_creation_tokens": 14922, + "prompt_cached_tokens": 0, "prompt_tokens": 6, "time_to_first_token": 0, "tokens": 14931 diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-wrapped.span-tree.txt b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-wrapped.span-tree.txt index c52aea7aa..d49b2d46e 100644 --- a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-latest-wrapped.span-tree.txt @@ -51,8 +51,8 @@ span_tree: │ metrics: { │ "completion_tokens": 2, │ "latency_ms": 0, - │ "prompt_cache_creation_tokens": 14921, - │ "prompt_cached_tokens": 0, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 14921, │ "prompt_tokens": 6, │ "tokens": 14929 │ } @@ -109,6 +109,35 @@ span_tree: │ "time_to_first_token": 0, │ "tokens": 14931 │ } + ├── bedrock-embedding-operation + │ metadata: { + │ "operation": "embedding", + │ "testRunId": "" + │ } + │ └── bedrock.invokeModel [llm] + │ input: { + │ "inputs": [ + │ { + │ "content": "Embed this short Braintrust test sentence." + │ } + │ ], + │ "output_dimensions": 256 + │ } + │ output: { + │ "count": 1 + │ } + │ metadata: { + │ "accept": "application/json", + │ "command": "InvokeModelCommand", + │ "contentType": "application/json", + │ "model": "amazon.titan-embed-text-v2:0", + │ "operation": "invokeModel", + │ "provider": "aws-bedrock" + │ } + │ metrics: { + │ "prompt_tokens": 9, + │ "tokens": 9 + │ } ├── bedrock-invoke-model-operation │ metadata: { │ "operation": "invoke-model", @@ -217,16 +246,16 @@ span_tree: "provider": "aws-bedrock", "trace": {}, "usage": { - "cacheReadInputTokenCount": 14922, - "cacheWriteInputTokenCount": 0, + "cacheReadInputTokenCount": 0, + "cacheWriteInputTokenCount": 14922, "inputTokens": 6, "outputTokens": 3 } } metrics: { "completion_tokens": 3, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 14922, + "prompt_cache_creation_tokens": 14922, + "prompt_cached_tokens": 0, "prompt_tokens": 6, "time_to_first_token": 0, "tokens": 14931 diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-wrapped.span-tree.json b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-wrapped.span-tree.json index 2559d106c..a98284ac8 100644 --- a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-wrapped.span-tree.json +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-wrapped.span-tree.json @@ -125,6 +125,43 @@ "testRunId": "" } }, + { + "name": "bedrock-embedding-operation", + "children": [ + { + "name": "bedrock.invokeModel", + "type": "llm", + "children": [], + "input": { + "inputs": [ + { + "content": "Embed this short Braintrust test sentence." + } + ], + "output_dimensions": 256 + }, + "output": { + "count": 1 + }, + "metadata": { + "accept": "application/json", + "command": "InvokeModelCommand", + "contentType": "application/json", + "model": "amazon.titan-embed-text-v2:0", + "operation": "invokeModel", + "provider": "aws-bedrock" + }, + "metrics": { + "prompt_tokens": 9, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "embedding", + "testRunId": "" + } + }, { "name": "bedrock-invoke-model-operation", "children": [ diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-wrapped.span-tree.txt b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-wrapped.span-tree.txt index 716d4fe5e..4cacfd581 100644 --- a/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-wrapped.span-tree.txt +++ b/e2e/scenarios/bedrock-runtime-instrumentation/__snapshots__/bedrock-runtime-v3-wrapped.span-tree.txt @@ -109,6 +109,35 @@ span_tree: │ "time_to_first_token": 0, │ "tokens": 14931 │ } + ├── bedrock-embedding-operation + │ metadata: { + │ "operation": "embedding", + │ "testRunId": "" + │ } + │ └── bedrock.invokeModel [llm] + │ input: { + │ "inputs": [ + │ { + │ "content": "Embed this short Braintrust test sentence." + │ } + │ ], + │ "output_dimensions": 256 + │ } + │ output: { + │ "count": 1 + │ } + │ metadata: { + │ "accept": "application/json", + │ "command": "InvokeModelCommand", + │ "contentType": "application/json", + │ "model": "amazon.titan-embed-text-v2:0", + │ "operation": "invokeModel", + │ "provider": "aws-bedrock" + │ } + │ metrics: { + │ "prompt_tokens": 9, + │ "tokens": 9 + │ } ├── bedrock-invoke-model-operation │ metadata: { │ "operation": "invoke-model", diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/assertions.ts b/e2e/scenarios/bedrock-runtime-instrumentation/assertions.ts index 406113a5b..1f760ca89 100644 --- a/e2e/scenarios/bedrock-runtime-instrumentation/assertions.ts +++ b/e2e/scenarios/bedrock-runtime-instrumentation/assertions.ts @@ -13,6 +13,7 @@ import { import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; import { CACHE_PROMPT_MARKER, + EMBEDDING_MODEL, MODEL, ROOT_NAME, SCENARIO_NAME, @@ -37,6 +38,7 @@ type RunBedrockRuntimeScenario = (harness: { const OPERATION_TO_SPAN_NAME = { "bedrock-converse-operation": "bedrock.converse", "bedrock-converse-stream-operation": "bedrock.converseStream", + "bedrock-embedding-operation": "bedrock.invokeModel", "bedrock-invoke-model-operation": "bedrock.invokeModel", "bedrock-invoke-model-stream-operation": "bedrock.invokeModelWithResponseStream", @@ -150,7 +152,10 @@ export function defineBedrockRuntimeInstrumentationAssertions(options: { const span = findBedrockSpan(events, operationName); expect(span, operationName).toBeDefined(); expect(span?.row.metadata).toMatchObject({ - model: MODEL, + model: + operationName === "bedrock-embedding-operation" + ? EMBEDDING_MODEL + : MODEL, provider: "aws-bedrock", }); expect(span?.output).toBeDefined(); @@ -180,6 +185,28 @@ export function defineBedrockRuntimeInstrumentationAssertions(options: { }); }); + test("captures canonical Bedrock embedding data", testConfig, () => { + const embeddingSpan = findBedrockSpan( + events, + "bedrock-embedding-operation", + ); + + expect(embeddingSpan?.input).toEqual({ + inputs: [ + { + content: "Embed this short Braintrust test sentence.", + }, + ], + output_dimensions: 256, + }); + expect(embeddingSpan?.output).toEqual({ count: 1 }); + expect(embeddingSpan?.metrics).toMatchObject({ + prompt_tokens: expect.any(Number), + tokens: expect.any(Number), + }); + expect(embeddingSpan?.metrics).not.toHaveProperty("completion_tokens"); + }); + test("captures Bedrock prompt cache metrics", testConfig, () => { const metrics = Object.keys(OPERATION_TO_SPAN_NAME) .map((operationName) => diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/constants.mjs b/e2e/scenarios/bedrock-runtime-instrumentation/constants.mjs index 33dac5d19..805df652a 100644 --- a/e2e/scenarios/bedrock-runtime-instrumentation/constants.mjs +++ b/e2e/scenarios/bedrock-runtime-instrumentation/constants.mjs @@ -15,6 +15,7 @@ function getModel() { } export const MODEL = getModel(); +export const EMBEDDING_MODEL = "amazon.titan-embed-text-v2:0"; export const REGION = process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION ?? "us-east-1"; export const CACHE_PROMPT_MARKER = "BRAINTRUST_BEDROCK_CACHE_CONTEXT"; diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/scenario.impl.mjs b/e2e/scenarios/bedrock-runtime-instrumentation/scenario.impl.mjs index 55af87dbc..0dca692f5 100644 --- a/e2e/scenarios/bedrock-runtime-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/bedrock-runtime-instrumentation/scenario.impl.mjs @@ -6,6 +6,7 @@ import { } from "../../helpers/provider-runtime.mjs"; import { CACHE_PROMPT_MARKER, + EMBEDDING_MODEL, MODEL, REGION, ROOT_NAME, @@ -156,6 +157,25 @@ export async function runBedrockRuntimeInstrumentationScenario(options) { }, ); + await runOperation( + "bedrock-embedding-operation", + "embedding", + async () => { + await client.send( + new options.InvokeModelCommand({ + accept: "application/json", + body: JSON.stringify({ + dimensions: 256, + inputText: "Embed this short Braintrust test sentence.", + normalize: true, + }), + contentType: "application/json", + modelId: EMBEDDING_MODEL, + }), + ); + }, + ); + await runOperation( "bedrock-invoke-model-stream-operation", "invoke-model-stream", diff --git a/e2e/scenarios/bedrock-runtime-instrumentation/scenario.test.ts b/e2e/scenarios/bedrock-runtime-instrumentation/scenario.test.ts index 6fc954e8b..ec533c955 100644 --- a/e2e/scenarios/bedrock-runtime-instrumentation/scenario.test.ts +++ b/e2e/scenarios/bedrock-runtime-instrumentation/scenario.test.ts @@ -40,8 +40,9 @@ describe.concurrent("variants", () => { entry: "scenario.ts", env: { BEDROCK_RUNTIME_PACKAGE_NAME: scenario.dependencyName }, runContext: { - variantKey: scenario.snapshotName, + cassette: { variantKey: scenario.snapshotName }, originalScenarioDir, + variantKey: `${scenario.snapshotName}-wrapped`, }, scenarioDir, timeoutMs: BEDROCK_RUNTIME_SCENARIO_TIMEOUT_MS, @@ -60,8 +61,9 @@ describe.concurrent("variants", () => { env: { BEDROCK_RUNTIME_PACKAGE_NAME: scenario.dependencyName }, nodeArgs: ["--import", "braintrust/hook.mjs"], runContext: { - variantKey: scenario.snapshotName, + cassette: { variantKey: scenario.snapshotName }, originalScenarioDir, + variantKey: `${scenario.snapshotName}-auto`, }, scenarioDir, timeoutMs: BEDROCK_RUNTIME_SCENARIO_TIMEOUT_MS, diff --git a/js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts b/js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts index fc74a9068..9b14e4b06 100644 --- a/js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts +++ b/js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts @@ -25,6 +25,10 @@ class InvokeModelWithBidirectionalStreamCommand { constructor(public input: Record) {} } +class InvokeModelCommand { + constructor(public input: Record) {} +} + class GetObjectCommand { constructor(public input: Record) {} } @@ -232,6 +236,176 @@ describe("BedrockRuntimePlugin", () => { ]), ); }); + + it("captures canonical Titan embedding data and token metrics", async () => { + await smithyCoreChannels.clientSend.tracePromise( + async () => ({ + body: new TextEncoder().encode( + JSON.stringify({ + embedding: [0.1, 0.2, 0.3], + embeddingsByType: { + float: [0.1, 0.2, 0.3], + }, + inputTextTokenCount: 4, + }), + ), + }), + { + arguments: [ + new InvokeModelCommand({ + body: JSON.stringify({ + dimensions: 3, + inputText: "Embed this sentence.", + normalize: true, + }), + modelId: "amazon.titan-embed-text-v2:0", + }) as any, + ], + }, + ); + + const spans = await backgroundLogger.drain(); + expect(spans).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + input: { + inputs: [{ content: "Embed this sentence." }], + output_dimensions: 3, + }, + metadata: expect.objectContaining({ + model: "amazon.titan-embed-text-v2:0", + provider: "aws-bedrock", + }), + metrics: expect.objectContaining({ + prompt_tokens: 4, + tokens: 4, + }), + output: { count: 1 }, + }), + ]), + ); + expect(spans).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + metrics: expect.objectContaining({ + completion_tokens: expect.anything(), + }), + output: { count: 1 }, + }), + ]), + ); + }); + + it("captures canonical Cohere batch embedding data without vectors", async () => { + await smithyCoreChannels.clientSend.tracePromise( + async () => ({ + body: new TextEncoder().encode( + JSON.stringify({ + embeddings: [ + [0.1, 0.2, 0.3], + [0.4, 0.5, 0.6], + ], + id: "response-id", + response_type: "embeddings_floats", + texts: ["First", "Second"], + }), + ), + }), + { + arguments: [ + new InvokeModelCommand({ + body: JSON.stringify({ + input_type: "search_document", + texts: ["First", "Second"], + }), + modelId: "cohere.embed-english-v3", + }) as any, + ], + }, + ); + + const spans = await backgroundLogger.drain(); + expect(spans).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + input: { + inputs: [{ content: "First" }, { content: "Second" }], + }, + output: { count: 2 }, + }), + ]), + ); + }); + + it("counts Cohere v4 multi-type embeddings once per input", async () => { + await smithyCoreChannels.clientSend.tracePromise( + async () => ({ + body: JSON.stringify({ + embeddings: { + float: [ + [0.1, 0.2], + [0.3, 0.4], + ], + int8: [ + [1, 2], + [3, 4], + ], + }, + response_type: "embeddings_by_type", + }), + }), + { + arguments: [ + new InvokeModelCommand({ + body: JSON.stringify({ + input_type: "search_document", + output_dimension: 2, + texts: ["First", "Second"], + }), + modelId: "us.cohere.embed-v4:0", + }) as any, + ], + }, + ); + + const spans = await backgroundLogger.drain(); + expect(spans).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + input: { + inputs: [{ content: "First" }, { content: "Second" }], + output_dimensions: 2, + }, + output: { count: 2 }, + }), + ]), + ); + }); + + it("never logs provider-native output for malformed embedding responses", async () => { + await smithyCoreChannels.clientSend.tracePromise( + async () => ({ + body: JSON.stringify({ unexpected: "response" }), + }), + { + arguments: [ + new InvokeModelCommand({ + body: JSON.stringify({ inputText: "Embed this." }), + modelId: "amazon.titan-embed-text-v2:0", + }) as any, + ], + }, + ); + + const spans = await backgroundLogger.drain(); + expect(spans).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + output: { count: 0 }, + }), + ]), + ); + }); }); describe("parseBedrockRuntimeMetrics", () => { diff --git a/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts b/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts index a759d1ffe..857f90e80 100644 --- a/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts +++ b/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts @@ -4,6 +4,7 @@ import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { getCurrentUnixTimestamp } from "../../util"; import type { Span } from "../../logger"; +import { processInputAttachments } from "../../wrappers/attachment-utils"; import type { AnyAsyncChannel } from "../core/channel-definitions"; import type { BedrockRuntimeConverseRequest, @@ -25,6 +26,9 @@ import { getBedrockRuntimeOperation, } from "./bedrock-runtime-common"; +const BEDROCK_EMBEDDING_MODEL_PATTERN = + /(?:^|[./])(?:amazon\.titan-embed-text|cohere\.embed-)/; + export class BedrockRuntimePlugin extends BasePlugin { protected onEnable(): void { this.unsubscribers.push( @@ -102,10 +106,11 @@ function extractBedrockRuntimeInput(command: unknown): { const invokeRequest = isObject(request) ? (request as BedrockRuntimeInvokeModelRequest) : undefined; + const parsedBody = parseJsonBody(invokeRequest?.body); return { - input: - parseJsonBody(invokeRequest?.body) ?? - summarizeBody(invokeRequest?.body), + input: isBedrockEmbeddingModel(invokeRequest?.modelId) + ? extractBedrockEmbeddingInput(parsedBody) + : (parsedBody ?? summarizeBody(invokeRequest?.body)), metadata, }; } @@ -161,7 +166,16 @@ function extractBedrockRuntimeOutput( if (operation === "invokeModel") { const response = isObject(result) ? result : undefined; - return parseJsonBody(response?.body) ?? summarizeBody(response?.body); + const parsedBody = parseJsonBody(response?.body); + const request = getBedrockRuntimeCommandInput(command); + const modelId = isObject(request) ? request.modelId : undefined; + if ( + isBedrockEmbeddingModel(modelId) || + isBedrockEmbeddingResponse(parsedBody) + ) { + return summarizeBedrockEmbeddingOutput(parsedBody); + } + return parsedBody ?? summarizeBody(response?.body); } return sanitizeBedrockValue(result); @@ -215,7 +229,9 @@ function extractBedrockRuntimeResponseMetrics( ? parsedBody.metadata : undefined; const metrics = parseBedrockRuntimeMetrics( - parsedBody.usage ?? metadata?.usage, + parsedBody.usage ?? + metadata?.usage ?? + (parsedBody.inputTextTokenCount !== undefined ? parsedBody : undefined), parsedBody.metrics ?? metadata?.metrics, ); if (Object.keys(metrics).length > 0) { @@ -238,6 +254,7 @@ export function parseBedrockRuntimeMetrics( const promptTokens = firstNumber( usageRecord.inputTokens, usageRecord.inputTokenCount, + usageRecord.inputTextTokenCount, usageRecord.input_tokens, usageRecord.prompt_tokens, ); @@ -308,6 +325,142 @@ export function parseBedrockRuntimeMetrics( return metrics; } +type BedrockEmbeddingContentPart = + | { type: "text"; text: string } + | { type: "image_url"; image_url: { url: unknown } }; + +type BedrockEmbeddingInput = { + inputs: Array<{ + content: string | BedrockEmbeddingContentPart[]; + }>; + output_dimensions?: number; +}; + +function isBedrockEmbeddingModel(modelId: unknown): boolean { + return ( + typeof modelId === "string" && BEDROCK_EMBEDDING_MODEL_PATTERN.test(modelId) + ); +} + +function extractBedrockEmbeddingInput(body: unknown): BedrockEmbeddingInput { + const request = isObject(body) ? body : undefined; + let inputs: BedrockEmbeddingInput["inputs"] = []; + + if (typeof request?.inputText === "string") { + inputs = [{ content: request.inputText }]; + } else if (Array.isArray(request?.inputs)) { + inputs = request.inputs.flatMap((input) => { + const content = isObject(input) ? input.content : undefined; + if (!Array.isArray(content)) { + return []; + } + + const parts = content.flatMap((part): BedrockEmbeddingContentPart[] => { + if (!isObject(part)) { + return []; + } + if (part.type === "text" && typeof part.text === "string") { + return [{ type: "text", text: part.text }]; + } + if ( + part.type === "image_url" && + isObject(part.image_url) && + typeof part.image_url.url === "string" + ) { + return [ + { + type: "image_url", + image_url: { url: part.image_url.url }, + }, + ]; + } + return []; + }); + return parts.length > 0 ? [{ content: parts }] : []; + }); + } else { + if (Array.isArray(request?.texts)) { + inputs.push( + ...request.texts.flatMap((text) => + typeof text === "string" ? [{ content: text }] : [], + ), + ); + } + if (Array.isArray(request?.images)) { + for (const image of request.images) { + if (typeof image === "string") { + inputs.push({ + content: [ + { + type: "image_url", + image_url: { url: image }, + }, + ], + }); + } + } + } + } + + const outputDimensions = request?.dimensions ?? request?.output_dimension; + return processInputAttachments({ + inputs, + ...(typeof outputDimensions === "number" && + Number.isFinite(outputDimensions) + ? { output_dimensions: outputDimensions } + : {}), + }); +} + +function isBedrockEmbeddingResponse(body: unknown): boolean { + return ( + isObject(body) && + (Array.isArray(body.embedding) || + Array.isArray(body.embeddings) || + isObject(body.embeddings) || + isObject(body.embeddingsByType)) + ); +} + +function summarizeBedrockEmbeddingOutput(body: unknown): { count: number } { + if (!isObject(body)) { + return { count: 0 }; + } + + if (Array.isArray(body.embedding)) { + return { count: body.embedding.length > 0 ? 1 : 0 }; + } + + if (Array.isArray(body.embeddings)) { + return { + count: + body.embeddings.length > 0 && !Array.isArray(body.embeddings[0]) + ? 1 + : body.embeddings.length, + }; + } + + for (const embeddingGroup of [body.embeddings, body.embeddingsByType]) { + if (!isObject(embeddingGroup)) { + continue; + } + const counts = Object.values(embeddingGroup).flatMap((embeddings) => + Array.isArray(embeddings) + ? [ + embeddings.length > 0 && !Array.isArray(embeddings[0]) + ? 1 + : embeddings.length, + ] + : [], + ); + if (counts.length > 0) { + return { count: Math.max(...counts) }; + } + } + + return { count: 0 }; +} + function firstNumber(...values: unknown[]): number | undefined { return values.find((value): value is number => typeof value === "number"); } diff --git a/js/src/vendor-sdk-types/bedrock-runtime.ts b/js/src/vendor-sdk-types/bedrock-runtime.ts index 158355f97..f4d2e3030 100644 --- a/js/src/vendor-sdk-types/bedrock-runtime.ts +++ b/js/src/vendor-sdk-types/bedrock-runtime.ts @@ -62,6 +62,7 @@ export interface BedrockRuntimeConverseRequest { export interface BedrockRuntimeTokenUsage { inputTokens?: number; + inputTextTokenCount?: number; outputTokens?: number; totalTokens?: number; cacheReadInputTokens?: number;