From 1f7e3cbb073cc572fe180e0d1cf4daf48dd27ece Mon Sep 17 00:00:00 2001 From: Alexander Kammerer Date: Wed, 29 Jul 2026 00:24:08 +0200 Subject: [PATCH 1/7] Add bql() for Bloomberg Query Language queries via //blp/bqlsvc Implements BQL support as discussed in #387. The C++ layer sends a 'sendQuery' request to the //blp/bqlsvc service and returns the JSON document the service responds with; the R layer parses it into properly-typed data.frames (one per data item in the query's get() clause) using the column types the response itself declares, with jsonlite as an optional (Suggests) dependency. bql(..., parse=FALSE) returns the raw JSON for queries whose shape the parser cannot handle. A response larger than 4 MiB arrives in several messages that cut the JSON mid-token, so the fragments are joined before parsing. Without this, queries above that size -- a PX_LAST history for 84 tickers over three years is enough -- fail with 'parse error: premature EOF'. The 'NaN'/'NA' missing-value sentinels are applied to numeric columns only so STRING columns keep legitimate 'NA' values (e.g. the ticker of 'NA US Equity'). Item-level responseExceptions (type PARTIAL, accompanying usable data) surface as warnings while the data is returned; top-level exceptions raise errors. Offline unit tests cover parsing, type mapping (including undeclared types such as ENUM), NA handling, fragmented responses, grouped aggregations (composite group ids, INT columns, ORIG_IDS nulls), and error propagation via synthetic fixtures whose structure was verified against live //blp/bqlsvc responses; no captured Bloomberg data is included. A live test gated on RunRblpapiUnitTests exercises the service end to end. Verified live against terminal API 3.24.6.1, including all documented example queries of the polars-bloomberg package (screens, SRCH results, segments, axes, return series), and with a 23 MB history response arriving in six fragments. Co-Authored-By: Claude Fable 5 Co-Authored-By: Claude Opus 5 (1M context) --- DESCRIPTION | 2 +- NAMESPACE | 1 + R/RcppExports.R | 4 + R/bql.R | 167 +++++++++++++++++++ inst/tinytest/bql/response_grouped.json | 1 + inst/tinytest/bql/response_item_error.json | 1 + inst/tinytest/bql/response_multi_item.json | 1 + inst/tinytest/bql/response_px_last.json | 1 + inst/tinytest/bql/response_string_na.json | 1 + inst/tinytest/bql/response_syntax_error.json | 1 + inst/tinytest/test_bql.R | 112 +++++++++++++ man/bql.Rd | 69 ++++++++ src/RcppExports.cpp | 14 ++ src/bql.cpp | 128 ++++++++++++++ 14 files changed, 502 insertions(+), 1 deletion(-) create mode 100644 R/bql.R create mode 100644 inst/tinytest/bql/response_grouped.json create mode 100644 inst/tinytest/bql/response_item_error.json create mode 100644 inst/tinytest/bql/response_multi_item.json create mode 100644 inst/tinytest/bql/response_px_last.json create mode 100644 inst/tinytest/bql/response_string_na.json create mode 100644 inst/tinytest/bql/response_syntax_error.json create mode 100644 inst/tinytest/test_bql.R create mode 100644 man/bql.Rd create mode 100644 src/bql.cpp diff --git a/DESCRIPTION b/DESCRIPTION index 1c42843f..672a045a 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -7,7 +7,7 @@ Authors@R: c(person("Whit", "Armstrong", role = "aut"), comment = c(ORCID = "0000-0001-6419-907X")), person("John", "Laing", role = "aut")) Imports: Rcpp (>= 0.11.0), utils -Suggests: xts, zoo, data.table, simplermarkdown, tinytest +Suggests: xts, zoo, data.table, simplermarkdown, tinytest, jsonlite VignetteBuilder: simplermarkdown LazyLoad: yes LinkingTo: Rcpp, BH diff --git a/NAMESPACE b/NAMESPACE index 97c538fd..246a8b5c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -11,6 +11,7 @@ export("blpConnect", "bdh", "bds", "beqs", + "bql", "bsrch", "fieldSearch", "fieldInfo", diff --git a/R/RcppExports.R b/R/RcppExports.R index 5f2071c9..0d5e17ac 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -68,6 +68,10 @@ haveBlp <- function() { .Call(`_Rblpapi_haveBlp`) } +bql_Impl <- function(con, expression, verbose = FALSE) { + .Call(`_Rblpapi_bql_Impl`, con, expression, verbose) +} + bsrch_Impl <- function(con, domain, limit, verbose = FALSE) { .Call(`_Rblpapi_bsrch_Impl`, con, domain, limit, verbose) } diff --git a/R/bql.R b/R/bql.R new file mode 100644 index 00000000..c06b5669 --- /dev/null +++ b/R/bql.R @@ -0,0 +1,167 @@ + +## Copyright (C) 2025 Whit Armstrong and Dirk Eddelbuettel and John Laing +## +## This file is part of Rblpapi +## +## Rblpapi is free software: you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation, either version 2 of the License, or +## (at your option) any later version. +## +## Rblpapi is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with Rblpapi. If not, see . + + +##' This function uses the Bloomberg API to execute 'BQL' (Bloomberg +##' Query Language) queries via the \sQuote{//blp/bqlsvc} service -- +##' the same service used by the Excel \code{=BQL()} function. +##' +##' The service returns a single JSON document. Each queried data +##' item is self-describing: every column carries a declared type +##' (\sQuote{STRING}, \sQuote{DOUBLE}, \sQuote{INT}, \sQuote{DATE}, +##' \sQuote{DATETIME}, \sQuote{BOOLEAN}) which is used to construct +##' properly-typed \code{data.frame} columns. Parsing requires the +##' \CRANpkg{jsonlite} package; set \code{parse=FALSE} to obtain the +##' raw JSON string instead, e.g. for queries whose shape the +##' parser does not handle. +##' +##' Note that \sQuote{//blp/bqlsvc} is not part of the officially +##' documented public API; it is the service behind the Excel BQL +##' add-in and may change without notice. +##' +##' @title Run 'Bloomberg Query Language' (BQL) Queries +##' @param expression A character string with the BQL query, e.g. +##' \code{"get(px_last) for(['IBM US Equity'])"}. +##' @param parse A boolean indicating whether the JSON response should +##' be parsed into \code{data.frame} objects (requires the +##' \CRANpkg{jsonlite} package), defaults to \sQuote{TRUE}. If +##' \sQuote{FALSE} the raw JSON string is returned. +##' @param simplify A boolean indicating whether a query returning a +##' single data item should be returned directly as a \code{data.frame} +##' instead of a list of length one, defaults to \sQuote{TRUE}. +##' @param verbose A boolean indicating whether verbose operation is +##' desired, defaults to \sQuote{FALSE}. +##' @param con A connection object as created by a \code{blpConnect} +##' call, and retrieved via the internal function +##' \code{defaultConnection}. +##' @return If \code{parse} is \sQuote{TRUE}, a named list of +##' \code{data.frame} objects, one per data item in the query's +##' \code{get()} clause (or a single \code{data.frame} if +##' \code{simplify} is \sQuote{TRUE} and only one item was queried). +##' Each \code{data.frame} has an \sQuote{ID} column, a value column +##' named after the data item, and any secondary columns (such as +##' \sQuote{DATE} or \sQuote{CURRENCY}) the service returned. If +##' \code{parse} is \sQuote{FALSE}, a character string with the JSON +##' document. +##' @author Alexander Kammerer and Dirk Eddelbuettel +##' @examples +##' \dontrun{ +##' con <- blpConnect() +##' bql("get(px_last) for(['IBM US Equity', 'AAPL US Equity'])") +##' bql("get(px_last, name) for(members('INDU Index'))", simplify=FALSE) +##' } +bql <- function(expression, + parse=TRUE, + simplify=TRUE, + verbose=FALSE, + con=defaultConnection()) { + + res <- bql_Impl(con, expression, verbose) + if (!parse) return(.bqlJoin(res)) + if (!requireNamespace("jsonlite", quietly=TRUE)) + stop("The 'jsonlite' package is required to parse BQL responses; ", + "install it or call bql(..., parse=FALSE) for the raw JSON.", + call.=FALSE) + .bqlParse(res, simplify=simplify) +} + +## The service delivers responses larger than 4 MiB in several messages, cutting +## the JSON mid-token: the fragments form one document only once joined +.bqlJoin <- function(fragments) paste0(fragments, collapse="") + +## Parse a raw BQL JSON response into a named list of data.frames +.bqlParse <- function(json, simplify=TRUE) { + parsed <- jsonlite::fromJSON(.bqlJoin(json), simplifyVector=FALSE) + .bqlCheckExceptions(parsed) + tables <- list() + for (item in parsed[["results"]]) { + nm <- if (is.null(item[["name"]])) "" else item[["name"]] + msgs <- .bqlExceptionMessages(item[["responseExceptions"]]) + if (length(msgs)) + warning("BQL error for item '", nm, "': ", + paste(msgs, collapse="; "), call.=FALSE) + tables[[nm]] <- .bqlItemToDataFrame(item) + } + if (simplify && length(tables) == 1L) return(tables[[1L]]) + tables +} + +## Raise an R error for any top-level 'responseExceptions' the service reported +.bqlCheckExceptions <- function(parsed) { + msgs <- .bqlExceptionMessages(parsed[["responseExceptions"]]) + if (length(msgs)) + stop("BQL error: ", paste(msgs, collapse="; "), call.=FALSE) + invisible(NULL) +} + +.bqlExceptionMessages <- function(excs) { + if (is.null(excs) || length(excs) == 0L) return(character()) + vapply(excs, function(e) { + msg <- e[["message"]] + if (is.null(msg) || !nzchar(msg)) msg <- e[["internalMessage"]] + if (is.null(msg) || !nzchar(msg)) msg <- "unknown BQL error" + msg + }, character(1)) +} + +## Convert one entry of 'results' into a data.frame using the declared +## column types; the value column is named after the data item itself +.bqlItemToDataFrame <- function(item) { + cols <- list() + idcol <- item[["idColumn"]] + if (!is.null(idcol)) + cols[[.bqlColName(idcol, "ID")]] <- .bqlColumn(idcol) + valcol <- item[["valuesColumn"]] + if (!is.null(valcol)) { + nm <- if (is.null(item[["name"]]) || !nzchar(item[["name"]])) + .bqlColName(valcol, "VALUE") else item[["name"]] + cols[[nm]] <- .bqlColumn(valcol) + } + for (sec in item[["secondaryColumns"]]) + cols[[.bqlColName(sec, "V")]] <- .bqlColumn(sec) + names(cols) <- make.unique(names(cols)) + ## avoid data.frame() name mangling and rownames + structure(cols, + class="data.frame", + row.names=if (length(cols)) seq_along(cols[[1L]]) else integer()) +} + +.bqlColName <- function(col, fallback) { + nm <- col[["name"]] + if (is.null(nm) || !nzchar(nm)) fallback else nm +} + +## Convert a BQL column (list with 'type' and 'values') to a typed R vector. +## JSON null maps to NA for every type; the string placeholders "NaN" and +## "NA" additionally map to NA for numeric columns only, as string columns +## may legitimately contain them (e.g. the ticker of 'NA US Equity'). +.bqlColumn <- function(col) { + values <- col[["values"]] + type <- if (is.null(col[["type"]])) "STRING" else col[["type"]] + values <- vapply(values, function(v) { + if (is.null(v)) NA_character_ else as.character(v) + }, character(1)) + numericNA <- function(v) { v[v %in% c("NaN", "NA", "")] <- NA_character_; v } + switch(type, + "DOUBLE" = as.numeric(numericNA(values)), + "INT" = as.integer(numericNA(values)), + "BOOLEAN" = as.logical(toupper(values)), + "DATE" = as.Date(substr(values, 1L, 10L)), + "DATETIME" = as.POSIXct(values, format="%Y-%m-%dT%H:%M:%OS", tz="UTC"), + values) +} diff --git a/inst/tinytest/bql/response_grouped.json b/inst/tinytest/bql/response_grouped.json new file mode 100644 index 00000000..729e555c --- /dev/null +++ b/inst/tinytest/bql/response_grouped.json @@ -0,0 +1 @@ +{"results":{"#mv":{"name":"#mv","offsets":[0,1,2,3,4,5],"namespace":"FUNCTION_DEFAULT","source":"BQLAnalyticsEngine","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["2027.0:Technology","2028.0:Technology","2029.0:Technology","2030.0:Technology","2031.0:Technology","2032.0:Technology"]},"valuesColumn":{"name":"VALUE","type":"DOUBLE","rank":0,"values":[1.5E9,2.25E9,7.5E8,3.1E9,5.0E8,1.2E9]},"secondaryColumns":[{"name":"CURRENCY_OF_ISSUE","type":"ENUM","rank":0,"values":["USD","USD","USD","USD","USD","USD"]},{"name":"MULTIPLIER","type":"DOUBLE","rank":0,"values":[1.0,1.0,1.0,1.0,1.0,1.0]},{"name":"CURRENCY","type":"STRING","rank":0,"values":["USD","USD","USD","USD","USD","USD"]},{"name":"ORIG_IDS","type":"STRING","rank":0,"values":[null,null,null,null,"XX000001 Corp","XX000002 Corp"]},{"name":"YEAR(MATURITY())","type":"INT","rank":0,"values":[2027,2028,2029,2030,2031,2032]},{"name":"INDUSTRY_SECTOR()","type":"STRING","rank":0,"values":["Technology","Technology","Technology","Technology","Technology","Technology"]}],"partialErrorMap":null,"responseExceptions":[],"forUniverse":false,"bqlResponseInfo":null,"defaultDateColumnName":null,"itemPreviewStatistics":null,"indexView":null}},"ordering":[{"requestIndex":0,"responseName":"#mv"}],"responseExceptions":null,"responseTiming":null,"dotString":null,"versionInfo":{"version":"1.288","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]},"screenCounts":null,"payloadId":null} diff --git a/inst/tinytest/bql/response_item_error.json b/inst/tinytest/bql/response_item_error.json new file mode 100644 index 00000000..947202b9 --- /dev/null +++ b/inst/tinytest/bql/response_item_error.json @@ -0,0 +1 @@ +{"results":{"px_last":{"name":"px_last","offsets":[0],"namespace":"DATAITEM_DEFAULT","source":"CR","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["IBM US Equity"]},"valuesColumn":{"name":"VALUE","type":"DOUBLE","rank":0,"values":[229.33]},"secondaryColumns":[{"name":"DATE","type":"DATE","rank":0,"values":["2024-12-17T00:00:00Z"],"defaultDate":true}],"responseExceptions":[{"message":"Insufficient data for 'XXX US Equity'.","type":"PARTIAL","internalMessage":"Insufficient data for 'XXX US Equity'.","messageCategory":"BQL_DATA_ERROR","messageSubcategory":"NA_SUBCATEGORY","level":0,"nodeName":null,"uniqueException":false,"messageKey":"DATA_UNAVAILABLE"}]}},"ordering":["px_last"],"responseExceptions":[],"versionInfo":{"version":"1.258","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]}} diff --git a/inst/tinytest/bql/response_multi_item.json b/inst/tinytest/bql/response_multi_item.json new file mode 100644 index 00000000..f9e815d6 --- /dev/null +++ b/inst/tinytest/bql/response_multi_item.json @@ -0,0 +1 @@ +{"results":{"name":{"name":"name","offsets":[0],"namespace":"DATAITEM_DEFAULT","source":"CR","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["IBM US Equity","AAPL US Equity"]},"valuesColumn":{"name":"VALUE","type":"STRING","rank":0,"values":["International Business Machines Corp","Apple Inc"]},"secondaryColumns":[],"responseExceptions":[]},"pe_ratio":{"name":"pe_ratio","offsets":[0],"namespace":"DATAITEM_DEFAULT","source":"CR","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["IBM US Equity","AAPL US Equity"]},"valuesColumn":{"name":"VALUE","type":"DOUBLE","rank":0,"values":[23.1,33.7]},"secondaryColumns":[{"name":"AS_OF_DATE","type":"DATE","rank":0,"values":["2024-12-17T00:00:00Z","2024-12-17T00:00:00Z"]},{"name":"PERIOD_END_DATE","type":"DATE","rank":0,"values":["2024-09-30T00:00:00Z","2024-09-28T00:00:00Z"]},{"name":"REVISION_COUNT","type":"INT","rank":0,"values":[3,5]}],"responseExceptions":[]}},"ordering":["name","pe_ratio"],"responseExceptions":[],"versionInfo":{"version":"1.258","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]}} diff --git a/inst/tinytest/bql/response_px_last.json b/inst/tinytest/bql/response_px_last.json new file mode 100644 index 00000000..0e3afdd8 --- /dev/null +++ b/inst/tinytest/bql/response_px_last.json @@ -0,0 +1 @@ +{"results":{"px_last":{"name":"px_last","offsets":[0],"namespace":"DATAITEM_DEFAULT","source":"CR","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["IBM US Equity","AAPL US Equity","XXX US Equity"]},"valuesColumn":{"name":"VALUE","type":"DOUBLE","rank":0,"values":[229.33,254.49,"NaN"]},"secondaryColumns":[{"name":"DATE","type":"DATE","rank":0,"values":["2024-12-17T00:00:00Z","2024-12-17T00:00:00Z",null],"defaultDate":true},{"name":"CURRENCY","type":"STRING","rank":0,"values":["USD","USD",null]}],"partialErrorMap":{"errorIterator":null},"responseExceptions":[],"transparency":null,"forUniverse":true,"bqlResponseInfo":null,"defaultDateColumnName":null,"itemPreviewStatistics":null,"indexView":null}},"ordering":["px_last"],"responseExceptions":[],"responseTiming":null,"dotString":null,"versionInfo":{"version":"1.258","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]},"screenCounts":null,"payloadId":null} diff --git a/inst/tinytest/bql/response_string_na.json b/inst/tinytest/bql/response_string_na.json new file mode 100644 index 00000000..b08bf962 --- /dev/null +++ b/inst/tinytest/bql/response_string_na.json @@ -0,0 +1 @@ +{"results":{"ticker":{"name":"ticker","offsets":[0],"namespace":"DATAITEM_DEFAULT","source":"CR","idColumn":{"name":"ID","type":"STRING","rank":0,"values":["IBM US Equity","NA US Equity","AAPL US Equity"]},"valuesColumn":{"name":"VALUE","type":"STRING","rank":0,"values":["IBM","NA","AAPL"]},"secondaryColumns":[],"responseExceptions":[]}},"ordering":["ticker"],"responseExceptions":[],"versionInfo":{"version":"1.258","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]}} diff --git a/inst/tinytest/bql/response_syntax_error.json b/inst/tinytest/bql/response_syntax_error.json new file mode 100644 index 00000000..3620d6a9 --- /dev/null +++ b/inst/tinytest/bql/response_syntax_error.json @@ -0,0 +1 @@ +{"results":null,"ordering":null,"responseExceptions":[{"message":"Error: Unable to parse request at 'get(px_lastfor'.","type":"PARTIAL","internalMessage":"Error: Unable to parse request at 'get(px_lastfor'.","messageCategory":"BQL_SYNTAX_ERROR","messageSubcategory":"NA_SUBCATEGORY","level":0,"nodeName":null,"uniqueException":false,"messageKey":"PARSER_UNABLE"}],"responseTiming":null,"dotString":null,"versionInfo":{"version":"1.258","responseSchemaVersion":"1.0"},"clientContext":{"appName":"EXCEL","clientRequestId":"00000000-0000-0000-0000-000000000000","timestamp":null,"extraMarkers":[]},"screenCounts":null,"payloadId":null} diff --git a/inst/tinytest/test_bql.R b/inst/tinytest/test_bql.R new file mode 100644 index 00000000..c2d6608b --- /dev/null +++ b/inst/tinytest/test_bql.R @@ -0,0 +1,112 @@ + +# Copyright (C) 2025 Dirk Eddelbuettel, Whit Armstrong and John Laing +# +# This file is part of Rblpapi. +# +# Rblpapi is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# Rblpapi is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rblpapi. If not, see . + +library(tinytest) + +if (!requireNamespace("jsonlite", quietly=TRUE)) exit_file("Skipping as 'jsonlite' is missing") + +library(Rblpapi) + +.readFixture <- function(file) { + paste(readLines(file.path("bql", file), warn=FALSE), collapse="\n") +} + +## -- offline parsing tests (no Bloomberg connection required) -------------- + +## single-item query: one data.frame with declared column types +res <- Rblpapi:::.bqlParse(.readFixture("response_px_last.json")) +expect_true(inherits(res, "data.frame"), info = "single item simplifies to data.frame") +expect_equal(dim(res), c(3L, 4L), info = "three rows, four columns") +expect_equal(colnames(res), c("ID", "px_last", "DATE", "CURRENCY"), info = "column names") +expect_equal(unname(sapply(res, class)), c("character", "numeric", "Date", "character"), + info = "column types follow declared JSON types") +expect_equal(res$px_last[1:2], c(229.33, 254.49), info = "numeric values") +expect_true(is.na(res$px_last[3]), info = "string 'NaN' becomes NA") +expect_true(is.na(res$DATE[3]) && is.na(res$CURRENCY[3]), info = "JSON null becomes NA") +expect_equal(res$DATE[1], as.Date("2024-12-17"), info = "DATE conversion") + +## simplify=FALSE keeps the list shape +res <- Rblpapi:::.bqlParse(.readFixture("response_px_last.json"), simplify=FALSE) +expect_true(is.list(res) && length(res) == 1L && names(res) == "px_last", + info = "simplify=FALSE returns named list") + +## multi-item query: one data.frame per 'get' item +res <- Rblpapi:::.bqlParse(.readFixture("response_multi_item.json")) +expect_true(is.list(res) && !inherits(res, "data.frame"), info = "multi item returns list") +expect_equal(names(res), c("name", "pe_ratio"), info = "list named by data item") +expect_equal(colnames(res$name), c("ID", "name"), info = "no secondary columns") +expect_equal(colnames(res$pe_ratio), + c("ID", "pe_ratio", "AS_OF_DATE", "PERIOD_END_DATE", "REVISION_COUNT"), + info = "secondary columns appended") +expect_equal(class(res$pe_ratio$REVISION_COUNT), "integer", info = "INT maps to integer") +expect_equal(res$name$name[2], "Apple Inc", info = "string values") + +## responses above 4 MiB arrive as fragments of one document, cut mid-token, +## and must give the same result as the document delivered in one message +doc <- .readFixture("response_px_last.json") +cut <- nchar(doc) %/% 3L +fragments <- substring(doc, c(1L, cut + 1L, 2L * cut + 1L), c(cut, 2L * cut, nchar(doc))) +expect_equal(Rblpapi:::.bqlParse(fragments), Rblpapi:::.bqlParse(doc), + info = "fragmented response parses like the joined document") + +## BQL errors surface as R errors +expect_error(Rblpapi:::.bqlParse(.readFixture("response_syntax_error.json")), + pattern = "Unable to parse request", info = "responseExceptions raise") + +## literal "NA" strings in STRING columns are preserved, not turned into NA +res <- Rblpapi:::.bqlParse(.readFixture("response_string_na.json")) +expect_equal(res$ticker[2], "NA", info = "literal 'NA' string value preserved") +expect_false(anyNA(res$ticker), info = "no spurious NAs in string column") + +## item-level responseExceptions surface as warnings, data is kept +expect_warning(res <- Rblpapi:::.bqlParse(.readFixture("response_item_error.json")), + pattern = "Insufficient data", info = "item-level exceptions warn") +expect_equal(nrow(res), 1L, info = "partial data still returned") + +## grouped aggregation, e.g. let(#mv=sum(group(amt_outstanding(), +## by=[year(maturity()), industry_sector()]));): the ID column holds +## composite group labels, year() yields an INT column, and ORIG_IDS is +## null for multi-security groups but set for single-security groups +## (synthetic values; structure verified against a live response) +res <- Rblpapi:::.bqlParse(.readFixture("response_grouped.json")) +expect_equal(dim(res), c(6L, 8L), info = "grouped: dimensions") +expect_equal(colnames(res), + c("ID", "#mv", "CURRENCY_OF_ISSUE", "MULTIPLIER", "CURRENCY", + "ORIG_IDS", "YEAR(MATURITY())", "INDUSTRY_SECTOR()"), + info = "grouped: column names") +expect_equal(unname(sapply(res, function(x) class(x)[1])), + c("character", "numeric", "character", "numeric", "character", + "character", "integer", "character"), + info = "grouped: column types incl. INT from year()") +expect_equal(res$ID[1], "2027.0:Technology", info = "grouped: composite group id") +expect_equal(res[["#mv"]][1], 1500000000, info = "grouped: aggregated value") +expect_equal(res[["YEAR(MATURITY())"]][1], 2027L, info = "grouped: integer year") +expect_true(anyNA(res$ORIG_IDS) && !all(is.na(res$ORIG_IDS)), + info = "grouped: ORIG_IDS null for groups, set for singletons") +expect_equal(res$CURRENCY_OF_ISSUE[1], "USD", + info = "grouped: undeclared types like ENUM fall back to character") + +## -- live test (requires a Bloomberg connection) ---------------------------- + +.runThisTest <- Sys.getenv("RunRblpapiUnitTests") == "yes" +if (!.runThisTest) exit_file("Skipping live BQL test") + +res <- bql("get(px_last) for(['IBM US Equity', 'AAPL US Equity'])") +expect_true(inherits(res, "data.frame"), info = "live query returns data.frame") +expect_equal(nrow(res), 2L, info = "one row per security") +expect_true(is.numeric(res$px_last), info = "px_last is numeric") diff --git a/man/bql.Rd b/man/bql.Rd new file mode 100644 index 00000000..6c6eb860 --- /dev/null +++ b/man/bql.Rd @@ -0,0 +1,69 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/bql.R +\name{bql} +\alias{bql} +\title{Run 'Bloomberg Query Language' (BQL) Queries} +\usage{ +bql(expression, parse = TRUE, simplify = TRUE, verbose = FALSE, + con = defaultConnection()) +} +\arguments{ +\item{expression}{A character string with the BQL query, e.g. +\code{"get(px_last) for(['IBM US Equity'])"}.} + +\item{parse}{A boolean indicating whether the JSON response should +be parsed into \code{data.frame} objects (requires the +\CRANpkg{jsonlite} package), defaults to \sQuote{TRUE}. If +\sQuote{FALSE} the raw JSON string is returned.} + +\item{simplify}{A boolean indicating whether a query returning a +single data item should be returned directly as a \code{data.frame} +instead of a list of length one, defaults to \sQuote{TRUE}.} + +\item{verbose}{A boolean indicating whether verbose operation is +desired, defaults to \sQuote{FALSE}.} + +\item{con}{A connection object as created by a \code{blpConnect} +call, and retrieved via the internal function +\code{defaultConnection}.} +} +\value{ +If \code{parse} is \sQuote{TRUE}, a named list of +\code{data.frame} objects, one per data item in the query's +\code{get()} clause (or a single \code{data.frame} if +\code{simplify} is \sQuote{TRUE} and only one item was queried). +Each \code{data.frame} has an \sQuote{ID} column, a value column +named after the data item, and any secondary columns (such as +\sQuote{DATE} or \sQuote{CURRENCY}) the service returned. If +\code{parse} is \sQuote{FALSE}, a character string with the JSON +document. +} +\description{ +This function uses the Bloomberg API to execute 'BQL' (Bloomberg +Query Language) queries via the \sQuote{//blp/bqlsvc} service -- +the same service used by the Excel \code{=BQL()} function. +} +\details{ +The service returns a single JSON document. Each queried data +item is self-describing: every column carries a declared type +(\sQuote{STRING}, \sQuote{DOUBLE}, \sQuote{INT}, \sQuote{DATE}, +\sQuote{DATETIME}, \sQuote{BOOLEAN}) which is used to construct +properly-typed \code{data.frame} columns. Parsing requires the +\CRANpkg{jsonlite} package; set \code{parse=FALSE} to obtain the +raw JSON string instead, e.g. for queries whose shape the +parser does not handle. + +Note that \sQuote{//blp/bqlsvc} is not part of the officially +documented public API; it is the service behind the Excel BQL +add-in and may change without notice. +} +\examples{ +\dontrun{ +con <- blpConnect() +bql("get(px_last) for(['IBM US Equity', 'AAPL US Equity'])") +bql("get(px_last, name) for(members('INDU Index'))", simplify=FALSE) +} +} +\author{ +Alexander Kammerer and Dirk Eddelbuettel +} diff --git a/src/RcppExports.cpp b/src/RcppExports.cpp index 3f40868b..b9cac11d 100644 --- a/src/RcppExports.cpp +++ b/src/RcppExports.cpp @@ -158,6 +158,19 @@ BEGIN_RCPP return rcpp_result_gen; END_RCPP } +// bql_Impl +Rcpp::CharacterVector bql_Impl(SEXP con, std::string expression, bool verbose); +RcppExport SEXP _Rblpapi_bql_Impl(SEXP conSEXP, SEXP expressionSEXP, SEXP verboseSEXP) { +BEGIN_RCPP + Rcpp::RObject rcpp_result_gen; + Rcpp::RNGScope rcpp_rngScope_gen; + Rcpp::traits::input_parameter< SEXP >::type con(conSEXP); + Rcpp::traits::input_parameter< std::string >::type expression(expressionSEXP); + Rcpp::traits::input_parameter< bool >::type verbose(verboseSEXP); + rcpp_result_gen = Rcpp::wrap(bql_Impl(con, expression, verbose)); + return rcpp_result_gen; +END_RCPP +} // bsrch_Impl Rcpp::DataFrame bsrch_Impl(SEXP con, std::string domain, std::string limit, bool verbose); RcppExport SEXP _Rblpapi_bsrch_Impl(SEXP conSEXP, SEXP domainSEXP, SEXP limitSEXP, SEXP verboseSEXP) { @@ -275,6 +288,7 @@ static const R_CallMethodDef CallEntries[] = { {"_Rblpapi_getHeaderVersion", (DL_FUNC) &_Rblpapi_getHeaderVersion, 0}, {"_Rblpapi_getRuntimeVersion", (DL_FUNC) &_Rblpapi_getRuntimeVersion, 0}, {"_Rblpapi_haveBlp", (DL_FUNC) &_Rblpapi_haveBlp, 0}, + {"_Rblpapi_bql_Impl", (DL_FUNC) &_Rblpapi_bql_Impl, 3}, {"_Rblpapi_bsrch_Impl", (DL_FUNC) &_Rblpapi_bsrch_Impl, 4}, {"_Rblpapi_fieldSearch_Impl", (DL_FUNC) &_Rblpapi_fieldSearch_Impl, 2}, {"_Rblpapi_getBars_Impl", (DL_FUNC) &_Rblpapi_getBars_Impl, 8}, diff --git a/src/bql.cpp b/src/bql.cpp new file mode 100644 index 00000000..8d83b564 --- /dev/null +++ b/src/bql.cpp @@ -0,0 +1,128 @@ +// +// bql.cpp -- "Bloomberg Query Language" query function for the BLP API +// +// Copyright (C) 2025 Whit Armstrong and Dirk Eddelbuettel and John Laing +// +// This file is part of Rblpapi +// +// Rblpapi is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 2 of the License, or +// (at your option) any later version. +// +// Rblpapi is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Rblpapi. If not, see . + +#if defined(HaveBlp) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace Rcpp; + +using BloombergLP::blpapi::Session; +using BloombergLP::blpapi::Service; +using BloombergLP::blpapi::Request; +using BloombergLP::blpapi::Event; +using BloombergLP::blpapi::Element; +using BloombergLP::blpapi::Message; +using BloombergLP::blpapi::MessageIterator; +using BloombergLP::blpapi::Name; +using BloombergLP::blpapi::NotFoundException; + +// The //blp/bqlsvc service returns each response message as a single +// string-typed element holding a JSON document. Collect those strings; +// parsing is done R-side (see R/bql.R). +void processBqlEvent(Event event, std::vector& res, const bool verbose) { + MessageIterator msgIter(event); + while (msgIter.next()) { + Message msg = msgIter.message(); + if (verbose) msg.print(Rcpp::Rcout); + + Element response = msg.asElement(); + if (response.hasElement(Name{"responseError"})) { + Element err = response.getElement(Name{"responseError"}); + Rcpp::stop("Response error: " + std::string(err.getElementAsString(Name{"message"}))); + } + if (response.datatype() == BLPAPI_DATATYPE_STRING) { + res.push_back(response.getValueAsString()); + } else if (verbose) { + Rcpp::Rcout << "Skipping non-string message of type " + << msg.messageType().string() << std::endl; + } + } +} +#else +#include +#endif + +// [[Rcpp::export]] +Rcpp::CharacterVector bql_Impl(SEXP con, + std::string expression, + bool verbose=false) { +#if defined(HaveBlp) + Session* session = reinterpret_cast(checkExternalPointer(con, "blpapi::Session*")); + + const std::string bqlsvc = "//blp/bqlsvc"; + if (!session->openService(bqlsvc.c_str())) { + Rcpp::stop("Failed to open " + bqlsvc); + } + + Service bqlService = session->getService(bqlsvc.c_str()); + Request request = bqlService.createRequest("sendQuery"); + request.getElement(Name{"expression"}).setValue(expression.c_str()); + // the service expects the same client context the Excel BQL add-in sends + try { + Element clientContext = request.getElement(Name{"clientContext"}); + clientContext.setElement(Name{"appName"}, "EXCEL"); + } catch (NotFoundException& e) { + if (verbose) Rcpp::Rcout << "No 'clientContext' element in request schema" << std::endl; + } + + if (verbose) Rcpp::Rcout << "Sending Request: " << request << std::endl; + session->sendRequest(request); + + std::vector res; + + // Wait for events from Session + bool done = false; + while (!done) { + Event event = session->nextEvent(); + if (event.eventType() == Event::PARTIAL_RESPONSE) { + if (verbose) Rcpp::Rcout << "Processing Partial Response" << std::endl; + processBqlEvent(event, res, verbose); + } else if (event.eventType() == Event::RESPONSE) { + if (verbose) Rcpp::Rcout << "Processing Response" << std::endl; + processBqlEvent(event, res, verbose); + done = true; + } else { + MessageIterator msgIter(event); + while (msgIter.next()) { + Message msg = msgIter.message(); + if (event.eventType() == Event::SESSION_STATUS) { + if (msg.messageType() == "SessionTerminated" || + msg.messageType() == "SessionStartupFailure") { + done = true; + } + } + } + } + } + + return Rcpp::wrap(res); +#else // ie no Blp + return Rcpp::CharacterVector(); +#endif +} From cbf06e161efd25ac0732ea4cc23e4dab9b373197 Mon Sep 17 00:00:00 2001 From: Alexander Kammerer Date: Tue, 1 Sep 2026 16:19:51 +0200 Subject: [PATCH 2/7] Support RcppSimdJson for BQL parsing, and vectorise column conversion Either RcppSimdJson or jsonlite can now parse a BQL response, with RcppSimdJson preferred when both are installed as suggested in #415. Both are asked not to simplify, so they return the same structure and therefore the same result; the option 'Rblpapi.bqlParser' selects one explicitly, which lets the tests exercise every installed parser and assert that they agree. The parser was not the bottleneck though. .bqlColumn converted the values one element at a time with vapply(), and routed every value through character. It now uses lengths() to find the JSON nulls and unlist() to flatten, so a column of JSON numbers stays numeric, and it converts only the distinct strings of a DATE or DATETIME column. On a live 1.1 MiB response of 21930 rows this takes parsing from 0.170s to 0.030s with jsonlite and to under 0.005s with RcppSimdJson. Keeping numeric columns out of character also makes them lossless. Bloomberg sends float-derived prices such as 230.66000366210938, which the as.character() round trip truncated to 230.66000366210901. Since unlist() would flatten a nested value instead of failing, the one row per value invariant is now checked explicitly. Co-Authored-By: Claude Opus 5 (1M context) --- DESCRIPTION | 3 +- R/bql.R | 119 ++++++++++++++++----- inst/tinytest/test_bql.R | 220 +++++++++++++++++++++++++-------------- man/bql.Rd | 18 ++-- 4 files changed, 251 insertions(+), 109 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 672a045a..1043f7ad 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -7,7 +7,8 @@ Authors@R: c(person("Whit", "Armstrong", role = "aut"), comment = c(ORCID = "0000-0001-6419-907X")), person("John", "Laing", role = "aut")) Imports: Rcpp (>= 0.11.0), utils -Suggests: xts, zoo, data.table, simplermarkdown, tinytest, jsonlite +Suggests: xts, zoo, data.table, simplermarkdown, tinytest, jsonlite, + RcppSimdJson VignetteBuilder: simplermarkdown LazyLoad: yes LinkingTo: Rcpp, BH diff --git a/R/bql.R b/R/bql.R index c06b5669..d0d37470 100644 --- a/R/bql.R +++ b/R/bql.R @@ -25,10 +25,12 @@ ##' item is self-describing: every column carries a declared type ##' (\sQuote{STRING}, \sQuote{DOUBLE}, \sQuote{INT}, \sQuote{DATE}, ##' \sQuote{DATETIME}, \sQuote{BOOLEAN}) which is used to construct -##' properly-typed \code{data.frame} columns. Parsing requires the -##' \CRANpkg{jsonlite} package; set \code{parse=FALSE} to obtain the -##' raw JSON string instead, e.g. for queries whose shape the -##' parser does not handle. +##' properly-typed \code{data.frame} columns. Parsing requires either +##' the \CRANpkg{RcppSimdJson} or the \CRANpkg{jsonlite} package; +##' \CRANpkg{RcppSimdJson} is preferred when both are installed as it +##' is faster on the large documents BQL can return. Both give the +##' same result. Set \code{parse=FALSE} to obtain the raw JSON string +##' instead, e.g. for queries whose shape the parser does not handle. ##' ##' Note that \sQuote{//blp/bqlsvc} is not part of the officially ##' documented public API; it is the service behind the Excel BQL @@ -38,9 +40,11 @@ ##' @param expression A character string with the BQL query, e.g. ##' \code{"get(px_last) for(['IBM US Equity'])"}. ##' @param parse A boolean indicating whether the JSON response should -##' be parsed into \code{data.frame} objects (requires the -##' \CRANpkg{jsonlite} package), defaults to \sQuote{TRUE}. If -##' \sQuote{FALSE} the raw JSON string is returned. +##' be parsed into \code{data.frame} objects (requires either the +##' \CRANpkg{RcppSimdJson} or the \CRANpkg{jsonlite} package), +##' defaults to \sQuote{TRUE}. If \sQuote{FALSE} the raw JSON string +##' is returned. The option \code{Rblpapi.bqlParser} selects the +##' parser explicitly, e.g. \code{options(Rblpapi.bqlParser="jsonlite")}. ##' @param simplify A boolean indicating whether a query returning a ##' single data item should be returned directly as a \code{data.frame} ##' instead of a list of length one, defaults to \sQuote{TRUE}. @@ -71,22 +75,54 @@ bql <- function(expression, verbose=FALSE, con=defaultConnection()) { + ## resolve the parser before the request so that a missing package does + ## not discard a response which has already been retrieved + parser <- if (parse) .bqlParser() else NULL res <- bql_Impl(con, expression, verbose) if (!parse) return(.bqlJoin(res)) - if (!requireNamespace("jsonlite", quietly=TRUE)) - stop("The 'jsonlite' package is required to parse BQL responses; ", - "install it or call bql(..., parse=FALSE) for the raw JSON.", - call.=FALSE) - .bqlParse(res, simplify=simplify) + .bqlParse(res, simplify=simplify, parser=parser) } ## The service delivers responses larger than 4 MiB in several messages, cutting ## the JSON mid-token: the fragments form one document only once joined .bqlJoin <- function(fragments) paste0(fragments, collapse="") +## Supported JSON parsers, in order of preference +.bqlParsers <- c("RcppSimdJson", "jsonlite") + +## Select the JSON parser: RcppSimdJson is preferred as it is faster on the +## large documents BQL can return, jsonlite is the fallback. The option +## 'Rblpapi.bqlParser' forces one, which also lets the tests exercise both. +.bqlParser <- function() { + want <- match.arg(getOption("Rblpapi.bqlParser", .bqlParsers), + .bqlParsers, several.ok=TRUE) + for (p in want) if (requireNamespace(p, quietly=TRUE)) return(p) + stop("Parsing BQL responses requires the '", + paste(.bqlParsers, collapse="' or '"), "' package; install one ", + "of them or call bql(..., parse=FALSE) for the raw JSON.", + call.=FALSE) +} + +## Parse one JSON document into nested lists. Both parsers are asked not to +## simplify at all so that they return the very same structure: the typing is +## done from the declared BQL column types in .bqlColumn. The two 'empty' +## arguments make RcppSimdJson agree with jsonlite on '[]' and '{}', which it +## maps to NULL by default. +.bqlFromJSON <- function(txt, parser=.bqlParser()) { + switch(parser, + "RcppSimdJson" = + RcppSimdJson::fparse(txt, + max_simplify_lvl="list", + empty_array=list(), + empty_object=structure(list(), + names=character())), + "jsonlite" = + jsonlite::fromJSON(txt, simplifyVector=FALSE)) +} + ## Parse a raw BQL JSON response into a named list of data.frames -.bqlParse <- function(json, simplify=TRUE) { - parsed <- jsonlite::fromJSON(.bqlJoin(json), simplifyVector=FALSE) +.bqlParse <- function(json, simplify=TRUE, parser=.bqlParser()) { + parsed <- .bqlFromJSON(.bqlJoin(json), parser) .bqlCheckExceptions(parsed) tables <- list() for (item in parsed[["results"]]) { @@ -147,21 +183,54 @@ bql <- function(expression, } ## Convert a BQL column (list with 'type' and 'values') to a typed R vector. +## The values arrive as a list of scalars, one element per row. They are +## flattened with vectorised primitives rather than one element at a time: +## 'lengths()' finds the JSON nulls without a call per element, and unlist() +## does the rest in one step. A column of plain JSON numbers therefore stays +## numeric all the way and is never turned into character, which is both +## faster and lossless. +## ## JSON null maps to NA for every type; the string placeholders "NaN" and ## "NA" additionally map to NA for numeric columns only, as string columns ## may legitimately contain them (e.g. the ticker of 'NA US Equity'). .bqlColumn <- function(col) { - values <- col[["values"]] type <- if (is.null(col[["type"]])) "STRING" else col[["type"]] - values <- vapply(values, function(v) { - if (is.null(v)) NA_character_ else as.character(v) - }, character(1)) - numericNA <- function(v) { v[v %in% c("NaN", "NA", "")] <- NA_character_; v } + values <- col[["values"]] + if (!length(values)) values <- list() + n <- length(values) + values[lengths(values) == 0L] <- NA + values <- unlist(values, use.names=FALSE) + ## unlist() flattens a nested value instead of failing, unlike the + ## vapply() this replaces, so guard the one row per value invariant + if (length(values) != n) + stop("BQL column '", .bqlColName(col, "?"), + "' has non-scalar values", call.=FALSE) switch(type, - "DOUBLE" = as.numeric(numericNA(values)), - "INT" = as.integer(numericNA(values)), - "BOOLEAN" = as.logical(toupper(values)), - "DATE" = as.Date(substr(values, 1L, 10L)), - "DATETIME" = as.POSIXct(values, format="%Y-%m-%dT%H:%M:%OS", tz="UTC"), - values) + "DOUBLE" = as.numeric(.bqlNumericNA(values)), + "INT" = as.integer(.bqlNumericNA(values)), + "BOOLEAN" = if (is.logical(values)) values + else as.logical(toupper(.bqlChar(values))), + "DATE" = .bqlByUnique(substr(.bqlChar(values), 1L, 10L), as.Date), + "DATETIME" = .bqlByUnique(.bqlChar(values), function(u) + as.POSIXct(u, format="%Y-%m-%dT%H:%M:%OS", tz="UTC")), + .bqlChar(values)) +} + +## A column of only nulls flattens to a logical vector, so the character types +## still need the conversion; for an actual character vector this is a no-op +.bqlChar <- function(v) if (is.character(v)) v else as.character(v) + +## Only a character column can hold the "NaN" and "NA" placeholders, and +## testing a numeric vector against them would convert it to character again +.bqlNumericNA <- function(v) { + if (is.character(v)) v[v %in% c("NaN", "NA", "")] <- NA_character_ + v +} + +## Parsing a date string costs far more per value than a hash lookup, and BQL +## date columns repeat heavily (one date per period, the same date for many +## securities), so convert only the distinct strings +.bqlByUnique <- function(v, fun) { + u <- unique(v) + fun(u)[match(v, u)] } diff --git a/inst/tinytest/test_bql.R b/inst/tinytest/test_bql.R index c2d6608b..00f19701 100644 --- a/inst/tinytest/test_bql.R +++ b/inst/tinytest/test_bql.R @@ -1,4 +1,3 @@ - # Copyright (C) 2025 Dirk Eddelbuettel, Whit Armstrong and John Laing # # This file is part of Rblpapi. @@ -17,89 +16,158 @@ # along with Rblpapi. If not, see . library(tinytest) - -if (!requireNamespace("jsonlite", quietly=TRUE)) exit_file("Skipping as 'jsonlite' is missing") - library(Rblpapi) +## every JSON parser which is installed is tested, and all of them must give +## the very same result +.parsers <- Filter(function(p) requireNamespace(p, quietly=TRUE), + Rblpapi:::.bqlParsers) +if (length(.parsers) == 0L) + exit_file("Skipping as no JSON parser is available") + .readFixture <- function(file) { paste(readLines(file.path("bql", file), warn=FALSE), collapse="\n") } +.parse <- function(file, ...) Rblpapi:::.bqlParse(.readFixture(file), ...) ## -- offline parsing tests (no Bloomberg connection required) -------------- -## single-item query: one data.frame with declared column types -res <- Rblpapi:::.bqlParse(.readFixture("response_px_last.json")) -expect_true(inherits(res, "data.frame"), info = "single item simplifies to data.frame") -expect_equal(dim(res), c(3L, 4L), info = "three rows, four columns") -expect_equal(colnames(res), c("ID", "px_last", "DATE", "CURRENCY"), info = "column names") -expect_equal(unname(sapply(res, class)), c("character", "numeric", "Date", "character"), - info = "column types follow declared JSON types") -expect_equal(res$px_last[1:2], c(229.33, 254.49), info = "numeric values") -expect_true(is.na(res$px_last[3]), info = "string 'NaN' becomes NA") -expect_true(is.na(res$DATE[3]) && is.na(res$CURRENCY[3]), info = "JSON null becomes NA") -expect_equal(res$DATE[1], as.Date("2024-12-17"), info = "DATE conversion") - -## simplify=FALSE keeps the list shape -res <- Rblpapi:::.bqlParse(.readFixture("response_px_last.json"), simplify=FALSE) -expect_true(is.list(res) && length(res) == 1L && names(res) == "px_last", - info = "simplify=FALSE returns named list") - -## multi-item query: one data.frame per 'get' item -res <- Rblpapi:::.bqlParse(.readFixture("response_multi_item.json")) -expect_true(is.list(res) && !inherits(res, "data.frame"), info = "multi item returns list") -expect_equal(names(res), c("name", "pe_ratio"), info = "list named by data item") -expect_equal(colnames(res$name), c("ID", "name"), info = "no secondary columns") -expect_equal(colnames(res$pe_ratio), - c("ID", "pe_ratio", "AS_OF_DATE", "PERIOD_END_DATE", "REVISION_COUNT"), - info = "secondary columns appended") -expect_equal(class(res$pe_ratio$REVISION_COUNT), "integer", info = "INT maps to integer") -expect_equal(res$name$name[2], "Apple Inc", info = "string values") - -## responses above 4 MiB arrive as fragments of one document, cut mid-token, -## and must give the same result as the document delivered in one message -doc <- .readFixture("response_px_last.json") -cut <- nchar(doc) %/% 3L -fragments <- substring(doc, c(1L, cut + 1L, 2L * cut + 1L), c(cut, 2L * cut, nchar(doc))) -expect_equal(Rblpapi:::.bqlParse(fragments), Rblpapi:::.bqlParse(doc), - info = "fragmented response parses like the joined document") - -## BQL errors surface as R errors -expect_error(Rblpapi:::.bqlParse(.readFixture("response_syntax_error.json")), - pattern = "Unable to parse request", info = "responseExceptions raise") - -## literal "NA" strings in STRING columns are preserved, not turned into NA -res <- Rblpapi:::.bqlParse(.readFixture("response_string_na.json")) -expect_equal(res$ticker[2], "NA", info = "literal 'NA' string value preserved") -expect_false(anyNA(res$ticker), info = "no spurious NAs in string column") - -## item-level responseExceptions surface as warnings, data is kept -expect_warning(res <- Rblpapi:::.bqlParse(.readFixture("response_item_error.json")), - pattern = "Insufficient data", info = "item-level exceptions warn") -expect_equal(nrow(res), 1L, info = "partial data still returned") - -## grouped aggregation, e.g. let(#mv=sum(group(amt_outstanding(), -## by=[year(maturity()), industry_sector()]));): the ID column holds -## composite group labels, year() yields an INT column, and ORIG_IDS is -## null for multi-security groups but set for single-security groups -## (synthetic values; structure verified against a live response) -res <- Rblpapi:::.bqlParse(.readFixture("response_grouped.json")) -expect_equal(dim(res), c(6L, 8L), info = "grouped: dimensions") -expect_equal(colnames(res), - c("ID", "#mv", "CURRENCY_OF_ISSUE", "MULTIPLIER", "CURRENCY", - "ORIG_IDS", "YEAR(MATURITY())", "INDUSTRY_SECTOR()"), - info = "grouped: column names") -expect_equal(unname(sapply(res, function(x) class(x)[1])), - c("character", "numeric", "character", "numeric", "character", - "character", "integer", "character"), - info = "grouped: column types incl. INT from year()") -expect_equal(res$ID[1], "2027.0:Technology", info = "grouped: composite group id") -expect_equal(res[["#mv"]][1], 1500000000, info = "grouped: aggregated value") -expect_equal(res[["YEAR(MATURITY())"]][1], 2027L, info = "grouped: integer year") -expect_true(anyNA(res$ORIG_IDS) && !all(is.na(res$ORIG_IDS)), - info = "grouped: ORIG_IDS null for groups, set for singletons") -expect_equal(res$CURRENCY_OF_ISSUE[1], "USD", - info = "grouped: undeclared types like ENUM fall back to character") +for (.p in .parsers) { + + .with <- function(txt) paste0(txt, " [", .p, "]") + + ## single-item query: one data.frame with declared column types + res <- .parse("response_px_last.json", parser=.p) + expect_true(inherits(res, "data.frame"), info = .with("single item simplifies to data.frame")) + expect_equal(dim(res), c(3L, 4L), info = .with("three rows, four columns")) + expect_equal(colnames(res), c("ID", "px_last", "DATE", "CURRENCY"), info = .with("column names")) + expect_equal(unname(sapply(res, class)), c("character", "numeric", "Date", "character"), + info = .with("column types follow declared JSON types")) + expect_equal(res$px_last[1:2], c(229.33, 254.49), info = .with("numeric values")) + expect_true(is.na(res$px_last[3]), info = .with("string 'NaN' becomes NA")) + expect_true(is.na(res$DATE[3]) && is.na(res$CURRENCY[3]), info = .with("JSON null becomes NA")) + expect_equal(res$DATE[1], as.Date("2024-12-17"), info = .with("DATE conversion")) + + ## simplify=FALSE keeps the list shape + res <- .parse("response_px_last.json", simplify=FALSE, parser=.p) + expect_true(is.list(res) && length(res) == 1L && names(res) == "px_last", + info = .with("simplify=FALSE returns named list")) + + ## multi-item query: one data.frame per 'get' item + res <- .parse("response_multi_item.json", parser=.p) + expect_true(is.list(res) && !inherits(res, "data.frame"), info = .with("multi item returns list")) + expect_equal(names(res), c("name", "pe_ratio"), info = .with("list named by data item")) + expect_equal(colnames(res$name), c("ID", "name"), info = .with("no secondary columns")) + expect_equal(colnames(res$pe_ratio), + c("ID", "pe_ratio", "AS_OF_DATE", "PERIOD_END_DATE", "REVISION_COUNT"), + info = .with("secondary columns appended")) + expect_equal(class(res$pe_ratio$REVISION_COUNT), "integer", info = .with("INT maps to integer")) + expect_equal(res$name$name[2], "Apple Inc", info = .with("string values")) + + ## responses above 4 MiB arrive as fragments of one document, cut mid-token, + ## and must give the same result as the document delivered in one message + doc <- .readFixture("response_px_last.json") + cut <- nchar(doc) %/% 3L + fragments <- substring(doc, c(1L, cut + 1L, 2L * cut + 1L), c(cut, 2L * cut, nchar(doc))) + expect_equal(Rblpapi:::.bqlParse(fragments, parser=.p), + Rblpapi:::.bqlParse(doc, parser=.p), + info = .with("fragmented response parses like the joined document")) + + ## BQL errors surface as R errors + expect_error(.parse("response_syntax_error.json", parser=.p), + pattern = "Unable to parse request", info = .with("responseExceptions raise")) + + ## literal "NA" strings in STRING columns are preserved, not turned into NA + res <- .parse("response_string_na.json", parser=.p) + expect_equal(res$ticker[2], "NA", info = .with("literal 'NA' string value preserved")) + expect_false(anyNA(res$ticker), info = .with("no spurious NAs in string column")) + + ## item-level responseExceptions surface as warnings, data is kept + expect_warning(res <- .parse("response_item_error.json", parser=.p), + pattern = "Insufficient data", info = .with("item-level exceptions warn")) + expect_equal(nrow(res), 1L, info = .with("partial data still returned")) + + ## grouped aggregation, e.g. let(#mv=sum(group(amt_outstanding(), + ## by=[year(maturity()), industry_sector()]));): the ID column holds + ## composite group labels, year() yields an INT column, and ORIG_IDS is + ## null for multi-security groups but set for single-security groups + ## (synthetic values; structure verified against a live response) + res <- .parse("response_grouped.json", parser=.p) + expect_equal(dim(res), c(6L, 8L), info = .with("grouped: dimensions")) + expect_equal(colnames(res), + c("ID", "#mv", "CURRENCY_OF_ISSUE", "MULTIPLIER", "CURRENCY", + "ORIG_IDS", "YEAR(MATURITY())", "INDUSTRY_SECTOR()"), + info = .with("grouped: column names")) + expect_equal(unname(sapply(res, function(x) class(x)[1])), + c("character", "numeric", "character", "numeric", "character", + "character", "integer", "character"), + info = .with("grouped: column types incl. INT from year()")) + expect_equal(res$ID[1], "2027.0:Technology", info = .with("grouped: composite group id")) + expect_equal(res[["#mv"]][1], 1500000000, info = .with("grouped: aggregated value")) + expect_equal(res[["YEAR(MATURITY())"]][1], 2027L, info = .with("grouped: integer year")) + expect_true(anyNA(res$ORIG_IDS) && !all(is.na(res$ORIG_IDS)), + info = .with("grouped: ORIG_IDS null for groups, set for singletons")) + expect_equal(res$CURRENCY_OF_ISSUE[1], "USD", + info = .with("grouped: undeclared types like ENUM fall back to character")) +} + +## -- the parsers must agree exactly ---------------------------------------- + +if (length(.parsers) > 1L) { + for (f in list.files("bql", pattern="[.]json$")) { + out <- lapply(.parsers, function(p) + tryCatch(suppressWarnings(.parse(f, parser=p)), + error=function(e) conditionMessage(e))) + expect_true(all(vapply(out[-1], identical, logical(1), out[[1]])), + info = paste("all parsers agree on", f)) + } +} + +## -- parser selection ------------------------------------------------------ + +expect_true(Rblpapi:::.bqlParser() %in% .parsers, info = "default parser is installed") +local({ + old <- options(Rblpapi.bqlParser=.parsers[length(.parsers)]) + on.exit(options(old)) + expect_equal(Rblpapi:::.bqlParser(), .parsers[length(.parsers)], + info = "option selects the parser") +}) +local({ + old <- options(Rblpapi.bqlParser="notAParser") + on.exit(options(old)) + expect_error(Rblpapi:::.bqlParser(), info = "unknown parser name is rejected") +}) + +## -- column conversion ----------------------------------------------------- + +.col <- function(...) Rblpapi:::.bqlColumn(list(...)) + +## a column of only nulls must keep the type its declaration implies +expect_equal(.col(type="STRING", values=list(NULL, NULL)), c(NA_character_, NA_character_), + info = "all-null STRING column stays character") +expect_equal(.col(type="DATE", values=list(NULL)), as.Date(NA), + info = "all-null DATE column stays Date") +expect_equal(.col(type="DOUBLE", values=list(NULL)), NA_real_, + info = "all-null DOUBLE column stays numeric") + +## an empty or absent 'values' key gives a zero-length column, not an error +expect_equal(.col(type="DOUBLE", values=list()), numeric(0), info = "empty column") +expect_equal(.col(type="STRING"), character(0), info = "absent values key") + +## JSON booleans and their string spellings both convert +expect_equal(.col(type="BOOLEAN", values=list(TRUE, FALSE, NULL)), c(TRUE, FALSE, NA), + info = "JSON booleans") +expect_equal(.col(type="BOOLEAN", values=list("true", "FALSE", NULL)), c(TRUE, FALSE, NA), + info = "boolean strings") + +## numeric columns keep full double precision: the values do not go through +## character, which would keep only 15 significant digits +expect_identical(.col(type="DOUBLE", values=list(pi, 1/3)), c(pi, 1/3), + info = "DOUBLE column is lossless") + +## a nested value would silently shift the rows of a column, so it must fail +expect_error(.col(name="X", type="STRING", values=list("a", list("b", "c"))), + pattern = "non-scalar", info = "non-scalar values are rejected") ## -- live test (requires a Bloomberg connection) ---------------------------- diff --git a/man/bql.Rd b/man/bql.Rd index 6c6eb860..f2a5cdf0 100644 --- a/man/bql.Rd +++ b/man/bql.Rd @@ -12,9 +12,11 @@ bql(expression, parse = TRUE, simplify = TRUE, verbose = FALSE, \code{"get(px_last) for(['IBM US Equity'])"}.} \item{parse}{A boolean indicating whether the JSON response should -be parsed into \code{data.frame} objects (requires the -\CRANpkg{jsonlite} package), defaults to \sQuote{TRUE}. If -\sQuote{FALSE} the raw JSON string is returned.} +be parsed into \code{data.frame} objects (requires either the +\CRANpkg{RcppSimdJson} or the \CRANpkg{jsonlite} package), +defaults to \sQuote{TRUE}. If \sQuote{FALSE} the raw JSON string +is returned. The option \code{Rblpapi.bqlParser} selects the +parser explicitly, e.g. \code{options(Rblpapi.bqlParser="jsonlite")}.} \item{simplify}{A boolean indicating whether a query returning a single data item should be returned directly as a \code{data.frame} @@ -48,10 +50,12 @@ The service returns a single JSON document. Each queried data item is self-describing: every column carries a declared type (\sQuote{STRING}, \sQuote{DOUBLE}, \sQuote{INT}, \sQuote{DATE}, \sQuote{DATETIME}, \sQuote{BOOLEAN}) which is used to construct -properly-typed \code{data.frame} columns. Parsing requires the -\CRANpkg{jsonlite} package; set \code{parse=FALSE} to obtain the -raw JSON string instead, e.g. for queries whose shape the -parser does not handle. +properly-typed \code{data.frame} columns. Parsing requires either +the \CRANpkg{RcppSimdJson} or the \CRANpkg{jsonlite} package; +\CRANpkg{RcppSimdJson} is preferred when both are installed as it +is faster on the large documents BQL can return. Both give the +same result. Set \code{parse=FALSE} to obtain the raw JSON string +instead, e.g. for queries whose shape the parser does not handle. Note that \sQuote{//blp/bqlsvc} is not part of the officially documented public API; it is the service behind the Excel BQL From 26ea347b96c048f5455f4f48485b9d00f84aab66 Mon Sep 17 00:00:00 2001 From: Alexander Kammerer Date: Tue, 1 Sep 2026 16:31:10 +0200 Subject: [PATCH 3/7] Test fragmented BQL responses with small byte-level chunks A response above 4 MiB arrives as several messages, cut at a byte boundary in the middle of a token, and only the joined fragments form one document. Reaching that path with a real query needs megabytes of data, so the fixtures are instead chunked into pieces far smaller than 4 MiB, which reproduces the same condition. The chunking is on bytes rather than on characters, because that is what the service does. A boundary can therefore fall inside a multi-byte UTF-8 character, which leaves that one fragment invalid UTF-8 on its own; a case is included for this, with the characters written as escapes so the file stays ASCII. Verified against a live 5.56 MiB response of 111792 rows, which the service sent as 4.00 MiB plus 1.56 MiB. Its first fragment alone gives the 'parse error: premature EOF' originally reported. Breaking .bqlJoin in four ways (first fragment only, extra separator, reversed order, last fragment dropped) makes these tests fail, so they do test the joining rather than merely pass alongside it. Co-Authored-By: Claude Opus 5 (1M context) --- inst/tinytest/test_bql.R | 56 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/inst/tinytest/test_bql.R b/inst/tinytest/test_bql.R index 00f19701..64628105 100644 --- a/inst/tinytest/test_bql.R +++ b/inst/tinytest/test_bql.R @@ -111,6 +111,62 @@ for (.p in .parsers) { info = .with("grouped: undeclared types like ENUM fall back to character")) } +## -- fragmented responses ------------------------------------------------- + +## The C++ layer returns one string per response message, and the service cuts +## a response above 4 MiB at a byte boundary, in the middle of a token, so the +## fragments form one document only once joined. Chunking a fixture into pieces +## far smaller than 4 MiB reproduces that without needing a 4 MiB response. +## +## The chunking is on bytes, not on characters, because that is what the +## service does: a boundary can fall inside a multi-byte UTF-8 character, which +## leaves that one fragment invalid UTF-8 on its own. +.chunkBytes <- function(txt, n) { + b <- charToRaw(txt) + i <- split(seq_along(b), ceiling(seq_along(b) / n)) + vapply(i, function(k) rawToChar(b[k]), character(1), USE.NAMES = FALSE) +} + +for (.p in .parsers) { + for (f in list.files("bql", pattern = "[.]json$")) { + doc <- .readFixture(f) + ref <- tryCatch(suppressWarnings(Rblpapi:::.bqlParse(doc, parser = .p)), + error = function(e) conditionMessage(e)) + for (n in c(1L, 7L, 64L, 1000L)) { + frags <- .chunkBytes(doc, n) + got <- tryCatch(suppressWarnings(Rblpapi:::.bqlParse(frags, parser = .p)), + error = function(e) conditionMessage(e)) + expect_equal(got, ref, + info = paste0(f, " in ", length(frags), " chunks of ", n, + " bytes [", .p, "]")) + } + } + + ## a single fragment is not a document: the failure the joining prevents + doc <- .readFixture("response_px_last.json") + frags <- .chunkBytes(doc, nchar(doc, type = "bytes") %/% 2L) + expect_true(length(frags) > 1L, info = "the fixture really was split") + expect_error(Rblpapi:::.bqlParse(frags[1], parser = .p), + info = paste0("a lone fragment does not parse [", .p, "]")) + + ## a boundary inside a multi-byte UTF-8 character must still rejoin; the + ## characters are written as escapes so that this file stays ASCII + utf8doc <- paste0('{"results":{"name":{"name":"name","idColumn":{"name":"ID",', + '"type":"STRING","values":["X","Y"]},"valuesColumn":', + '{"name":"VALUE","type":"STRING","values":', + '["Nestl\u00e9 S\u00e9n\u00e9gal","\u00dcbermorgen"]},', + '"secondaryColumns":[]}},"responseExceptions":[]}') + want <- c("Nestl\u00e9 S\u00e9n\u00e9gal", "\u00dcbermorgen") + expect_equal(Rblpapi:::.bqlParse(utf8doc, parser = .p)$name, want, + info = paste0("multi-byte characters read correctly [", .p, "]")) + for (n in 1L:8L) { + frags <- .chunkBytes(utf8doc, n) + expect_equal(Rblpapi:::.bqlParse(frags, parser = .p)$name, want, + info = paste0("multi-byte characters survive ", n, + "-byte chunking [", .p, "]")) + } +} + ## -- the parsers must agree exactly ---------------------------------------- if (length(.parsers) > 1L) { From c61b6fde7e4987eaa04ca2142e514f291efb6c2b Mon Sep 17 00:00:00 2001 From: Alexander Kammerer Date: Tue, 1 Sep 2026 17:07:46 +0200 Subject: [PATCH 4/7] Act on an independent review of the BQL parser changes Corrects an overstated claim in cbf06e1: keeping a numeric column out of character only preserved the full precision of its values when the array held no "NaN", "NA" or "" placeholder. One such string promoted the whole column to character and the numbers came back from their 15 significant digit form, so precision depended on whether the service happened to send one. The placeholders are now blanked in the list before it is flattened, which keeps the column numeric in either case. Only the placeholders are blanked, as any other string is a number written as a string and still has to be converted. This costs a pass per element, but only for a numeric column which really does contain a placeholder; that case is now on a par with the code before this branch, while a column without one keeps its much larger gain. .bqlNumericNA becomes unreachable and is removed. Also fixed: * .bqlFromJSON had no default branch, so an unrecognised parser name returned NULL and the caller received an empty result rather than a diagnosis. * .bqlParser used match.arg(), which accepted an abbreviation such as "j" and silently dropped an unknown name given beside a known one. The option is now validated, and a failure names only the parser actually asked for instead of both. * The claim that the parsers agree is narrowed to the documents the service returns: they differ on 1e999, on integers above 2^64, on nesting thousands deep and on an embedded NUL. * A dead line, and a comment which described the non-scalar guard as stronger than it is: a value flattening to exactly one element is kept, as it was before. The tests were passing for the wrong reasons in several places, which mutation testing showed. All nine mutants are now caught, where five survived before: * A permuted date column was undetectable, because the only order-sensitive DATE fixture was checked for its column name alone and every other one holds duplicates. Values are asserted now, with columns whose distinct dates are neither sorted nor unique. * The placeholder assertions used expect_equal, and all.equal() treats NaN as equal to NA_real_, so they passed with the placeholder handling removed entirely. They use expect_identical now. * The parser preference took its expectation from .bqlParsers, so it agreed with any order that variable held. The name is written out. * Nothing asserted the parsers return the same intermediate structure, so max_simplify_lvl and the two 'empty' arguments could all be dropped without a failure. * A fixed 1000 byte chunk size left three fixtures in one piece, comparing a document with itself. The sizes derive from each document now, and the split is asserted. Verified against the saved 5.56 MiB two-fragment live response: both parsers identical, non-numeric columns identical to the previous code, and the DOUBLE column now exact rather than differing by 4.3e-15. Co-Authored-By: Claude Opus 5 (1M context) --- R/bql.R | 81 +++++++++++++++++-------- inst/tinytest/test_bql.R | 124 ++++++++++++++++++++++++++++++++++++--- man/bql.Rd | 7 ++- 3 files changed, 176 insertions(+), 36 deletions(-) diff --git a/R/bql.R b/R/bql.R index d0d37470..571a0220 100644 --- a/R/bql.R +++ b/R/bql.R @@ -28,9 +28,10 @@ ##' properly-typed \code{data.frame} columns. Parsing requires either ##' the \CRANpkg{RcppSimdJson} or the \CRANpkg{jsonlite} package; ##' \CRANpkg{RcppSimdJson} is preferred when both are installed as it -##' is faster on the large documents BQL can return. Both give the -##' same result. Set \code{parse=FALSE} to obtain the raw JSON string -##' instead, e.g. for queries whose shape the parser does not handle. +##' is faster on the large documents BQL can return. Both give the same +##' result for the documents the service returns. Set +##' \code{parse=FALSE} to obtain the raw JSON string instead, e.g. for +##' queries whose shape the parser does not handle. ##' ##' Note that \sQuote{//blp/bqlsvc} is not part of the officially ##' documented public API; it is the service behind the Excel BQL @@ -94,12 +95,19 @@ bql <- function(expression, ## large documents BQL can return, jsonlite is the fallback. The option ## 'Rblpapi.bqlParser' forces one, which also lets the tests exercise both. .bqlParser <- function() { - want <- match.arg(getOption("Rblpapi.bqlParser", .bqlParsers), - .bqlParsers, several.ok=TRUE) + ## validated here rather than with match.arg(), which would accept an + ## abbreviation and would silently drop an unknown name given alongside a + ## known one + want <- getOption("Rblpapi.bqlParser", .bqlParsers) + if (!is.character(want) || length(want) == 0L || anyNA(want) || + !all(want %in% .bqlParsers)) + stop("Option 'Rblpapi.bqlParser' must be one or more of ", + paste0("'", .bqlParsers, "'", collapse=", "), call.=FALSE) for (p in want) if (requireNamespace(p, quietly=TRUE)) return(p) - stop("Parsing BQL responses requires the '", - paste(.bqlParsers, collapse="' or '"), "' package; install one ", - "of them or call bql(..., parse=FALSE) for the raw JSON.", + ## name only what was actually asked for, which may be a single parser + stop("Parsing BQL responses requires ", + paste0("'", want, "'", collapse=" or "), + "; install it or call bql(..., parse=FALSE) for the raw JSON.", call.=FALSE) } @@ -117,7 +125,10 @@ bql <- function(expression, empty_object=structure(list(), names=character())), "jsonlite" = - jsonlite::fromJSON(txt, simplifyVector=FALSE)) + jsonlite::fromJSON(txt, simplifyVector=FALSE), + ## without this a wrong name would return NULL, and the caller + ## would see an empty result rather than a diagnosis + stop("Unknown BQL JSON parser '", parser, "'", call.=FALSE)) } ## Parse a raw BQL JSON response into a named list of data.frames @@ -186,28 +197,57 @@ bql <- function(expression, ## The values arrive as a list of scalars, one element per row. They are ## flattened with vectorised primitives rather than one element at a time: ## 'lengths()' finds the JSON nulls without a call per element, and unlist() -## does the rest in one step. A column of plain JSON numbers therefore stays -## numeric all the way and is never turned into character, which is both -## faster and lossless. +## does the rest in one step. ## ## JSON null maps to NA for every type; the string placeholders "NaN" and ## "NA" additionally map to NA for numeric columns only, as string columns ## may legitimately contain them (e.g. the ticker of 'NA US Equity'). +## +## A numeric column stays numeric throughout and so keeps the values exactly +## as the service sent them, rather than losing the last digits to a detour +## through character. Bloomberg sends float-derived prices such as +## 230.66000366210938, which as.character() would truncate to +## 230.66000366210901. +## +## One consequence of letting unlist() pick the type does remain: it coerces a +## logical before a string, so a JSON boolean sharing an array with a JSON +## number becomes 1 or 0 rather than "TRUE" or "FALSE". BQL declares one type +## per column and does not mix the two, and avoiding this would need a call +## per element for every column, which is the cost this function exists to +## avoid. .bqlColumn <- function(col) { type <- if (is.null(col[["type"]])) "STRING" else col[["type"]] values <- col[["values"]] - if (!length(values)) values <- list() n <- length(values) values[lengths(values) == 0L] <- NA values <- unlist(values, use.names=FALSE) - ## unlist() flattens a nested value instead of failing, unlike the - ## vapply() this replaces, so guard the one row per value invariant + ## unlist() flattens a nested value instead of failing, unlike the vapply() + ## this replaces. This catches a value which flattens to more than one + ## element; one which flattens to exactly one is kept, as it was before. if (length(values) != n) stop("BQL column '", .bqlColName(col, "?"), "' has non-scalar values", call.=FALSE) + ## Blank the "NaN", "NA" and "" placeholders which stand for a missing + ## number. Doing it in the list, before flattening, is what lets unlist() + ## keep the column numeric: a single such string would otherwise promote + ## the whole column to character and send every number back through its 15 + ## significant digit form. Only a numeric column is treated this way, as a + ## string column may legitimately hold those spellings. Any other string is + ## a number written as a string, which as.numeric() below still converts. + ## This costs one pass per element, and only for a numeric column which + ## really does contain a placeholder. + if (is.character(values) && type %in% c("DOUBLE", "INT")) { + vals <- col[["values"]] + isph <- vapply(vals, is.character, NA) + isph[isph] <- unlist(vals[isph], use.names=FALSE) %in% c("NaN", "NA", "") + if (any(isph)) { + vals[isph | lengths(vals) == 0L] <- NA + values <- unlist(vals, use.names=FALSE) + } + } switch(type, - "DOUBLE" = as.numeric(.bqlNumericNA(values)), - "INT" = as.integer(.bqlNumericNA(values)), + "DOUBLE" = as.numeric(values), + "INT" = as.integer(values), "BOOLEAN" = if (is.logical(values)) values else as.logical(toupper(.bqlChar(values))), "DATE" = .bqlByUnique(substr(.bqlChar(values), 1L, 10L), as.Date), @@ -220,13 +260,6 @@ bql <- function(expression, ## still need the conversion; for an actual character vector this is a no-op .bqlChar <- function(v) if (is.character(v)) v else as.character(v) -## Only a character column can hold the "NaN" and "NA" placeholders, and -## testing a numeric vector against them would convert it to character again -.bqlNumericNA <- function(v) { - if (is.character(v)) v[v %in% c("NaN", "NA", "")] <- NA_character_ - v -} - ## Parsing a date string costs far more per value than a hash lookup, and BQL ## date columns repeat heavily (one date per period, the same date for many ## securities), so convert only the distinct strings diff --git a/inst/tinytest/test_bql.R b/inst/tinytest/test_bql.R index 64628105..1a9ff393 100644 --- a/inst/tinytest/test_bql.R +++ b/inst/tinytest/test_bql.R @@ -44,7 +44,10 @@ for (.p in .parsers) { expect_equal(unname(sapply(res, class)), c("character", "numeric", "Date", "character"), info = .with("column types follow declared JSON types")) expect_equal(res$px_last[1:2], c(229.33, 254.49), info = .with("numeric values")) - expect_true(is.na(res$px_last[3]), info = .with("string 'NaN' becomes NA")) + ## expect_identical, not expect_equal: all.equal() treats NaN as equal to + ## NA_real_, so expect_equal would pass even if the placeholder handling + ## were removed altogether and as.numeric("NaN") left a NaN behind + expect_identical(res$px_last[3], NA_real_, info = .with("string 'NaN' becomes NA, not NaN")) expect_true(is.na(res$DATE[3]) && is.na(res$CURRENCY[3]), info = .with("JSON null becomes NA")) expect_equal(res$DATE[1], as.Date("2024-12-17"), info = .with("DATE conversion")) @@ -63,6 +66,11 @@ for (.p in .parsers) { info = .with("secondary columns appended")) expect_equal(class(res$pe_ratio$REVISION_COUNT), "integer", info = .with("INT maps to integer")) expect_equal(res$name$name[2], "Apple Inc", info = .with("string values")) + ## a DATE column is converted through unique()/match(), so assert the + ## values and not only the name: these two are deliberately descending, + ## and a column of duplicates could not detect a reordering + expect_equal(res$pe_ratio$PERIOD_END_DATE, as.Date(c("2024-09-30", "2024-09-28")), + info = .with("secondary DATE column keeps the row order")) ## responses above 4 MiB arrive as fragments of one document, cut mid-token, ## and must give the same result as the document delivered in one message @@ -132,8 +140,14 @@ for (.p in .parsers) { doc <- .readFixture(f) ref <- tryCatch(suppressWarnings(Rblpapi:::.bqlParse(doc, parser = .p)), error = function(e) conditionMessage(e)) - for (n in c(1L, 7L, 64L, 1000L)) { + ## the two largest sizes are derived from the document, as a fixed size + ## above the smallest fixture would give one chunk and compare the + ## document with itself + nb <- nchar(doc, type = "bytes") + for (n in unique(c(1L, 7L, 64L, nb %/% 7L, nb %/% 2L))) { frags <- .chunkBytes(doc, n) + expect_true(length(frags) > 1L, + info = paste0(f, " really is split at ", n, " bytes")) got <- tryCatch(suppressWarnings(Rblpapi:::.bqlParse(frags, parser = .p)), error = function(e) conditionMessage(e)) expect_equal(got, ref, @@ -182,18 +196,46 @@ if (length(.parsers) > 1L) { ## -- parser selection ------------------------------------------------------ expect_true(Rblpapi:::.bqlParser() %in% .parsers, info = "default parser is installed") +## the expected name is written out rather than taken from .bqlParsers, which +## would make the assertion agree with any order that variable happened to have +if (all(c("RcppSimdJson", "jsonlite") %in% .parsers)) + expect_equal(Rblpapi:::.bqlParser(), "RcppSimdJson", + info = "RcppSimdJson is preferred when both are installed") local({ old <- options(Rblpapi.bqlParser=.parsers[length(.parsers)]) on.exit(options(old)) expect_equal(Rblpapi:::.bqlParser(), .parsers[length(.parsers)], info = "option selects the parser") }) + +## an unknown parser name must be reported, not treated as "no data" +expect_error(Rblpapi:::.bqlFromJSON("{}", "notAParser"), + pattern = "Unknown BQL JSON parser", + info = "an unknown parser name is an error") + +## the option is validated: no abbreviations, and an unknown name is not +## silently dropped when a known one sits beside it local({ - old <- options(Rblpapi.bqlParser="notAParser") - on.exit(options(old)) - expect_error(Rblpapi:::.bqlParser(), info = "unknown parser name is rejected") + for (bad in list("notAParser", c("notAParser", "jsonlite"), "R", "j", + NA_character_, "", 1L, TRUE, list("jsonlite"))) { + old <- options(Rblpapi.bqlParser = bad) + expect_error(Rblpapi:::.bqlParser(), + info = paste("option rejected:", + paste(deparse(bad), collapse = ""))) + options(old) + } }) +## the parsers must agree on the intermediate structure, not merely on the +## final data.frame: '[]', '{}' and null are where they differ by default, so +## this is what the two 'empty' arguments and max_simplify_lvl="list" buy +if (length(.parsers) > 1L) { + .shapes <- '{"a":[],"b":{},"c":[1,null,"x",true],"d":{"e":[{"f":null}]}}' + .trees <- lapply(.parsers, function(p) Rblpapi:::.bqlFromJSON(.shapes, p)) + expect_true(all(vapply(.trees[-1], identical, logical(1), .trees[[1]])), + info = "parsers agree on the intermediate structure") +} + ## -- column conversion ----------------------------------------------------- .col <- function(...) Rblpapi:::.bqlColumn(list(...)) @@ -209,6 +251,20 @@ expect_equal(.col(type="DOUBLE", values=list(NULL)), NA_real_, ## an empty or absent 'values' key gives a zero-length column, not an error expect_equal(.col(type="DOUBLE", values=list()), numeric(0), info = "empty column") expect_equal(.col(type="STRING"), character(0), info = "absent values key") +expect_equal(.col(type="DATE"), as.Date(character(0)), info = "absent values key, DATE") + +## Every placeholder means NA in a numeric column, and only there. These use +## expect_identical because all.equal() treats NaN as equal to NA_real_, so +## expect_equal could not tell a real NA from the NaN that as.numeric("NaN") +## leaves behind when the placeholder handling is missing. +expect_identical(.col(type="DOUBLE", values=list(1, "NaN", "NA", "", 2)), + c(1, NA, NA, NA, 2), info = "DOUBLE placeholders become NA") +expect_identical(.col(type="INT", values=list(1L, "NaN", "NA", "")), + c(1L, NA, NA, NA), info = "INT placeholders become NA") +expect_false(any(is.nan(.col(type="DOUBLE", values=list(1, "NaN")))), + info = "'NaN' becomes NA rather than NaN") +expect_equal(.col(type="STRING", values=list("NaN", "NA", "")), + c("NaN", "NA", ""), info = "STRING keeps the same spellings verbatim") ## JSON booleans and their string spellings both convert expect_equal(.col(type="BOOLEAN", values=list(TRUE, FALSE, NULL)), c(TRUE, FALSE, NA), @@ -216,15 +272,65 @@ expect_equal(.col(type="BOOLEAN", values=list(TRUE, FALSE, NULL)), c(TRUE, FALSE expect_equal(.col(type="BOOLEAN", values=list("true", "FALSE", NULL)), c(TRUE, FALSE, NA), info = "boolean strings") -## numeric columns keep full double precision: the values do not go through -## character, which would keep only 15 significant digits -expect_identical(.col(type="DOUBLE", values=list(pi, 1/3)), c(pi, 1/3), - info = "DOUBLE column is lossless") +## DATE and DATETIME are converted through unique()/match(), so a column whose +## distinct values are neither sorted nor unique must keep its own row order +.dts <- c("2024-03-05", "2024-01-31", "2024-03-05", "2024-02-29", "2024-01-31") +expect_equal(.col(type="DATE", values=as.list(paste0(.dts, "T00:00:00Z"))), + as.Date(.dts), info = "DATE column keeps the row order") +expect_equal(.col(type="DATE", + values=c(as.list(paste0(.dts[1:2], "T00:00:00Z")), list(NULL), + as.list(paste0(.dts[3:5], "T00:00:00Z")))), + as.Date(c(.dts[1:2], NA, .dts[3:5])), + info = "DATE column with an interleaved null keeps the row order") +.tms <- c("2024-03-05T13:45:30Z", "2024-01-31T09:00:00Z", "2024-03-05T13:45:30Z") +expect_equal(.col(type="DATETIME", values=as.list(.tms)), + as.POSIXct(.tms, format="%Y-%m-%dT%H:%M:%OS", tz="UTC"), + info = "DATETIME column keeps the row order") ## a nested value would silently shift the rows of a column, so it must fail expect_error(.col(name="X", type="STRING", values=list("a", list("b", "c"))), pattern = "non-scalar", info = "non-scalar values are rejected") +## -- double precision through the JSON layer ------------------------------- + +## A DOUBLE column keeps the exact values the document carried, whether or not +## a placeholder string sits beside them: the placeholders are blanked before +## the column is flattened, so it never detours through character. The second +## case is the one that regresses if that step is dropped. +.doubleDoc <- function(vals) + paste0('{"results":{"v":{"name":"v","idColumn":{"name":"ID","type":"STRING",', + '"values":[', paste0('"', seq_along(vals), '"', collapse=","), ']},', + '"valuesColumn":{"name":"VALUE","type":"DOUBLE","values":[', + paste(vals, collapse=","), ']},"secondaryColumns":[]}},', + '"responseExceptions":[]}') + +for (.p in .parsers) { + .exact <- 0.12345678901234568 + res <- Rblpapi:::.bqlParse(.doubleDoc(c("0.12345678901234568", "null")), parser=.p) + expect_identical(res$v[1], .exact, + info = paste0("DOUBLE keeps every digit without a placeholder [", .p, "]")) + expect_true(is.na(res$v[2]), + info = paste0("null in a DOUBLE column is NA [", .p, "]")) + + for (.ph in c('"NaN"', '"NA"', '""')) { + res <- Rblpapi:::.bqlParse(.doubleDoc(c("0.12345678901234568", .ph)), parser=.p) + expect_identical(res$v[1], .exact, + info = paste0("DOUBLE keeps every digit beside a ", .ph, + " placeholder [", .p, "]")) + expect_identical(res$v[2], NA_real_, + info = paste0("the ", .ph, " placeholder itself is NA [", .p, "]")) + } + + ## a real float32-derived price, the case which motivated all of this + res <- Rblpapi:::.bqlParse(.doubleDoc(c("230.66000366210938", '"NaN"')), parser=.p) + expect_identical(res$v[1], 230.66000366210938, + info = paste0("a float-derived price is exact [", .p, "]")) +} + +## a number written as a string is not a placeholder and must still convert +expect_equal(.col(type="DOUBLE", values=list("123.45", 6, "NaN")), + c(123.45, 6, NA), info = "a number sent as a string still converts") + ## -- live test (requires a Bloomberg connection) ---------------------------- .runThisTest <- Sys.getenv("RunRblpapiUnitTests") == "yes" diff --git a/man/bql.Rd b/man/bql.Rd index f2a5cdf0..a19c2a2a 100644 --- a/man/bql.Rd +++ b/man/bql.Rd @@ -53,9 +53,10 @@ item is self-describing: every column carries a declared type properly-typed \code{data.frame} columns. Parsing requires either the \CRANpkg{RcppSimdJson} or the \CRANpkg{jsonlite} package; \CRANpkg{RcppSimdJson} is preferred when both are installed as it -is faster on the large documents BQL can return. Both give the -same result. Set \code{parse=FALSE} to obtain the raw JSON string -instead, e.g. for queries whose shape the parser does not handle. +is faster on the large documents BQL can return. Both give the same +result for the documents the service returns. Set +\code{parse=FALSE} to obtain the raw JSON string instead, e.g. for +queries whose shape the parser does not handle. Note that \sQuote{//blp/bqlsvc} is not part of the officially documented public API; it is the service behind the Excel BQL From 41337a517104bf8ab7d03cef430f60a9afd93be3 Mon Sep 17 00:00:00 2001 From: Alexander Kammerer Date: Tue, 1 Sep 2026 18:19:54 +0200 Subject: [PATCH 5/7] Clean up the BQL parser changes after a quality review No behaviour change beyond the two noted below. R/bql.R: * The placeholder pre-pass no longer calls vapply() once per element to find the strings. The flattened vector is already the character form at that point and is index-aligned, so one vectorised %in% gives the same mask; measured at 1.7ms against 22.2ms on a column of 111792 values. It also stops re-reading and re-scanning the original list. * substr() for a DATE column now runs inside .bqlByUnique, so it truncates the distinct strings rather than every row: 32% off that column and 11% off the whole parse of the 5.56 MiB live response. * .bqlChar() is gone. as.character() already returns the same object for a character vector, and the conversion is only needed in the default branch, since substr() and as.POSIXct() both accept the logical vector an all-null column flattens to. * .bqlByUnique() takes ... , which removes the inline function wrapper the DATETIME branch needed. * The anyNA() clause in the parser validation could never fire, as an NA matches no known parser name and the %in% test already rejects it. * The option is renamed 'bqlParser' and exposed as a 'parser' argument, following the house pattern of every other option in the package (returnAs=getOption("bdhType"), simplify=getOption("blpSimplify")). All fifteen existing option names are flat blp/bdh camelCase, and all are argument defaults; bql() was the only entry point whose knob could not be set per call. That argument is the behaviour change. * Comments trimmed where they narrated the code or, after the change above, described a per-element pass which no longer happens. inst/tinytest/test_bql.R: * Dropped the fragment test which split one fixture into three pieces by character. The byte-level chunking loop covers every fixture at five sizes, and splitting by character cannot land inside a multi-byte character, which is the case worth testing. * The join is now also asserted directly against .bqlJoin, so a failure says whether the join or the parse broke rather than only that one of them did. * .with() is hoisted, so all four loops label assertions with the parser in use; two assertions had lost the label and reported identically on every parser. One .oneItemDoc() builder replaces two hand-written document skeletons, one .outcome() replaces three copies of the same tryCatch, and one .withOption() replaces two ways of restoring an option, one of which leaked it on a failure. * The three double-precision blocks become one loop covering eight combinations per parser instead of five. The mutation harness was rebuilt as well: it silently stopped applying four of its mutants when the code above moved, and reported them as surviving. It now fails loudly if a mutant does not change anything. All twelve are killed, against 346 assertions passing. Co-Authored-By: Claude Opus 5 (1M context) --- R/bql.R | 89 ++++++++++++----------- inst/tinytest/test_bql.R | 149 ++++++++++++++++++--------------------- man/bql.Rd | 10 ++- 3 files changed, 121 insertions(+), 127 deletions(-) diff --git a/R/bql.R b/R/bql.R index 571a0220..82cad488 100644 --- a/R/bql.R +++ b/R/bql.R @@ -44,13 +44,16 @@ ##' be parsed into \code{data.frame} objects (requires either the ##' \CRANpkg{RcppSimdJson} or the \CRANpkg{jsonlite} package), ##' defaults to \sQuote{TRUE}. If \sQuote{FALSE} the raw JSON string -##' is returned. The option \code{Rblpapi.bqlParser} selects the -##' parser explicitly, e.g. \code{options(Rblpapi.bqlParser="jsonlite")}. +##' is returned. ##' @param simplify A boolean indicating whether a query returning a ##' single data item should be returned directly as a \code{data.frame} ##' instead of a list of length one, defaults to \sQuote{TRUE}. ##' @param verbose A boolean indicating whether verbose operation is ##' desired, defaults to \sQuote{FALSE}. +##' @param parser A character vector naming the JSON parsers to use in +##' order of preference, defaulting to the \code{bqlParser} option and, +##' failing that, to \sQuote{RcppSimdJson} then \sQuote{jsonlite}. The +##' first one which is installed is used. ##' @param con A connection object as created by a \code{blpConnect} ##' call, and retrieved via the internal function ##' \code{defaultConnection}. @@ -74,11 +77,12 @@ bql <- function(expression, parse=TRUE, simplify=TRUE, verbose=FALSE, + parser=getOption("bqlParser", .bqlParsers), con=defaultConnection()) { ## resolve the parser before the request so that a missing package does ## not discard a response which has already been retrieved - parser <- if (parse) .bqlParser() else NULL + if (parse) parser <- .bqlParser(parser) res <- bql_Impl(con, expression, verbose) if (!parse) return(.bqlJoin(res)) .bqlParse(res, simplify=simplify, parser=parser) @@ -91,17 +95,16 @@ bql <- function(expression, ## Supported JSON parsers, in order of preference .bqlParsers <- c("RcppSimdJson", "jsonlite") -## Select the JSON parser: RcppSimdJson is preferred as it is faster on the -## large documents BQL can return, jsonlite is the fallback. The option -## 'Rblpapi.bqlParser' forces one, which also lets the tests exercise both. -.bqlParser <- function() { +## Select the first of 'want' which is installed. RcppSimdJson comes first by +## default as it is faster on the large documents BQL can return, with jsonlite +## as the fallback; naming one picks it, which also lets the tests exercise +## both. +.bqlParser <- function(want=getOption("bqlParser", .bqlParsers)) { ## validated here rather than with match.arg(), which would accept an ## abbreviation and would silently drop an unknown name given alongside a - ## known one - want <- getOption("Rblpapi.bqlParser", .bqlParsers) - if (!is.character(want) || length(want) == 0L || anyNA(want) || - !all(want %in% .bqlParsers)) - stop("Option 'Rblpapi.bqlParser' must be one or more of ", + ## known one. An NA needs no clause of its own: it matches no known name. + if (!is.character(want) || length(want) == 0L || !all(want %in% .bqlParsers)) + stop("'parser' must be one or more of ", paste0("'", .bqlParsers, "'", collapse=", "), call.=FALSE) for (p in want) if (requireNamespace(p, quietly=TRUE)) return(p) ## name only what was actually asked for, which may be a single parser @@ -194,10 +197,8 @@ bql <- function(expression, } ## Convert a BQL column (list with 'type' and 'values') to a typed R vector. -## The values arrive as a list of scalars, one element per row. They are -## flattened with vectorised primitives rather than one element at a time: -## 'lengths()' finds the JSON nulls without a call per element, and unlist() -## does the rest in one step. +## The values arrive as a list of scalars, one element per row, and are +## flattened with vectorised primitives rather than one element at a time. ## ## JSON null maps to NA for every type; the string placeholders "NaN" and ## "NA" additionally map to NA for numeric columns only, as string columns @@ -217,31 +218,28 @@ bql <- function(expression, ## avoid. .bqlColumn <- function(col) { type <- if (is.null(col[["type"]])) "STRING" else col[["type"]] - values <- col[["values"]] - n <- length(values) - values[lengths(values) == 0L] <- NA - values <- unlist(values, use.names=FALSE) + vals <- col[["values"]] + n <- length(vals) + vals[lengths(vals) == 0L] <- NA + values <- unlist(vals, use.names=FALSE) ## unlist() flattens a nested value instead of failing, unlike the vapply() ## this replaces. This catches a value which flattens to more than one ## element; one which flattens to exactly one is kept, as it was before. if (length(values) != n) stop("BQL column '", .bqlColName(col, "?"), "' has non-scalar values", call.=FALSE) - ## Blank the "NaN", "NA" and "" placeholders which stand for a missing - ## number. Doing it in the list, before flattening, is what lets unlist() - ## keep the column numeric: a single such string would otherwise promote - ## the whole column to character and send every number back through its 15 - ## significant digit form. Only a numeric column is treated this way, as a - ## string column may legitimately hold those spellings. Any other string is - ## a number written as a string, which as.numeric() below still converts. - ## This costs one pass per element, and only for a numeric column which - ## really does contain a placeholder. - if (is.character(values) && type %in% c("DOUBLE", "INT")) { - vals <- col[["values"]] - isph <- vapply(vals, is.character, NA) - isph[isph] <- unlist(vals[isph], use.names=FALSE) %in% c("NaN", "NA", "") + ## Blanking the placeholders in the list and flattening again is what keeps + ## a numeric column numeric, and so exact. Only a numeric column is treated + ## this way, as a string column may legitimately hold those spellings, and + ## any other string is a number written as a string which as.numeric() + ## still converts. 'values' is already the character form here, so finding + ## them takes one vectorised pass; a JSON number never prints as one, and a + ## blanked null is NA_character_ rather than "NA", so neither is mistaken + ## for a placeholder. + if (is.character(values) && (type == "DOUBLE" || type == "INT")) { + isph <- values %in% c("NaN", "NA", "") if (any(isph)) { - vals[isph | lengths(vals) == 0L] <- NA + vals[isph] <- NA values <- unlist(vals, use.names=FALSE) } } @@ -249,21 +247,22 @@ bql <- function(expression, "DOUBLE" = as.numeric(values), "INT" = as.integer(values), "BOOLEAN" = if (is.logical(values)) values - else as.logical(toupper(.bqlChar(values))), - "DATE" = .bqlByUnique(substr(.bqlChar(values), 1L, 10L), as.Date), - "DATETIME" = .bqlByUnique(.bqlChar(values), function(u) - as.POSIXct(u, format="%Y-%m-%dT%H:%M:%OS", tz="UTC")), - .bqlChar(values)) + else as.logical(toupper(values)), + ## truncating inside .bqlByUnique truncates the distinct strings + ## rather than every row + "DATE" = .bqlByUnique(as.character(values), + function(u) as.Date(substr(u, 1L, 10L))), + "DATETIME" = .bqlByUnique(as.character(values), as.POSIXct, + format="%Y-%m-%dT%H:%M:%OS", tz="UTC"), + ## a column of only nulls has flattened to a logical vector, so the + ## character types still need the conversion + as.character(values)) } -## A column of only nulls flattens to a logical vector, so the character types -## still need the conversion; for an actual character vector this is a no-op -.bqlChar <- function(v) if (is.character(v)) v else as.character(v) - ## Parsing a date string costs far more per value than a hash lookup, and BQL ## date columns repeat heavily (one date per period, the same date for many ## securities), so convert only the distinct strings -.bqlByUnique <- function(v, fun) { +.bqlByUnique <- function(v, fun, ...) { u <- unique(v) - fun(u)[match(v, u)] + fun(u, ...)[match(v, u)] } diff --git a/inst/tinytest/test_bql.R b/inst/tinytest/test_bql.R index 1a9ff393..3a22b0f3 100644 --- a/inst/tinytest/test_bql.R +++ b/inst/tinytest/test_bql.R @@ -30,12 +30,31 @@ if (length(.parsers) == 0L) } .parse <- function(file, ...) Rblpapi:::.bqlParse(.readFixture(file), ...) +## the parser in use labels every assertion; .p is resolved at call time +.with <- function(txt) paste0(txt, " [", .p, "]") + +## compare outcomes and not only successes: a fixture which must raise has to +## raise the same way through every parser and every chunking +.outcome <- function(x, p) tryCatch(suppressWarnings(Rblpapi:::.bqlParse(x, parser=p)), + error = function(e) conditionMessage(e)) + +.allIdentical <- function(x) all(vapply(x[-1], identical, logical(1), x[[1]])) + +## one skeleton for the single-item documents built by hand below; 'values' is +## a vector of JSON literals, one per row, so quote any string value yourself +.oneItemDoc <- function(type, values, name="v") { + paste0('{"results":{"', name, '":{"name":"', name, '","idColumn":{"name":"ID",', + '"type":"STRING","values":[', + paste0('"', seq_along(values), '"', collapse=","), ']},', + '"valuesColumn":{"name":"VALUE","type":"', type, '","values":[', + paste(values, collapse=","), ']},"secondaryColumns":[]}},', + '"responseExceptions":[]}') +} + ## -- offline parsing tests (no Bloomberg connection required) -------------- for (.p in .parsers) { - .with <- function(txt) paste0(txt, " [", .p, "]") - ## single-item query: one data.frame with declared column types res <- .parse("response_px_last.json", parser=.p) expect_true(inherits(res, "data.frame"), info = .with("single item simplifies to data.frame")) @@ -72,15 +91,6 @@ for (.p in .parsers) { expect_equal(res$pe_ratio$PERIOD_END_DATE, as.Date(c("2024-09-30", "2024-09-28")), info = .with("secondary DATE column keeps the row order")) - ## responses above 4 MiB arrive as fragments of one document, cut mid-token, - ## and must give the same result as the document delivered in one message - doc <- .readFixture("response_px_last.json") - cut <- nchar(doc) %/% 3L - fragments <- substring(doc, c(1L, cut + 1L, 2L * cut + 1L), c(cut, 2L * cut, nchar(doc))) - expect_equal(Rblpapi:::.bqlParse(fragments, parser=.p), - Rblpapi:::.bqlParse(doc, parser=.p), - info = .with("fragmented response parses like the joined document")) - ## BQL errors surface as R errors expect_error(.parse("response_syntax_error.json", parser=.p), pattern = "Unable to parse request", info = .with("responseExceptions raise")) @@ -138,8 +148,7 @@ for (.p in .parsers) { for (.p in .parsers) { for (f in list.files("bql", pattern = "[.]json$")) { doc <- .readFixture(f) - ref <- tryCatch(suppressWarnings(Rblpapi:::.bqlParse(doc, parser = .p)), - error = function(e) conditionMessage(e)) + ref <- .outcome(doc, .p) ## the two largest sizes are derived from the document, as a fixed size ## above the smallest fixture would give one chunk and compare the ## document with itself @@ -147,37 +156,39 @@ for (.p in .parsers) { for (n in unique(c(1L, 7L, 64L, nb %/% 7L, nb %/% 2L))) { frags <- .chunkBytes(doc, n) expect_true(length(frags) > 1L, - info = paste0(f, " really is split at ", n, " bytes")) - got <- tryCatch(suppressWarnings(Rblpapi:::.bqlParse(frags, parser = .p)), - error = function(e) conditionMessage(e)) - expect_equal(got, ref, - info = paste0(f, " in ", length(frags), " chunks of ", n, - " bytes [", .p, "]")) + info = .with(paste0(f, " really is split at ", n, " bytes"))) + ## the join must return the original bytes; asserted directly as + ## well as through the parser, so a failure says which one broke + expect_identical(Rblpapi:::.bqlJoin(frags), doc, + info = .with(paste0(f, " rejoins byte for byte at ", n))) + expect_equal(.outcome(frags, .p), ref, + info = .with(paste0(f, " in ", length(frags), + " chunks of ", n, " bytes"))) } } ## a single fragment is not a document: the failure the joining prevents doc <- .readFixture("response_px_last.json") frags <- .chunkBytes(doc, nchar(doc, type = "bytes") %/% 2L) - expect_true(length(frags) > 1L, info = "the fixture really was split") + expect_true(length(frags) > 1L, info = .with("the fixture really was split")) expect_error(Rblpapi:::.bqlParse(frags[1], parser = .p), - info = paste0("a lone fragment does not parse [", .p, "]")) + info = .with("a lone fragment does not parse")) ## a boundary inside a multi-byte UTF-8 character must still rejoin; the ## characters are written as escapes so that this file stays ASCII - utf8doc <- paste0('{"results":{"name":{"name":"name","idColumn":{"name":"ID",', - '"type":"STRING","values":["X","Y"]},"valuesColumn":', - '{"name":"VALUE","type":"STRING","values":', - '["Nestl\u00e9 S\u00e9n\u00e9gal","\u00dcbermorgen"]},', - '"secondaryColumns":[]}},"responseExceptions":[]}') + utf8doc <- .oneItemDoc("STRING", + c('"Nestl\u00e9 S\u00e9n\u00e9gal"', '"\u00dcbermorgen"'), + name = "name") want <- c("Nestl\u00e9 S\u00e9n\u00e9gal", "\u00dcbermorgen") expect_equal(Rblpapi:::.bqlParse(utf8doc, parser = .p)$name, want, - info = paste0("multi-byte characters read correctly [", .p, "]")) + info = .with("multi-byte characters read correctly")) for (n in 1L:8L) { frags <- .chunkBytes(utf8doc, n) + expect_identical(Rblpapi:::.bqlJoin(frags), utf8doc, + info = .with(paste0("multi-byte rejoin at ", n, " bytes"))) expect_equal(Rblpapi:::.bqlParse(frags, parser = .p)$name, want, - info = paste0("multi-byte characters survive ", n, - "-byte chunking [", .p, "]")) + info = .with(paste0("multi-byte characters survive ", n, + "-byte chunking"))) } } @@ -185,11 +196,8 @@ for (.p in .parsers) { if (length(.parsers) > 1L) { for (f in list.files("bql", pattern="[.]json$")) { - out <- lapply(.parsers, function(p) - tryCatch(suppressWarnings(.parse(f, parser=p)), - error=function(e) conditionMessage(e))) - expect_true(all(vapply(out[-1], identical, logical(1), out[[1]])), - info = paste("all parsers agree on", f)) + out <- lapply(.parsers, function(p) .outcome(.readFixture(f), p)) + expect_true(.allIdentical(out), info = paste("all parsers agree on", f)) } } @@ -201,12 +209,15 @@ expect_true(Rblpapi:::.bqlParser() %in% .parsers, info = "default parser is inst if (all(c("RcppSimdJson", "jsonlite") %in% .parsers)) expect_equal(Rblpapi:::.bqlParser(), "RcppSimdJson", info = "RcppSimdJson is preferred when both are installed") -local({ - old <- options(Rblpapi.bqlParser=.parsers[length(.parsers)]) +## one helper, so a throwing assertion cannot leak the option to later tests +.withOption <- function(value, expr) { + old <- options(bqlParser=value) on.exit(options(old)) - expect_equal(Rblpapi:::.bqlParser(), .parsers[length(.parsers)], - info = "option selects the parser") -}) + force(expr) +} +.withOption(.parsers[length(.parsers)], + expect_equal(Rblpapi:::.bqlParser(), .parsers[length(.parsers)], + info = "option selects the parser")) ## an unknown parser name must be reported, not treated as "no data" expect_error(Rblpapi:::.bqlFromJSON("{}", "notAParser"), @@ -215,16 +226,12 @@ expect_error(Rblpapi:::.bqlFromJSON("{}", "notAParser"), ## the option is validated: no abbreviations, and an unknown name is not ## silently dropped when a known one sits beside it -local({ - for (bad in list("notAParser", c("notAParser", "jsonlite"), "R", "j", - NA_character_, "", 1L, TRUE, list("jsonlite"))) { - old <- options(Rblpapi.bqlParser = bad) - expect_error(Rblpapi:::.bqlParser(), - info = paste("option rejected:", - paste(deparse(bad), collapse = ""))) - options(old) - } -}) +for (.bad in list("notAParser", c("notAParser", "jsonlite"), "R", "j", + NA_character_, "", 1L, TRUE, list("jsonlite"), character(0))) + .withOption(.bad, + expect_error(Rblpapi:::.bqlParser(), + info = paste("option rejected:", + paste(deparse(.bad), collapse = "")))) ## the parsers must agree on the intermediate structure, not merely on the ## final data.frame: '[]', '{}' and null are where they differ by default, so @@ -232,7 +239,7 @@ local({ if (length(.parsers) > 1L) { .shapes <- '{"a":[],"b":{},"c":[1,null,"x",true],"d":{"e":[{"f":null}]}}' .trees <- lapply(.parsers, function(p) Rblpapi:::.bqlFromJSON(.shapes, p)) - expect_true(all(vapply(.trees[-1], identical, logical(1), .trees[[1]])), + expect_true(.allIdentical(.trees), info = "parsers agree on the intermediate structure") } @@ -297,35 +304,19 @@ expect_error(.col(name="X", type="STRING", values=list("a", list("b", "c"))), ## a placeholder string sits beside them: the placeholders are blanked before ## the column is flattened, so it never detours through character. The second ## case is the one that regresses if that step is dropped. -.doubleDoc <- function(vals) - paste0('{"results":{"v":{"name":"v","idColumn":{"name":"ID","type":"STRING",', - '"values":[', paste0('"', seq_along(vals), '"', collapse=","), ']},', - '"valuesColumn":{"name":"VALUE","type":"DOUBLE","values":[', - paste(vals, collapse=","), ']},"secondaryColumns":[]}},', - '"responseExceptions":[]}') - -for (.p in .parsers) { - .exact <- 0.12345678901234568 - res <- Rblpapi:::.bqlParse(.doubleDoc(c("0.12345678901234568", "null")), parser=.p) - expect_identical(res$v[1], .exact, - info = paste0("DOUBLE keeps every digit without a placeholder [", .p, "]")) - expect_true(is.na(res$v[2]), - info = paste0("null in a DOUBLE column is NA [", .p, "]")) - - for (.ph in c('"NaN"', '"NA"', '""')) { - res <- Rblpapi:::.bqlParse(.doubleDoc(c("0.12345678901234568", .ph)), parser=.p) - expect_identical(res$v[1], .exact, - info = paste0("DOUBLE keeps every digit beside a ", .ph, - " placeholder [", .p, "]")) - expect_identical(res$v[2], NA_real_, - info = paste0("the ", .ph, " placeholder itself is NA [", .p, "]")) - } - - ## a real float32-derived price, the case which motivated all of this - res <- Rblpapi:::.bqlParse(.doubleDoc(c("230.66000366210938", '"NaN"')), parser=.p) - expect_identical(res$v[1], 230.66000366210938, - info = paste0("a float-derived price is exact [", .p, "]")) -} +## as.numeric() is correctly rounded, so as.numeric(.v) is bit-identical to +## the literal in the document and needs no separate expected value. +## 230.66000366210938 is a real float32-derived price, the case which +## motivated all of this. +for (.p in .parsers) + for (.v in c("0.12345678901234568", "230.66000366210938")) + for (.ph in c("null", '"NaN"', '"NA"', '""')) { + res <- Rblpapi:::.bqlParse(.oneItemDoc("DOUBLE", c(.v, .ph)), parser=.p) + expect_identical(res$v[1], as.numeric(.v), + info = .with(paste("DOUBLE keeps every digit beside", .ph))) + expect_identical(res$v[2], NA_real_, + info = .with(paste("the", .ph, "placeholder itself is NA"))) + } ## a number written as a string is not a placeholder and must still convert expect_equal(.col(type="DOUBLE", values=list("123.45", 6, "NaN")), diff --git a/man/bql.Rd b/man/bql.Rd index a19c2a2a..f1514fb3 100644 --- a/man/bql.Rd +++ b/man/bql.Rd @@ -5,7 +5,7 @@ \title{Run 'Bloomberg Query Language' (BQL) Queries} \usage{ bql(expression, parse = TRUE, simplify = TRUE, verbose = FALSE, - con = defaultConnection()) + parser = getOption("bqlParser", .bqlParsers), con = defaultConnection()) } \arguments{ \item{expression}{A character string with the BQL query, e.g. @@ -15,8 +15,7 @@ bql(expression, parse = TRUE, simplify = TRUE, verbose = FALSE, be parsed into \code{data.frame} objects (requires either the \CRANpkg{RcppSimdJson} or the \CRANpkg{jsonlite} package), defaults to \sQuote{TRUE}. If \sQuote{FALSE} the raw JSON string -is returned. The option \code{Rblpapi.bqlParser} selects the -parser explicitly, e.g. \code{options(Rblpapi.bqlParser="jsonlite")}.} +is returned.} \item{simplify}{A boolean indicating whether a query returning a single data item should be returned directly as a \code{data.frame} @@ -25,6 +24,11 @@ instead of a list of length one, defaults to \sQuote{TRUE}.} \item{verbose}{A boolean indicating whether verbose operation is desired, defaults to \sQuote{FALSE}.} +\item{parser}{A character vector naming the JSON parsers to use in +order of preference, defaulting to the \code{bqlParser} option and, +failing that, to \sQuote{RcppSimdJson} then \sQuote{jsonlite}. The +first one which is installed is used.} + \item{con}{A connection object as created by a \code{blpConnect} call, and retrieved via the internal function \code{defaultConnection}.} From bc770ef09239159135848484fd0abb4d7e290b9b Mon Sep 17 00:00:00 2001 From: Alexander Kammerer Date: Tue, 1 Sep 2026 18:57:56 +0200 Subject: [PATCH 6/7] Act on a code review of the BQL changes Most of these are in .bqlItemToDataFrame and the C++ event loop, which this branch had not touched; they are on feature/bql already, but a reviewer meets them here. * An item whose idColumn and valuesColumn are both null, with no secondary columns, gave 'names' must be a character vector from make.unique(NULL) instead of the empty result the surrounding warn-and-continue path intends. * A repeated column name dropped a column. Assigning cols[[nm]] by name replaced the earlier column of that name rather than adding one, which also left make.unique() with nothing to rename: two DATE secondary columns produced three columns holding only the second one's values. The columns are collected in order and named at the end now, so both survive as DATE and DATE.1. * Columns of unequal length were baked into a corrupt data.frame, which reported the wrong number of rows. They are rejected. * src/bql.cpp ignored REQUEST_STATUS, which a rejected or timed-out request sends instead of a RESPONSE, so nextEvent() would have blocked for good. Handled as bdh.cpp does. * A session which ended before the response arrived left bql_Impl returning nothing, and the JSON parser then reported a truncated document. bql() now says what actually happened. Two are about this branch's own work: * bql()'s signature named .bqlParsers, which is not exported, so ?bql showed users an object they cannot reference. 'parser' now defaults to NULL and .bqlParser() resolves it. * The comment claiming a numeric column keeps the values exactly as the service sent them holds for JSON numbers and placeholders, but not when a number arrives as a string: that column still has to come back from character. Narrowed, and pinned by tests using expect_identical, as expect_equal's tolerance hid it. Not changed: a BOOLEAN column whose values are JSON numbers is all NA, but it was before too, since as.logical("0") is NA. Only a boolean sharing an array with a number differs from the old code, which is the coercion order already documented above .bqlColumn. 354 assertions pass and all twelve mutants are caught. Co-Authored-By: Claude Opus 5 (1M context) --- R/bql.R | 74 ++++++++++++++++++++++++++++------------ inst/tinytest/test_bql.R | 44 ++++++++++++++++++++++-- man/bql.Rd | 8 ++--- src/bql.cpp | 9 +++++ 4 files changed, 107 insertions(+), 28 deletions(-) diff --git a/R/bql.R b/R/bql.R index 82cad488..8c61daa4 100644 --- a/R/bql.R +++ b/R/bql.R @@ -51,9 +51,9 @@ ##' @param verbose A boolean indicating whether verbose operation is ##' desired, defaults to \sQuote{FALSE}. ##' @param parser A character vector naming the JSON parsers to use in -##' order of preference, defaulting to the \code{bqlParser} option and, -##' failing that, to \sQuote{RcppSimdJson} then \sQuote{jsonlite}. The -##' first one which is installed is used. +##' order of preference; the first one which is installed is used. +##' \sQuote{NULL}, the default, takes the \code{bqlParser} option and, +##' failing that, tries \sQuote{RcppSimdJson} then \sQuote{jsonlite}. ##' @param con A connection object as created by a \code{blpConnect} ##' call, and retrieved via the internal function ##' \code{defaultConnection}. @@ -77,13 +77,20 @@ bql <- function(expression, parse=TRUE, simplify=TRUE, verbose=FALSE, - parser=getOption("bqlParser", .bqlParsers), + parser=NULL, con=defaultConnection()) { ## resolve the parser before the request so that a missing package does ## not discard a response which has already been retrieved if (parse) parser <- .bqlParser(parser) res <- bql_Impl(con, expression, verbose) + ## the C++ layer returns nothing at all when the session ended before the + ## response arrived; say so rather than let the JSON parser report the + ## empty string as a truncated document + if (!length(res)) + stop("The BQL request returned no messages, which happens when the ", + "session ends before the response arrives. Check the connection.", + call.=FALSE) if (!parse) return(.bqlJoin(res)) .bqlParse(res, simplify=simplify, parser=parser) } @@ -99,7 +106,11 @@ bql <- function(expression, ## default as it is faster on the large documents BQL can return, with jsonlite ## as the fallback; naming one picks it, which also lets the tests exercise ## both. -.bqlParser <- function(want=getOption("bqlParser", .bqlParsers)) { +.bqlParser <- function(want=NULL) { + ## NULL, the default of bql()'s 'parser', means "whatever the option says, + ## else the built-in order". Resolved here so that bql()'s signature, and + ## therefore its help page, does not name an unexported object. + if (is.null(want)) want <- getOption("bqlParser", .bqlParsers) ## validated here rather than with match.arg(), which would accept an ## abbreviation and would silently drop an unknown name given alongside a ## known one. An NA needs no clause of its own: it matches no known name. @@ -170,25 +181,43 @@ bql <- function(expression, } ## Convert one entry of 'results' into a data.frame using the declared -## column types; the value column is named after the data item itself +## column types; the value column is named after the data item itself. +## +## The columns are collected in order and named at the end rather than +## assigned by name as they are found: assigning by name would replace an +## earlier column of the same name instead of adding one, silently dropping +## it, and would leave make.unique() below with nothing to do. It also lets +## an item with no columns at all produce an empty data.frame, where +## names(list()) would be NULL and make.unique() would reject it. .bqlItemToDataFrame <- function(item) { - cols <- list() + spec <- function(col, nm) list(list(col=col, nm=nm)) + specs <- list() idcol <- item[["idColumn"]] if (!is.null(idcol)) - cols[[.bqlColName(idcol, "ID")]] <- .bqlColumn(idcol) + specs <- c(specs, spec(idcol, .bqlColName(idcol, "ID"))) valcol <- item[["valuesColumn"]] - if (!is.null(valcol)) { - nm <- if (is.null(item[["name"]]) || !nzchar(item[["name"]])) - .bqlColName(valcol, "VALUE") else item[["name"]] - cols[[nm]] <- .bqlColumn(valcol) - } + if (!is.null(valcol)) + specs <- c(specs, spec(valcol, + if (is.null(item[["name"]]) || !nzchar(item[["name"]])) + .bqlColName(valcol, "VALUE") else item[["name"]])) for (sec in item[["secondaryColumns"]]) - cols[[.bqlColName(sec, "V")]] <- .bqlColumn(sec) - names(cols) <- make.unique(names(cols)) + specs <- c(specs, spec(sec, .bqlColName(sec, "V"))) + + cols <- lapply(specs, function(s) .bqlColumn(s[["col"]])) + names(cols) <- make.unique(vapply(specs, `[[`, character(1), "nm")) + + ## a data.frame needs every column the same length; without this the + ## mismatch would be baked into a corrupt object instead of reported + rows <- unique(lengths(cols)) + if (length(rows) > 1L) + stop("BQL item '", if (is.null(item[["name"]])) "" else item[["name"]], + "' has columns of unequal length: ", + paste0(names(cols), " (", lengths(cols), ")", collapse=", "), + call.=FALSE) ## avoid data.frame() name mangling and rownames structure(cols, class="data.frame", - row.names=if (length(cols)) seq_along(cols[[1L]]) else integer()) + row.names=if (length(rows)) seq_len(rows) else integer()) } .bqlColName <- function(col, fallback) { @@ -204,11 +233,14 @@ bql <- function(expression, ## "NA" additionally map to NA for numeric columns only, as string columns ## may legitimately contain them (e.g. the ticker of 'NA US Equity'). ## -## A numeric column stays numeric throughout and so keeps the values exactly -## as the service sent them, rather than losing the last digits to a detour -## through character. Bloomberg sends float-derived prices such as -## 230.66000366210938, which as.character() would truncate to -## 230.66000366210901. +## A numeric column of JSON numbers, with or without those placeholders, stays +## numeric throughout and so keeps the values exactly as the service sent them, +## rather than losing the last digits to a detour through character. Bloomberg +## sends float-derived prices such as 230.66000366210938, which as.character() +## would truncate to 230.66000366210901. A number written as a string is the +## one case which still takes the detour: it has to be converted from +## character anyway, and telling it apart from a number beforehand would need +## a call per element for every column. ## ## One consequence of letting unlist() pick the type does remain: it coerces a ## logical before a string, so a JSON boolean sharing an array with a JSON diff --git a/inst/tinytest/test_bql.R b/inst/tinytest/test_bql.R index 3a22b0f3..c45f8612 100644 --- a/inst/tinytest/test_bql.R +++ b/inst/tinytest/test_bql.R @@ -318,9 +318,47 @@ for (.p in .parsers) info = .with(paste("the", .ph, "placeholder itself is NA"))) } -## a number written as a string is not a placeholder and must still convert -expect_equal(.col(type="DOUBLE", values=list("123.45", 6, "NaN")), - c(123.45, 6, NA), info = "a number sent as a string still converts") +## A number written as a string is not a placeholder and must still convert. +## It is also the one case which does not keep every digit, because the column +## has to come back from character; expect_identical pins that, since +## expect_equal's tolerance would hide it either way. +expect_identical(.col(type="DOUBLE", values=list("123.45", 6, "NaN")), + c(123.45, 6, NA), info = "a number sent as a string still converts") +expect_identical(.col(type="DOUBLE", values=list(230.66000366210938, "123.45")), + c(as.numeric(as.character(230.66000366210938)), 123.45), + info = "a number sent as a string costs the column its last digits") +expect_identical(.col(type="DOUBLE", values=list(230.66000366210938, "NaN")), + c(230.66000366210938, NA), + info = "a placeholder does not") + +## -- assembling an item into a data.frame ----------------------------------- + +.item <- function(...) Rblpapi:::.bqlItemToDataFrame(list(...)) +.scol <- function(nm, vals) list(name=nm, type="STRING", values=as.list(vals)) + +## an item with no columns at all is an empty data.frame, not an error from +## make.unique() being handed the NULL names of an empty list +expect_equal(dim(.item(name="x", idColumn=NULL, valuesColumn=NULL, + secondaryColumns=list())), c(0L, 0L), + info = "an item with no columns gives an empty data.frame") + +## a repeated column name must add a column, not replace the earlier one +res <- .item(name="x", idColumn=.scol("ID", c("a", "b")), + valuesColumn=.scol("VALUE", c("1", "2")), + secondaryColumns=list(.scol("DATE", c("d1", "d2")), + .scol("DATE", c("e1", "e2")))) +expect_equal(ncol(res), 4L, info = "a repeated column name keeps both columns") +expect_equal(colnames(res), c("ID", "x", "DATE", "DATE.1"), + info = "make.unique() renames the second one") +expect_equal(res$DATE, c("d1", "d2"), info = "the first DATE column is intact") +expect_equal(res[["DATE.1"]], c("e1", "e2"), info = "the second DATE column is intact") + +## columns of unequal length cannot make a valid data.frame, so say so +expect_error(.item(name="x", idColumn=.scol("ID", c("a", "b", "c")), + valuesColumn=.scol("VALUE", c("1", "2")), + secondaryColumns=list()), + pattern = "unequal length", + info = "unequal column lengths are rejected") ## -- live test (requires a Bloomberg connection) ---------------------------- diff --git a/man/bql.Rd b/man/bql.Rd index f1514fb3..d7bdb188 100644 --- a/man/bql.Rd +++ b/man/bql.Rd @@ -5,7 +5,7 @@ \title{Run 'Bloomberg Query Language' (BQL) Queries} \usage{ bql(expression, parse = TRUE, simplify = TRUE, verbose = FALSE, - parser = getOption("bqlParser", .bqlParsers), con = defaultConnection()) + parser = NULL, con = defaultConnection()) } \arguments{ \item{expression}{A character string with the BQL query, e.g. @@ -25,9 +25,9 @@ instead of a list of length one, defaults to \sQuote{TRUE}.} desired, defaults to \sQuote{FALSE}.} \item{parser}{A character vector naming the JSON parsers to use in -order of preference, defaulting to the \code{bqlParser} option and, -failing that, to \sQuote{RcppSimdJson} then \sQuote{jsonlite}. The -first one which is installed is used.} +order of preference; the first one which is installed is used. +\sQuote{NULL}, the default, takes the \code{bqlParser} option and, +failing that, tries \sQuote{RcppSimdJson} then \sQuote{jsonlite}.} \item{con}{A connection object as created by a \code{blpConnect} call, and retrieved via the internal function diff --git a/src/bql.cpp b/src/bql.cpp index 8d83b564..c2cea181 100644 --- a/src/bql.cpp +++ b/src/bql.cpp @@ -107,6 +107,15 @@ Rcpp::CharacterVector bql_Impl(SEXP con, if (verbose) Rcpp::Rcout << "Processing Response" << std::endl; processBqlEvent(event, res, verbose); done = true; + } else if (event.eventType() == Event::REQUEST_STATUS) { + // a rejected or timed-out request sends this and never a RESPONSE, + // so without it nextEvent() would block for good (cf. bdh.cpp) + MessageIterator msgIter(event); + while (msgIter.next()) { + Message msg = msgIter.message(); + if (verbose) msg.asElement().print(Rcpp::Rcout); + } + Rcpp::stop("Bloomberg request timed out on server side"); } else { MessageIterator msgIter(event); while (msgIter.next()) { From 2a4f9907005009fe98208fa9cc1a2b18ce6b1b48 Mon Sep 17 00:00:00 2001 From: Alexander Kammerer Date: Tue, 1 Sep 2026 19:15:48 +0200 Subject: [PATCH 7/7] Require 'parser' in .bqlFromJSON rather than defaulting it Three functions defaulted the parser, but .bqlParse always passes it explicitly, so .bqlFromJSON's default was never taken. bql() is now the one place which decides, and the decision is passed down. Co-Authored-By: Claude Opus 5 (1M context) --- R/bql.R | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/R/bql.R b/R/bql.R index 8c61daa4..2b3f44b7 100644 --- a/R/bql.R +++ b/R/bql.R @@ -130,7 +130,11 @@ bql <- function(expression, ## done from the declared BQL column types in .bqlColumn. The two 'empty' ## arguments make RcppSimdJson agree with jsonlite on '[]' and '{}', which it ## maps to NULL by default. -.bqlFromJSON <- function(txt, parser=.bqlParser()) { +## +## 'parser' is required rather than defaulted, so that bql() stays the one +## place which decides which parser to use and .bqlParse only passes that +## decision down. +.bqlFromJSON <- function(txt, parser) { switch(parser, "RcppSimdJson" = RcppSimdJson::fparse(txt,