diff --git a/ES9_PRODUCTION_RUNBOOK.md b/ES9_PRODUCTION_RUNBOOK.md new file mode 100644 index 0000000..61b251b --- /dev/null +++ b/ES9_PRODUCTION_RUNBOOK.md @@ -0,0 +1,178 @@ +# Production runbook: SBOLExplorer on Elasticsearch 9 + +For the people who run the SynBioHub production VM and maintain +`synbiohub/synbiohub-docker`. Background and code details are in +[ES9_UPGRADE.md](ES9_UPGRADE.md); this page is only what to change and run. + +## What changes and why + +- Elasticsearch goes from **6.3.2** (end of life since 2020) to **9.5.3**. +- The SBOLExplorer image gets a **9.x Elasticsearch client**. It cannot talk to + ES 6.3, and the old image cannot talk to ES 9. **Both switch together.** +- New: faceted search counts (`/facets`), and a fix that keeps SBOLExplorer + from crashing when it starts before Elasticsearch is ready. + +**Search downtime: none.** While the new index is built (~30 min), +SBOLExplorer does not answer, and SynBioHub falls back to querying Virtuoso +directly (`SBOLExplorerFallback` defaults to `true`). Search is slower in that +window, but it works. + +## The one ordering rule + +Production runs `myersresearchgroup/sbolexplorer:snapshot-synbiohub`, which CI +rebuilds on **every merge to SBOLExplorer master**. When the ES 9 branch +(`upgrade_ES`) is merged, that tag immediately becomes the ES 9 version, and +the next time the VM pulls it, it will not work against ES 6.3. + +So: **merge `upgrade_ES` only when you are ready to do steps 2–4 right away**, +or do not pull/recreate the explorer on the VM until then. + +## 1. Before the window + +On the VM: + +```bash +sysctl vm.max_map_count # must be >= 262144 (ES 9 refuses to start otherwise) +free -g; df -h # ~2 GB RAM for ES, a few GB disk for the new index +curl -s localhost:13162/config # check the live SBOLExplorer config: + # pagerank_tolerance should be 0.0001 (not 1) + # uclust_identity should be 0.8 (not 1.8) +``` + +If either config value is wrong, fix it now (`POST /config` or edit +`/mnt/config/config.json` in the `explorer` volume). The rebuild in step 4 uses +them; with the wrong values PageRank stops after one iteration and clustering +silently reuses stale data. + +Record the current explorer image digest, for rollback: + +```bash +docker image inspect myersresearchgroup/sbolexplorer:snapshot-synbiohub --format '{{range .RepoDigests}}{{.}}{{end}}' +# as of 2026-10-09 this is ...@sha256:ef904541fe6a... (master 9a56562, ES 6 client) +``` + +## 2. Change `docker-compose.explorer.yml` (synbiohub-docker PR) + +```yaml +version: '3.2' + +services: + explorer: + image: myersresearchgroup/sbolexplorer:snapshot-synbiohub + restart: unless-stopped # NEW: a crash at startup no longer leaves it down + privileged: true + ports: + - "13162:13162" + depends_on: + - elasticsearch + - synbiohub + dns: 8.8.8.8 + volumes: + - type: volume + source: explorer + target: /mnt + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:9.5.3 # was 6.3.2 + restart: unless-stopped + environment: # NEW: ES 8+ enables TLS + passwords by default + - discovery.type=single-node + - xpack.security.enabled=false + - ES_JAVA_OPTS=-Xms1g -Xmx1g + depends_on: + - virtuoso + ports: + - "127.0.0.1:9200:9200" # was "9200:9200": see Security below + volumes: + - type: volume + source: esdata9 # NEW volume: ES 9 cannot open the 6.x data + target: /usr/share/elasticsearch/data + +volumes: + esdata: # keep: the old 6.3 index, for rollback + esdata9: + explorer: +``` + +**Security.** With `xpack.security.enabled=false`, anyone who can reach port +9200 can read and delete the index, which includes users' **private** parts. +Only the `explorer` container needs ES, and it connects over the compose +network (`elasticsearch:9200`). Bind the port to loopback as above, or remove +the `ports:` entry entirely. Also check that the Azure network security group +does not expose 9200. + +`depends_on: condition: service_healthy` is **not** used: it is not supported +by the `version: '3.2'` file format with docker-compose v1, which the VM uses. +`restart: unless-stopped` plus the new startup retry in SBOLExplorer cover the +same problem. + +## 3. Switch over (on the VM, in the synbiohub-docker directory) + +Use the same `-f` list as the normal start command: + +```bash +F="-f docker-compose.yml -f docker-compose.explorer.yml -f docker-compose.pluginVisualComponentUse.yml -f docker-compose.pluginSubmitExcel2SBOL.yml" + +git pull # the compose change from step 2 +docker pull myersresearchgroup/sbolexplorer:snapshot-synbiohub # the ES 9 build (after upgrade_ES is merged) +docker pull docker.elastic.co/elasticsearch/elasticsearch:9.5.3 + +/usr/local/bin/docker-compose $F up -d --no-deps elasticsearch +curl -s localhost:9200 | grep '"number"' # wait ~30-60 s; expect "9.5.3" + +/usr/local/bin/docker-compose $F up -d --no-deps explorer +docker logs --tail 20 synbiohub-docker_explorer_1 # expect "Index not found, creating new index." +``` + +`--no-deps` restarts only these two services; SynBioHub, Virtuoso and the +plugins keep running. + +## 4. Build the index + +With an empty ES, SBOLExplorer builds the index **during startup** ("Index not +found, creating new index"). It does not open port 13162 until the build is +done (~30 min; the clustering step prints no progress), so its HTTP endpoints +do not answer yet. Follow progress inside the container: + +```bash +docker exec synbiohub-docker_explorer_1 tail -5 /SBOLExplorer/flask/indexing_log.txt +``` + +It is done when the last lines are five `Facet ...: updated N parts` lines and +`Successfully updated index`, and `curl localhost:13162/info` answers. +To rebuild again later (with the service running): `curl -s localhost:13162/update`. + +## 5. Verify + +```bash +curl -s 'localhost:9200/_cat/indices/part?v' # health green; docs.count in the 100k+ range +curl -s -o /dev/null -w '%{http_code}\n' localhost:13162/info # 200 +curl -s 'localhost:13162/facets?query=promoter&facet_by=role&size=3' # non-empty role counts +``` + +In the SynBioHub UI: +- search `gfp`, `rbs`, `terminator`: results appear +- a typo (`terminatr`) still finds terminators +- a logged-in user sees their private parts at the top +- search text plus a Collections filter (e.g. iGEM) returns results + (this failed on ES 9 before commit 24a2a32) + +## Rollback + +```bash +git checkout HEAD~1 -- docker-compose.explorer.yml # back to ES 6.3.2 + esdata +docker tag myersresearchgroup/sbolexplorer@sha256:ef904541fe6a \ + myersresearchgroup/sbolexplorer:snapshot-synbiohub +/usr/local/bin/docker-compose $F up -d --no-deps elasticsearch explorer +``` + +The old `esdata` volume still holds the 6.3 index, so no rebuild is needed. The +explorer image and ES must go back **together**. + +## Related changes (independent of this window) + +These can be deployed before or after; both fall back to the current behavior +when `/facets` is not available: +- synbiohub `facets-via-explorer`: `/facets` proxy that scopes counts to the + logged-in user's graphs. +- synbiohub3 `facets-via-explorer`: the search sidebar and result count use + `/facets`. diff --git a/ES9_UPGRADE.md b/ES9_UPGRADE.md new file mode 100644 index 0000000..65deaf2 --- /dev/null +++ b/ES9_UPGRADE.md @@ -0,0 +1,273 @@ +# Upgrading Elasticsearch 6.3 → 9.x + +SBOLExplorer previously ran on Elasticsearch **6.3.2** (EOL) with the Python +client `elasticsearch==6.3.0`. This branch (`upgrade_ES`) moves it to +Elasticsearch **9.5.3** with the `elasticsearch>=9,<10` client. + +The index is always rebuilt from Virtuoso, so **no data migration is needed**: +start an empty ES 9 node and run `/update`. + +> The ES server and the Python client must be upgraded **together**. The 9.x +> client cannot talk to a 6.x server, and the old code cannot run on a 9.x +> server. Never deploy one without the other. + +--- + +## 1. Code changes (branch `upgrade_ES`) + +| File | Change | +| --- | --- | +| `flask/requirements.txt` | `elasticsearch>=9,<10`; removed the unused `elasticsearch-dsl`; `requests>=2.32`; dropped the 2018 pins of `certifi`, `chardet`, `idna`, `urllib3` (they conflicted with the new client) | +| `flask/index.py` | Removed mapping types (ES 8+ rejects them): the `index_name:` level inside `mappings`, `_type` in bulk actions, and `doc_type=` in `delete_by_query` / `index`. `indices.exists` now uses a keyword argument (the 9.x client rejects positional args). `number_of_replicas: 0`, so a single-node index is green instead of yellow. `max_terms_count: 1000000`, for the subject restriction in `search.py` (default is 65,536) | +| `flask/search.py` | All `search()` calls pass `rest_total_hits_as_int=True`, so `hits.total` stays an integer (ES 7+ returns an object). The search code and the JSON returned to SynBioHub are unchanged. `empty_search_es` sets `track_total_hits: True`, so counts above 10,000 stay exact. Searches restricted to the parts Virtuoso allowed (text + filter) use a `terms` query on `subject` instead of an `ids` query (`subjects_query()`), see §4 | +| `flask/explorer.py` | Health checks call `cat.indices(index=index_name)` instead of taking `[0]`. ES 9 creates hidden system indices, so `[0]` is not guaranteed to be `part` | +| `flask/docker/docker-compose.yml` | ES image `9.5.3` + `discovery.type=single-node`, `xpack.security.enabled=false`, `ES_JAVA_OPTS=-Xms1g -Xmx1g` | + +Unchanged and still working on 9.x: `function_score` + Painless `script_score` +(including the #158 private-graph boost), `multi_match` with +`fuzziness: AUTO`, and `from + size <= 10000`. `ids` queries still work, but +only up to 10,000 ids (see §4). + +--- + +## 2. Local development setup (what was done) + +### 2.1 Start ES 9 next to the old 6.3 container + +Rename the old container so it stays available for rollback and comparison: + +```bash +docker rename es01 es01-6.3 + +docker run -d --name es01 -p 9201:9200 \ + -e discovery.type=single-node \ + -e xpack.security.enabled=false \ + -e "ES_JAVA_OPTS=-Xms1g -Xmx1g" \ + -v es9-data:/usr/share/elasticsearch/data \ + -m 2g \ + docker.elastic.co/elasticsearch/elasticsearch:9.5.3 +``` + +| Flag | Why | +| --- | --- | +| `-p 9201:9200` | `host:container`. The host port is 9201, so it can run alongside the old 6.3 container on 9200 | +| `discovery.type=single-node` | One-node cluster, no discovery | +| `xpack.security.enabled=false` | ES 8+ enables TLS and auth by default. Disabled to keep `http://` without credentials. **Only safe if the port is not publicly reachable** (see §3.4) | +| `-v es9-data:...` | Named volume, so the index survives recreating the container | +| `-m 2g` | A 1 GB heap needs about 2 GB of container memory | + +Verify. First startup takes 30–60 s; until then `curl` fails with +`Connection reset by peer`. + +```bash +curl localhost:9201 # "number" : "9.5.3" +docker ps -a --filter "name=es" --format "table {{.Names}}\t{{.Image}}\t{{.Status}}" +docker logs --tail 50 es01 # if it does not come up +``` + +The WSL host needs `vm.max_map_count >= 262144` +(`sysctl vm.max_map_count`). It already was. + +### 2.2 Upgrade the Python environment + +SBOLExplorer runs from the `jammy/` virtualenv (Python 3.11). Use its `pip`, not +the system one or pyenv: + +```bash +cd ~/git_repo/SBOLExplorer +jammy/bin/pip uninstall -y elasticsearch-dsl +jammy/bin/pip install "elasticsearch>=9,<10" +jammy/bin/pip install -U "requests>=2.32" certifi # fixes the idna/urllib3 conflict +jammy/bin/pip check # "No broken requirements found." +``` + +Resulting versions: `elasticsearch 9.5.1`, `elastic-transport 9.4.2`, +`requests 2.34.2`, `urllib3 2.8.0`, `idna 3.20`. + +### 2.3 Point SBOLExplorer at ES 9 + +In `flask/config.json`: + +```json +"elasticsearch_endpoint": "http://localhost:9201/" +``` + +### 2.4 Run Flask from the virtualenv + +```bash +cd ~/git_repo/SBOLExplorer/flask +../jammy/bin/flask --app explorer run --port 13162 +``` + +Running plain `flask` picks up pyenv's Python 3.6, which fails with +`ModuleNotFoundError: No module named 'elasticsearch'`. In VS Code, select +`./jammy/bin/python` as the interpreter. + +### 2.5 Rebuild the index + +``` +GET http://localhost:13162/update +``` + +Before this, `/` and `/search` return 503 because the `part` index does not +exist yet. The local rebuild took about 8 minutes (2026-10-04). The clustering +step can be much slower on a full corpus and prints no progress. + +Result: + +```bash +curl 'localhost:9201/_cat/indices/part?v' +# health green, docs.count 132920 +curl -s -o /dev/null -w '%{http_code}\n' 'localhost:13162/search?query=gfp' # 200 +``` + +### 2.6 Rollback (local) + +```bash +docker stop es01 +docker start es01-6.3 # back on port 9200 +# config.json: "elasticsearch_endpoint": "http://localhost:9200/" +# jammy: pip install -r +git checkout master +``` + +--- + +## 3. Deploying to production + +### 3.1 How production is wired + +- SynBioHub's production stack is defined in the **`synbiohub/synbiohub-docker`** + repo (`docker-compose.explorer.yml`, `docker-compose.version.yml`), not in + this repo. `flask/docker/docker-compose.yml` here is only a standalone + example. +- Pushing to `master` → `.github/workflows/build.yml` builds + `myersresearchgroup/sbolexplorer:snapshot` and `:snapshot-synbiohub` from + `flask/docker/Dockerfile-source` (local `requirements.txt`). +- Publishing a release → `.github/workflows/release.yml` builds + `myersresearchgroup/sbolexplorer:-standalone` and **auto-commits the new + SBOLExplorer tag** into `synbiohub-docker`. +- **That workflow does not change the Elasticsearch image.** The ES version in + `synbiohub-docker` must be bumped by hand, in the same rollout. + +### 3.2 Pre-deployment checklist + +- [ ] `upgrade_ES` merged to `master` and the CI image built +- [ ] Coordinated with the `synbiohub-docker` maintainers; the ES image bump is + ready in the same change set +- [ ] Production host has `vm.max_map_count >= 262144` + (`sudo sysctl -w vm.max_map_count=262144`, persist in `/etc/sysctl.conf`) +- [ ] At least ~2 GB RAM free for ES (1 GB heap) +- [ ] Production `config.json` values are sane. In June 2026 the Azure instance + had `pagerank_tolerance: 1` (PageRank stops after 1 iteration) and + `uclust_identity: 1.8` (vsearch fails, clustering silently reuses a stale + `.uc`). They should be `0.0001` and `0.8`. A full `/update` runs both + steps. +- [ ] `elasticsearch_endpoint` still matches the ES service name in compose + (`http://elasticsearch:9200/` in `config-synbiohub.json`) +- [ ] A maintenance window is planned. Search is unavailable (503) from the + switch until `/update` finishes + +### 3.3 Elasticsearch service in `synbiohub-docker` + +Replace the ES service definition with something equivalent to: + +```yaml + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:9.5.3 + environment: + - discovery.type=single-node + - xpack.security.enabled=false + - ES_JAVA_OPTS=-Xms1g -Xmx1g + volumes: + - es9-data:/usr/share/elasticsearch/data + # no "ports:" — only SBOLExplorer needs to reach ES, over the compose network + +volumes: + es9-data: +``` + +Use a **new** volume name. A 9.x node cannot start on a 6.x data directory, and +a fresh volume also keeps the old one intact for rollback. + +### 3.4 Security note + +With `xpack.security.enabled=false`, anyone who can reach port 9200 can read and +delete the index, and the index contains users' **private** parts. In +production, either: + +- do not publish ES's port at all (recommended; containers on the same compose + network can still reach `elasticsearch:9200`), or bind it to loopback + (`"127.0.0.1:9200:9200"`); or +- enable security and give `elasticsearchManager.py` credentials. This requires + a code change. + +### 3.5 Rollout steps + +```bash +# on the production host, in the synbiohub-docker checkout +git pull # ES 9 image + new SBOLExplorer tag +docker compose -f pull +docker compose -f stop sbolexplorer elasticsearch +docker compose -f up -d elasticsearch +curl :9200 # wait for "number" : "9.5.3" +docker compose -f up -d sbolexplorer +curl http://:13162/update # rebuild the index +curl http://:13162/indexinginfo +``` + +Then verify (§3.7). + +**Lower-downtime alternative:** bring up the new ES and the new SBOLExplorer +under different service names/ports, run `/update` there, check results, then +switch SynBioHub's SBOLExplorer endpoint over. Search keeps working on the old +stack during the rebuild. + +### 3.6 Rollback (production) + +1. Revert the `synbiohub-docker` change: old ES 6.3.2 image, old volume, old + SBOLExplorer tag. +2. `docker compose ... up -d`. The old volume still holds the 6.x index, so no + rebuild is needed. + +The old SBOLExplorer image and the old ES must go back **together** (see the +note at the top). + +### 3.7 Verification + +- [ ] `curl :9200` reports `9.5.3` +- [ ] `curl ':9200/_cat/indices/part?v'` is `green`. Compare + `docs.count` with the old 6.3 index. Locally the ES 9 rebuild gave + 132,920 docs; a June 2026 6.3 index had 220,935. Check whether the gap + comes from the dedup fix on master or is a regression. +- [ ] SynBioHub search for common terms (`gfp`, `rbs`, `terminator`) returns + results; typo queries (e.g. `terminatr`) still match +- [ ] A logged-in user sees their private parts first (#158) +- [ ] Count queries (result totals in SynBioHub) look right +- [ ] Optional: run `evaluation/` on the old and the new index and compare + NDCG@10 + +--- + +## 4. Behavior differences to expect + +- **Score scale.** Lucene 8+ dropped the `(k1+1)` factor from BM25, so raw + `_score` values are smaller. Rankings within a query are unaffected, because + ES scores are only compared with each other. Fuzzy matching and the standard + analyzer were also updated, so near-tied results can swap order. +- **System indices.** ES 9 creates hidden `.ds-*` indices. Code must address the + `part` index by name, never by position. +- **`ids` queries are capped at 10,000 ids.** ES 8+ rejects an `ids` query with + more ids than `max_result_window` (`Too many ids specified, allowed max result + window is [10000]`); ES 6.3 accepted any number. A text search combined with a + broad filter (e.g. the iGEM collection: 75,333 parts) hit this. Fixed by + restricting with a `terms` query on the keyword `subject` field, whose limit is + `index.max_terms_count` (set to 1,000,000 at index creation). Same constant + score per match, so ranking is unchanged. On an index created before this + change, raise it in place (a dynamic setting, no rebuild): + `curl -X PUT localhost:9200/part/_settings -H 'Content-Type: application/json' -d '{"index":{"max_terms_count":1000000}}'` +- **License log line.** `license state changed, now [not valid]` in the ES logs + comes from a paid-only feature. Ignore it; the `basic` license is `valid`. +- **New capabilities** now available: `dense_vector` / kNN (with filters), + aggregations for facets, and the `retriever` API. Some fusion features (RRF) + may require a paid license; check before relying on them. diff --git a/flask/FACETS_API.md b/flask/FACETS_API.md new file mode 100644 index 0000000..855000f --- /dev/null +++ b/flask/FACETS_API.md @@ -0,0 +1,173 @@ +# Faceted Search API — a guide (for the undergrad) + +This explains **faceted search** in SBOLExplorer: what a facet is, why counting +facets does not need SPARQL at query time, the `/facets` API, and how the +frontend should use it. + +--- + +## 1. What is a "facet"? (the big picture) + +Look at the search UI: you type `rbs`, get results, and on the left there are +dropdowns — **Part Type**, **Part Role**, **Object Type**, **Collections**, +**Creator** — each value with a number: + +``` +Select Part Role + engineered_region (3789) + Composite (1552) + Generator (493) + ... +``` + +A **facet** is one of those category fields. A **facet count** is "how many of the +*current* results fall under each value." Click one, and the results narrow to +that value. + +Two different things: +- **Facet counts**: the numbers, used to *build* the dropdown. +- **Facet filtering**: clicking a value *narrows* the result list. + +--- + +## 2. Does this need SPARQL? + +### Index build (on `/update`): SPARQL required +Every part's data is pulled out of **Virtuoso** with SPARQL and stored as one +**document** per part in **Elasticsearch**. After the parts are indexed, the facet +values are added in a second pass (`index.index_facet_fields`), one SPARQL query +stream per facet, because a part can have **several** roles or collections. + +### Serving a search: no SPARQL +Elasticsearch already has everything. It computes the facet counts itself with an +**aggregation**, in the same request that applies the search text and the +selected filters. + +``` +/update (once): Virtuoso --SPARQL--> facet_role, facet_type, ... --> ES docs +/facets (each q): ES counts its own docs (Virtuoso not touched) +``` + +### What the current SynBioHub 3 UI does +Today `synbiohub3/frontend` sends one SPARQL query **per facet** to Virtuoso +(`sparql/getRoles.js`, `getTypes.js`, ...) plus one for the total. Those match +text with `CONTAINS` (substring, every word required) while the result list comes +from Elasticsearch (fuzzy, any word), so the numbers and the list disagree. Using +`/facets` fixes that: the counts come from the same query as the results. + +--- + +## 3. The backend API + +### Fields + +| facet name (API) | what it is | ES field | filled from (SPARQL) | +|---|---|---|---| +| `role` | Part Role (SO term or iGEM part type) | `facet_role` | `?subject sbol2:role ?value` | +| `sboltype` | Part Type (DnaRegion, Protein, ...) | `facet_sboltype` | `?subject sbol2:type ?value` | +| `type` | Object Type (ComponentDefinition, Collection, ...) | `facet_type` | `?subject a ?value` | +| `creator` | Creator | `facet_creator` | `?subject dc:creator ?value` | +| `collections` | Collections the part belongs to | `facet_collections` | `?value sbol2:member ?subject` | + +All of them are `keyword` arrays: a part with two roles is counted under both. +Definitions live in `facets.py` (`FACETS`). The fields only exist after a full +`/update`. + +### Endpoint: `GET /facets` + +``` +GET /facets?query=rbs&graph=https://synbiohub.org/public&facet_by=role,type +``` + +| param | meaning | default | +|---|---|---| +| `query` | search text, matched exactly like `/search` | '' (everything) | +| `graph` | graph the caller may see; **repeat** for several (public + the user's private graph) | public graphs only | +| `facet_by` | comma-separated facets to count | all five | +| `size` | max values per facet (1–500) | 50 | +| `role`, `sboltype`, `type`, `creator`, `collections` | selected filter values; **repeat** a param to select several | none | + +Filter semantics: **OR within a facet, AND across facets.** +`role=A&role=B&creator=X` means "(role A or B) and creator X". + +Response: +```json +{ + "found": 4908, + "facets": { + "role": [ + {"value": "http://identifiers.org/so/SO:0000804", "label": "engineered_region", "count": 3789}, + {"value": "http://wiki.synbiohub.org/wiki/Terms/igem#partType/Composite", "label": "Composite", "count": 1552} + ], + "collections": [ + {"value": "https://synbiohub.org/public/igem/igem_collection/1", "label": "iGEM Parts Registry", "count": 4863} + ] + } +} +``` + +- `found`: number of results with **all** selected filters applied. Use it for + "Showing 1–50 of N". +- `facets.`: one entry per value, sorted by count. Send `value` back as the + filter; show `label`. +- Each facet's counts ignore **its own** selection but honour all others + ("self-exclusion"). After picking role A, the role list still shows how many + results B, C, ... would give, instead of collapsing to just A. +- Values are exact keyword matches. Pass `value` back exactly as received. + +### Privacy +- Without `graph`, only public documents are counted. +- Collections are listed only if the collection itself is in a visible graph, so a + user's private collection never shows up for others. +- `graph` is trusted as given. The browser must **not** call SBOLExplorer + directly; the SynBioHub backend should forward the request and add the logged-in + user's graph, as it already does for search. + +### The code (where to look) +- `facets.py`: `FACETS`, `facet_search()` (builds the single ES request), labels. +- `explorer.py::facets_by_string()`: the `/facets` route. +- `index.py`: `index_facet_fields()` (full update) and `add_facet_fields()` + (incremental updates). +- `query.py::query_subject_values()`: streams `(subject, values)` from Virtuoso. +- `tests/test_facets.py`: unit tests. + +### Not supported (yet) +- The UI's free-form "additional filters" (any predicate = value). Keep the + existing SPARQL path for those. +- Private parts are counted from the ES index. Private results in the list come + from Virtuoso (#158), so a private part added since the last index update can + appear in the list before it is counted. + +--- + +## 4. What the frontend should do + +Two columns: the **facet sidebar** (left, from `/facets`) and the **results list** +(right, from the existing search). Send the **same** query and selections to both. + +```js +const BASE = "/api/explorer"; // via the SynBioHub backend, not SBOLExplorer directly + +async function runSearch(query, selected) { // selected = {role: [...], creator: [...]} + const params = new URLSearchParams({ query }); + for (const [facet, values] of Object.entries(selected)) + for (const v of values) params.append(facet, v); + + const facets = await fetch(`${BASE}/facets?${params}`).then(r => r.json()); + renderCount(facets.found); // "Showing 1-50 of N" + renderFacets(facets.facets); // label + (count); clicking toggles value in `selected` + // results: the existing search request with the same query + filters +} +``` + +`facets_frontend_example.html` is a self-contained demo of the layout. It was +written for the earlier Typesense prototype, so update its response handling to +the shape above (`found`, `facets.[].label`). + +--- + +## 5. TL;DR +- **Facet counts at query time come from Elasticsearch only. No SPARQL.** +- **SPARQL runs at index build**, to load every value of each facet. +- **One request** returns the total and all facet counts, consistent with the result list. +- Run one full `/update` before using `/facets`. diff --git a/flask/docker/docker-compose.yml b/flask/docker/docker-compose.yml index 9af118a..6331ff1 100644 --- a/flask/docker/docker-compose.yml +++ b/flask/docker/docker-compose.yml @@ -7,6 +7,11 @@ services: depends_on: - elasticsearch elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:6.3.2 + image: docker.elastic.co/elasticsearch/elasticsearch:9.5.3 + environment: + - discovery.type=single-node + - xpack.security.enabled=false + - ES_JAVA_OPTS=-Xms1g -Xmx1g ports: - "9200:9200" + diff --git a/flask/explorer.py b/flask/explorer.py index b3d3682..0c39cb3 100644 --- a/flask/explorer.py +++ b/flask/explorer.py @@ -10,6 +10,7 @@ import cluster import pagerank import index +import facets import search import query from configManager import ConfigManager @@ -172,7 +173,7 @@ def sparql_search_endpoint(): try: es = elasticsearch_manager.get_es() index_name = config_manager.load_config().get('elasticsearch_index_name') - if not es.indices.exists(index=index_name) or es.cat.indices(format='json')[0]['health'] == 'red': + if not es.indices.exists(index=index_name) or es.cat.indices(index=index_name, format='json')[0]['health'] == 'red': abort(503, 'Elasticsearch is not working or the index does not exist.') sparql_query = request.args.get('query') @@ -202,7 +203,7 @@ def search_by_string(): try: es = elasticsearch_manager.get_es() index_name = config_manager.load_config().get('elasticsearch_index_name') - if not es.indices.exists(index=index_name) or es.cat.indices(format='json')[0]['health'] == 'red': + if not es.indices.exists(index=index_name) or es.cat.indices(index=index_name, format='json')[0]['health'] == 'red': abort(503, 'Elasticsearch is not working or the index does not exist.') query = request.args.get('query') @@ -212,6 +213,40 @@ def search_by_string(): log.error(f'Error during search by string: {e}') raise +@app.route('/facets', methods=['GET']) +def facets_by_string(): + """ + Facet counts for a text search; see FACETS_API.md. + + GET /facets?query=rbs&graph=&graph=&role=&facet_by=role,type + """ + try: + es = elasticsearch_manager.get_es() + index_name = config_manager.load_config().get('elasticsearch_index_name') + if not es.indices.exists(index=index_name) or es.cat.indices(index=index_name, format='json')[0]['health'] == 'red': + abort(503, 'Elasticsearch is not working or the index does not exist.') + + facet_by = [f.strip() for f in request.args.get('facet_by', ','.join(facets.DEFAULT_FACETS)).split(',') if f.strip()] + unknown = [f for f in facet_by if f not in facets.FACETS] + if unknown: + abort(400, f'Unknown facet(s): {", ".join(unknown)}. Valid: {", ".join(facets.FACETS)}') + try: + size = min(max(int(request.args.get('size', 50)), 1), facets.MAX_FACET_SIZE) + except ValueError: + abort(400, 'size must be an integer') + + selected = {name: request.args.getlist(name) for name in facets.FACETS} + return jsonify(facets.facet_search( + request.args.get('query', '').strip(), + request.args.getlist('graph'), + selected, + facet_by, + size + )) + except Exception as e: + log.error(f'Error during facet search: {e}') + raise + # Initialize Explorer after all startup dependencies, including update_index, # have been defined. startup() diff --git a/flask/facets.py b/flask/facets.py new file mode 100644 index 0000000..ad9ad8b --- /dev/null +++ b/flask/facets.py @@ -0,0 +1,176 @@ +import json +import re +from functools import lru_cache +from typing import Dict, List, Optional + +import search +from configManager import ConfigManager +from elasticsearchManager import ElasticsearchManager + +config_manager = ConfigManager() +elasticsearch_manager = ElasticsearchManager(config_manager) + +# Facet name (the /facets API name) -> the keyword-array field it is stored in, +# and the SPARQL pattern (binding ?subject and ?value) that fills that field at +# index time. These fields are separate from the single-valued 'role', 'type' +# and 'sboltype' fields the text search uses, which keep only one value per part. +FACETS = { + 'role': { + 'field': 'facet_role', + 'pattern': '?subject sbol2:role ?value .' + }, + 'sboltype': { + 'field': 'facet_sboltype', + 'pattern': '?subject sbol2:type ?value .' + }, + 'type': { + 'field': 'facet_type', + 'pattern': '?subject a ?value .' + }, + 'creator': { + 'field': 'facet_creator', + 'pattern': '?subject dc:creator ?value .' + }, + 'collections': { + 'field': 'facet_collections', + 'pattern': '?value a sbol2:Collection . ?value sbol2:member ?subject .' + }, +} + +DEFAULT_FACETS = list(FACETS) +MAX_FACET_SIZE = 500 + +SO_TERM_PATTERN = re.compile(r'SO[:_](\d{7})') + + +@lru_cache(maxsize=1) +def so_labels() -> Dict[str, str]: + """Maps 'SO_0000167' -> 'promoter' from the bundled SO term list.""" + with open('so-simplified.json', 'r') as so_json: + terms = json.load(so_json) + return {term['id'].rsplit('/', 1)[-1]: term['lbl'] for term in terms if term.get('lbl')} + + +def short_name(uri: str) -> str: + return re.split(r'[#/]', uri.rstrip('/'))[-1] + + +def role_label(uri: str) -> str: + match = SO_TERM_PATTERN.search(uri) + if match: + label = so_labels().get(f'SO_{match.group(1)}') + if label: + return label + return short_name(uri) + + +def graph_filter(graphs: List[str]) -> Dict: + """ + Restricts documents to the caller's graphs. With no graphs given, only + public documents are counted, so a caller that forgets to pass its graphs + can never see counts (or collection names) from someone's private graph. + """ + if graphs: + return {'terms': {'graph': graphs}} + return {'bool': {'must_not': {'wildcard': {'graph': '*/user/*'}}}} + + +def selected_filters(selected: Dict[str, List[str]], exclude: Optional[str] = None) -> List[Dict]: + """OR within a facet, AND across facets; optionally leaving one facet out.""" + return [ + {'terms': {FACETS[name]['field']: values}} + for name, values in selected.items() + if values and name != exclude + ] + + +def facet_search(es_query: str, graphs: List[str], selected: Dict[str, List[str]], + facet_by: List[str], size: int = 50) -> Dict: + """ + Facet counts for a text search, in a single Elasticsearch request. + + Every facet's counts honour the selections in all OTHER facets but not its + own (self-exclusion), so picking one role still shows the counts of the + other roles. 'found' honours every selection: it is the size of the + filtered result set. + + Arguments: + es_query -- search text; '' counts every document in the graphs + graphs -- graph URIs the caller may see (public + their own private graph) + selected -- {facet name: [selected values]} + facet_by -- facets to return counts for + size -- maximum values returned per facet + + Returns: + {'found': int, 'facets': {name: [{'value', 'label', 'count'}]}} + """ + index_name = config_manager.load_config()['elasticsearch_index_name'] + es = elasticsearch_manager.get_es() + visible = graph_filter(graphs) + + aggs = {'_found': {'filter': {'bool': {'filter': selected_filters(selected)}}}} + for name in facet_by: + aggs[name] = { + 'filter': {'bool': {'filter': selected_filters(selected, exclude=name)}}, + 'aggs': {'values': {'terms': {'field': FACETS[name]['field'], 'size': size}}} + } + + body = { + 'size': 0, + 'query': { + 'bool': { + 'must': [search.text_query(es_query) if es_query else {'match_all': {}}], + 'filter': [visible] + } + }, + 'aggs': aggs + } + aggregations = es.search(index=index_name, body=body)['aggregations'] + + facets = {} + for name in facet_by: + buckets = aggregations[name]['values']['buckets'] + facets[name] = [{'value': b['key'], 'count': b['doc_count']} for b in buckets] + for name, values in facets.items(): + if name == 'collections': + facets[name] = visible_collections(es, index_name, values, visible) + else: + for value in values: + value['label'] = label_for(name, value['value']) + + return {'found': aggregations['_found']['doc_count'], 'facets': facets} + + +def label_for(name: str, value: str) -> str: + if name == 'role': + return role_label(value) + if name == 'creator': + return value + return short_name(value) + + +def visible_collections(es, index_name: str, values: List[Dict], visible: Dict) -> List[Dict]: + """ + Keeps only collections the caller may see, labelled with their name. + + A public part can be a member of someone's private collection; the counts + are over visible parts either way, but the private collection itself must + not appear as an option. Collections are top-level documents, so their own + 'graph' field decides visibility. + """ + if not values: + return [] + response = es.search(index=index_name, body={ + 'size': len(values), + '_source': ['subject', 'name', 'displayId'], + 'query': {'bool': {'filter': [{'terms': {'subject': [v['value'] for v in values]}}, visible]}} + }) + names = {} + for hit in response['hits']['hits']: + source = hit['_source'] + names[source['subject']] = source.get('name') or source.get('displayId') or short_name(source['subject']) + return [ + {**value, 'label': names[value['value']]} + for value in values + if value['value'] in names + ] diff --git a/flask/index.py b/flask/index.py index e4cb6ca..23cd643 100644 --- a/flask/index.py +++ b/flask/index.py @@ -1,6 +1,9 @@ +import threading + from elasticsearch import helpers from configManager import ConfigManager from elasticsearchManager import ElasticsearchManager +import facets import query import json from logger import Logger @@ -78,6 +81,11 @@ def add_sbol_type(parts_response): part['keywords'] += ' ' + type_ +# Upper bound on the subjects one filtered search may restrict to; above the +# total number of top-level objects (~180k). +MAX_TERMS_COUNT = 1000000 + + def create_parts_index(index_name): """ Creates a new index. @@ -86,25 +94,28 @@ def create_parts_index(index_name): index_name {String} -- Name of the new index """ es = elasticsearch_manager.get_es() - if es.indices.exists(index_name): + if es.indices.exists(index=index_name): logger_.log('Index already exists -> deleting', True) es.indices.delete(index=index_name) body = { 'mappings': { - index_name: { - 'properties': { - 'subject': { - 'type': 'keyword' - }, - 'graph': { - 'type': 'keyword' - } + 'properties': { + 'subject': { + 'type': 'keyword' }, - } + 'graph': { + 'type': 'keyword' + }, + **{facet['field']: {'type': 'keyword'} for facet in facets.FACETS.values()} + }, }, 'settings': { - 'number_of_shards': 1 + 'number_of_shards': 1, + 'number_of_replicas': 0, # single-node: a replica can never be allocated, leaving the index yellow + # search.subjects_query sends every part a filter allows (a whole + # collection is 100k+); the default limit is 65,536 terms. + 'max_terms_count': MAX_TERMS_COUNT } } es.indices.create(index=index_name, body=body) @@ -143,7 +154,6 @@ def actions(): continue yield { '_index': index_name, - '_type': index_name, '_id': part['subject'], '_source': part } @@ -158,6 +168,134 @@ def actions(): f'(ES _id limit); first: {skipped[0][:120]}', True) +def index_facet_fields(es, index_name): + """ + Fills the facet_* keyword-array fields of documents already in the index. + + The main parts query returns one row per (role, type, ...) combination and + later rows overwrite earlier ones under the same _id, so the documents keep + only one role. Facets need every value, so each facet is streamed from + Virtuoso grouped by subject and written with partial updates. Parts not in + the index (e.g. skipped for an over-long URI) are ignored. + """ + for name, facet in facets.FACETS.items(): + def actions(): + for subject, values in query.query_subject_values(facet['pattern']): + if len(subject.encode('utf-8')) > 512: # ES _id limit, see index_page + continue + yield { + '_op_type': 'update', + '_index': index_name, + '_id': subject, + 'doc': {facet['field']: sorted(values)} + } + + updated = missing = failed = 0 + for ok, item in helpers.streaming_bulk(es, actions(), raise_on_error=False): + if ok: + updated += 1 + elif item['update'].get('status') == 404: + missing += 1 + else: + failed += 1 + if failed <= 3: + logger_.log(f'[ERROR] facet {name} update failed: {item}', True) + logger_.log(f'Facet {name}: updated {updated} parts ' + f'({missing} not in index, {failed} failed)', True) + + +# Seconds after an incremental update at which the facet fields that only +# Virtuoso knows (collections, creator) are fetched again; see +# schedule_facet_refresh. +FACET_REFRESH_DELAYS = (30, 120, 600) + +# Facets whose value SynBioHub already sends in the /incrementalupdate payload. +PAYLOAD_FACETS = {'role': 'role', 'sboltype': 'sboltype', 'type': 'type'} + + +def add_facet_fields_from_payload(parts): + """ + Sets facet_role / facet_sboltype / facet_type from the fields SynBioHub + sends with each part, so they are right even before the part is in Virtuoso. + """ + for part in parts: + for name, key in PAYLOAD_FACETS.items(): + field = facets.FACETS[name]['field'] + values = set(part.get(field) or []) + if part.get(key): + values.add(part[key]) + part[field] = sorted(values) + + +def add_facet_fields(parts): + """ + Adds the facet values Virtuoso has for these parts to their facet_* fields + (incremental update / refresh), querying Virtuoso for just those subjects. + Existing values are kept, so values taken from the payload are not wiped + when Virtuoso does not have the part yet. + """ + subjects = [part['subject'] for part in parts] + for facet in facets.FACETS.values(): + values = query.query_values_for_subjects(facet['pattern'], subjects) + for part in parts: + field = facet['field'] + part[field] = sorted(set(part.get(field) or []) | values.get(part['subject'], set())) + + +def add_facet_fields_safely(parts): + # An incremental update must not fail because facets could not be fetched; + # the part is still searchable and the next full /update fills its facets. + try: + add_facet_fields(parts) + except Exception as e: + logger_.log(f'[ERROR] Error while adding facet fields: {str(e)}', True) + + +def schedule_facet_refresh(subjects, delays=FACET_REFRESH_DELAYS): + """ + Fetches the parts' facet values from Virtuoso again after delays[0] seconds. + + SynBioHub sends /incrementalupdate while it prepares a submission, before + it uploads the submission to Virtuoso, so collections and creator (which + only Virtuoso has) are usually missing at that point. Each attempt keeps + the parts that still have no collection for the next delay. + """ + if not subjects or not delays: + return + timer = threading.Timer(delays[0], refresh_facet_fields, args=(list(subjects), tuple(delays[1:]))) + timer.daemon = True + timer.start() + + +def refresh_facet_fields(subjects, remaining_delays=()): + """ + Writes the facet values Virtuoso now has for these indexed parts (partial + updates), then reschedules the parts still without a collection. + """ + collections_field = facets.FACETS['collections']['field'] + try: + found = {subject: {} for subject in subjects} + for facet in facets.FACETS.values(): + values = query.query_values_for_subjects(facet['pattern'], subjects) + for subject, subject_values in values.items(): + found[subject][facet['field']] = sorted(subject_values) + + # Virtuoso's values are complete once the part is uploaded (the same + # source a full /update uses); fields it has no value for are left as is. + actions = [ + {'_op_type': 'update', '_index': config['elasticsearch_index_name'], '_id': subject, 'doc': fields} + for subject, fields in found.items() if fields + ] + if actions: + helpers.bulk(elasticsearch_manager.get_es(), actions, raise_on_error=False) + pending = [subject for subject, fields in found.items() if not fields.get(collections_field)] + logger_.log(f'Facet refresh: {len(subjects) - len(pending)} of {len(subjects)} parts complete', True) + except Exception as e: + logger_.log(f'[ERROR] Facet refresh failed: {str(e)}', True) + pending = subjects + schedule_facet_refresh(pending, remaining_delays) + + def load_term_list(): """ Loads the SO-Ontologies term list that add_roles matches roles against. @@ -216,6 +354,14 @@ def update_index(uri2rank): create_parts_index(index_name) logger_.log(f'******** Finished adding {total} parts to index ********', True) + + # Search works without facets, so a failure here is logged, not raised. + try: + logger_.log('******** Adding facet fields ********', True) + index_facet_fields(es, index_name) + except Exception as e: + logger_.log(f'[ERROR] Error while adding facet fields: {str(e)}', True) + logger_.log('------------ Successfully updated index ------------\n', True) @@ -239,14 +385,14 @@ def delete_subject(subject): }, 'conflicts': 'proceed' } - es.delete_by_query(index=index_name, doc_type=index_name, body=body) + es.delete_by_query(index=index_name, body=body) def index_part(part): delete_subject(part['subject']) index_name = config['elasticsearch_index_name'] es = elasticsearch_manager.get_es() - es.index(index=index_name, doc_type=index_name, id=part['subject'], body=part) + es.index(index=index_name, id=part['subject'], body=part) def refresh_index(subject, uri2rank): @@ -256,6 +402,7 @@ def refresh_index(subject, uri2rank): if len(part_response) == 1: add_pagerank(part_response, uri2rank) add_keywords(part_response) + add_facet_fields_safely(part_response) index_part(part_response[0]) @@ -271,10 +418,16 @@ def incremental_update(updates, uri2rank): add_keywords(parts_to_add) add_roles(parts_to_add, load_term_list()) add_sbol_type(parts_to_add) + add_facet_fields_from_payload(parts_to_add) + add_facet_fields_safely(parts_to_add) for part in parts_to_add: index_part(part) + # Usually not in Virtuoso yet (see schedule_facet_refresh). + collections_field = facets.FACETS['collections']['field'] + schedule_facet_refresh([part['subject'] for part in parts_to_add if not part.get(collections_field)]) + def incremental_remove(subject): delete_subject(subject) diff --git a/flask/query.py b/flask/query.py index 08fc8a9..8fb5a34 100644 --- a/flask/query.py +++ b/flask/query.py @@ -113,6 +113,71 @@ def query_parts_paged(_from='', criteria='', indexing=False, page_size=10000): break offset += page_size +def _sparql_string(value): + """Escapes a value for use inside a SPARQL "..." string literal.""" + return value.replace('\\', '\\\\').replace('"', '\\"') + +def query_subject_values(pattern, page_size=10000): + """ + Streams (subject, values) for every top-level subject that matches + `pattern`, which binds ?subject and ?value (e.g. '?subject sbol2:role ?value .'). + Each subject is yielded once with ALL of its values, so multi-valued + properties (several roles, several collections) are not truncated. + + Pages by keyset on (subject, value) rather than OFFSET: Virtuoso refuses + ORDER BY + OFFSET once more than 10,000 rows would have to be sorted. Rows + arrive sorted by subject, so a subject's values may span pages; its group is + only yielded once a different subject (or the end) is reached. + """ + endpoint = config['sparql_endpoint'] + last = None # (subject, value) of the previous page's last row + current, values = None, set() + while True: + keyset = '' + if last is not None: + s, v = (_sparql_string(x) for x in last) + keyset = (f'FILTER(STR(?subject) > "{s}" || ' + f'(STR(?subject) = "{s}" && STR(?value) > "{v}"))') + page_query_ = f'''{QUERY_PREFIX} + SELECT DISTINCT ?subject ?value WHERE {{ + ?subject sbh:topLevel ?subject . + {pattern} + {keyset} + }} ORDER BY STR(?subject) STR(?value) LIMIT {page_size}''' + rows = send_query(page_query_, endpoint) + + for row in rows: + if row['subject'] != current: + if current is not None: + yield current, values + current, values = row['subject'], set() + values.add(row['value']) + + if len(rows) < page_size: + if current is not None: + yield current, values + return + last = (rows[-1]['subject'], rows[-1]['value']) + +def query_values_for_subjects(pattern, subjects, chunk_size=50): + """ + Like query_subject_values, but only for the given subjects (incremental + indexing). Returns {subject: set(values)}; subjects without values are absent. + """ + endpoint = config['sparql_endpoint'] + values = {} + subjects = list(subjects) + for start in range(0, len(subjects), chunk_size): + chunk = ' '.join(f'<{s}>' for s in subjects[start:start + chunk_size]) + rows = send_query(f'''{QUERY_PREFIX} + SELECT DISTINCT ?subject ?value WHERE {{ + VALUES ?subject {{ {chunk} }} + {pattern} + }}''', endpoint) + for row in rows: + values.setdefault(row['subject'], set()).add(row['value']) + return values + @lru_cache(maxsize=32) def memoized_query_sparql(query): """ diff --git a/flask/requirements.txt b/flask/requirements.txt index 828e708..eb332c6 100644 --- a/flask/requirements.txt +++ b/flask/requirements.txt @@ -1,18 +1,12 @@ -certifi==2018.4.16 -chardet==3.0.4 click==8.1.7 -elasticsearch==6.3.0 -elasticsearch-dsl==6.1.0 +elasticsearch>=9,<10 Flask==3.0.3 -idna==2.7 ipaddress==1.0.22 itsdangerous==2.2.0 Jinja2==3.1.4 MarkupSafe==2.1.5 numpy==2.1.1 -python-dateutil==2.7.3 -requests==2.19.1 +requests>=2.32 six==1.11.0 -urllib3==1.23 Werkzeug==3.0.4 apscheduler==3.10.4 diff --git a/flask/search.py b/flask/search.py index 86d6706..3aedc62 100644 --- a/flask/search.py +++ b/flask/search.py @@ -26,6 +26,41 @@ def extract_offset(sparql_query): offset_match = OFFSET_PATTERN.search(sparql_query) return int(offset_match.group(1)) if offset_match else 0 +def text_query(es_query: str) -> Dict: + """ + The free-text match shared by the string searches and /facets, so facet + counts are computed over exactly the documents a text search can return. + """ + return { + 'multi_match': { + 'query': es_query, + 'fields': [ + 'subject', + 'displayId^3', # caret indicates displayId is 3 times as important during search + 'version', + 'name', + 'description', + 'type', + 'keywords' + ], + 'operator': 'or', + 'fuzziness': 'AUTO' + } + } + +def subjects_query(subjects: List[str]) -> Dict: + """ + Restricts a search to the given subject URIs (the parts Virtuoso allowed). + + Uses a terms query on the keyword 'subject' field (equal to the document + _id) rather than an ids query: since ES 8 an ids query is capped at + max_result_window (10,000) ids, and a broad filter such as a whole + collection allows far more parts. The terms limit is the index's + max_terms_count, raised in index.create_parts_index. Both give every + match the same constant score, so ranking is unchanged. + """ + return {'terms': {'subject': list(subjects)}} + def search_es(es_query: str) -> Dict: """ String query for ES searches. @@ -33,22 +68,7 @@ def search_es(es_query: str) -> Dict: body = { 'query': { 'function_score': { - 'query': { - 'multi_match': { - 'query': es_query, - 'fields': [ - 'subject', - 'displayId^3', # caret indicates displayId is 3 times as important during search - 'version', - 'name', - 'description', - 'type', - 'keywords' - ], - 'operator': 'or', - 'fuzziness': 'AUTO' - } - }, + 'query': text_query(es_query), 'script_score': { 'script': { 'source': "_score * Math.log(doc['pagerank'].value + 1)" # Math.log is a natural log @@ -60,7 +80,7 @@ def search_es(es_query: str) -> Dict: 'size': 10000 } try: - return elasticsearch_manager.get_es().search(index=config_manager.load_config()['elasticsearch_index_name'], body=body) + return elasticsearch_manager.get_es().search(index=config_manager.load_config()['elasticsearch_index_name'], body=body, rest_total_hits_as_int=True) except: logger_.log("search_es(es_query: str)") raise @@ -96,10 +116,11 @@ def empty_search_es(offset: int, limit: int, allowed_graphs: List[str]) -> Dict: } }, 'from': offset, - 'size': limit + 'size': limit, + 'track_total_hits': True # ES 7+ stops counting at 10,000 by default; the total is returned as the result count } try: - return elasticsearch_manager.get_es().search(index=config_manager.load_config()['elasticsearch_index_name'], body=body) + return elasticsearch_manager.get_es().search(index=config_manager.load_config()['elasticsearch_index_name'], body=body, rest_total_hits_as_int=True) except: logger_.log("empty_search_es(offset: int, limit: int, allowed_graphs: List[str])") raise @@ -120,21 +141,8 @@ def search_es_allowed_subjects(es_query: str, allowed_subjects: List[str]) -> Di 'query': { 'bool': { 'must': [ - {'multi_match': { - 'query': es_query, - 'fields': [ - 'subject', - 'displayId^3', - 'version', - 'name', - 'description', - 'type', - 'keywords' - ], - 'operator': 'or', - 'fuzziness': 'AUTO' - }}, - {'ids': {'values': list(allowed_subjects)}} + text_query(es_query), + subjects_query(allowed_subjects) ] } }, @@ -149,7 +157,7 @@ def search_es_allowed_subjects(es_query: str, allowed_subjects: List[str]) -> Di 'size': 10000 } try: - return elasticsearch_manager.get_es().search(index=config_manager.load_config()['elasticsearch_index_name'], body=body) + return elasticsearch_manager.get_es().search(index=config_manager.load_config()['elasticsearch_index_name'], body=body, rest_total_hits_as_int=True) except: logger_.log("search_es_allowed_subjects(es_query: str, allowed_subjects: List[str])") raise @@ -169,7 +177,7 @@ def search_es_allowed_subjects_empty_string(allowed_subjects: List[str]): 'query': { 'bool': { 'must': [ - {'ids': {'values': list(allowed_subjects)}} + subjects_query(allowed_subjects) ] } }, @@ -184,7 +192,7 @@ def search_es_allowed_subjects_empty_string(allowed_subjects: List[str]): 'size': 10000 } try: - return elasticsearch_manager.get_es().search(index=config_manager.load_config()['elasticsearch_index_name'], body=body) + return elasticsearch_manager.get_es().search(index=config_manager.load_config()['elasticsearch_index_name'], body=body, rest_total_hits_as_int=True) except: logger_.log("search_es_allowed_subjects_empty_string") raise diff --git a/flask/tests/test_facets.py b/flask/tests/test_facets.py new file mode 100644 index 0000000..d02487d --- /dev/null +++ b/flask/tests/test_facets.py @@ -0,0 +1,82 @@ +import re +import unittest +from unittest import mock + +import facets +import query + + +def fake_virtuoso(pairs): + """ + Serves keyset-paged queries from an in-memory list of (subject, value) + pairs, honouring the (subject, value) keyset FILTER and LIMIT like Virtuoso. + """ + rows = sorted(set(pairs)) + + def send_query(sparql, endpoint): + limit = int(re.search(r'LIMIT (\d+)', sparql).group(1)) + keyset = re.search(r'STR\(\?subject\) > "([^"]*)".*STR\(\?value\) > "([^"]*)"', sparql) + selected = [r for r in rows if r > keyset.groups()] if keyset else rows + return [{'subject': s, 'value': v} for s, v in selected[:limit]] + return send_query + + +class QuerySubjectValuesTests(unittest.TestCase): + def collect(self, pairs, page_size): + with mock.patch('query.send_query', side_effect=fake_virtuoso(pairs)): + result = list(query.query_subject_values('?subject sbol2:role ?value .', page_size=page_size)) + subjects = [s for s, _ in result] + self.assertEqual(len(subjects), len(set(subjects)), 'a subject was yielded twice') + return dict(result) + + def expected(self, pairs): + grouped = {} + for s, v in pairs: + grouped.setdefault(s, set()).add(v) + return grouped + + def test_values_split_across_page_boundary_are_merged(self): + pairs = [('a', 'r1'), ('b', 'r1'), ('b', 'r2'), ('b', 'r3'), ('c', 'r1'), ('d', 'r1'), ('d', 'r2')] + for page_size in (2, 3, 4, 5, 100): + with self.subTest(page_size=page_size): + self.assertEqual(self.collect(pairs, page_size), self.expected(pairs)) + + def test_subject_with_more_values_than_a_page(self): + pairs = [('a', 'r1')] + [('b', f'r{i}') for i in range(7)] + [('c', 'r1')] + for page_size in (2, 3): + with self.subTest(page_size=page_size): + self.assertEqual(self.collect(pairs, page_size), self.expected(pairs)) + + def test_page_size_exact_multiple(self): + pairs = [('a', 'r1'), ('b', 'r1'), ('c', 'r1'), ('d', 'r1')] + self.assertEqual(self.collect(pairs, page_size=2), self.expected(pairs)) + + def test_empty(self): + self.assertEqual(self.collect([], page_size=3), {}) + + +class FacetQueryTests(unittest.TestCase): + def test_selected_filters_and_across_or_within(self): + selected = {'role': ['r1', 'r2'], 'creator': ['alice'], 'type': []} + self.assertEqual(facets.selected_filters(selected), [ + {'terms': {'facet_role': ['r1', 'r2']}}, + {'terms': {'facet_creator': ['alice']}}, + ]) + + def test_selected_filters_excludes_own_facet(self): + selected = {'role': ['r1'], 'creator': ['alice']} + self.assertEqual(facets.selected_filters(selected, exclude='role'), + [{'terms': {'facet_creator': ['alice']}}]) + + def test_no_graphs_means_public_only(self): + self.assertEqual(facets.graph_filter([]), + {'bool': {'must_not': {'wildcard': {'graph': '*/user/*'}}}}) + self.assertEqual(facets.graph_filter(['g1', 'g2']), {'terms': {'graph': ['g1', 'g2']}}) + + def test_role_label(self): + self.assertEqual(facets.role_label('http://identifiers.org/so/SO:0000167'), 'promoter') + self.assertEqual(facets.role_label('http://wiki.synbiohub.org/wiki/Terms/igem#partType/Composite'), 'Composite') + + +if __name__ == '__main__': + unittest.main() diff --git a/flask/tests/test_incremental_facets.py b/flask/tests/test_incremental_facets.py new file mode 100644 index 0000000..58aa858 --- /dev/null +++ b/flask/tests/test_incremental_facets.py @@ -0,0 +1,82 @@ +import unittest +from unittest import mock + +import index + +ROLE = 'http://identifiers.org/so/SO:0000167' +DNA = 'http://www.biopax.org/release/biopax-level3.owl#DnaRegion' +CD = 'http://sbols.org/v2#ComponentDefinition' +COLL = 'https://synbiohub.org/user/alice/c/c_collection/1' + + +def payload_part(subject): + # The shape SynBioHub's PrepareSubmissionJob sends to /incrementalupdate. + return {'subject': subject, 'displayId': 'p', 'role': ROLE, 'sboltype': DNA, + 'type': CD, 'graph': 'https://synbiohub.org/user/alice'} + + +class IncrementalUpdateTests(unittest.TestCase): + @mock.patch('index.schedule_facet_refresh') + @mock.patch('index.index_part') + @mock.patch('index.load_term_list', return_value=[]) + @mock.patch('index.query.query_values_for_subjects', return_value={}) + def test_payload_facets_kept_when_virtuoso_has_nothing_yet(self, values, _terms, index_part, schedule): + # SynBioHub sends the update before uploading to Virtuoso. + part = payload_part('s1') + index.incremental_update({'partsToAdd': [part]}, {}) + + indexed = index_part.call_args.args[0] + self.assertEqual(indexed['facet_role'], [ROLE]) + self.assertEqual(indexed['facet_sboltype'], [DNA]) + self.assertEqual(indexed['facet_type'], [CD]) + self.assertEqual(indexed['facet_collections'], []) + schedule.assert_called_once_with(['s1']) + + @mock.patch('index.schedule_facet_refresh') + @mock.patch('index.index_part') + @mock.patch('index.load_term_list', return_value=[]) + @mock.patch('index.query.query_values_for_subjects') + def test_no_refresh_when_virtuoso_already_has_collections(self, values, _terms, index_part, schedule): + values.side_effect = lambda pattern, subjects: ( + {'s1': {COLL}} if 'sbol2:member' in pattern else {'s1': {ROLE, 'http://identifiers.org/so/SO:0000804'}} + if 'sbol2:role' in pattern else {}) + index.incremental_update({'partsToAdd': [payload_part('s1')]}, {}) + + indexed = index_part.call_args.args[0] + self.assertEqual(indexed['facet_collections'], [COLL]) + self.assertEqual(len(indexed['facet_role']), 2) # Virtuoso adds the second role + schedule.assert_called_once_with([]) + + +class RefreshFacetFieldsTests(unittest.TestCase): + @mock.patch('index.schedule_facet_refresh') + @mock.patch('index.helpers.bulk') + @mock.patch('index.elasticsearch_manager') + @mock.patch('index.query.query_values_for_subjects') + def test_updates_found_parts_and_reschedules_the_rest(self, values, _es, bulk, schedule): + values.side_effect = lambda pattern, subjects: ( + {'s1': {COLL}} if 'sbol2:member' in pattern else {'s1': {'Alice'}} if 'dc:creator' in pattern else {}) + index.refresh_facet_fields(['s1', 's2'], (120, 600)) + + actions = bulk.call_args.args[1] + self.assertEqual([a['_id'] for a in actions], ['s1']) + self.assertEqual(actions[0]['doc'], {'facet_collections': [COLL], 'facet_creator': ['Alice']}) + schedule.assert_called_once_with(['s2'], (120, 600)) + + @mock.patch('index.schedule_facet_refresh') + @mock.patch('index.query.query_values_for_subjects', side_effect=Exception('Virtuoso down')) + def test_failure_retries_everything_later(self, _values, schedule): + index.refresh_facet_fields(['s1', 's2'], (600,)) + schedule.assert_called_once_with(['s1', 's2'], (600,)) + + @mock.patch('index.threading.Timer') + def test_schedule_stops_when_nothing_left(self, timer): + index.schedule_facet_refresh([], (30,)) + index.schedule_facet_refresh(['s1'], ()) + timer.assert_not_called() + index.schedule_facet_refresh(['s1'], (30, 120)) + timer.assert_called_once_with(30, index.refresh_facet_fields, args=(['s1'], (120,))) + + +if __name__ == '__main__': + unittest.main()