diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 639e136f..fa07cafd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -120,6 +120,7 @@ jobs: sql-server, label-studio, ocr-extraction, + caption-image, dataset-ingestion-movies ] runs-on: ubuntu-latest diff --git a/apps/caption-image/Dockerfile b/apps/caption-image/Dockerfile new file mode 100644 index 00000000..e3f8a8a0 --- /dev/null +++ b/apps/caption-image/Dockerfile @@ -0,0 +1,15 @@ +# Pull base image. +ARG VERSION=latest +FROM aperturedata/workflows-base:${VERSION} + +ENV APP_NAME=workflows-caption-image + +COPY requirements.txt / +RUN pip install -U pip +RUN pip install --no-cache-dir -r /requirements.txt + +COPY warmup_validate.py /warmup_validate.py +ARG PRELOAD_MODEL=false +RUN if [ "$PRELOAD_MODEL" = "true" ]; then python /warmup_validate.py; fi && rm /warmup_validate.py + +COPY app /app/ diff --git a/apps/caption-image/README.md b/apps/caption-image/README.md new file mode 100644 index 00000000..8f64a9d2 --- /dev/null +++ b/apps/caption-image/README.md @@ -0,0 +1,66 @@ +# Caption Image App + +This workflow retrieves all images from ApertureDB that have not been +analyzed before, and runs them through a +[BLIP (Bootstrapping Language-Image Pre-training)](https://github.com/salesforce/BLIP) +model to generate a caption for each image. + +The workflow runs continuously by default, periodically checking for and processing new uncaptioned images. To run it only once, set `RUN_ONCE=true`. + +## Database details + +```mermaid +sequenceDiagram + participant W as Caption Image + participant A as ApertureDB instance + + W->>A: FindImage + A-->>W: count + loop Until done + W->>A: FindImage + A-->>W: images + W->>A: UpdateImage + end +``` + +Each image is updated with a caption property (`wf_caption_image`) containing the generated caption text. The BLIP model processes each image to generate descriptive text that describes the visual content of the image. + +## Running in Docker + +``` +docker run \ + -e RUN_NAME=my_testing_run \ + -e DB_HOST=workflowstesting.gcp.cloud.aperturedata.dev \ + -e DB_PASS="password" \ + -e NUM_WORKERS=4 \ + -e BATCH_SIZE=32 \ + -e WF_LOG_LEVEL=INFO \ + aperturedata/workflows-caption-image +``` + +Parameters: +* **`NUM_WORKERS`**: Specifies the number of worker threads that will be running simultaneously, +retrieving and processing images in parallel. Default is `1`. Note that the BLIP model inference is serialized with a lock to prevent PyTorch intra-op threading conflicts on CPU, so increasing `NUM_WORKERS` only parallelizes image fetching and preprocessing, not inference itself. +* **`BATCH_SIZE`**: Specifies the batch size for processing images. Default is `1`. +* **`WF_LOG_LEVEL`**: Set log level for workflow code. Available options: DEBUG, INFO, WARNING, ERROR. Default is `WARNING`. `LOG_LEVEL` is also supported as a legacy alias. + +See [Common Parameters](../../README.md#common-parameters) for common parameters. + +## Cleaning up + +To clean all captions generated by this workflow, simply run the following query: + +``` +q = [ + { + "UpdateImage": { + "constraints": { + "wf_caption_image": ["!=", None] + }, + "remove_props": ["wf_caption_image", "wf_caption_image_done", "wf_caption_image_failed", "wf_caption_image_error"] + } + } + ] +``` + +or manually remove the `wf_caption_image`, `wf_caption_image_done`, `wf_caption_image_failed`, and `wf_caption_image_error` properties from images that have been processed. \ No newline at end of file diff --git a/apps/caption-image/app/app.sh b/apps/caption-image/app/app.sh new file mode 100644 index 00000000..87e219cd --- /dev/null +++ b/apps/caption-image/app/app.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +SLEEPING_TIME=$(/app/wf_argparse.py --type non_negative_int --envar SLEEPING_TIME --default 30) +RUN_ONCE=$(/app/wf_argparse.py --type bool --envar RUN_ONCE --default false) + +python3 status_tools.py --completed 0 --phases processing --phases sleeping --phase processing +while true; do + python3 status_tools.py --completed 0 --phase processing + python3 log_processor.py "python3 caption_images.py" + + if [ "$RUN_ONCE" = "true" ]; then + break + fi + python3 status_tools.py --completed 0 --phase sleeping + sleep $SLEEPING_TIME +done diff --git a/apps/caption-image/app/caption_images.py b/apps/caption-image/app/caption_images.py new file mode 100644 index 00000000..4d8ba1a3 --- /dev/null +++ b/apps/caption-image/app/caption_images.py @@ -0,0 +1,42 @@ +import logging + +import typer + +from images import FindImageQueryGenerator +from aperturedb import ParallelQuery +from connection_pool import ConnectionPool + +CAPTION_IMAGE_PROPERTY = 'wf_caption_image' + +def caption_images( + num_workers: int = typer.Option(1, envvar="NUM_WORKERS", help="Number of concurrent workers"), + batch_size: int = typer.Option(1, envvar="BATCH_SIZE", help="Batch size for fetching images"), + log_level: str = typer.Option("WARNING", envvar=["WF_LOG_LEVEL", "LOG_LEVEL"], help="Logging level") +): + num_workers = int(num_workers) + if num_workers <= 0: + raise ValueError("num_workers must be > 0") + + batch_size = int(batch_size) + if batch_size <= 0: + raise ValueError("batch_size must be > 0") + + logging.basicConfig(level=log_level.upper(), force=True) + logger = logging.getLogger(__name__) + pool = ConnectionPool() + data = FindImageQueryGenerator( + pool, + batch_size=batch_size, + caption_image_property=CAPTION_IMAGE_PROPERTY) + + logger.info("Running Caption Image...") + with pool.get_connection() as db: + querier = ParallelQuery.ParallelQuery(db) + querier.query(data, batchsize=1, numthreads=num_workers, stats=True) + + +def main(): + typer.run(caption_images) + +if __name__ == "__main__": + main() diff --git a/apps/caption-image/app/images.py b/apps/caption-image/app/images.py new file mode 100644 index 00000000..38d17137 --- /dev/null +++ b/apps/caption-image/app/images.py @@ -0,0 +1,250 @@ +import io +import math +import logging +import threading + +from PIL import Image + +from aperturedb import QueryGenerator + + +logger = logging.getLogger(__name__) + +# Lazy-loaded globals +_processor = None +_model = None +_model_lock = threading.Lock() +_inference_lock = threading.Lock() + +def get_model_and_processor(): + global _processor, _model + with _model_lock: + if _processor is None or _model is None: + from transformers import AutoProcessor, BlipForConditionalGeneration + _processor = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + _model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base") + _model.eval() + return _processor, _model + + +class FindImageQueryGenerator(QueryGenerator.QueryGenerator): + + """ + Generates n FindImage Queries + """ + + def __init__(self, pool, caption_image_property: str, batch_size: int = 32): + + self.pool = pool + self.caption_image_property = caption_image_property + + try: + self.batch_size = int(batch_size) + except ValueError: + raise ValueError(f"batch_size must be a positive integer, got {batch_size}") + + if self.batch_size <= 0: + raise ValueError(f"batch_size must be a positive integer, got {batch_size}") + + query = [{ + "FindImage": { + "constraints": { + self.caption_image_property + "_done": ["!=", True] + }, + "results": { + "count": True + } + } + }] + + status, response, _ = self.pool.execute_query(query) + if status != 0: + raise RuntimeError(f"Error executing query to find images: {response}") + + try: + total_images = response[0]["FindImage"]["count"] + except Exception as e: + logger.exception(f"error parsing count from response: {response}") + raise RuntimeError(f"error parsing count from response: {response}") from e + + if total_images == 0: + logger.warning("No images to be processed. Continuing!") + self.total_batches = 0 + self.len = 0 + return + + logger.info(f"Total images to process: {total_images}") + + self.total_batches = int(math.ceil(total_images / self.batch_size)) + self.len = self.total_batches + + def __len__(self): + return self.len + + def getitem(self, idx): + + if idx < 0 or self.len <= idx: + return None + + query = [{ + "FindImage": { + "blobs": True, + "constraints": { + self.caption_image_property + "_done": ["!=", True] + }, + "batch": { + "batch_size": self.batch_size, + "batch_id": idx + }, + "operations": [ + { + "type": "resize", + "width": 224, + "height": 224 + } + ], + "results": { + "list": ["_uniqueid"] + } + } + }] + + return query, [] + + def response_handler(self, query, blobs, response, r_blobs): + + try: + uniqueids = [i["_uniqueid"] + for i in response[0]["FindImage"]["entities"]] + except Exception as e: + logger.exception(f"error parsing uniqueids from response: {response}") + raise RuntimeError(f"error parsing uniqueids from response: {response}") from e + + if len(uniqueids) != len(r_blobs): + logger.error(f"Mismatch in response: {len(uniqueids)} uniqueids vs {len(r_blobs)} blobs") + query_fail = [] + ref_idx = 1 + for uid in uniqueids: + query_fail.append({ + "FindImage": {"_ref": ref_idx, "constraints": {"_uniqueid": ["==", uid]}} + }) + query_fail.append({ + "UpdateImage": { + "ref": ref_idx, + "properties": { + self.caption_image_property + "_done": True, + self.caption_image_property + "_failed": True, + self.caption_image_property + "_error": "Mismatch in blob response" + } + } + }) + ref_idx += 1 + status, r, _ = self.pool.execute_query(query_fail) + if status != 0: + logger.error(f"Failed to update images on mismatch: {r}") + return 0 + + processor, model = get_model_and_processor() + import torch + + valid_uniqueids = [] + captions = [] + failed_uniqueids = [] + failed_reasons = [] + + images_to_process = [] + texts = [] + uids_to_process = [] + + for uid, b in zip(uniqueids, r_blobs): + try: + image = Image.open(io.BytesIO(b)).convert("RGB") + images_to_process.append(image) + texts.append("A picture of") + uids_to_process.append(uid) + except Exception as e: + logger.error(f"Failed to load image {uid}: {e}") + failed_uniqueids.append(uid) + failed_reasons.append(str(e)) + + if images_to_process: + try: + inputs = processor(images=images_to_process, text=texts, return_tensors="pt", padding=True) + with _inference_lock: + with torch.no_grad(): + outputs = model.generate(**inputs) + batch_captions = processor.batch_decode(outputs, skip_special_tokens=True) + for uid, caption in zip(uids_to_process, batch_captions): + valid_uniqueids.append(uid) + captions.append(caption) + except Exception as e: + logger.error(f"Failed to process batch, falling back to per-image: {e}") + for uid, img, txt in zip(uids_to_process, images_to_process, texts): + try: + inputs = processor(images=img, text=txt, return_tensors="pt", padding=True) + with _inference_lock: + with torch.no_grad(): + outputs = model.generate(**inputs) + caption = processor.decode(outputs[0], skip_special_tokens=True) + valid_uniqueids.append(uid) + captions.append(caption) + except Exception as single_e: + logger.error(f"Failed to process image {uid} individually: {single_e}") + failed_uniqueids.append(uid) + failed_reasons.append(str(single_e)) + + if not valid_uniqueids and not failed_uniqueids: + return 0 + + query = [] + ref_idx = 1 + + for uniqueid, caption in zip(valid_uniqueids, captions): + query.append({ + "FindImage": { + "_ref": ref_idx, + "constraints": { + "_uniqueid": ["==", uniqueid] + }, + } + }) + + query.append({ + "UpdateImage": { + "ref": ref_idx, + "properties": { + self.caption_image_property: caption, + self.caption_image_property + "_done": True + }, + } + }) + ref_idx += 1 + + for uniqueid, reason in zip(failed_uniqueids, failed_reasons): + query.append({ + "FindImage": { + "_ref": ref_idx, + "constraints": { + "_uniqueid": ["==", uniqueid] + }, + } + }) + + query.append({ + "UpdateImage": { + "ref": ref_idx, + "properties": { + self.caption_image_property + "_done": True, + self.caption_image_property + "_failed": True, + self.caption_image_property + "_error": reason + }, + } + }) + ref_idx += 1 + + status, r, _ = self.pool.execute_query(query) + if status != 0: + logger.error(f"Query failed: {r}") + raise RuntimeError(f"Query failed: {r}") + + return len(valid_uniqueids) diff --git a/apps/caption-image/requirements.txt b/apps/caption-image/requirements.txt new file mode 100644 index 00000000..ca4b89c8 --- /dev/null +++ b/apps/caption-image/requirements.txt @@ -0,0 +1,5 @@ +--index-url https://download.pytorch.org/whl/cpu +--extra-index-url https://pypi.org/simple +torch>=2.0 +pillow +transformers>=4.38.0 diff --git a/apps/caption-image/test.sh b/apps/caption-image/test.sh new file mode 100755 index 00000000..1dfce121 --- /dev/null +++ b/apps/caption-image/test.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -x +set -euo pipefail +cd $(dirname "$(readlink -f "$0")") +source ../../.commonrc +run_pytest diff --git a/apps/caption-image/test/Dockerfile b/apps/caption-image/test/Dockerfile new file mode 100644 index 00000000..f975eb06 --- /dev/null +++ b/apps/caption-image/test/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.10-slim +WORKDIR /app + +RUN pip install --upgrade pip +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY *.py /app/ diff --git a/apps/caption-image/test/docker-compose.yml b/apps/caption-image/test/docker-compose.yml new file mode 100644 index 00000000..b0c160e0 --- /dev/null +++ b/apps/caption-image/test/docker-compose.yml @@ -0,0 +1,55 @@ +services: + test-base: + build: + context: apps/caption-image/test + dockerfile: Dockerfile + image: aperturedata/workflows-caption-image-tests:${VERSION:-latest} + deploy: + replicas: 0 + + seed: + image: aperturedata/workflows-caption-image-tests:${VERSION:-latest} + depends_on: + lenz: + condition: service_healthy + working_dir: /app + environment: + DB_HOST: lenz + DB_PORT: 55551 + DB_USER: admin + DB_PASS: admin + CA_CERT: /ca/ca.crt + volumes: + - ./ca:/ca + command: ["python", "/app/seed.py"] + + caption-image: + image: aperturedata/workflows-caption-image:${VERSION:-latest} + depends_on: + seed: + condition: service_completed_successfully + environment: + DB_HOST: lenz + DB_PORT: 55551 + DB_USER: admin + DB_PASS: admin + CA_CERT: /ca/ca.crt + RUN_ONCE: "true" + volumes: + - ./ca:/ca + + tests: + image: aperturedata/workflows-caption-image-tests:${VERSION:-latest} + depends_on: + caption-image: + condition: service_completed_successfully + environment: + DB_HOST: lenz + DB_PORT: 55551 + DB_USER: admin + DB_PASS: admin + CA_CERT: /ca/ca.crt + working_dir: /app + volumes: + - ./ca:/ca + command: ["pytest", "-vv", "-s", "-rA", "--log-cli-level=DEBUG"] diff --git a/apps/caption-image/test/requirements.txt b/apps/caption-image/test/requirements.txt new file mode 100644 index 00000000..aefbf66c --- /dev/null +++ b/apps/caption-image/test/requirements.txt @@ -0,0 +1,3 @@ +aperturedb +pytest +Pillow diff --git a/apps/caption-image/test/seed.py b/apps/caption-image/test/seed.py new file mode 100644 index 00000000..a13b4561 --- /dev/null +++ b/apps/caption-image/test/seed.py @@ -0,0 +1,47 @@ +import os +import sys +import io +from PIL import Image +from aperturedb.CommonLibrary import execute_query +from aperturedb.Connector import Connector + +def db_connection(): + DB_HOST = os.getenv("DB_HOST", "lenz") + DB_PORT = int(os.getenv("DB_PORT", "55551")) + DB_USER = os.getenv("DB_USER", "admin") + DB_PASS = os.getenv("DB_PASS", "admin") + CA_CERT = os.getenv("CA_CERT", None) + return Connector(host=DB_HOST, user=DB_USER, port=DB_PORT, password=DB_PASS, ca_cert=CA_CERT) + +def main(): + print("Starting caption-image test data seeding...") + client = db_connection() + + try: + # Create a simple test image + img = Image.new('RGB', (100, 100), color = 'red') + img_byte_arr = io.BytesIO() + img.save(img_byte_arr, format='JPEG') + img_data = img_byte_arr.getvalue() + + query = [{ + "AddImage": { + "properties": { + "filename": "test_red_square.jpg" + } + } + }] + + status, response, _ = execute_query(client, query, [img_data]) + if status != 0: + print(f"Failed to create test image: {response}") + sys.exit(1) + + print("Created test image successfully.") + + except Exception as e: + print(f"Error during seeding: {e}") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/apps/caption-image/test/test_caption.py b/apps/caption-image/test/test_caption.py new file mode 100644 index 00000000..cdadc35e --- /dev/null +++ b/apps/caption-image/test/test_caption.py @@ -0,0 +1,34 @@ +import os +from aperturedb.CommonLibrary import execute_query +from aperturedb.Connector import Connector + +def db_connection(): + DB_HOST = os.getenv("DB_HOST", "lenz") + DB_PORT = int(os.getenv("DB_PORT", "55551")) + DB_USER = os.getenv("DB_USER", "admin") + DB_PASS = os.getenv("DB_PASS", "admin") + CA_CERT = os.getenv("CA_CERT", None) + return Connector(host=DB_HOST, user=DB_USER, port=DB_PORT, password=DB_PASS, ca_cert=CA_CERT) + +def test_caption_added(): + client = db_connection() + query = [{ + "FindImage": { + "constraints": { + "filename": ["==", "test_red_square.jpg"] + }, + "results": { + "list": ["wf_caption_image_done", "wf_caption_image"] + } + } + }] + status, response, _ = execute_query(client, query) + assert status == 0, f"Query failed: {response}" + + entities = response[0].get("FindImage", {}).get("entities", []) + assert len(entities) > 0, "Image not found" + + props = entities[0] + assert props.get("wf_caption_image_done") == True, f"Image not marked as done: {props}" + assert "wf_caption_image" in props, f"Caption missing: {props}" + print(f"Caption generated: {props['wf_caption_image']}") diff --git a/apps/caption-image/warmup_validate.py b/apps/caption-image/warmup_validate.py new file mode 100644 index 00000000..0d17cff6 --- /dev/null +++ b/apps/caption-image/warmup_validate.py @@ -0,0 +1,20 @@ +import torch +from PIL import Image +from transformers import AutoProcessor, BlipForConditionalGeneration + +# This serves as a warmup for the model to load into memory +# It also validates that the model is working correctly +processor = AutoProcessor.from_pretrained("Salesforce/blip-image-captioning-base") +model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base") +model.eval() + +# Use a dummy image instead of fetching from external network +image = Image.new("RGB", (224, 224), color="red") +text = "A picture of" + +inputs = processor(images=image, text=text, return_tensors="pt") + +with torch.no_grad(): + output = model.generate(**inputs) +caption = processor.decode(output[0], skip_special_tokens=True) +print("Warmup complete. Dummy image caption:", caption) diff --git a/apps/embeddings-extraction/test/docker-compose.yml b/apps/embeddings-extraction/test/docker-compose.yml index 8920b22c..6f38a6d5 100644 --- a/apps/embeddings-extraction/test/docker-compose.yml +++ b/apps/embeddings-extraction/test/docker-compose.yml @@ -14,7 +14,7 @@ services: image: aperturedata/workflows-embeddings-extraction-tests:${VERSION:-latest} depends_on: lenz: - condition: service_started + condition: service_healthy working_dir: /app environment: DB_HOST: lenz diff --git a/apps/ingest-croissant/test.sh b/apps/ingest-croissant/test.sh index 6412f087..1469fca7 100755 --- a/apps/ingest-croissant/test.sh +++ b/apps/ingest-croissant/test.sh @@ -8,6 +8,9 @@ if [ $CI_RUN -eq 0 ]; then $COMMAND build base fi +# Override the Croissant URL for tests to use the local dummy data +export WF_CROISSANT_URL="/test_data/croissant.json" + # This log file is useful for debugging test failures TEST_LOG=$BIN_DIR/test.log echo "Writing logs to $TEST_LOG" diff --git a/apps/ingest-croissant/test_data/croissant.json b/apps/ingest-croissant/test_data/croissant.json new file mode 100644 index 00000000..1f9d564f --- /dev/null +++ b/apps/ingest-croissant/test_data/croissant.json @@ -0,0 +1,96 @@ +{ + "@context": { + "@language": "en", + "@vocab": "https://schema.org/", + "citeAs": "cr:citeAs", + "column": "cr:column", + "conformsTo": "dct:conformsTo", + "cr": "http://mlcommons.org/croissant/", + "data": { + "@id": "cr:data", + "@type": "@json" + }, + "dataBiases": "cr:dataBiases", + "dataCollection": "cr:dataCollection", + "dataType": { + "@id": "cr:dataType", + "@type": "@vocab" + }, + "dct": "http://purl.org/dc/terms/", + "extract": "cr:extract", + "field": "cr:field", + "fileProperty": "cr:fileProperty", + "fileObject": "cr:fileObject", + "fileSet": "cr:fileSet", + "format": "cr:format", + "includes": "cr:includes", + "isLiveDataset": "cr:isLiveDataset", + "jsonPath": "cr:jsonPath", + "key": "cr:key", + "md5": "cr:md5", + "parentField": "cr:parentField", + "path": "cr:path", + "personalSensitiveInformation": "cr:personalSensitiveInformation", + "recordSet": "cr:recordSet", + "references": "cr:references", + "regex": "cr:regex", + "repeated": "cr:repeated", + "replace": "cr:replace", + "sc": "https://schema.org/", + "separator": "cr:separator", + "source": "cr:source", + "subField": "cr:subField", + "transform": "cr:transform" + }, + "@type": "sc:Dataset", + "conformsTo": "http://mlcommons.org/croissant/1.0", + "name": "Dummy", + "description": "Dummy dataset", + "url": "https://dummy.co", + "distribution": [ + { + "@type": "cr:FileObject", + "@id": "dummy.csv", + "name": "dummy.csv", + "contentUrl": "/test_data/dummy.csv", + "encodingFormat": "text/csv", + "md5": "c3c6bc2ae8ece4bd2510dca21225c041" + } + ], + "recordSet": [ + { + "@type": "cr:RecordSet", + "@id": "dummy_records", + "name": "dummy_records", + "description": "Dummy records", + "field": [ + { + "@type": "cr:Field", + "@id": "dummy_records/a", + "dataType": "sc:Integer", + "source": { + "fileObject": { + "@id": "dummy.csv" + }, + "extract": { + "column": "a" + } + } + }, + { + "@type": "cr:Field", + "@id": "dummy_records/b", + "dataType": "sc:Integer", + "source": { + "fileObject": { + "@id": "dummy.csv" + }, + "extract": { + "column": "b" + } + } + } + ] + } + ] +} diff --git a/apps/ingest-croissant/test_data/dummy.csv b/apps/ingest-croissant/test_data/dummy.csv new file mode 100644 index 00000000..0099ae93 --- /dev/null +++ b/apps/ingest-croissant/test_data/dummy.csv @@ -0,0 +1,3 @@ +a,b +1,2 +3,4 diff --git a/apps/mcp-server/test/docker-compose.yml b/apps/mcp-server/test/docker-compose.yml index aa498a16..f4edfbc5 100644 --- a/apps/mcp-server/test/docker-compose.yml +++ b/apps/mcp-server/test/docker-compose.yml @@ -13,8 +13,10 @@ services: seed: image: aperturedata/workflows-mcp-server-tests:${VERSION:-latest} depends_on: + aperturedb: + condition: service_healthy lenz: - condition: service_started + condition: service_healthy working_dir: /app environment: DB_HOST: lenz @@ -24,7 +26,7 @@ services: CA_CERT: /ca/ca.crt volumes: - ./ca:/ca - command: ["python", "/app/seed.py"] + command: ["sh", "-c", "for i in $(seq 1 10); do python /app/seed.py && exit 0; sleep 5; done; exit 1"] mcp-server: depends_on: diff --git a/apps/mcp-server/test/test_find_similar.py b/apps/mcp-server/test/test_find_similar.py index 33de5e02..79061ada 100644 --- a/apps/mcp-server/test/test_find_similar.py +++ b/apps/mcp-server/test/test_find_similar.py @@ -20,7 +20,7 @@ def mcp_auth(): @pytest_asyncio.fixture async def client(mcp_url, mcp_auth): """Get a FastMCP client.""" - async with Client(mcp_url, auth=mcp_auth, timeout=30) as aclient: + async with Client(mcp_url, auth=mcp_auth, timeout=300) as aclient: yield aclient diff --git a/apps/ocr-extraction/test/docker-compose.yml b/apps/ocr-extraction/test/docker-compose.yml index 13e84934..0ac5efea 100644 --- a/apps/ocr-extraction/test/docker-compose.yml +++ b/apps/ocr-extraction/test/docker-compose.yml @@ -14,7 +14,7 @@ services: image: aperturedata/workflows-ocr-extraction-tests:${VERSION:-latest} depends_on: lenz: - condition: service_started + condition: service_healthy working_dir: /app environment: DB_HOST: lenz diff --git a/apps/sql-server/test/docker-compose.yml b/apps/sql-server/test/docker-compose.yml index 49c3ced1..00d14b13 100644 --- a/apps/sql-server/test/docker-compose.yml +++ b/apps/sql-server/test/docker-compose.yml @@ -14,7 +14,7 @@ services: image: aperturedata/workflows-sql-server-tests:${VERSION:-latest} depends_on: lenz: - condition: service_started + condition: service_healthy working_dir: /app environment: DB_HOST: lenz diff --git a/docker-compose.yml b/docker-compose.yml index 01f119d8..b90ecfb3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,6 +50,13 @@ services: # ports: # - 55555:55551 restart: always + healthcheck: + test: + - CMD-SHELL + - "bash -lc 'echo > /dev/tcp/127.0.0.1/$${LNZ_TCP_PORT}'" + interval: 2s + timeout: 1s + retries: 60 environment: LNZ_HEALTH_PORT: 58085 LNZ_TCP_PORT: 55551 @@ -124,7 +131,7 @@ services: image: aperturedata/workflows-crawl-website:${VERSION} depends_on: lenz: - condition: service_started + condition: service_healthy environment: <<: *common-env WF_CLEAN: "${WF_CLEAN:-true}" @@ -146,7 +153,7 @@ services: image: aperturedata/workflows-embeddings-extraction:${VERSION} depends_on: lenz: - condition: service_started + condition: service_healthy # Don't let this service kill your dev box # It's OK, but lets' not over compensate for it. deploy: @@ -202,7 +209,7 @@ services: image: aperturedata/workflows-jupyterlab:${VERSION} depends_on: lenz: - condition: service_started + condition: service_healthy ports: - "8888:8888" healthcheck: @@ -383,9 +390,10 @@ services: image: aperturedata/workflows-ingest-croissant:${VERSION} depends_on: lenz: - condition: service_started + condition: service_healthy volumes: - ./ca:/ca + - ./apps/ingest-croissant/test_data:/test_data environment: <<: *common-env WF_LOG_LEVEL: "${WF_LOG_LEVEL:-DEBUG}" @@ -404,7 +412,7 @@ services: image: aperturedata/workflows-dataset-ingestion:${VERSION} depends_on: lenz: - condition: service_started + condition: service_healthy volumes: - ./ca:/ca environment: @@ -428,7 +436,7 @@ services: image: aperturedata/workflows-dataset-ingestion-movies:${VERSION} depends_on: lenz: - condition: service_started + condition: service_healthy volumes: - ./ca:/ca environment: @@ -440,7 +448,7 @@ services: image: aperturedata/wf-add-image:latest depends_on: lenz: - condition: service_started + condition: service_healthy volumes: - ./ca:/ca environment: @@ -457,7 +465,7 @@ services: image: aperturedata/workflows-face-detection:${VERSION} depends_on: lenz: - condition: service_started + condition: service_healthy add-image: condition: service_completed_successfully volumes: @@ -477,7 +485,7 @@ services: image: aperturedata/workflows-object-detection:${VERSION} depends_on: lenz: - condition: service_started + condition: service_healthy add-image: condition: service_completed_successfully volumes: @@ -487,6 +495,28 @@ services: RUN_ONCE: "${RUN_ONCE:-true}" MODEL_NAME: "${MODEL_NAME:-frcnn-mobilenet}" + caption-image: + build: + context: ./apps/caption-image + args: + <<: *build-args + PRELOAD_MODEL: "${PRELOAD_MODEL:-false}" + labels: + <<: *build-labels + org.opencontainers.image.ref.name: "docker.io/aperturedata/workflows-caption-image:${VERSION}" + image: aperturedata/workflows-caption-image:${VERSION} + depends_on: + lenz: + condition: service_healthy + volumes: + - ./ca:/ca + environment: + <<: *common-env + RUN_ONCE: "${RUN_ONCE:-true}" + NUM_WORKERS: "${NUM_WORKERS:-1}" + BATCH_SIZE: "${BATCH_SIZE:-1}" + WF_LOG_LEVEL: "${WF_LOG_LEVEL:-WARNING}" + ingest-from-sql: image: aperturedata/workflows-ingest-from-sql:${VERSION} build: