diff --git a/ARGO/oceancurrent/README.md b/ARGO/oceancurrent/README.md index 3e9194ff..25e775ca 100644 --- a/ARGO/oceancurrent/README.md +++ b/ARGO/oceancurrent/README.md @@ -167,3 +167,55 @@ Currently, the script is adaptive to these products: `currentMetersPlot-48` | `\timeseries\currentMetersPlot-48.json` `currentMetersPlot-49` | `\timeseries\currentMetersPlot-49.json` + +## Gliders asset indexes + +The scanner discovers assets under `/mnt/oceancurrent/website/gliders/`. +Registering these proposed product IDs in `ocean-current-api` is a separate +future task; this change does not integrate Gliders into the API or frontend. + +| Product ID | JSON path relative to the website root | Assets | +| --- | --- | --- | +| `gliders-mission` | `/gliders/gliders-mission.json` | Standard mission GIFs | +| `gliders-mission-video` | `/gliders/gliders-mission-video.json` | Standard mission MP4s | +| `gliders-navigation` | `/gliders/gliders-navigation.json` | Navigation GIFs | +| `gliders-navigation-video` | `/gliders/gliders-navigation-video.json` | Navigation MP4s | +| `gliders-ts` | `/gliders/gliders-ts.json` | GIFs directly under `missionTS/` | +| `gliders-yearmaps` | `/gliders/gliders-yearmaps.json` | GIFs directly under `yearmaps/` | + +Mission directory names start with a letter and contain letters, digits, +underscores or hyphens, followed by an eight-digit mission date. The date can +have a single-letter suffix or a `_1` / `_2` mission leg. Supported periods are +`_4d`, `_12d` and `_30d`, with optional `_nrt` before or after the period: + +- `TasEastCoast20260922_nrt_4d` +- `Brisbane20180306A_4d` +- `JervisBay20180503a_nrt_12d` +- `Perth20110626_1_4d` +- `Cooktown20160217_4d_nrt` + +Navigation directories use the same format with a `nav` prefix and are excluded +from standard mission indexes. Only files directly inside matching directories +are indexed. GIF and MP4 extensions are case insensitive; HTML, JSON, PHP, +unsupported periods, scientific variant suffixes and auxiliary folders are excluded. +Missing periods or animations are allowed. An index is written only when matching +assets exist; the existing scanner does not remove stale indexes when assets disappear. + +Each mission directory is preserved in `path` and used verbatim as `region`, +so identical filenames from different missions remain distinguishable. `depth` +is null; T-S and annual-map records also have null `region`. Example record: + +```json +{ + "path": "/gliders/TasEastCoast20260922_nrt_4d", + "productId": "gliders-mission", + "region": "TasEastCoast20260922_nrt_4d", + "depth": null, + "files": [{"name": "2026101500.gif"}] +} +``` + +Merging activates these configurations on the next scheduled run because the +production job downloads the scanner from `master`. Gliders runs after existing +products and writes up to six indexes under `gliders/`; check the existing runtime +and memory logs after deployment. No cron or deployment changes are required. diff --git a/ARGO/oceancurrent/oceancurrent_file_server_api.py b/ARGO/oceancurrent/oceancurrent_file_server_api.py index b7447817..4f10d8e8 100644 --- a/ARGO/oceancurrent/oceancurrent_file_server_api.py +++ b/ARGO/oceancurrent/oceancurrent_file_server_api.py @@ -546,6 +546,99 @@ def _load_api_endpoint(): "filetype": ".gif", "max_layer": 2, "save_in_product_folder": True + }, + # Keep Gliders last so existing products are processed first. + { + "productId": "gliders-mission", + "include": [ + {"path": "^gliders$", "layer": 1}, + { + "path": ( + "^(?!nav)[A-Za-z][A-Za-z0-9_-]*[0-9]{8}" + "(?:[A-Za-z]|_[12])?" + "(?:_nrt_(?:4|12|30)d|_(?:4|12|30)d(?:_nrt)?)$" + ), + "layer": 2 + } + ], + "filetype": ".gif", + "region_layer": 2, + "max_layer": 2, + "save_in_product_folder": True + }, + { + "productId": "gliders-mission-video", + "include": [ + {"path": "^gliders$", "layer": 1}, + { + "path": ( + "^(?!nav)[A-Za-z][A-Za-z0-9_-]*[0-9]{8}" + "(?:[A-Za-z]|_[12])?" + "(?:_nrt_(?:4|12|30)d|_(?:4|12|30)d(?:_nrt)?)$" + ), + "layer": 2 + } + ], + "filetype": ".mp4", + "region_layer": 2, + "max_layer": 2, + "save_in_product_folder": True + }, + { + "productId": "gliders-navigation", + "include": [ + {"path": "^gliders$", "layer": 1}, + { + "path": ( + "^nav[A-Za-z][A-Za-z0-9_-]*[0-9]{8}" + "(?:[A-Za-z]|_[12])?" + "(?:_nrt_(?:4|12|30)d|_(?:4|12|30)d(?:_nrt)?)$" + ), + "layer": 2 + } + ], + "filetype": ".gif", + "region_layer": 2, + "max_layer": 2, + "save_in_product_folder": True + }, + { + "productId": "gliders-navigation-video", + "include": [ + {"path": "^gliders$", "layer": 1}, + { + "path": ( + "^nav[A-Za-z][A-Za-z0-9_-]*[0-9]{8}" + "(?:[A-Za-z]|_[12])?" + "(?:_nrt_(?:4|12|30)d|_(?:4|12|30)d(?:_nrt)?)$" + ), + "layer": 2 + } + ], + "filetype": ".mp4", + "region_layer": 2, + "max_layer": 2, + "save_in_product_folder": True + }, + { + "productId": "gliders-ts", + "include": [ + {"path": "^gliders$", "layer": 1}, + {"path": "^missionTS$", "layer": 2} + ], + "filetype": ".gif", + "max_layer": 2, + "save_in_product_folder": True + }, + { + "productId": "gliders-yearmaps", + "include": [ + {"path": "^gliders$", "layer": 1}, + {"path": "^yearmaps$", "layer": 2} + ], + "filetype": ".gif", + "max_layer": 2, + "save_in_product_folder": True } ] diff --git a/ARGO/oceancurrent/test_oceancurrent_file_server_api.py b/ARGO/oceancurrent/test_oceancurrent_file_server_api.py index e669047d..10fcd64c 100644 --- a/ARGO/oceancurrent/test_oceancurrent_file_server_api.py +++ b/ARGO/oceancurrent/test_oceancurrent_file_server_api.py @@ -478,6 +478,137 @@ def prepare_test_cases(self): ] } ], + "gliders-mission": [ + { + "path": "/gliders/TasEastCoast20260922_nrt_4d", + "productId": "gliders-mission", + "region": "TasEastCoast20260922_nrt_4d", + "depth": None, + "files": [{"name": "2026101500.gif"}, {"name": "2026101512.gif"}] + }, + { + "path": "/gliders/TasEastCoast20260922_nrt_12d", + "productId": "gliders-mission", + "region": "TasEastCoast20260922_nrt_12d", + "depth": None, + "files": [{"name": "2026101500.gif"}] + }, + { + "path": "/gliders/TasEastCoast20260922_nrt_30d", + "productId": "gliders-mission", + "region": "TasEastCoast20260922_nrt_30d", + "depth": None, + "files": [{"name": "2026101500.gif"}] + }, + { + "path": "/gliders/TasEastCoast20260811_4d", + "productId": "gliders-mission", + "region": "TasEastCoast20260811_4d", + "depth": None, + "files": [{"name": "2026101500.gif"}] + }, + { + "path": "/gliders/Brisbane20180306A_4d", + "productId": "gliders-mission", + "region": "Brisbane20180306A_4d", + "depth": None, + "files": [{"name": "2018031500.gif"}] + }, + { + "path": "/gliders/Perth20110626_1_4d", + "productId": "gliders-mission", + "region": "Perth20110626_1_4d", + "depth": None, + "files": [{"name": "2011070100.gif"}] + }, + { + "path": "/gliders/Cooktown20160217_4d_nrt", + "productId": "gliders-mission", + "region": "Cooktown20160217_4d_nrt", + "depth": None, + "files": [{"name": "2016022000.gif"}] + }, + { + "path": "/gliders/JervisBay20180503a_nrt_12d", + "productId": "gliders-mission", + "region": "JervisBay20180503a_nrt_12d", + "depth": None, + "files": [{"name": "2018051000.gif"}] + }, + ], + "gliders-mission-video": [ + { + "path": "/gliders/TasEastCoast20260922_nrt_4d", + "productId": "gliders-mission-video", + "region": "TasEastCoast20260922_nrt_4d", + "depth": None, + "files": [{"name": "TasEastCoast20260922_nrt_4d.mp4"}] + }, + ], + "gliders-navigation": [ + { + "path": "/gliders/navTasEastCoast20260922_nrt_4d", + "productId": "gliders-navigation", + "region": "navTasEastCoast20260922_nrt_4d", + "depth": None, + "files": [{"name": "2026101500.gif"}] + }, + { + "path": "/gliders/navBrisbane20180306A_nrt_4d", + "productId": "gliders-navigation", + "region": "navBrisbane20180306A_nrt_4d", + "depth": None, + "files": [{"name": "2018031500.gif"}] + }, + { + "path": "/gliders/navPerth20110626_1_4d", + "productId": "gliders-navigation", + "region": "navPerth20110626_1_4d", + "depth": None, + "files": [{"name": "2011070100.gif"}] + }, + { + "path": "/gliders/navCooktown20160217_4d_nrt", + "productId": "gliders-navigation", + "region": "navCooktown20160217_4d_nrt", + "depth": None, + "files": [{"name": "2016022000.gif"}] + }, + { + "path": "/gliders/navJervisBay20180503a_nrt_12d", + "productId": "gliders-navigation", + "region": "navJervisBay20180503a_nrt_12d", + "depth": None, + "files": [{"name": "2018051000.gif"}] + }, + ], + "gliders-navigation-video": [ + { + "path": "/gliders/navTasEastCoast20260922_nrt_4d", + "productId": "gliders-navigation-video", + "region": "navTasEastCoast20260922_nrt_4d", + "depth": None, + "files": [{"name": "navTasEastCoast20260922_nrt_4d.mp4"}] + }, + ], + "gliders-ts": [ + { + "path": "/gliders/missionTS", + "productId": "gliders-ts", + "region": None, + "depth": None, + "files": [{"name": "sl_TasEastCoast20260922_nrt.gif"}] + }, + ], + "gliders-yearmaps": [ + { + "path": "/gliders/yearmaps", + "productId": "gliders-yearmaps", + "region": None, + "depth": None, + "files": [{"name": "2026_sl_temp_nrt.gif"}] + }, + ], } def load_and_normalize_json(self, file_path): @@ -574,6 +705,20 @@ def test_file_structure_explorer(self): self.verify_json("fishSOOP-profiles", "fishsoop", "fishSOOP-profiles") self.verify_json("fishSOOP-quarterlyAnomalies", "fishsoop", "fishSOOP-quarterlyAnomalies") self.verify_json("fishSOOP-depthAnomalies", "fishsoop", "fishSOOP-depthAnomalies") + # Exact Gliders paths exclude navigation from standard indexes and + # unrelated directories. Exact filenames also exclude HTML/JSON/PHP. + for product_id in ("gliders-mission", "gliders-mission-video", + "gliders-navigation", "gliders-navigation-video", + "gliders-ts", "gliders-yearmaps"): + self.verify_json(product_id, "gliders", product_id) + output = os.path.join(self.file_test_dir, "gliders", f"{product_id}.json") + actual = self.load_and_normalize_json(output) + expected = self.prepare_test_cases()[product_id] + self.assertEqual( + {item["path"]: sorted(f["name"] for f in item["files"]) for item in actual}, + {item["path"]: sorted(f["name"] for f in item["files"]) for item in expected}, + f"Unexpected Gliders filenames in {product_id}", + ) # Verify the fishsoop finder maps folder is excluded by the region whitelist fishsoop_json_path = os.path.join(self.file_test_dir, "fishsoop", "fishSOOP-profiles.json") fishsoop_regions = {item["region"] for item in self.load_and_normalize_json(fishsoop_json_path)} @@ -586,7 +731,6 @@ def test_file_structure_explorer(self): not_existed_path = os.path.join(self.file_test_dir, "timeseries", "currentMetersCalendar-48.json") self.assertFalse(os.path.exists(not_existed_path)) - class TestMonitoringFunctions(unittest.TestCase): """Tests for EC2 monitoring and fatal log notification functions.""" diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/Brisbane20180306AB_4d/ignored.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/Brisbane20180306AB_4d/ignored.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/Brisbane20180306A_4d/2018031500.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/Brisbane20180306A_4d/2018031500.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/Brisbane20180306A_4d_tsfoa/ignored.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/Brisbane20180306A_4d_tsfoa/ignored.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/Cooktown20160217_4d_nrt/2016022000.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/Cooktown20160217_4d_nrt/2016022000.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/Cooktown20160217_nrt_4d_nrt/ignored.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/Cooktown20160217_nrt_4d_nrt/ignored.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/JervisBay20180503a_nrt_12d/2018051000.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/JervisBay20180503a_nrt_12d/2018051000.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/Perth20110626_1_4d/2011070100.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/Perth20110626_1_4d/2011070100.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/Perth20110626_3_4d/ignored.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/Perth20110626_3_4d/ignored.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260811_4d/2026101500.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260811_4d/2026101500.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_12d/2026101500.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_12d/2026101500.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_30d/2026101500.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_30d/2026101500.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_40d/ignored.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_40d/ignored.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/2026101500.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/2026101500.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/2026101512.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/2026101512.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/TasEastCoast20260922_nrt_4d.mp4 b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/TasEastCoast20260922_nrt_4d.mp4 new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/data.JSON b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/data.JSON new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/fake.gif.html b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/fake.gif.html new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/index.html b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/index.html new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/index.php b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/index.php new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/nested/ignored.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d/nested/ignored.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d_backup/ignored.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/TasEastCoast20260922_nrt_4d_backup/ignored.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/missionTS/ignored.mp4 b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/missionTS/ignored.mp4 new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/missionTS/sl_TasEastCoast20260922_nrt.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/missionTS/sl_TasEastCoast20260922_nrt.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/navBrisbane20180306AB_4d/ignored.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/navBrisbane20180306AB_4d/ignored.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/navBrisbane20180306A_nrt_4d/2018031500.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/navBrisbane20180306A_nrt_4d/2018031500.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/navCooktown20160217_4d_nrt/2016022000.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/navCooktown20160217_4d_nrt/2016022000.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/navJervisBay20180503a_nrt_12d/2018051000.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/navJervisBay20180503a_nrt_12d/2018051000.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/navPerth20110626_1_4d/2011070100.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/navPerth20110626_1_4d/2011070100.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/navTasEastCoast20260922_nrt_4d/2026101500.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/navTasEastCoast20260922_nrt_4d/2026101500.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/navTasEastCoast20260922_nrt_4d/navTasEastCoast20260922_nrt_4d.mp4 b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/navTasEastCoast20260922_nrt_4d/navTasEastCoast20260922_nrt_4d.mp4 new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/yearmaps/2026_sl_temp_nrt.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/yearmaps/2026_sl_temp_nrt.gif new file mode 100644 index 00000000..e69de29b diff --git a/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/yearmaps_junk/ignored.gif b/ARGO/oceancurrent/tests/mnt/oceancurrent/website/gliders/yearmaps_junk/ignored.gif new file mode 100644 index 00000000..e69de29b