Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion felt_python/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
# Deprecated
get_map_details,
)
from .exceptions import AuthError
from .exceptions import AuthError, ExportFailedError, ExportTimeoutError
from .layers import (
list_layers,
upload_file,
Expand Down Expand Up @@ -167,6 +167,8 @@
"get_current_user",
# Exceptions
"AuthError",
"ExportFailedError",
"ExportTimeoutError",
# Deprecated
"post_elements",
"post_element_group",
Expand Down
20 changes: 20 additions & 0 deletions felt_python/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,23 @@

class AuthError(Exception):
"""Class for authentication errors"""


class ExportFailedError(Exception):
"""A layer export failed"""

def __init__(self, poll_endpoint: str):
super().__init__(f"The export failed: {poll_endpoint}")
self.poll_endpoint = poll_endpoint


class ExportTimeoutError(Exception):
"""A layer export didn't finish in time; poll `poll_endpoint` to keep waiting"""

def __init__(self, poll_endpoint: str, timeout: float):
super().__init__(
f"The export didn't finish within {timeout:g} seconds. "
f"Poll {poll_endpoint} to keep waiting."
)
self.poll_endpoint = poll_endpoint
self.timeout = timeout
44 changes: 41 additions & 3 deletions felt_python/layers.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,15 @@
import json
import os
import tempfile
import time
import typing
import urllib.request
import uuid

from urllib.parse import urljoin

from .api import make_request, BASE_URL
from .exceptions import ExportFailedError, ExportTimeoutError
from .util import deprecated


Expand All @@ -27,6 +29,10 @@
)
LAYER_DUPLICATE = urljoin(BASE_URL, "duplicate_layers")

EXPORT_WAIT_TIMEOUT_SECONDS = 600
EXPORT_POLL_INITIAL_INTERVAL_SECONDS = 2.0
EXPORT_POLL_MAX_INTERVAL_SECONDS = 15.0


def list_layers(map_id: str, api_token: str | None = None):
"""List layers on a map"""
Expand Down Expand Up @@ -252,32 +258,64 @@ def get_export_link(
map_id: str,
layer_id: str,
api_token: str | None = None,
timeout: float = EXPORT_WAIT_TIMEOUT_SECONDS,
):
"""Get a download link for a layer

Vector layers will be downloaded in GPKG format. Raster layers will be GeoTIFFs.

A large layer can take longer to export than the API waits. In that case
this keeps polling the export until it finishes, for up to `timeout` seconds.

Raises:
ExportFailedError: The export failed.
ExportTimeoutError: The export didn't finish within `timeout` seconds.
The export keeps running; the error's `poll_endpoint` reports it.
"""
response = make_request(
url=LAYER_EXPORT_LINK.format(map_id=map_id, layer_id=layer_id),
method="GET",
api_token=api_token,
)
return json.load(response)["export_link"]
body = json.load(response)
if "export_link" in body:
return body["export_link"]
return _wait_for_export(body["poll_endpoint"], api_token, timeout)


def _wait_for_export(poll_endpoint: str, api_token: str | None, timeout: float) -> str:
deadline = time.monotonic() + timeout
interval = EXPORT_POLL_INITIAL_INTERVAL_SECONDS
while True:
status = json.load(
make_request(url=poll_endpoint, method="GET", api_token=api_token)
)
if status["status"] == "completed" and status.get("download_url"):
return status["download_url"]
if status["status"] == "failed":
raise ExportFailedError(poll_endpoint)
remaining = deadline - time.monotonic()
if remaining <= 0:
raise ExportTimeoutError(poll_endpoint, timeout)
time.sleep(min(interval, remaining))
interval = min(interval * 1.5, EXPORT_POLL_MAX_INTERVAL_SECONDS)


def download_layer(
map_id: str,
layer_id: str,
file_name: str | None = None,
api_token: str | None = None,
timeout: float = EXPORT_WAIT_TIMEOUT_SECONDS,
) -> str:
"""Download a layer to a file.

Vector layers will be downloaded in GPKG format. Raster layers will be GeoTIFFs.
If a specific file name is not provided, the default file name will be saved to
the current working directory.
the current working directory. Waits up to `timeout` seconds for a slow export,
as `get_export_link` does.
"""
export_link = get_export_link(map_id, layer_id, api_token)
export_link = get_export_link(map_id, layer_id, api_token, timeout)
with urllib.request.urlopen(export_link) as response:
if file_name is None:
parsed_url = urllib.parse.urlparse(response.url)
Expand Down
4 changes: 2 additions & 2 deletions felt_python/projects.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
from .api import make_request, BASE_URL


PROJECTS = urljoin(BASE_URL, "projects/")
PROJECT = urljoin(BASE_URL, "projects/{project_id}/")
PROJECTS = urljoin(BASE_URL, "projects")
PROJECT = urljoin(BASE_URL, "projects/{project_id}")
PROJECT_UPDATE = urljoin(BASE_URL, "projects/{project_id}/update")


Expand Down
87 changes: 87 additions & 0 deletions tests/export_link_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
"""
Unit tests for get_export_link's wait on a slow export. No API token needed.
"""

import io
import json
import os
import sys
import unittest
from unittest import mock

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from felt_python import ExportFailedError, ExportTimeoutError, get_export_link

POLL_ENDPOINT = "https://felt.com/api/v2/maps/m/layers/l/custom_exports/e"


def respond(*bodies):
return mock.patch(
"felt_python.layers.make_request",
side_effect=[io.BytesIO(json.dumps(body).encode()) for body in bodies],
)


class Clock:
def __init__(self):
self.now = 0.0
self.sleeps = []

def monotonic(self):
return self.now

def sleep(self, seconds):
self.sleeps.append(seconds)
self.now += seconds


class FeltExportLinkTest(unittest.TestCase):
def setUp(self):
self.clock = Clock()
for name in ("monotonic", "sleep"):
patcher = mock.patch(
f"felt_python.layers.time.{name}", getattr(self.clock, name)
)
patcher.start()
self.addCleanup(patcher.stop)

def test_returns_the_link_when_the_export_is_ready(self):
with respond({"export_link": "https://files/layer.gpkg"}) as request:
self.assertEqual(get_export_link("m", "l"), "https://files/layer.gpkg")
self.assertEqual(request.call_count, 1)

def test_polls_a_slow_export_until_it_completes(self):
with respond(
{"status": "processing", "poll_endpoint": POLL_ENDPOINT},
{"status": "pending", "download_url": None},
{"status": "in_progress", "download_url": None},
{"status": "completed", "download_url": "https://files/layer.gpkg"},
) as request:
self.assertEqual(get_export_link("m", "l"), "https://files/layer.gpkg")
self.assertEqual(request.call_args.kwargs["url"], POLL_ENDPOINT)
self.assertEqual(self.clock.sleeps, [2.0, 3.0])

def test_raises_when_the_export_fails(self):
with respond(
{"status": "processing", "poll_endpoint": POLL_ENDPOINT},
{"status": "failed", "download_url": None},
):
with self.assertRaises(ExportFailedError) as raised:
get_export_link("m", "l")
self.assertEqual(raised.exception.poll_endpoint, POLL_ENDPOINT)

def test_raises_with_the_poll_endpoint_after_the_timeout(self):
in_progress = {"status": "in_progress", "download_url": None}
with respond(
{"status": "processing", "poll_endpoint": POLL_ENDPOINT},
*[in_progress] * 5,
):
with self.assertRaises(ExportTimeoutError) as raised:
get_export_link("m", "l", timeout=5)
self.assertEqual(raised.exception.poll_endpoint, POLL_ENDPOINT)
self.assertEqual(self.clock.now, 5)


if __name__ == "__main__":
unittest.main()
2 changes: 2 additions & 0 deletions tests/tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from sources_test import FeltSourcesTest
from components_test import FeltComponentsTest
from delete_test import FeltDeleteTest
from export_link_test import FeltExportLinkTest


if __name__ == "__main__":
Expand All @@ -42,6 +43,7 @@
FeltSourcesTest,
FeltComponentsTest,
FeltDeleteTest,
FeltExportLinkTest,
]

for test_case in test_cases:
Expand Down
Loading