diff --git a/docs/Config.md b/docs/Config.md index e0db539..9143ea4 100644 --- a/docs/Config.md +++ b/docs/Config.md @@ -42,6 +42,7 @@ Each target may have the following properties: * `path`: Path to the root * `buffer_size`: Size of chunks (in bytes) when streaming file content (default: 8192) * `calculate_etags`: If true, then the etags will be calculated by hashing the content of each file. This is much more expensive and may not be needed for all use cases. + * `virtual_prefix`: Mounts the target at this key prefix inside the bucket. The prefix does not exist on disk: keys are reported with it, and it is stripped from incoming keys. This is the inverse of the aioboto `prefix` option, which hides a prefix that does exist upstream. For example, `virtual_prefix: shared/my-data` exposes `/README.md` as `shared/my-data/README.md`. ### Botocore Config Options diff --git a/pyproject.toml b/pyproject.toml index 27756c0..e09e02e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "x2s3" -version = "1.4.3" +version = "1.5.0" description = "RESTful web service which makes any storage system X available as an S3-compatible REST API" readme = "README.md" license = { file = "LICENSE" } diff --git a/tests/test_file.py b/tests/test_file.py index 5ef3e59..0ab04d2 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -27,6 +27,14 @@ def get_settings(): 'calculate_etags':'true' } ), + Target( + name='mounted-files', + client='file', + options={ + 'path':'.', + 'virtual_prefix':'abc123/my-data/' + } + ), Target( name='hidden-files', browseable=False, @@ -271,3 +279,92 @@ def test_unbrowseable_head(app): assert response.status_code == 200 response = client.head("/local-files/missing") assert response.status_code == 404 + + +VIRTUAL_PREFIX = 'abc123/my-data/' + + +def test_virtual_prefix_list(app): + # Keys are reported under the virtual prefix, which is not on disk + with TestClient(app) as client: + response = client.get(f"/mounted-files?list-type=2&prefix={VIRTUAL_PREFIX}tests/&delimiter=/") + assert response.status_code == 200 + root = parse_xml(response.text) + assert root.find('Prefix').text == f"{VIRTUAL_PREFIX}tests/" + keys = [c.find('Key').text for c in root.findall('Contents')] + prefixes = [cp.find('Prefix').text for cp in root.findall('CommonPrefixes')] + assert keys and prefixes + assert f"{VIRTUAL_PREFIX}tests/test_file.py" in keys + assert f"{VIRTUAL_PREFIX}tests/java/" in prefixes + assert all(k.startswith(VIRTUAL_PREFIX) for k in keys + prefixes) + + +def test_virtual_prefix_list_within_prefix(app): + # A prefix that stops inside the virtual prefix sees only the next segment + with TestClient(app) as client: + for prefix, expected in [('', 'abc123/'), + ('abc', 'abc123/'), + ('abc123/', VIRTUAL_PREFIX), + ('abc123/my-', VIRTUAL_PREFIX)]: + response = client.get(f"/mounted-files?list-type=2&prefix={prefix}&delimiter=/") + assert response.status_code == 200 + root = parse_xml(response.text) + assert [cp.find('Prefix').text for cp in root.findall('CommonPrefixes')] == [expected] + assert root.findall('Contents') == [] + assert root.find('KeyCount').text == '1' + + +def test_virtual_prefix_list_no_match(app): + # Anything outside the virtual prefix matches nothing + with TestClient(app) as client: + for prefix in ['zzz/', 'abc123/other/', 'tests/']: + response = client.get(f"/mounted-files?list-type=2&prefix={prefix}&delimiter=/") + assert response.status_code == 200 + root = parse_xml(response.text) + assert root.findall('Contents') == [] + assert root.findall('CommonPrefixes') == [] + assert root.find('KeyCount').text == '0' + + +def test_virtual_prefix_list_recursive(app): + # Without a delimiter, a prefix inside the virtual prefix lists everything + with TestClient(app) as client: + response = client.get("/mounted-files?list-type=2&prefix=abc123/&max-keys=5") + assert response.status_code == 200 + root = parse_xml(response.text) + keys = [c.find('Key').text for c in root.findall('Contents')] + assert len(keys) == 5 + assert all(k.startswith(VIRTUAL_PREFIX) for k in keys) + + +def test_virtual_prefix_objects(app): + with TestClient(app) as client: + response = client.get(f"/mounted-files/{VIRTUAL_PREFIX}README.md") + assert response.status_code == 200 + assert 'x2s3' in response.text + assert client.head(f"/mounted-files/{VIRTUAL_PREFIX}README.md").status_code == 200 + # The real path, without the virtual prefix, is not addressable + response = client.get("/mounted-files/README.md") + assert response.status_code == 404 + assert parse_xml(response.text).find('Code').text == 'NoSuchKey' + assert client.head("/mounted-files/README.md").status_code == 404 + + +def test_virtual_prefix_continuation(app): + # Continuation tokens are reported and accepted in the client's key space + with TestClient(app) as client: + url = f"/mounted-files?list-type=2&prefix={VIRTUAL_PREFIX}x2s3/&delimiter=/" + root = parse_xml(client.get(url).text) + all_keys = [c.find('Key').text for c in root.findall('Contents')] + assert len(all_keys) > 3 + + paged, token = [], None + while True: + page_url = url + "&max-keys=2" + (f"&continuation-token={token}" if token else "") + root = parse_xml(client.get(page_url).text) + paged += [c.find('Key').text for c in root.findall('Contents')] + if root.find('IsTruncated').text != 'true': + break + token = root.find('NextContinuationToken').text + assert token.startswith(VIRTUAL_PREFIX) + assert paged == all_keys diff --git a/x2s3/client_file.py b/x2s3/client_file.py index 50c3859..04f6d34 100644 --- a/x2s3/client_file.py +++ b/x2s3/client_file.py @@ -167,6 +167,12 @@ def __init__(self, proxy_kwargs, **kwargs): self.root_path = str(Path(kwargs['path']).resolve()) self.calculate_etags = kwargs.get('calculate_etags', False) self.buffer_size = kwargs.get('buffer_size', DEFAULT_BUFFER_SIZE) + # Optional virtual prefix. The target is mounted at this key prefix + # inside the bucket, even though the prefix does not exist on disk: + # keys are reported with it, and it is stripped from incoming keys. + # This is the inverse of the aioboto client's 'prefix' option, which + # hides a prefix that does exist upstream. + self.virtual_prefix = dir_path((kwargs.get('virtual_prefix') or '').lstrip('/')) def _safe_path(self, key: str) -> Optional[str]: """Resolve key to absolute path and validate it's within root_path. @@ -181,10 +187,43 @@ def _safe_path(self, key: str) -> Optional[str]: return None return path + def _strip_virtual_prefix(self, key: str) -> Optional[str]: + """Remove the virtual prefix from an incoming key. + + Returns None if the key does not fall under the virtual prefix. + """ + if not self.virtual_prefix: + return key + if key and key.startswith(self.virtual_prefix): + return key[len(self.virtual_prefix):] + return None + + def _map_prefix(self, prefix: str, delimiter: str): + """Map a requested key prefix into this target's own key space. + + Returns (real_prefix, virtual_common_prefix). Exactly one is not None: + either a prefix to walk on disk, or a common prefix that exists only + within the virtual prefix. Both are None when nothing can match. + """ + if not self.virtual_prefix or prefix.startswith(self.virtual_prefix): + return remove_prefix(self.virtual_prefix, prefix), None + if self.virtual_prefix.startswith(prefix): + # The prefix stops inside the virtual prefix, which has nothing on + # disk behind it. With a delimiter, all the client can see is the + # next virtual segment; without one, it sees the whole target. + i = self.virtual_prefix.find(delimiter, len(prefix)) if delimiter else -1 + if i >= 0: + return None, self.virtual_prefix[:i + len(delimiter)] + return '', None + return None, None + @override async def head_object(self, key: str): try: - path = self._safe_path(key) + real_key = self._strip_virtual_prefix(key) + if real_key is None: + return get_nosuchkey_response(key) + path = self._safe_path(real_key) if path is None: return get_nosuchkey_response(key) if not os.path.isfile(path): @@ -214,7 +253,10 @@ async def open_object(self, key: str, range_header: str = None): """Open a file object and return a handle for streaming.""" file_handle = None try: - path = self._safe_path(key) + real_key = self._strip_virtual_prefix(key) + if real_key is None: + return get_nosuchkey_response(key) + path = self._safe_path(real_key) if path is None: return get_nosuchkey_response(key) if not os.path.isfile(path): @@ -319,39 +361,56 @@ async def list_objects_v2(self, prefix: str, start_after: str): - # prefix user-supplied prefix with configured prefix - real_prefix = prefix - - # ensure the prefix ends with a slash - if real_prefix and not real_prefix.endswith('/'): - real_prefix += '/' - + prefix = prefix or '' try: - path = str(self.root_path) - if real_prefix: - path = self._safe_path(real_prefix) - if path is None: - # Path traversal attempt - return empty listing - return Response(content=get_list_xml([], [], Name=self.target_name), - media_type="application/xml") - - logger.debug(f"root_path: {self.root_path}, real_prefix: {real_prefix}, path: {path}") - - res = self.walk_path(path, continuation_token, delimiter, max_keys) - contents = res['contents'] - is_truncated = res['is_truncated'] - common_prefixes = sorted(res['common_prefixes']) + contents = [] + common_prefixes = [] + is_truncated = 'false' + next_token = None + + real_prefix, virtual_common = self._map_prefix(prefix, delimiter) + if virtual_common is not None: + common_prefixes = [virtual_common] + elif real_prefix is not None: + + # ensure the prefix ends with a slash + if real_prefix and not real_prefix.endswith('/'): + real_prefix += '/' + + path = str(self.root_path) + if real_prefix: + # None means a path traversal attempt, leaving the listing empty + path = self._safe_path(real_prefix) + + logger.debug(f"root_path: {self.root_path}, real_prefix: {real_prefix}, path: {path}") + + if path is not None: + res = self.walk_path(path, + remove_prefix(self.virtual_prefix, continuation_token), + delimiter, max_keys) + contents = res['contents'] + common_prefixes = sorted(res['common_prefixes']) + is_truncated = res['is_truncated'] + next_token = res['next_token'] + + if self.virtual_prefix: + # Report keys as the client sees them, under the prefix + for obj in contents: + obj['Key'] = self.virtual_prefix + obj['Key'] + common_prefixes = [self.virtual_prefix + cp for cp in common_prefixes] + if next_token: + next_token = self.virtual_prefix + next_token kwargs = { 'Name': self.target_name, - 'Prefix': prefix or '', + 'Prefix': prefix, 'Delimiter': delimiter, 'MaxKeys': max_keys, 'EncodingType': encoding_type, 'KeyCount': len(contents) + len(common_prefixes), 'IsTruncated': is_truncated, 'ContinuationToken': continuation_token, - 'NextContinuationToken': res['next_token'], + 'NextContinuationToken': next_token, 'StartAfter': start_after }