Add virtual_prefix option to the file client

docs/Config.md

@@ -42,6 +42,7 @@ Each target may have the following properties:
         * `path`: Path to the root
         * `buffer_size`: Size of chunks (in bytes) when streaming file content (default: 8192)
         * `calculate_etags`: If true, then the etags will be calculated by hashing the content of each file. This is much more expensive and may not be needed for all use cases.
+        * `virtual_prefix`: Mounts the target at this key prefix inside the bucket. The prefix does not exist on disk: keys are reported with it, and it is stripped from incoming keys. This is the inverse of the aioboto `prefix` option, which hides a prefix that does exist upstream. For example, `virtual_prefix: shared/my-data` exposes `<root>/README.md` as `shared/my-data/README.md`.
 
 ### Botocore Config Options
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "x2s3"
-version = "1.4.3"
+version = "1.5.0"
 description = "RESTful web service which makes any storage system X available as an S3-compatible REST API"
 readme = "README.md"
 license = { file = "LICENSE" }

tests/test_file.py

@@ -27,6 +27,14 @@ def get_settings():
                 'calculate_etags':'true'
             }
         ),
+        Target(
+            name='mounted-files',
+            client='file',
+            options={
+                'path':'.',
+                'virtual_prefix':'abc123/my-data/'
+            }
+        ),
         Target(
             name='hidden-files',
             browseable=False,
@@ -271,3 +279,92 @@ def test_unbrowseable_head(app):
         assert response.status_code == 200
         response = client.head("/local-files/missing")
         assert response.status_code == 404
+
+
+VIRTUAL_PREFIX = 'abc123/my-data/'
+
+
+def test_virtual_prefix_list(app):
+    # Keys are reported under the virtual prefix, which is not on disk
+    with TestClient(app) as client:
+        response = client.get(f"/mounted-files?list-type=2&prefix={VIRTUAL_PREFIX}tests/&delimiter=/")
+        assert response.status_code == 200
+        root = parse_xml(response.text)
+        assert root.find('Prefix').text == f"{VIRTUAL_PREFIX}tests/"
+        keys = [c.find('Key').text for c in root.findall('Contents')]
+        prefixes = [cp.find('Prefix').text for cp in root.findall('CommonPrefixes')]
+        assert keys and prefixes
+        assert f"{VIRTUAL_PREFIX}tests/test_file.py" in keys
+        assert f"{VIRTUAL_PREFIX}tests/java/" in prefixes
+        assert all(k.startswith(VIRTUAL_PREFIX) for k in keys + prefixes)
+
+
+def test_virtual_prefix_list_within_prefix(app):
+    # A prefix that stops inside the virtual prefix sees only the next segment
+    with TestClient(app) as client:
+        for prefix, expected in [('', 'abc123/'),
+                                 ('abc', 'abc123/'),
+                                 ('abc123/', VIRTUAL_PREFIX),
+                                 ('abc123/my-', VIRTUAL_PREFIX)]:
+            response = client.get(f"/mounted-files?list-type=2&prefix={prefix}&delimiter=/")
+            assert response.status_code == 200
+            root = parse_xml(response.text)
+            assert [cp.find('Prefix').text for cp in root.findall('CommonPrefixes')] == [expected]
+            assert root.findall('Contents') == []
+            assert root.find('KeyCount').text == '1'
+
+
+def test_virtual_prefix_list_no_match(app):
+    # Anything outside the virtual prefix matches nothing
+    with TestClient(app) as client:
+        for prefix in ['zzz/', 'abc123/other/', 'tests/']:
+            response = client.get(f"/mounted-files?list-type=2&prefix={prefix}&delimiter=/")
+            assert response.status_code == 200
+            root = parse_xml(response.text)
+            assert root.findall('Contents') == []
+            assert root.findall('CommonPrefixes') == []
+            assert root.find('KeyCount').text == '0'
+
+
+def test_virtual_prefix_list_recursive(app):
+    # Without a delimiter, a prefix inside the virtual prefix lists everything
+    with TestClient(app) as client:
+        response = client.get("/mounted-files?list-type=2&prefix=abc123/&max-keys=5")
+        assert response.status_code == 200
+        root = parse_xml(response.text)
+        keys = [c.find('Key').text for c in root.findall('Contents')]
+        assert len(keys) == 5
+        assert all(k.startswith(VIRTUAL_PREFIX) for k in keys)
+
+
+def test_virtual_prefix_objects(app):
+    with TestClient(app) as client:
+        response = client.get(f"/mounted-files/{VIRTUAL_PREFIX}README.md")
+        assert response.status_code == 200
+        assert 'x2s3' in response.text
+        assert client.head(f"/mounted-files/{VIRTUAL_PREFIX}README.md").status_code == 200
+        # The real path, without the virtual prefix, is not addressable
+        response = client.get("/mounted-files/README.md")
+        assert response.status_code == 404
+        assert parse_xml(response.text).find('Code').text == 'NoSuchKey'
+        assert client.head("/mounted-files/README.md").status_code == 404
+
+
+def test_virtual_prefix_continuation(app):
+    # Continuation tokens are reported and accepted in the client's key space
+    with TestClient(app) as client:
+        url = f"/mounted-files?list-type=2&prefix={VIRTUAL_PREFIX}x2s3/&delimiter=/"
+        root = parse_xml(client.get(url).text)
+        all_keys = [c.find('Key').text for c in root.findall('Contents')]
+        assert len(all_keys) > 3
+
+        paged, token = [], None
+        while True:
+            page_url = url + "&max-keys=2" + (f"&continuation-token={token}" if token else "")
+            root = parse_xml(client.get(page_url).text)
+            paged += [c.find('Key').text for c in root.findall('Contents')]
+            if root.find('IsTruncated').text != 'true':
+                break
+            token = root.find('NextContinuationToken').text
+            assert token.startswith(VIRTUAL_PREFIX)
+        assert paged == all_keys

x2s3/client_file.py

@@ -167,6 +167,12 @@ def __init__(self, proxy_kwargs, **kwargs):
         self.root_path = str(Path(kwargs['path']).resolve())
         self.calculate_etags = kwargs.get('calculate_etags', False)
         self.buffer_size = kwargs.get('buffer_size', DEFAULT_BUFFER_SIZE)
+        # Optional virtual prefix. The target is mounted at this key prefix
+        # inside the bucket, even though the prefix does not exist on disk:
+        # keys are reported with it, and it is stripped from incoming keys.
+        # This is the inverse of the aioboto client's 'prefix' option, which
+        # hides a prefix that does exist upstream.
+        self.virtual_prefix = dir_path((kwargs.get('virtual_prefix') or '').lstrip('/'))
 
     def _safe_path(self, key: str) -> Optional[str]:
         """Resolve key to absolute path and validate it's within root_path.
@@ -181,10 +187,43 @@ def _safe_path(self, key: str) -> Optional[str]:
             return None
         return path
 
+    def _strip_virtual_prefix(self, key: str) -> Optional[str]:
+        """Remove the virtual prefix from an incoming key.
+
+        Returns None if the key does not fall under the virtual prefix.
+        """
+        if not self.virtual_prefix:
+            return key
+        if key and key.startswith(self.virtual_prefix):
+            return key[len(self.virtual_prefix):]
+        return None
+
+    def _map_prefix(self, prefix: str, delimiter: str):
+        """Map a requested key prefix into this target's own key space.
+
+        Returns (real_prefix, virtual_common_prefix). Exactly one is not None:
+        either a prefix to walk on disk, or a common prefix that exists only
+        within the virtual prefix. Both are None when nothing can match.
+        """
+        if not self.virtual_prefix or prefix.startswith(self.virtual_prefix):
+            return remove_prefix(self.virtual_prefix, prefix), None
+        if self.virtual_prefix.startswith(prefix):
+            # The prefix stops inside the virtual prefix, which has nothing on
+            # disk behind it. With a delimiter, all the client can see is the
+            # next virtual segment; without one, it sees the whole target.
+            i = self.virtual_prefix.find(delimiter, len(prefix)) if delimiter else -1
+            if i >= 0:
+                return None, self.virtual_prefix[:i + len(delimiter)]
+            return '', None
+        return None, None
+
     @override
     async def head_object(self, key: str):
         try:
-            path = self._safe_path(key)
+            real_key = self._strip_virtual_prefix(key)
+            if real_key is None:
+                return get_nosuchkey_response(key)
+            path = self._safe_path(real_key)
             if path is None:
                 return get_nosuchkey_response(key)
             if not os.path.isfile(path):
@@ -214,7 +253,10 @@ async def open_object(self, key: str, range_header: str = None):
         """Open a file object and return a handle for streaming."""
         file_handle = None
         try:
-            path = self._safe_path(key)
+            real_key = self._strip_virtual_prefix(key)
+            if real_key is None:
+                return get_nosuchkey_response(key)
+            path = self._safe_path(real_key)
             if path is None:
                 return get_nosuchkey_response(key)
             if not os.path.isfile(path):
@@ -319,39 +361,56 @@ async def list_objects_v2(self,
                             prefix: str,
                             start_after: str):
 
-        # prefix user-supplied prefix with configured prefix
-        real_prefix = prefix
-
-        # ensure the prefix ends with a slash
-        if real_prefix and not real_prefix.endswith('/'):
-            real_prefix += '/'
-
+        prefix = prefix or ''
         try:
-            path = str(self.root_path)
-            if real_prefix:
-                path = self._safe_path(real_prefix)
-                if path is None:
-                    # Path traversal attempt - return empty listing
-                    return Response(content=get_list_xml([], [], Name=self.target_name),
-                                    media_type="application/xml")
-
-            logger.debug(f"root_path: {self.root_path}, real_prefix: {real_prefix}, path: {path}")
-
-            res = self.walk_path(path, continuation_token, delimiter, max_keys)
-            contents = res['contents']
-            is_truncated = res['is_truncated']
-            common_prefixes = sorted(res['common_prefixes'])
+            contents = []
+            common_prefixes = []
+            is_truncated = 'false'
+            next_token = None
+
+            real_prefix, virtual_common = self._map_prefix(prefix, delimiter)
+            if virtual_common is not None:
+                common_prefixes = [virtual_common]
+            elif real_prefix is not None:
+
+                # ensure the prefix ends with a slash
+                if real_prefix and not real_prefix.endswith('/'):
+                    real_prefix += '/'
+
+                path = str(self.root_path)
+                if real_prefix:
+                    # None means a path traversal attempt, leaving the listing empty
+                    path = self._safe_path(real_prefix)
+
+                logger.debug(f"root_path: {self.root_path}, real_prefix: {real_prefix}, path: {path}")
+
+                if path is not None:
+                    res = self.walk_path(path,
+                                         remove_prefix(self.virtual_prefix, continuation_token),
+                                         delimiter, max_keys)
+                    contents = res['contents']
+                    common_prefixes = sorted(res['common_prefixes'])
+                    is_truncated = res['is_truncated']
+                    next_token = res['next_token']
+
+                    if self.virtual_prefix:
+                        # Report keys as the client sees them, under the prefix
+                        for obj in contents:
+                            obj['Key'] = self.virtual_prefix + obj['Key']
+                        common_prefixes = [self.virtual_prefix + cp for cp in common_prefixes]
+                        if next_token:
+                            next_token = self.virtual_prefix + next_token
 
             kwargs = {
                 'Name': self.target_name,
-                'Prefix': prefix or '',
+                'Prefix': prefix,
                 'Delimiter': delimiter,
                 'MaxKeys': max_keys,
                 'EncodingType': encoding_type,
                 'KeyCount': len(contents) + len(common_prefixes),
                 'IsTruncated': is_truncated,
                 'ContinuationToken': continuation_token,
-                'NextContinuationToken': res['next_token'],
+                'NextContinuationToken': next_token,
                 'StartAfter': start_after
             }