diff --git a/examples/README-sandbox.md b/examples/README-sandbox.md index d95eafe9..4f91424a 100644 --- a/examples/README-sandbox.md +++ b/examples/README-sandbox.md @@ -14,6 +14,81 @@ python examples/sandbox_injection_rules.py 示例会创建资源并在结束时清理。保存的注入规则还需要 AK/SK;直接设置运行时 HTTP 注入可使用 API key。 +## 沙箱列表分页 + +`client.list_sandboxes(limit=100, nextToken=cursor)` 返回 +`{'items': [...], 'nextToken': ...}`,游标来自 `X-Next-Token` 响应头, +`nextToken` 为空表示没有下一页。 + +从 v7.19.0 升级时,把已移除的 `list_sandboxes_v2()`/`listSandboxesV2()` 改为 +`list_sandboxes()`。原来直接遍历 `client.list()` 的单页代码改为遍历 `items`: + +```python +for item in client.list_sandboxes(limit=100)['items']: + print(item) +``` + +完整遍历可让 `Sandbox.list()` 分页器自动传递游标。 + +```python +from qiniu.services.sandbox import Sandbox + +pages = Sandbox.list(client=client, limit=100, metadata={'app': 'example'}) +while pages.has_next: + for sandbox in pages.next_items(): + print(sandbox.sandbox_id) +``` + +元数据字典会按服务端过滤协议编码,`+`、`&`、`=`、`%` 和中文均保留原值。 +直接传入 `metadata` 字符串时,调用方应先分别编码每个键和值,再将这些项编码为查询字符串。 + +Envd 文件和进程请求不自动跟随 HTTP 重定向;收到 30x 时抛出 `SandboxError`, +避免把访问 token 或签名 URL 带给跳转目标。 + +## PTY 实时输出与断连 + +`sandbox.pty.create(on_data=callback)` 和 +`sandbox.pty.connect(pid, on_data=callback)` 返回进程句柄。回调接收原始 `bytes`。 +首帧含有 PTY 输出时,`create()` 或 `connect()` 会在调用线程执行回调,回调抛错时 +会关闭流并直接抛出异常。后续输出由后台读流线程调用回调,无需先调用 `wait()`。 +回调应快速返回,可把数据放入队列交给应用处理。 + +`handle.wait()` 等待进程结束,返回带字符串 `stdout` 的 `CommandResult`, +并抛出后台读流或回调异常。未提供 `on_data` 时,首次调用 `wait()` 会启动后台读流, +传给该次 `wait()` 的 `on_stdout` 和 `on_stderr` 仍在调用线程执行。 + +`handle.disconnect()` 只关闭本地连接,不终止远端 PTY,可重复调用。 +断连后的 `wait()` 返回已收到的结果;未收到退出事件时 `exit_code` 保持 `-1`。 +断连会冻结已处理的输出并释放等待者,包括另一个线程正在同步 `wait()` 的情况; +后台任务负责清理连接,之后重复 `wait()` 返回同一份冻结结果。 +部分操作系统的读操作或关闭操作会延迟到既有读取时限才退出;`timeout=None` +的语义不变,不保证读线程立即结束。输出回调在状态锁中确认调度后,在锁外执行。 +断连不会等待已经调度的回调,它们可能在 `disconnect()` 返回后才开始或结束; +断连后不再调度后续回调,晚到的输出或异常也不再修改冻结结果。 +之后可用 `sandbox.pty.connect(handle.pid, on_data=callback)` 重新连接。 +需要终止进程时显式调用 `handle.kill()`;任务结束时清理沙箱。 + +## 私有 Git 克隆 + +`clone()` 支持 `username` 和 `password`,与 `pull()` / `push()` 一样使用临时 +askpass 文件和进程环境,不把凭据写入远程 URL 或持久 credential store: + +```python +import os + +result = sandbox.git.clone( + 'https://github.com/owner/private.git', '/home/user/private', + username=os.environ['GIT_USERNAME'], password=os.environ['GIT_PASSWORD']) +``` + +两项凭据必须同时提供,且只适用于 HTTP(S) URL。前台执行完成后删除临时文件; +使用 `background=True` 时,在句柄 `wait()` 结束后删除。该方式会把凭据交给 +沙箱中的 Git 进程,应仅用于允许沙箱访问这些凭据的任务。 + +显式用户名在本次 Git 调用中优先于 `credential.username`,已配置的 credential helper +不会保存本次凭据。每次调用使用独立的私有临时目录,准备、上传、 +执行及清理均使用本次 `user` 指定的身份;结束后同时删除 askpass 和临时目录。 + ## 暂停与恢复 `Sandbox.connect()`、实例 `connect()` 和 `SandboxClient.connect_sandbox()` 使用 Connect v2。省略 `timeout` 使用服务端默认的 300 秒;显式值必须至少为 1 秒。连接只延长 TTL,不会缩短已有 TTL;需要主动调整过期时间时使用 `set_timeout()`。 diff --git a/examples/sandbox_git.py b/examples/sandbox_git.py index 8cc4c5be..43e7366f 100644 --- a/examples/sandbox_git.py +++ b/examples/sandbox_git.py @@ -8,12 +8,6 @@ from sandbox_common import cleanup_sandbox, create_sandbox, run_example -try: - from urllib.parse import quote, urlparse, urlunparse -except ImportError: - from urllib import quote - from urlparse import urlparse, urlunparse - def is_git_ok(result): message = result.stderr or result.stdout or result.error or '' @@ -75,36 +69,17 @@ def remote_git_config(): return repo_url, username, password -def credentialed_repo_url(repo_url, username, password): - parsed = urlparse(repo_url) - if parsed.scheme not in ('http', 'https'): - raise RuntimeError( - 'remote git push: only http(s) URLs support credentials') - netloc = parsed.netloc.split('@', 1)[-1] - auth = '{0}:{1}@'.format( - quote(username, safe=''), - quote(password, safe=''), - ) - return urlunparse(( - parsed.scheme, - auth + netloc, - parsed.path, - parsed.params, - parsed.query, - parsed.fragment, - )) - - def clone_remote_with_credentials(sandbox, repo_url, repo_path, username, password, attempts=5): - auth_url = credentialed_repo_url(repo_url, username, password) result = None for attempt in range(attempts): sandbox.commands.run('rm -rf {0}'.format(repo_path)) result = sandbox.git.clone( - auth_url, + repo_url, repo_path, depth=1, + username=username, + password=password, timeout=180, request_timeout=180, ) diff --git a/examples/sandbox_runtime_configuration.py b/examples/sandbox_runtime_configuration.py index e89fbea5..5204c058 100644 --- a/examples/sandbox_runtime_configuration.py +++ b/examples/sandbox_runtime_configuration.py @@ -9,10 +9,10 @@ def main(): sandbox = create_sandbox(timeout=300) try: - listed = sandbox.client.list_sandboxes_v2( + listed = sandbox.client.list_sandboxes( template=[sandbox.template_id], ) - print('sandboxes using template:', listed) + print('sandboxes using template:', listed['items']) print('current injections:', sandbox.get_injections()) sandbox.update_injections([{ diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 9bc7c697..2f4fc291 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -7,6 +7,11 @@ import requests +try: + from urllib.parse import quote +except ImportError: + from urllib import quote + from qiniu.auth import QiniuMacAuth, QiniuMacRequestsAuth from qiniu.compat import ( basestring, @@ -203,6 +208,14 @@ def _require_sandbox_id(sandbox_id): return sandbox_id +def _encode_metadata_component(value): + if not isinstance(value, (bytes_type, text_type)): + value = text_type(value) + if isinstance(value, text_type): + value = value.encode('utf-8') + return quote(value, safe='') + + def _normalize_list_options(opts): opts = dict(opts or {}) query = opts.pop('query', None) or {} @@ -210,7 +223,11 @@ def _normalize_list_options(opts): if metadata is not None and 'metadata' not in opts: opts['metadata'] = metadata if isinstance(opts.get('metadata'), dict): - opts['metadata'] = urlencode(opts.get('metadata')) + # The API unescapes the whole filter, then each key/value separately. + opts['metadata'] = urlencode([ + (_encode_metadata_component(key), _encode_metadata_component(value)) + for key, value in opts['metadata'].items() + ]) for key, value in query.items(): if ( key != 'metadata' and @@ -392,8 +409,12 @@ def _request(self, method, path, params=None, body=_UNSET, return None data = parse_json_response(response) if page: - return {'items': data, 'nextToken': response.headers.get( - 'X-Next-Token')} + next_token = response.headers.get('X-Next-Token') + if isinstance(data, dict): + if next_token is None: + next_token = data.get('nextToken') or data.get('next_token') + data = data.get('items') or data.get('sandboxes') or [] + return {'items': data, 'nextToken': next_token} return data def _is_retryable(self, err): @@ -438,18 +459,14 @@ def _retry_call(self, fn, retryable=None, retry_after=False, raise def list_sandboxes(self, **opts): - return self._request('GET', '/sandboxes', params=opts) - - listSandboxes = list_sandboxes - - def list_sandboxes_v2(self, **opts): + """Return {'items': [...], 'nextToken': ...} from the sandbox list.""" return self._request( 'GET', '/v2/sandboxes', - params=_normalize_list_options(opts)) + params=_normalize_list_options(opts), page=True) - listSandboxesV2 = list_sandboxes_v2 - list = list_sandboxes_v2 + listSandboxes = list_sandboxes + list = list_sandboxes def create_sandbox(self, template=None, **opts): body = _normalize_sandbox_create_options(template, **opts) diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index 1558dd59..2202e401 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -65,7 +65,7 @@ def _decode_bytes(value): def command_result_from_events(events, on_stdout=None, on_stderr=None, - on_pty=None): + on_pty=None, on_data=None): pid = 0 stdout = '' stderr = '' @@ -88,6 +88,15 @@ def command_result_from_events(events, on_stdout=None, on_stderr=None, on_stderr(stderr_chunk) if pty_chunk and on_pty: on_pty(pty_chunk) + if data.get('pty') is not None and on_data: + value = data['pty'] + if isinstance(value, bytes_type): + pass + elif isinstance(value, basestring): + value = base64.b64decode(value) + else: + value = bytes_type(bytearray(value)) + on_data(value) stdout += stdout_chunk stderr += stderr_chunk stdout += pty_chunk @@ -109,7 +118,7 @@ def command_result_from_events(events, on_stdout=None, on_stderr=None, class CommandHandle(object): def __init__(self, commands, result=None, throw_on_error=False, - events=None, on_stdout=None, on_stderr=None): + events=None, on_stdout=None, on_stderr=None, on_data=None): self.commands = commands result = result or CommandResult() self.result = result @@ -122,6 +131,7 @@ def __init__(self, commands, result=None, throw_on_error=False, self._events = events self._on_stdout = on_stdout self._on_stderr = on_stderr + self._on_data = on_data def wait(self, on_stdout=None, on_stderr=None): if self._events is not None: @@ -129,6 +139,7 @@ def wait(self, on_stdout=None, on_stderr=None): self._events, on_stdout=on_stdout or self._on_stdout, on_stderr=on_stderr or self._on_stderr, + on_data=self._on_data, ) if not result.pid: result.pid = self.pid @@ -143,7 +154,9 @@ def wait(self, on_stdout=None, on_stderr=None): self.stdout = result.stdout self.stderr = result.stderr self._events = None - if self.throw_on_error and self.result.exit_code: + if (self.throw_on_error and self.result.exit_code and not ( + self.result.exit_code == -1 and + getattr(self, '_disconnected', False))): raise CommandExitError(self.result) return self.result @@ -253,7 +266,7 @@ def connect(self, pid, tag=None, user=None, timeout=None, events = connect_stream_rpc( self.sandbox, '/process.Process/Connect', - {'process': {'selector': selector}}, + {'process': selector}, user=user, timeout=timeout, stream=True, @@ -280,7 +293,7 @@ def send_stdin(self, pid, data, user=None, timeout=None): else: data = str(data).encode('utf-8') return connect_rpc(self.sandbox, '/process.Process/SendInput', { - 'process': {'selector': {'pid': pid}}, + 'process': {'pid': pid}, 'input': {'stdin': base64.b64encode(data).decode('ascii')}, }, user=user, timeout=timeout) @@ -288,7 +301,7 @@ def send_stdin(self, pid, data, user=None, timeout=None): def close_stdin(self, pid, user=None, timeout=None): return connect_rpc(self.sandbox, '/process.Process/CloseStdin', { - 'process': {'selector': {'pid': pid}}, + 'process': {'pid': pid}, }, user=user, timeout=timeout) closeStdin = close_stdin @@ -296,7 +309,7 @@ def close_stdin(self, pid, user=None, timeout=None): def kill(self, pid, user=None, timeout=None): try: connect_rpc(self.sandbox, '/process.Process/SendSignal', { - 'process': {'selector': {'pid': pid}}, + 'process': {'pid': pid}, 'signal': 'SIGNAL_SIGKILL', }, user=user, timeout=timeout) return True diff --git a/qiniu/services/sandbox/envd.py b/qiniu/services/sandbox/envd.py index e75acaa0..9ad80b17 100644 --- a/qiniu/services/sandbox/envd.py +++ b/qiniu/services/sandbox/envd.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import json +import socket import struct +import threading import requests @@ -34,13 +36,11 @@ def connect_rpc(sandbox, procedure, body=None, user=None, timeout=None): data=json.dumps(body or {}, separators=(',', ':')), headers=headers, timeout=timeout, + allow_redirects=False, ) except requests.RequestException as err: raise SandboxError('Sandbox envd request failed: {0}'.format(err)) - if response.status_code < 200 or response.status_code >= 300: - raise SandboxError( - 'Sandbox envd request failed with status {0}'.format( - response.status_code), response, response.text, ) + _check_envd_response(response) if not response.content: return None data = response.json() @@ -144,6 +144,105 @@ def iter_connect_envelopes(chunks, response=None): pass +class _ConnectStream(object): + """An event iterator whose transport can be interrupted during a read.""" + + def __init__(self, response): + self._response = response + self._chunks = response.iter_content(chunk_size=8192) + self._events = iter_connect_envelopes(self._chunks) + self._closed = False + self._reading = False + self._state_lock = threading.Lock() + + def __del__(self): + # Background commands can be dropped after the first event. + try: + self.close() + except Exception: + pass + + def __iter__(self): + return self + + def __next__(self): + with self._state_lock: + if self._closed: + raise StopIteration + if self._reading: + raise ValueError('Connect stream is already being read') + self._reading = True + + try: + event = next(self._events) + except BaseException as err: + if self._finish_read(terminal=True) and isinstance(err, Exception): + raise StopIteration + raise + if self._finish_read(): + raise StopIteration + return event + + next = __next__ + + def _finish_read(self, terminal=False): + with self._state_lock: + self._reading = False + interrupted = self._closed + if not (terminal or interrupted): + return False + self._closed = True + try: + self._close_resources() + except Exception: + pass + return interrupted + + def _close_resources(self): + try: + self._events.close() + close_chunks = getattr(self._chunks, 'close', None) + if close_chunks is not None: + close_chunks() + finally: + self._response.close() + + def close(self): + with self._state_lock: + if self._closed: + return + self._closed = True + if self._reading: + # Only wake the read. Closing the response here would race + # urllib3 releasing the connection as the read unwinds. + self._interrupt_read() + return + self._close_resources() + + def _interrupt_read(self): + raw = getattr(self._response, 'raw', None) + if getattr(raw, 'closed', False): + return + shutdown = getattr(raw, 'shutdown', None) + try: + if shutdown is not None: + shutdown() + except (IOError, OSError, RuntimeError, ValueError): + shutdown = None + if shutdown is None: + # A closing HTTP connection can detach its socket while the + # current response's file object still owns the active read. + fp = getattr(getattr(raw, '_fp', None), 'fp', None) + sock = getattr(fp, '_sock', None) + if sock is None: + sock = getattr(getattr(fp, 'raw', None), '_sock', None) + try: + if sock is not None: + sock.shutdown(socket.SHUT_RDWR) + except (IOError, OSError, RuntimeError, ValueError): + pass + + def connect_stream_rpc(sandbox, procedure, body=None, user=None, timeout=None, stream=False): if timeout is None: @@ -157,6 +256,7 @@ def connect_stream_rpc(sandbox, procedure, body=None, user=None, timeout=None, 'data': encode_connect_envelope(body), 'headers': headers, 'timeout': timeout, + 'allow_redirects': False, } if stream: request_opts['stream'] = True @@ -168,16 +268,10 @@ def connect_stream_rpc(sandbox, procedure, body=None, user=None, timeout=None, response = sandbox.client.session.post(url, **request_opts) except requests.RequestException as err: raise SandboxError('Sandbox envd request failed: {0}'.format(err)) - if response.status_code < 200 or response.status_code >= 300: - raise SandboxError( - 'Sandbox envd request failed with status {0}'.format( - response.status_code), response, response.text, ) + _check_envd_response(response) if stream and hasattr(response, 'iter_content'): - return iter_connect_envelopes( - response.iter_content(chunk_size=8192), - response=response, - ) + return _ConnectStream(response) content_type = response.headers.get('Content-Type', '') if 'application/connect+json' in content_type: @@ -198,12 +292,22 @@ def connect_stream_rpc(sandbox, procedure, body=None, user=None, timeout=None, def raw_envd_request(sandbox, method, url, **kwargs): if kwargs.get('timeout') is None: kwargs['timeout'] = sandbox.client.timeout + # Envd credentials and signed URLs must not be forwarded to redirect targets. + kwargs['allow_redirects'] = False try: response = sandbox.client.session.request(method, url, **kwargs) except requests.RequestException as err: raise SandboxError('Sandbox envd request failed: {0}'.format(err)) + _check_envd_response(response) + return response + + +def _check_envd_response(response): if response.status_code < 200 or response.status_code >= 300: + try: + data = response.text + finally: + response.close() raise SandboxError( 'Sandbox envd request failed with status {0}'.format( - response.status_code), response, response.text, ) - return response + response.status_code), response, data) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index cccf5e6c..8a187f73 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -30,45 +30,48 @@ def _normalize_paths(paths): RESET_MODES = set(['soft', 'mixed', 'hard', 'merge', 'keep']) -def _remove_credential_file(filesystem, path): - if not path: - return - try: - filesystem.remove(path) - except Exception: - pass +def _remove_credential_files(filesystem, path, directory, user): + opts = {} if user is None else {'user': user} + for item in (path, directory): + try: + filesystem.remove(item, **opts) + except Exception: + pass class _CredentialCleanupHandle(object): - def __init__(self, handle, filesystem, path): + def __init__(self, handle, filesystem, path, directory, user): self._handle = handle self._filesystem = filesystem self._path = path + self._directory = directory + self._user = user self._cleaned = [False] self.wait = self._wrap_wait( - handle.wait, filesystem, path, self._cleaned) + handle.wait, filesystem, path, directory, user, self._cleaned) def __getattr__(self, name): return getattr(self._handle, name) @staticmethod - def _cleanup(filesystem, path, cleaned): + def _cleanup(filesystem, path, directory, user, cleaned): if not cleaned[0]: cleaned[0] = True - _remove_credential_file(filesystem, path) + _remove_credential_files(filesystem, path, directory, user) @classmethod - def _wrap_wait(cls, wait, filesystem, path, cleaned): + def _wrap_wait(cls, wait, filesystem, path, directory, user, cleaned): def wrapped_wait(*args, **kwargs): try: return wait(*args, **kwargs) finally: - cls._cleanup(filesystem, path, cleaned) + cls._cleanup(filesystem, path, directory, user, cleaned) return wrapped_wait def __del__(self): try: - self._cleanup(self._filesystem, self._path, self._cleaned) + self._cleanup(self._filesystem, self._path, self._directory, + self._user, self._cleaned) except Exception: pass @@ -357,6 +360,12 @@ def __init__(self, commands): def _run_git(self, repo_path, args, **opts): if repo_path: opts['cwd'] = repo_path + username = opts.pop('_credential_username', None) + if username is not None: + args = [ + '-c', 'credential.helper=', + '-c', shell_quote('credential.username={0}'.format(username)), + ] + args return self.commands.run('git {0}'.format(' '.join(args)), **opts) def _get_remote_url(self, repo_path, remote, **opts): @@ -396,34 +405,37 @@ def _resolve_remote_name(self, repo_path, remote=None, **opts): def _with_remote_credentials(self, repo_path, remote, username, password, operation, **opts): original_url = self._get_remote_url(repo_path, remote, **opts) - _validate_git_url_credentials(original_url, username, password) + return self._with_credentials( + original_url, username, password, operation, **opts) + + def _with_credentials(self, url, username, password, operation, **opts): + _validate_git_url_credentials(url, username, password) sandbox = getattr(self.commands, 'sandbox', None) filesystem = getattr(sandbox, 'files', None) if filesystem is None: raise SandboxError( 'Sandbox filesystem is required for credentialed Git ' 'operations.') - temp_dir = '/tmp/qiniu-git-auth' + temp_dir = '/tmp/qiniu-git-auth-{0}'.format(uuid.uuid4().hex) + user = opts.get('user') + file_opts = {} if user is None else {'user': user} setup_opts = dict(opts) setup_opts.pop('background', None) prepare_result = self.commands.run( - 'install -d -m 700 {0}'.format(shell_quote(temp_dir)), + 'mkdir -m 700 {0}'.format(shell_quote(temp_dir)), **setup_opts ) if getattr(prepare_result, 'exit_code', 0): return prepare_result - askpass_path = '{0}/qiniu-git-askpass-{1}'.format( - temp_dir, - uuid.uuid4().hex, - ) + askpass_path = temp_dir + '/askpass' try: - filesystem.write(askpass_path, _askpass_script()) + filesystem.write(askpass_path, _askpass_script(), **file_opts) chmod_result = self.commands.run( 'chmod 700 {0}'.format(shell_quote(askpass_path)), **setup_opts ) if getattr(chmod_result, 'exit_code', 0): - _remove_credential_file(filesystem, askpass_path) + _remove_credential_files(filesystem, askpass_path, temp_dir, user) return chmod_result auth_opts = dict(opts) @@ -435,16 +447,17 @@ def _with_remote_credentials(self, repo_path, remote, username, password, 'GIT_PASSWORD': password, }) auth_opts['envs'] = envs + auth_opts['_credential_username'] = username result = operation(auth_opts) if opts.get('background') and hasattr(result, 'wait'): result = _CredentialCleanupHandle( - result, filesystem, askpass_path) + result, filesystem, askpass_path, temp_dir, user) else: - _remove_credential_file(filesystem, askpass_path) + _remove_credential_files(filesystem, askpass_path, temp_dir, user) return result except BaseException as err: - _remove_credential_file(filesystem, askpass_path) + _remove_credential_files(filesystem, askpass_path, temp_dir, user) raise err def _raise_known_result_error( @@ -476,7 +489,19 @@ def wrapped_wait(*args, **kwargs): return result return wrapped_wait - def clone(self, repo_url, path=None, branch=None, depth=None, **opts): + def clone(self, repo_url, path=None, branch=None, depth=None, + username=None, password=None, **opts): + """Clone with optional temporary HTTP(S) username/password credentials.""" + explicit = username is not None or password is not None + if explicit: + if not username or not password: + raise InvalidArgumentException( + 'Both username and password are required when using Git ' + 'credentials.') + url = _validate_git_url_credentials(repo_url, username, password) + if '@' in url.netloc: + # Git takes the URL userinfo before command-scoped credentials. + repo_url = url._replace(netloc=url.netloc.rsplit('@', 1)[1]).geturl() args = ['clone'] if depth: args.extend(['--depth', shell_quote(depth)]) @@ -485,7 +510,18 @@ def clone(self, repo_url, path=None, branch=None, depth=None, **opts): args.append(shell_quote(repo_url)) if path: args.append(shell_quote(path)) - return self._run_git(None, args, **opts) + if not explicit: + return self._run_git(None, args, **opts) + throw_on_error = opts.pop('throw_on_error', False) + result = self._with_credentials( + repo_url, username, password, + lambda auth_opts: self._run_git( + None, args, throw_on_error=False, **auth_opts), + **opts + ) + self._raise_known_result_error( + result, 'clone', throw_on_error=throw_on_error) + return result def init(self, repo_path, bare=False, initial_branch=None, **opts): args = ['init'] diff --git a/qiniu/services/sandbox/pty.py b/qiniu/services/sandbox/pty.py index 500c7a52..fb8ee233 100644 --- a/qiniu/services/sandbox/pty.py +++ b/qiniu/services/sandbox/pty.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- import base64 +import threading from .commands import CommandHandle, command_result_from_events from .envd import connect_rpc, connect_stream_rpc -from .errors import SandboxError +from .errors import CommandExitError, SandboxError class PtySize(object): @@ -33,12 +34,200 @@ def _normalize_size(size=None, rows=None, cols=None): } +class _PtyHandle(CommandHandle): + def __init__(self, pty, result, events, on_data=None, + throw_on_error=False): + super(_PtyHandle, self).__init__( + pty, result, events=events, on_data=on_data, + throw_on_error=throw_on_error) + self._reader_error = None + self._done = False + self._state = threading.Condition() + self._callback_waiter = None + self._pending_callback = None + self._disconnected = False + self._stdout_chunks = [result.stdout] + self._stderr_chunks = [result.stderr] + self._result_frozen = False + self._reader = None + self._closer = None + if on_data is not None: + self._start_reader() + + def _start_reader(self, on_stdout=None, on_stderr=None): + self._reader = threading.Thread( + target=self._read, args=(self._events, on_stdout, on_stderr)) + self._reader.daemon = True + self._reader.start() + + def _read(self, events, on_stdout, on_stderr): + try: + for event in events: + callbacks = [] + part = command_result_from_events( + [event], + on_stdout=(lambda data: callbacks.append((on_stdout, data))) + if on_stdout is not None else None, + on_stderr=(lambda data: callbacks.append((on_stderr, data))) + if on_stderr is not None else None, + on_data=(lambda data: callbacks.append((self._on_data, data))) + if self._on_data is not None else None, + ) + with self._state: + if self._disconnected: + break + if part.pid: + self.result.pid = part.pid + self.pid = part.pid + if part.stdout: + self._stdout_chunks.append(part.stdout) + if part.stderr: + self._stderr_chunks.append(part.stderr) + if part.exit_code != -1: + self.result.exit_code = part.exit_code + if part.error: + self.result.error = part.error + for callback, chunk in callbacks: + with self._state: + if self._disconnected: + break + if self._callback_waiter is not None: + # One callback at a time, on the first waiter's thread. + self._pending_callback = (callback, chunk) + self._state.notify_all() + while self._pending_callback is not None: + self._state.wait() + if self._reader_error is not None: + break + continue + # Dispatch is committed here; user code runs unlocked. + callback(chunk) + with self._state: + if self._disconnected or self._reader_error is not None: + break + except BaseException as err: + with self._state: + if not self._disconnected: + self._reader_error = err + finally: + try: + self._close_stream(events) + finally: + with self._state: + self._freeze_result() + self._events = None + self._done = True + self._state.notify_all() + + def _close_stream(self, events): + try: + close = getattr(events, 'close', None) + if close is not None: + close() + except BaseException as err: + with self._state: + if not self._disconnected and self._reader_error is None: + self._reader_error = err + + def _freeze_result(self): + if self._result_frozen: + return + self.result.stdout = ''.join(self._stdout_chunks) + self.result.stderr = ''.join(self._stderr_chunks) + self.stdout = self.result.stdout + self.stderr = self.result.stderr + self._stdout_chunks = [] + self._stderr_chunks = [] + self.exit_code = self.result.exit_code + self.exitCode = self.result.exit_code + self.result.exitCode = self.result.exit_code + if (self._reader_error is None and self.throw_on_error and + self.result.exit_code and not ( + self._disconnected and self.result.exit_code == -1)): + self._reader_error = CommandExitError(self.result) + self._result_frozen = True + + def wait(self, on_stdout=None, on_stderr=None): + current = threading.current_thread() + with self._state: + if not self._done: + if self._reader is None: + if on_stdout is not None or on_stderr is not None: + self._callback_waiter = current + self._start_reader(on_stdout, on_stderr) + elif current in (self._reader, self._callback_waiter): + raise RuntimeError( + 'Cannot wait for a PTY from its output callback') + + dispatch_callbacks = self._callback_waiter is current + try: + return self._wait_for_result(dispatch_callbacks) + except BaseException: + with self._state: + interrupted_dispatch = dispatch_callbacks and not self._done + if interrupted_dispatch: + # No waiter remains to acknowledge the reader's callbacks. + self.disconnect() + raise + + def _wait_for_result(self, dispatch_callbacks): + while True: + with self._state: + if self._done: + if self._reader_error is not None: + raise self._reader_error + return self.result + if not dispatch_callbacks or self._pending_callback is None: + self._state.wait() + continue + callback, chunk = self._pending_callback + + error = None + try: + callback(chunk) + except BaseException as err: + error = err + with self._state: + if not self._disconnected and error is not None: + self._reader_error = error + self._pending_callback = None + self._state.notify_all() + + def disconnect(self): + """Detach immediately; already-dispatched callbacks may still run.""" + with self._state: + if self._disconnected: + return + self._disconnected = True + events = self._events + self._events = None + self._freeze_result() + self._pending_callback = None + self._done = True + self._state.notify_all() + if events is not None: + self._closer = threading.Thread( + target=self._close_stream, args=(events,)) + self._closer.daemon = True + closer = self._closer + else: + closer = None + if closer is not None: + closer.start() + + class Pty(object): def __init__(self, sandbox): self.sandbox = sandbox def create(self, size=None, user=None, cwd=None, envs=None, timeout=None, - rows=None, cols=None, **opts): + rows=None, cols=None, on_data=None, **opts): + """Create a PTY and pass raw output bytes to on_data. + + Output in the first event calls on_data inline. A callback error there + closes the stream and propagates from create(). Later events call + on_data from the reader thread and report callback errors via wait(). + """ envs = dict(envs or {}) envs.setdefault('TERM', 'xterm-256color') envs.setdefault('LANG', 'C.UTF-8') @@ -66,31 +255,57 @@ def create(self, size=None, user=None, cwd=None, envs=None, timeout=None, ) events = iter(events) try: - first_event = next(events) - except StopIteration: - first_event = None - result = command_result_from_events([first_event]) - return CommandHandle(self, result, events=events) + try: + first_event = next(events) + except StopIteration: + first_event = None + result = command_result_from_events([first_event], on_data=on_data) + except BaseException: + close = getattr(events, 'close', None) + if close is not None: + try: + close() + except BaseException: + pass + raise + return _PtyHandle(self, result, events, on_data=on_data) - def connect(self, pid, user=None, timeout=None, throw_on_error=False): + def connect(self, pid, user=None, timeout=None, throw_on_error=False, + on_data=None): + """Connect to a PTY and pass raw output bytes to on_data. + + Output in the first event calls on_data inline. A callback error there + closes the stream and propagates from connect(). Later events call + on_data from the reader thread and report callback errors via wait(). + """ events = connect_stream_rpc( self.sandbox, '/process.Process/Connect', - {'process': {'selector': {'pid': pid}}}, + {'process': {'pid': pid}}, user=user, timeout=timeout, stream=True, ) events = iter(events) try: - first_event = next(events) - except StopIteration: - first_event = None - result = command_result_from_events([first_event]) - return CommandHandle( + try: + first_event = next(events) + except StopIteration: + first_event = None + result = command_result_from_events([first_event], on_data=on_data) + except BaseException: + close = getattr(events, 'close', None) + if close is not None: + try: + close() + except BaseException: + pass + raise + return _PtyHandle( self, result, - events=events, + events, + on_data=on_data, throw_on_error=throw_on_error, ) @@ -103,7 +318,7 @@ def send_stdin(self, pid, data, user=None, timeout=None): else: data = str(data).encode('utf-8') return connect_rpc(self.sandbox, '/process.Process/SendInput', { - 'process': {'selector': {'pid': pid}}, + 'process': {'pid': pid}, 'input': {'pty': base64.b64encode(data).decode('ascii')}, }, user=user, timeout=timeout) @@ -114,14 +329,14 @@ def send_stdin(self, pid, data, user=None, timeout=None): def resize(self, pid, size=None, user=None, timeout=None, rows=None, cols=None): return connect_rpc(self.sandbox, '/process.Process/Update', { - 'process': {'selector': {'pid': pid}}, + 'process': {'pid': pid}, 'pty': {'size': _normalize_size(size, rows=rows, cols=cols)}, }, user=user, timeout=timeout) def kill(self, pid, user=None, timeout=None): try: connect_rpc(self.sandbox, '/process.Process/SendSignal', { - 'process': {'selector': {'pid': pid}}, + 'process': {'pid': pid}, 'signal': 'SIGNAL_SIGKILL', }, user=user, timeout=timeout) return True diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index 024848a7..3378e0e6 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -70,17 +70,15 @@ def has_next(self): hasNext = has_next def next_items(self, **opts): + if not self.has_next and opts.get('nextToken') is None: + return [] request_opts = dict(self.opts) request_opts.update(opts) if self.next_token and request_opts.get('nextToken') is None: request_opts['nextToken'] = self.next_token - data = self.client.list_sandboxes_v2(**request_opts) or {} - items = data if isinstance(data, list) else ( - data.get('items') or data.get('sandboxes') or [] - ) - self.next_token = None if isinstance(data, list) else ( - data.get('nextToken') or data.get('next_token') - ) + data = self.client.list_sandboxes(**request_opts) + items = data.get('items') or [] + self.next_token = data.get('nextToken') self._has_next = bool(self.next_token) return [Sandbox(client=self.client, info=item) for item in items] diff --git a/qiniu/services/sandbox/util.py b/qiniu/services/sandbox/util.py index 3139c871..f70d814a 100644 --- a/qiniu/services/sandbox/util.py +++ b/qiniu/services/sandbox/util.py @@ -61,12 +61,13 @@ def basic_auth(user=None): def file_signature(path, operation, user, access_token, expiration): - if expiration is None: - expiration = '' - components = [path, operation, user, access_token, expiration] + components = [path, operation, user, access_token] + if expiration is not None: + components.append(expiration) raw = b':'.join([_to_utf8_bytes(component) for component in components]) - return 'v1_{0}'.format(hashlib.sha256(raw).hexdigest()) + encoded = base64.b64encode(hashlib.sha256(raw).digest()).rstrip(b'=') + return 'v1_{0}'.format(encoded.decode('ascii')) def _to_utf8_bytes(value): diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 9e4a067c..8b55537b 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import gc import json +import posixpath import threading import pytest @@ -9,9 +10,10 @@ import qiniu.services.sandbox.sandbox as sandbox_module try: - from urllib.parse import parse_qs, urlparse + from urllib.parse import parse_qs, unquote_plus, urlparse except ImportError: from urlparse import parse_qs, urlparse + from urllib import unquote_plus try: from socketserver import StreamRequestHandler, TCPServer @@ -342,13 +344,14 @@ def test_client_reads_qiniu_mac_credentials_from_env(monkeypatch): def test_util_helpers_encode_unicode_values_safely(): assert encode_path(u'目录/文件.txt') + # Fixed envd vectors generated with Go crypto/sha256 and base64.RawStdEncoding. assert file_signature( u'/tmp/文件.txt', 'read', u'用户', u'token', 1893456000, - ).startswith('v1_') + ) == 'v1_iiDuVS1ZoEJWJwcmaMLXBJSUfap4iuJG1WlhzAi81fU' def test_create_with_kodo_resource_requires_qiniu_credentials(monkeypatch): @@ -794,7 +797,7 @@ def test_sandbox_envd_and_file_urls_are_signed_when_token_is_available(): assert query['path'] == ['/tmp/hello.txt'] assert query['username'] == ['user'] assert query['signature_expiration'] == ['1893456000'] - assert query['signature'][0] + assert query['signature'][0] == 'v1_UeNL+5BVc/nTtL8kVKm6r9cV9MHKKK+ONfF8DqRHmQM' def test_envd_url_requires_domain_without_override(): @@ -814,14 +817,7 @@ def test_file_url_accepts_none_signature_expiration(): url = sandbox.download_url('/tmp/hello.txt', signatureExpiration=None) query = parse_qs(urlparse(url).query) - assert query['signature'][0] - assert query['signature'][0] == file_signature( - '/tmp/hello.txt', - 'read', - 'user', - 'token', - '', - ) + assert query['signature'][0] == 'v1_Ts3v6XNhCOn8v/BAzyGbrgPhWwKFdEvLtchOSN7BqbU' assert 'signature_expiration' not in query @@ -932,13 +928,83 @@ class RecordingSandboxListClient(object): def __init__(self): self.calls = [] - def list_sandboxes_v2(self, **opts): + def list_sandboxes(self, **opts): self.calls.append(opts) if len(self.calls) == 1: return {'items': [], 'nextToken': 'next-page'} return {'items': []} +@pytest.mark.parametrize('method_name', [ + 'list_sandboxes', 'listSandboxes', 'list', +]) +def test_client_sandbox_list_returns_items_and_header_cursor(method_name): + first = DummyResponse(200, [{'sandboxID': 'first'}]) + first.headers['X-Next-Token'] = 'page-2' + session = RecordingSession([ + first, DummyResponse(200, [{'sandboxID': 'second'}]), + ]) + client = SandboxClient(api_key='api-key', session=session) + list_sandboxes = getattr(client, method_name) + + page = list_sandboxes(limit=1, query={'metadata': {'app': 'tests'}}) + assert page == {'items': [{'sandboxID': 'first'}], 'nextToken': 'page-2'} + last = list_sandboxes(limit=1, nextToken=page['nextToken']) + assert last == {'items': [{'sandboxID': 'second'}], 'nextToken': None} + assert urlparse(session.requests[0].url).path == '/v2/sandboxes' + assert parse_qs(urlparse(session.requests[0].url).query) == { + 'limit': ['1'], 'metadata': ['app=tests'], + } + assert parse_qs(urlparse(session.requests[1].url).query) == { + 'limit': ['1'], 'nextToken': ['page-2'], + } + + +def test_sandbox_paginator_follows_header_cursor_and_stops_at_last_page(): + first = DummyResponse(200, [{'sandboxID': 'first'}]) + first.headers['X-Next-Token'] = 'page-2' + session = RecordingSession([ + first, DummyResponse(200, [{'sandboxID': 'second'}]), + ]) + client = SandboxClient(api_key='api-key', session=session) + paginator = Sandbox.list(client=client, limit=1, + metadata={'app': 'tests'}) + + assert [item.sandbox_id for item in paginator.next_items()] == ['first'] + assert paginator.has_next is True + assert paginator.next_token == 'page-2' + assert [item.sandbox_id for item in paginator.next_items()] == ['second'] + assert paginator.has_next is False + assert paginator.next_items() == [] + assert len(session.requests) == 2 + query = parse_qs(urlparse(session.requests[1].url).query) + assert query == {'limit': ['1'], 'metadata': ['app=tests'], + 'nextToken': ['page-2']} + + +@pytest.mark.parametrize('body', [ + {'items': [{'sandboxID': 'first'}], 'nextToken': 'body-page'}, + {'sandboxes': [{'sandboxID': 'first'}], 'next_token': 'body-page'}, +]) +@pytest.mark.parametrize('header_cursor', [None, 'header-page']) +def test_sandbox_lists_normalize_json_pages_and_prefer_header_cursor( + body, header_cursor): + responses = [DummyResponse(200, body), DummyResponse(200, body)] + if header_cursor is not None: + for response in responses: + response.headers['X-Next-Token'] = header_cursor + client = SandboxClient(api_key='api-key', + session=RecordingSession(responses)) + expected_cursor = header_cursor or 'body-page' + + assert client.list_sandboxes() == { + 'items': [{'sandboxID': 'first'}], 'nextToken': expected_cursor, + } + paginator = Sandbox.list(client=client) + assert [item.sandbox_id for item in paginator.next_items()] == ['first'] + assert paginator.next_token == expected_cursor + + def test_sandbox_paginator_does_not_reuse_initial_next_token(): client = RecordingSandboxListClient() paginator = SandboxPaginator(client=client, next_token='saved-page') @@ -966,11 +1032,11 @@ def test_sandbox_paginator_does_not_send_client_credentials_as_filters(): assert client.calls[0] == {'metadata': {'app': 'tests'}} -def test_list_sandboxes_v2_serializes_metadata_dict_as_query_string(): - session = RecordingSession([DummyResponse(200, {'items': []})]) +def test_list_sandboxes_serializes_metadata_dict_as_query_string(): + session = RecordingSession([DummyResponse(200, [])]) client = SandboxClient(api_key='api-key', session=session) - client.list_sandboxes_v2(query={ + client.list_sandboxes(query={ 'metadata': {'app': 'tests'}, 'state': 'running', }) @@ -980,31 +1046,97 @@ def test_list_sandboxes_v2_serializes_metadata_dict_as_query_string(): assert query['state'] == ['running'] -def test_list_sandboxes_v2_accepts_metadata_string(): - session = RecordingSession([DummyResponse(200, {'items': []})]) +def test_list_sandboxes_accepts_metadata_string(): + session = RecordingSession([DummyResponse(200, [])]) client = SandboxClient(api_key='api-key', session=session) - client.list_sandboxes_v2(metadata='user=abc&app=prod') + client.list_sandboxes(metadata='user=abc&app=prod') query = parse_qs(urlparse(session.requests[0].url).query) assert query['metadata'] == ['user=abc&app=prod'] -def test_list_sandboxes_v2_serializes_template_filters_as_comma_string(): - session = RecordingSession([DummyResponse(200, {'items': []})]) +@pytest.mark.parametrize('value', [ + u'plain', u'中文 空格', u'a+b', u'100%', u'a=b', u'a&b', + u'feat/a&b=1+2%', +]) +@pytest.mark.parametrize('nested', [False, True]) +def test_list_metadata_survives_server_filter_decoding(value, nested): + # The server unescapes the complete filter before splitting each key/value. + # A regular query decoder alone would miss '+' corruption and '&'/'=' errors. + metadata = {u'key+&=% 中文': value} + opts = {'query': {'metadata': metadata}} if nested else {'metadata': metadata} + session = RecordingSession([DummyResponse(200, [])]) + client = SandboxClient(api_key='api-key', session=session) + + client.list_sandboxes(**opts) + + query = parse_qs(urlparse(session.requests[0].url).query) + filters = unquote_plus(query['metadata'][0]) + pairs = [item.split('=') for item in filters.split('&')] + assert all(len(pair) == 2 for pair in pairs) + decoded = dict((_decode_filter_component(key), _decode_filter_component(item)) + for key, item in pairs) + assert decoded == metadata + + +def test_list_metadata_normalizes_utf8_bytes_and_non_string_values(): + session = RecordingSession([DummyResponse(200, [])]) + client = SandboxClient(api_key='api-key', session=session) + metadata = {u'字节+'.encode('utf-8'): u'值&+'.encode('utf-8'), 7: 42, + 'flag': True, 'optional': None} + + client.list_sandboxes(metadata=metadata) + + query = parse_qs(urlparse(session.requests[0].url).query) + filters = unquote_plus(query['metadata'][0]) + decoded = dict((_decode_filter_component(key), _decode_filter_component(value)) + for key, value in (part.split('=') for part in filters.split('&'))) + assert decoded == {u'字节+': u'值&+', '7': '42', 'flag': 'True', + 'optional': 'None'} + + +def test_list_metadata_preserves_preformatted_filter(): + session = RecordingSession([DummyResponse(200, [])]) + client = SandboxClient(api_key='api-key', session=session) + formatted = 'branch=a%252Bb%2526c%253Dd%2525' + + client.list_sandboxes(metadata=formatted) + + query = parse_qs(urlparse(session.requests[0].url).query) + assert query['metadata'] == [formatted] + + +def test_list_metadata_preserves_raw_byte_values(): + session = RecordingSession([DummyResponse(200, [])]) + client = SandboxClient(api_key='api-key', session=session) + + client.list_sandboxes(metadata={'raw': b'\xff+'}) + + query = parse_qs(urlparse(session.requests[0].url).query) + assert query['metadata'] == ['raw=%25FF%252B'] + + +def _decode_filter_component(value): + decoded = unquote_plus(value) + return decoded if isinstance(decoded, type(u'')) else decoded.decode('utf-8') + + +def test_list_sandboxes_serializes_template_filters_as_comma_string(): + session = RecordingSession([DummyResponse(200, [])]) client = SandboxClient(api_key='api-key', session=session) - client.list_sandboxes_v2(template=['python', 'team/node']) + client.list_sandboxes(template=['python', 'team/node']) query = parse_qs(urlparse(session.requests[0].url).query) assert query['template'] == ['python,team/node'] -def test_list_sandboxes_v2_serializes_array_filters_from_query_options(): - session = RecordingSession([DummyResponse(200, {'items': []})]) +def test_list_sandboxes_serializes_array_filters_from_query_options(): + session = RecordingSession([DummyResponse(200, [])]) client = SandboxClient(api_key='api-key', session=session) - client.list_sandboxes_v2(query={ + client.list_sandboxes(query={ 'template': ['python', 'team/node'], 'state': ['running', 'paused'], 'limit': 10, @@ -1018,11 +1150,11 @@ def test_list_sandboxes_v2_serializes_array_filters_from_query_options(): assert query['nextToken'] == ['next-page'] -def test_list_sandboxes_v2_explicit_options_override_extra_query_values(): - session = RecordingSession([DummyResponse(200, {'items': []})]) +def test_list_sandboxes_explicit_options_override_extra_query_values(): + session = RecordingSession([DummyResponse(200, [])]) client = SandboxClient(api_key='api-key', session=session) - client.list_sandboxes_v2( + client.list_sandboxes( limit=20, state=['running'], template=['python'], @@ -1039,11 +1171,11 @@ def test_list_sandboxes_v2_explicit_options_override_extra_query_values(): assert query['template'] == ['python'] -def test_list_sandboxes_v2_explicit_none_options_override_query_values(): - session = RecordingSession([DummyResponse(200, {'items': []})]) +def test_list_sandboxes_explicit_none_options_override_query_values(): + session = RecordingSession([DummyResponse(200, [])]) client = SandboxClient(api_key='api-key', session=session) - client.list_sandboxes_v2( + client.list_sandboxes( metadata=None, state=None, template=None, @@ -1060,32 +1192,32 @@ def test_list_sandboxes_v2_explicit_none_options_override_query_values(): assert 'template' not in query -def test_list_sandboxes_v2_serializes_non_string_filter_values(): - session = RecordingSession([DummyResponse(200, {'items': []})]) +def test_list_sandboxes_serializes_non_string_filter_values(): + session = RecordingSession([DummyResponse(200, [])]) client = SandboxClient(api_key='api-key', session=session) - client.list_sandboxes_v2(template=[123, 456], state=7) + client.list_sandboxes(template=[123, 456], state=7) query = parse_qs(urlparse(session.requests[0].url).query) assert query['template'] == ['123,456'] assert query['state'] == ['7'] -def test_list_sandboxes_v2_does_not_join_mapping_keys(): - session = RecordingSession([DummyResponse(200, {'items': []})]) +def test_list_sandboxes_does_not_join_mapping_keys(): + session = RecordingSession([DummyResponse(200, [])]) client = SandboxClient(api_key='api-key', session=session) - client.list_sandboxes_v2(template={'template': 'python'}) + client.list_sandboxes(template={'template': 'python'}) query = parse_qs(urlparse(session.requests[0].url).query) assert query['template'] == ["{'template': 'python'}"] -def test_list_sandboxes_v2_preserves_bytes_and_unicode_filter_values(): - session = RecordingSession([DummyResponse(200, {'items': []})]) +def test_list_sandboxes_preserves_bytes_and_unicode_filter_values(): + session = RecordingSession([DummyResponse(200, [])]) client = SandboxClient(api_key='api-key', session=session) - client.list_sandboxes_v2( + client.list_sandboxes( template=[u'团队/模板', b'team/node'], state=[b'running'], ) @@ -1949,18 +2081,19 @@ def test_git_push_with_credentials_uses_askpass_without_remote_url_leak(): ) assert commands.calls[0][0] == 'git remote get-url origin' - assert commands.calls[1][0] == 'install -d -m 700 /tmp/qiniu-git-auth' - assert files.writes[0][0].startswith( - '/tmp/qiniu-git-auth/qiniu-git-askpass-') - assert commands.calls[2][0].startswith( - 'chmod 700 /tmp/qiniu-git-auth/qiniu-git-askpass-') - assert commands.calls[3][0] == 'git push --set-upstream origin main' + directory = posixpath.dirname(files.writes[0][0]) + assert commands.calls[1][0] == 'mkdir -m 700 ' + directory + assert directory.startswith('/tmp/qiniu-git-auth-') + assert commands.calls[2][0] == 'chmod 700 ' + files.writes[0][0] + assert commands.calls[3][0] == ( + 'git -c credential.helper= -c credential.username=git:user push --set-upstream origin main') assert 'secret:%@token' not in commands.calls[3][0] assert commands.calls[3][1]['request_timeout'] == 7 assert commands.calls[3][1]['envs']['GIT_ASKPASS'] == files.writes[0][0] assert commands.calls[3][1]['envs']['GIT_USERNAME'] == 'git:user' assert commands.calls[3][1]['envs']['GIT_PASSWORD'] == 'secret:%@token' - assert files.removes == [(files.writes[0][0], {})] + assert files.removes == [(files.writes[0][0], {}), + (posixpath.dirname(files.writes[0][0]), {})] def test_git_pull_with_credentials_resolves_single_remote(): @@ -2008,11 +2141,13 @@ def test_git_pull_with_credentials_resolves_single_remote(): ) assert commands.calls[0][0] == 'git remote' - assert commands.calls[2][0] == 'install -d -m 700 /tmp/qiniu-git-auth' - assert commands.calls[4][0] == 'git pull origin main' + assert commands.calls[2][0] == 'mkdir -m 700 ' + posixpath.dirname(files.writes[0][0]) + assert commands.calls[4][0] == ( + "git -c credential.helper= -c credential.username=git-user pull origin main") assert 'secret-token' not in commands.calls[4][0] assert commands.calls[4][1]['envs']['GIT_ASKPASS'] == files.writes[0][0] - assert files.removes == [(files.writes[0][0], {})] + assert files.removes == [(files.writes[0][0], {}), + (posixpath.dirname(files.writes[0][0]), {})] def test_git_credential_helpers_ignore_background_when_reading_remote(): @@ -2065,7 +2200,8 @@ def test_git_credential_helpers_ignore_background_when_reading_remote(): assert commands.calls[4][1]['background'] is True assert files.removes == [] handle.wait() - assert files.removes == [(files.writes[0][0], {})] + assert files.removes == [(files.writes[0][0], {}), + (posixpath.dirname(files.writes[0][0]), {})] def test_git_background_throw_on_error_waits_before_raising(): @@ -2117,7 +2253,8 @@ def test_git_background_throw_on_error_waits_before_raising(): assert files.removes == [] with pytest.raises(CommandExitError): handle.wait() - assert files.removes == [(files.writes[0][0], {})] + assert files.removes == [(files.writes[0][0], {}), + (posixpath.dirname(files.writes[0][0]), {})] def test_git_background_credentials_clean_up_when_handle_is_discarded(): @@ -2180,7 +2317,8 @@ def test_git_background_credentials_clean_up_when_handle_is_discarded(): del handle gc.collect() - assert files.removes == [(credential_path, {})] + assert files.removes == [(credential_path, {}), + (posixpath.dirname(credential_path), {})] def test_git_push_with_credentials_cleans_askpass_on_auth_failure(): @@ -2228,8 +2366,10 @@ def test_git_push_with_credentials_cleans_askpass_on_auth_failure(): password='bad-token', ) - assert commands.calls[-1][0] == 'git push --set-upstream origin' - assert files.removes == [(files.writes[0][0], {})] + assert commands.calls[-1][0] == ( + "git -c credential.helper= -c credential.username=git-user push --set-upstream origin") + assert files.removes == [(files.writes[0][0], {}), + (posixpath.dirname(files.writes[0][0]), {})] def test_git_push_cleans_askpass_after_operation_exception(): @@ -2262,7 +2402,8 @@ def test_git_push_cleans_askpass_after_operation_exception(): ) assert 'rpc timed out' in str(err.value) - assert files.removes == [(files.writes[0][0], {})] + assert files.removes == [(files.writes[0][0], {}), + (posixpath.dirname(files.writes[0][0]), {})] def test_git_credentials_clean_askpass_when_chmod_raises(): @@ -2298,7 +2439,8 @@ def run(self, cmd, **opts): password='secret-token', ) - assert files.removes == [(files.writes[0][0], {})] + assert files.removes == [(files.writes[0][0], {}), + (posixpath.dirname(files.writes[0][0]), {})] def test_git_credentials_clean_askpass_when_write_raises(): @@ -2334,7 +2476,8 @@ def write(self, path, data, **opts): password='secret-token', ) - assert files.removes == [(files.writes[0][0], {})] + assert files.removes == [(files.writes[0][0], {}), + (posixpath.dirname(files.writes[0][0]), {})] def test_git_helpers_accept_e2b_style_signatures(): diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 3b838fea..0a9ab064 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -1,27 +1,65 @@ # -*- coding: utf-8 -*- import base64 +import gc +import hashlib +import inspect from io import BytesIO, StringIO import json +import os +import posixpath +import shlex +import shutil import struct +import subprocess +import sys +import tempfile +import time +import traceback +from threading import Event, Thread, Timer, current_thread + +try: + from http.server import BaseHTTPRequestHandler, HTTPServer, SimpleHTTPRequestHandler +except ImportError: + from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer + from SimpleHTTPServer import SimpleHTTPRequestHandler + +try: + from queue import Queue +except ImportError: + from Queue import Queue + +try: + from urllib.parse import parse_qs, urlencode, urlparse +except ImportError: + from urlparse import parse_qs, urlparse + from urllib import urlencode import pytest import requests from qiniu.services.sandbox import ( + CommandExitError, + CommandResult, EntryInfo, FileType, FilesystemEventType, + GitAuthException, + Git, + InvalidArgumentException, PtySize, Sandbox, SandboxClient, SandboxError, ) from qiniu.services.sandbox.envd import ( + connect_rpc, + connect_stream_rpc, decode_connect_envelopes, encode_connect_envelope, iter_connect_envelopes, ) from qiniu.services.sandbox.commands import command_result_from_events +from qiniu.services.sandbox.pty import _PtyHandle import qiniu.services.sandbox.util as sandbox_util @@ -54,7 +92,9 @@ def __init__(self): self.post_exception = None self.request_exception = None - def post(self, url, data=None, headers=None, timeout=None, stream=False): + def post(self, url, data=None, headers=None, timeout=None, stream=False, + allow_redirects=False): + assert allow_redirects is False if self.post_exception is not None: raise self.post_exception if headers.get('Content-Type') == 'application/connect+json': @@ -63,6 +103,11 @@ def post(self, url, data=None, headers=None, timeout=None, stream=False): decoded = json.loads( data.decode('utf-8') if isinstance(data, bytes) else data ) + # Protobuf JSON flattens the ProcessSelector oneof: no selector wrapper. + if '/process.Process/' in url and not url.endswith('/Start'): + process = decoded.get('process') + if process is not None: + assert set(process) in ({'pid'}, {'tag'}) self.posts.append({ 'url': url, 'data': decoded, @@ -325,6 +370,100 @@ def test_raw_envd_request_wraps_transport_errors(): assert 'timed out' in str(err.value) +@pytest.mark.parametrize('status', [301, 302, 303, 307, 308]) +@pytest.mark.parametrize('operation', [ + 'read', 'read_stream', 'write', 'write_octet', 'rpc', 'stream_rpc', +]) +def test_envd_redirects_never_forward_credentials(status, operation): + class RecordingSession(requests.Session): + def __init__(self): + super(RecordingSession, self).__init__() + self.responses = [] + + def send(self, request, **kwargs): + response = super(RecordingSession, self).send(request, **kwargs) + original_close = response.close + response.close_called = False + + def close(): + response.close_called = True + original_close() + + response.close = close + self.responses.append(response) + return response + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.respond() + + def do_POST(self): + self.rfile.read(int(self.headers.get('Content-Length', '0'))) + self.respond() + + def respond(self): + self.server.received.append( + requests.structures.CaseInsensitiveDict(self.headers.items())) + target = getattr(self.server, 'redirect_target', None) + body = b'redirect' if target else b'public target' + self.send_response(status if target else 200) + if target: + self.send_header('Location', target) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + target = HTTPServer(('127.0.0.1', 0), Handler) + origin = HTTPServer(('127.0.0.1', 0), Handler) + target.received = [] + origin.received = [] + origin.redirect_target = 'http://127.0.0.1:{0}/target'.format( + target.server_port) + threads = [] + for server in (target, origin): + thread = Thread(target=server.serve_forever, + kwargs={'poll_interval': 0.01}) + thread.daemon = True + thread.start() + threads.append(thread) + session = RecordingSession() + client = SandboxClient(api_key='api-key', session=session, timeout=2) + sandbox = Sandbox(client=client, sandbox_id='redirect-test', + envd_url='http://127.0.0.1:{0}'.format(origin.server_port), + envdAccessToken='not-a-real-token') + try: + with pytest.raises(SandboxError) as failure: + if operation == 'read': + sandbox.files.read('/file') + elif operation == 'read_stream': + sandbox.files.read('/file', format='stream') + elif operation == 'write': + sandbox.files.write('/file', b'hello') + elif operation == 'write_octet': + sandbox.files.write('/file', b'hello', use_octet_stream=True) + elif operation == 'rpc': + connect_rpc(sandbox, '/filesystem.Filesystem/Stat', {}) + else: + connect_stream_rpc(sandbox, '/process.Process/Start', {}, + stream=True) + assert failure.value.status_code == status + assert len(origin.received) == 1 + assert origin.received[0]['X-Access-Token'] == 'not-a-real-token' + assert target.received == [] + assert len(session.responses) == 1 + assert session.responses[0].close_called + finally: + session.close() + for server in (origin, target): + server.shutdown() + server.server_close() + for thread in threads: + thread.join(1) + + def test_filesystem_write_accepts_unicode_text(): sandbox, session = sandbox_with_envd_session() @@ -406,7 +545,7 @@ def test_commands_connect_returns_handle_for_running_process(): assert session.posts[0]['timeout'] == 30 assert session.posts[0]['url'].endswith('/process.Process/Connect') assert session.posts[0]['data'] == { - 'process': {'selector': {'pid': 12}}, + 'process': {'pid': 12}, } @@ -529,6 +668,1169 @@ def test_pty_create_and_connect_handle_empty_event_streams(): assert connected.wait().stdout == '' +class StreamingPtyResponse(DummyResponse): + def __init__(self): + super(StreamingPtyResponse, self).__init__( + raw=b'', content_type='application/connect+json') + self.chunks = Queue() + self.closed_event = Event() + self.chunks.put(encode_connect_envelope( + {'event': {'start': {'pid': 12}}})) + + def iter_content(self, chunk_size=8192): + del chunk_size + while True: + chunk = self.chunks.get() + if chunk is None: + return + if isinstance(chunk, Exception): + raise chunk + yield chunk + + def send(self, data): + self.chunks.put(encode_connect_envelope({'event': {'data': { + 'pty': base64.b64encode(data).decode('ascii'), + }}})) + + def finish(self, exit_code=0): + self.chunks.put(encode_connect_envelope( + {'event': {'end': {'exitCode': exit_code}}})) + self.chunks.put(None) + + def close(self): + self.closed = True + self.closed_event.set() + self.chunks.put(None) + + +class StreamingPtySession(EnvdSession): + def __init__(self, responses): + super(StreamingPtySession, self).__init__() + self.responses = list(responses) + + def post(self, url, **opts): + response = super(StreamingPtySession, self).post(url, **opts) + if url.endswith(('/process.Process/Start', '/process.Process/Connect')): + return self.responses.pop(0) + return response + + +@pytest.mark.parametrize('reconnect', [False, True]) +@pytest.mark.parametrize('late_error', [False, True]) +@pytest.mark.parametrize('with_wait_callback', [False, True]) +def test_pty_disconnect_freezes_output_consumed_by_synchronous_wait( + reconnect, late_error, with_wait_callback): + class PausedReaderResponse(StreamingPtyResponse): + def __init__(self): + super(PausedReaderResponse, self).__init__() + self.consumed = Event() + self.release = Event() + + def iter_content(self, chunk_size=8192): + for index, chunk in enumerate( + super(PausedReaderResponse, self).iter_content(chunk_size)): + yield chunk + if index == 1: + self.consumed.set() + self.release.wait() + + def close(self): + self.closed = True + self.closed_event.set() + + response = PausedReaderResponse() + session = StreamingPtySession([response]) + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + handle = sandbox.pty.connect(12) if reconnect else sandbox.pty.create() + results = [] + errors = [] + + def wait(): + try: + on_stdout = (lambda data: None) if with_wait_callback else None + results.append(handle.wait(on_stdout=on_stdout)) + except BaseException as err: + errors.append(err) + + waiting = Thread(target=wait) + waiting.daemon = True + try: + response.send(b'before disconnect') + waiting.start() + assert response.consumed.wait(1) + handle.disconnect() + snapshot = handle.wait() + assert snapshot.stdout == 'before disconnect' + assert snapshot.exit_code == -1 + assert handle.wait() is snapshot + waiting.join(1) + assert not waiting.is_alive(), 'wait blocked on the detached reader' + assert errors == [] + assert results == [snapshot] + if late_error: + response.chunks.put(requests.ConnectionError('late read error')) + else: + response.send(b'late output') + response.finish(exit_code=7) + response.release.set() + handle._reader.join(1) + assert not handle._reader.is_alive() + assert handle.wait() is snapshot + assert snapshot.stdout == 'before disconnect' + assert snapshot.exit_code == -1 + assert response.closed_event.wait(1) + finally: + response.release.set() + response.chunks.put(None) + response.close() + waiting.join(1) + + +def test_pty_disconnect_before_wait_does_not_block_on_response_close(): + class BlockingCloseResponse(StreamingPtyResponse): + def __init__(self): + super(BlockingCloseResponse, self).__init__() + self.closing = Event() + self.release = Event() + + def close(self): + self.closing.set() + self.release.wait() + super(BlockingCloseResponse, self).close() + + response = BlockingCloseResponse() + session = StreamingPtySession([response]) + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + handle = sandbox.pty.create() + detached = Event() + + def disconnect(): + handle.disconnect() + detached.set() + + closing = Thread(target=disconnect) + closing.daemon = True + try: + closing.start() + assert detached.wait(1), 'disconnect waited for response.close()' + assert response.closing.wait(1) + assert handle.wait().stdout == '' + assert handle.wait().exit_code == -1 + assert not response.closed_event.is_set() + finally: + response.release.set() + closing.join(1) + handle._closer.join(1) + assert response.closed_event.is_set() + + +def test_pty_synchronous_wait_keeps_callback_channels_and_caller_thread(): + response = StreamingPtyResponse() + session = StreamingPtySession([response]) + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + handle = sandbox.pty.create() + response.chunks.put(encode_connect_envelope({'event': {'data': { + 'stdout': base64.b64encode(b'out').decode('ascii'), + 'stderr': base64.b64encode(b'err').decode('ascii'), + 'pty': base64.b64encode(b'terminal').decode('ascii'), + }}})) + response.finish() + received = [] + try: + result = handle.wait( + on_stdout=lambda data: received.append( + ('stdout', data, current_thread())), + on_stderr=lambda data: received.append( + ('stderr', data, current_thread())), + ) + assert received == [('stdout', 'out', current_thread()), + ('stderr', 'err', current_thread())] + assert result.stdout == 'outterminal' + assert result.stderr == 'err' + assert result.exit_code == 0 + assert handle.wait() is result + finally: + response.close() + + +def test_pty_wait_callback_error_reaches_later_waiters(): + response = StreamingPtyResponse() + session = StreamingPtySession([response]) + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + handle = sandbox.pty.create() + response.chunks.put(encode_connect_envelope({'event': {'data': { + 'stdout': base64.b64encode(b'out').decode('ascii'), + }}})) + error = ValueError('callback failed') + + def callback(data): + with pytest.raises(RuntimeError, match='Cannot wait for a PTY'): + handle.wait() + raise error + + try: + with pytest.raises(ValueError) as first: + handle.wait(on_stdout=callback) + with pytest.raises(ValueError) as repeated: + handle.wait() + assert first.value is repeated.value is error + assert response.closed_event.is_set() + finally: + response.close() + + +@pytest.mark.parametrize('reconnect', [False, True]) +@pytest.mark.parametrize('blocked_close', [False, True]) +@pytest.mark.parametrize('late_error', [False, True]) +def test_pty_disconnect_releases_waiters_before_closed_reader_unwinds( + reconnect, blocked_close, late_error): + class DelayedCloseResponse(StreamingPtyResponse): + def __init__(self): + super(DelayedCloseResponse, self).__init__() + self.reading = Event() + self.released = Event() + self.close_started = Event() + self.close_released = Event() + self.reader_unwound = Event() + if not blocked_close: + self.close_released.set() + + def iter_content(self, chunk_size=8192): + try: + for chunk in super(DelayedCloseResponse, self).iter_content( + chunk_size): + yield chunk + self.reading.set() + self.released.wait() + finally: + self.reader_unwound.set() + + def close(self): + self.close_started.set() + self.close_released.wait() + self.closed = True + self.closed_event.set() + + response = DelayedCloseResponse() + response.send(b'before disconnect') + session = StreamingPtySession([response]) + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + received = [] + available = Event() + waited = Event() + results = [] + + def on_data(data): + received.append(data) + available.set() + + handle = None + waiter = None + try: + if reconnect: + handle = sandbox.pty.connect(12, on_data=on_data) + else: + handle = sandbox.pty.create(on_data=on_data) + response.released.set() + assert available.wait(1) + response.released.clear() + assert response.reading.wait(1) + + def wait(): + results.append(handle.wait()) + waited.set() + + waiter = Thread(target=wait) + waiter.daemon = True + waiter.start() + handle.disconnect() + assert waited.wait(1), 'wait still depends on a closed socket read' + handle._closer.join(1) + assert not handle._closer.is_alive() + assert not response.close_started.is_set() + assert results[0].stdout == 'before disconnect' + assert results[0].exit_code == -1 + assert handle.wait() is results[0] + + if late_error: + response.chunks.put(requests.ConnectionError('late close error')) + else: + response.send(b'late output') + response.released.set() + assert response.reader_unwound.wait(1) + assert response.close_started.wait(1) + if blocked_close: + assert not response.closed_event.is_set() + response.close_released.set() + assert response.closed_event.wait(1) + assert handle.wait() is results[0] + assert received == [b'before disconnect'] + assert results[0].stdout == 'before disconnect' + finally: + response.released.set() + response.close_released.set() + response.chunks.put(None) + if handle is not None: + handle.disconnect() + if waiter is not None: + waiter.join(1) + + +def test_pty_output_callback_can_disconnect_its_own_handle(): + response = StreamingPtyResponse() + session = StreamingPtySession([response]) + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + handles = [] + disconnected = Event() + + def on_data(data): + handles[0].disconnect() + disconnected.set() + + handle = sandbox.pty.create(on_data=on_data) + handles.append(handle) + try: + response.send(b'last output') + assert disconnected.wait(1) + assert handle.wait().stdout == 'last output' + assert handle.wait().exit_code == -1 + assert response.closed_event.wait(1) + finally: + response.close() + + +@pytest.mark.parametrize('reconnect', [False, True]) +def test_pty_disconnect_allows_already_dispatched_callback_to_start_later( + monkeypatch, reconnect): + original_read = _PtyHandle._read + lines, first_line = inspect.getsourcelines(original_read) + dispatch_line = next(first_line + index for index, line in enumerate(lines) + if line.strip() == 'callback(chunk)') + function = getattr(original_read, 'im_func', original_read) + code = getattr(function, '__code__', None) or function.func_code + dispatched = Event() + release = Event() + + def trace(frame, event, arg): + if (event == 'line' and frame.f_code is code and + frame.f_lineno == dispatch_line): + dispatched.set() + release.wait() + return trace + + def traced_read(self, *args, **kwargs): + sys.settrace(trace) + try: + return original_read(self, *args, **kwargs) + finally: + sys.settrace(None) + + monkeypatch.setattr(_PtyHandle, '_read', traced_read) + response = StreamingPtyResponse() + session = StreamingPtySession([response]) + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + received = [] + handle = (sandbox.pty.connect(12, on_data=received.append) if reconnect + else sandbox.pty.create(on_data=received.append)) + try: + response.send(b'committed output') + assert dispatched.wait(1) + handle.disconnect() + snapshot = handle.wait() + assert received == [] + assert snapshot.stdout == 'committed output' + response.send(b'late output') + release.set() + handle._reader.join(1) + assert not handle._reader.is_alive() + assert received == [b'committed output'] + assert handle.wait() is snapshot + assert snapshot.stdout == 'committed output' + finally: + release.set() + response.close() + handle._reader.join(1) + + +def test_pty_disconnect_does_not_wait_for_an_inflight_output_callback(): + response = StreamingPtyResponse() + session = StreamingPtySession([response]) + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + entered = Event() + release = Event() + returned = Event() + detached = Event() + + def on_data(data): + entered.set() + release.wait() + returned.set() + raise ValueError('inflight callback failed after disconnect') + + handle = sandbox.pty.create(on_data=on_data) + try: + response.send(b'inflight output') + assert entered.wait(1) + + def disconnect(): + handle.disconnect() + detached.set() + + closing = Thread(target=disconnect) + closing.daemon = True + closing.start() + assert detached.wait(1), 'disconnect waited for user callback code' + assert not returned.is_set() + assert handle.wait().stdout == 'inflight output' + finally: + release.set() + response.close() + assert returned.wait(1) + handle._reader.join(1) + assert not handle._reader.is_alive() + assert handle.wait().stdout == 'inflight output' + + +@pytest.mark.parametrize('reconnect', [False, True]) +def test_pty_callback_receives_raw_bytes_before_process_exit(reconnect): + response = StreamingPtyResponse() + session = StreamingPtySession([response]) + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + received = [] + available = Event() + + def on_data(data): + received.append(data) + available.set() + + try: + if reconnect: + handle = sandbox.pty.connect(12, on_data=on_data) + else: + handle = sandbox.pty.create(on_data=on_data) + response.send(b'\xff\x00hello') + assert available.wait(1), 'PTY output was not delivered before exit' + assert received == [b'\xff\x00hello'] + response.finish() + assert handle.wait().stdout == u'\ufffd\x00hello' + assert handle.wait().exit_code == 0 + assert handle.wait().exitCode == 0 + finally: + response.close() + + +@pytest.mark.parametrize('reconnect', [False, True]) +def test_pty_first_output_callback_error_closes_stream_before_traceback_release( + reconnect): + response = StreamingPtyResponse() + response.chunks.get() + response.send(b'first event is data') + session = StreamingPtySession([response]) + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + error = ValueError('first output callback failed') + retained = [] + + def on_data(data): + raise error + + try: + try: + if reconnect: + sandbox.pty.connect(12, on_data=on_data) + else: + sandbox.pty.create(on_data=on_data) + except ValueError: + retained.append(sys.exc_info()) + else: + pytest.fail('callback error was not raised') + assert retained[0][1] is error + gc.collect() + assert response.closed is True + finally: + retained[:] = [] + response.close() + + +@pytest.mark.parametrize('with_callback', [False, True]) +def test_pty_disconnect_closes_stream_without_killing_and_allows_reconnect( + with_callback): + class InterruptibleResponse(StreamingPtyResponse): + @property + def raw(self): + return self + + def shutdown(self): + self.chunks.put(None) + + response = InterruptibleResponse() + reconnected_response = StreamingPtyResponse() + session = StreamingPtySession([response, reconnected_response]) + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + available = Event() + callback = (lambda data: available.set()) if with_callback else None + handle = sandbox.pty.create(on_data=callback) + try: + if with_callback: + response.send(b'before disconnect') + assert available.wait(1) + handle.disconnect() + handle.disconnect() + assert response.closed_event.wait(1) + assert handle.wait().pid == 12 + assert handle.wait().exit_code == -1 + assert len(session.posts) == 1 + + received = [] + connected = sandbox.pty.connect(handle.pid, on_data=received.append) + reconnected_response.send(b'after reconnect') + reconnected_response.finish() + assert connected.wait().stdout == 'after reconnect' + assert received == [b'after reconnect'] + assert session.posts[1]['url'].endswith('/process.Process/Connect') + assert session.posts[1]['data']['process'] == {'pid': 12} + finally: + response.close() + reconnected_response.close() + + +@pytest.mark.parametrize('with_callback', [False, True]) +def test_pty_disconnect_with_throw_on_error_preserves_unknown_exit_status( + with_callback): + response = StreamingPtyResponse() + session = StreamingPtySession([response]) + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + received = Event() + callback = (lambda data: received.set()) if with_callback else None + handle = sandbox.pty.connect(12, on_data=callback, throw_on_error=True) + try: + if with_callback: + response.send(b'still running') + assert received.wait(1) + handle.disconnect() + for unused in range(2): + result = handle.wait() + assert result.pid == 12 + assert result.exit_code == -1 + assert len(session.posts) == 1 + finally: + response.close() + + +@pytest.mark.parametrize('with_callback', [False, True]) +def test_pty_throw_on_error_keeps_real_nonzero_exit_failure(with_callback): + response = StreamingPtyResponse() + session = StreamingPtySession([response]) + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + callback = (lambda data: None) if with_callback else None + handle = sandbox.pty.connect(12, on_data=callback, throw_on_error=True) + try: + response.finish(exit_code=7) + for unused in range(2): + with pytest.raises(CommandExitError) as caught: + handle.wait() + assert caught.value.result.exit_code == 7 + assert caught.value.result.exitCode == 7 + handle.disconnect() + finally: + response.close() + + +@pytest.mark.parametrize('callback_error,with_callback', [ + (False, False), (False, True), (True, True), +]) +def test_pty_wait_repeats_stream_errors_and_closes_response( + callback_error, with_callback): + response = StreamingPtyResponse() + session = StreamingPtySession([response]) + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + error = ValueError('callback failed') if callback_error else ( + requests.ConnectionError('stream failed')) + + def on_data(data): + if callback_error: + raise error + + handle = sandbox.pty.create(on_data=on_data if with_callback else None) + try: + if callback_error: + response.send(b'output') + else: + response.chunks.put(error) + for unused in range(2): + with pytest.raises(type(error)) as caught: + handle.wait() + assert caught.value is error + assert response.closed is True + finally: + response.close() + + +def test_dropped_background_command_returns_connection_pool_slot(monkeypatch): + first = {'event': {'start': {'pid': 12}}, 'padding': ''} + first['padding'] = 'x' * (8192 - len(encode_connect_envelope(first))) + body = encode_connect_envelope(first) + encode_connect_envelope({'padding': 'x' * 8192}) + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + self.rfile.read(int(self.headers['Content-Length'])) + self.send_response(200) + self.send_header('Content-Type', 'application/connect+json') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + server = HTTPServer(('127.0.0.1', 0), Handler) + serving = Thread(target=server.serve_forever) + serving.daemon = True + serving.start() + endpoint = 'http://127.0.0.1:{0}'.format(server.server_port) + session = requests.Session() + session.trust_env = False + adapter = requests.adapters.HTTPAdapter(pool_maxsize=1, pool_block=True) + session.mount(endpoint, adapter) + pools = [] + post = session.post + + def observe_post(*args, **kwargs): + response = post(*args, **kwargs) + pools.append(response.raw._pool) + return response + + monkeypatch.setattr(session, 'post', observe_post) + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + sandbox_id='sbx123', envd_url=endpoint) + try: + handle = sandbox.commands.start('echo example') + assert handle.pid == 12 + pool = pools[0] + assert pool.pool.qsize() == 0 + del handle + gc.collect() + assert pool.pool.qsize() == 1 + finally: + session.close() + server.shutdown() + server.server_close() + serving.join(1) + + +@pytest.mark.parametrize('connection_close', [False, True]) +def test_pty_disconnect_releases_http_connection_once(connection_close, monkeypatch): + release = Event() + connection_closed = Event() + disconnected = Event() + errors = [] + reading = Event() + returning_connection = Event() + release_connection = Event() + returned = [] + + class Handler(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def do_POST(self): + self.rfile.read(int(self.headers['Content-Length'])) + self.send_response(200) + self.send_header('Content-Type', 'application/connect+json') + self.send_header('Transfer-Encoding', 'chunked') + if connection_close: + self.send_header('Connection', 'close') + self.end_headers() + frame = encode_connect_envelope({'event': {'start': {'pid': 12}}}) + self.wfile.write(('{0:x}\r\n'.format(len(frame))).encode('ascii') + + frame + b'\r\n') + self.wfile.flush() + release.wait(5) + try: + self.wfile.write(b'0\r\n\r\n') + except IOError: + pass + finally: + self.close_connection = True + + def log_message(self, *args): + pass + + server = HTTPServer(('127.0.0.1', 0), Handler) + serving = Thread(target=server.serve_forever) + serving.daemon = True + serving.start() + sandbox = Sandbox(client=SandboxClient(api_key='api-key', timeout=5), + sandbox_id='sbx123', + envd_url='http://127.0.0.1:{0}'.format(server.server_port)) + post = sandbox.client.session.post + + def observe_response(*args, **kwargs): + response = post(*args, **kwargs) + pool = response.raw._pool + connection = response.raw._connection + put_conn = pool._put_conn + iter_content = response.iter_content + + def pause_first_release(conn): + if conn is connection: + returned.append(conn) + if len(returned) == 1: + returning_connection.set() + release_connection.wait(5) + return put_conn(conn) + + def observe_read(*args, **kwargs): + for chunk in iter_content(*args, **kwargs): + yield chunk + reading.set() + + monkeypatch.setattr(pool, '_put_conn', pause_first_release) + monkeypatch.setattr(response, 'iter_content', observe_read) + return response + + monkeypatch.setattr(sandbox.client.session, 'post', observe_response) + handle = None + try: + handle = sandbox.pty.create(on_data=lambda data: None) + assert reading.wait(1), 'reader did not start the next HTTP read' + + def disconnect(): + try: + handle.disconnect() + connection_closed.set() + handle.wait() + except BaseException as error: + errors.append(error) + finally: + disconnected.set() + + closing = Thread(target=disconnect) + closing.daemon = True + closing.start() + if not disconnected.wait(1): + frames = sys._current_frames() + stacks = [] + for thread in (closing, handle._reader): + frame = frames.get(thread.ident) + if frame is not None: + stacks.append('{0}:\n{1}'.format( + thread.name, ''.join(traceback.format_stack(frame)))) + pytest.fail('disconnect blocked on an idle HTTP stream; ' + 'connection close returned: {0}\n{1}'.format( + connection_closed.is_set(), '\n'.join(stacks))) + if errors: + raise errors[0] + assert handle.wait().exit_code == -1 + handle._closer.join(1) + assert not handle._closer.is_alive(), 'disconnect raced the reader to release the response' + # Some platforms keep the active read blocked after socket shutdown. + # End the stream on the server so pool cleanup can be checked independently. + release.set() + assert returning_connection.wait(2), 'the reader did not release the connection' + assert len(returned) == 1, 'the same connection was released twice' + release_connection.set() + handle._reader.join(2) + assert not handle._reader.is_alive() + assert len(returned) == 1, 'the same connection was released twice' + finally: + release_connection.set() + release.set() + if handle is not None: + handle.disconnect() + sandbox.client.session.close() + server.shutdown() + server.server_close() + serving.join(1) + + +@pytest.mark.parametrize('background', [False, True]) +def test_git_clone_uses_temporary_credentials_and_cleans_them(background): + sandbox, session = sandbox_with_envd_session() + result = sandbox.git.clone( + 'https://github.com/qiniu/private.git', '/workspace/private', + username='demo-user', password='private-token', + envs={'CUSTOM': 'preserved'}, background=background) + starts = [post['data']['process'] for post in session.posts + if post['url'].endswith('/process.Process/Start')] + clone = next(process for process in starts + if (process.get('envs') or {}).get('GIT_ASKPASS')) + assert clone['args'][-1] == ( + 'git -c credential.helper= -c credential.username=demo-user ' + 'clone https://github.com/qiniu/private.git /workspace/private') + assert clone['envs']['GIT_USERNAME'] == 'demo-user' + assert clone['envs']['GIT_PASSWORD'] == 'private-token' + assert clone['envs']['CUSTOM'] == 'preserved' + askpass_path = clone['envs']['GIT_ASKPASS'] + assert clone['envs']['GIT_TERMINAL_PROMPT'] == '0' + assert all('config --global' not in process['args'][-1] + for process in starts) + if background: + assert not any(post['url'].endswith('/filesystem.Filesystem/Remove') + for post in session.posts) + result = result.wait() + assert result.exit_code == 0 + removed = [post['data']['path'] for post in session.posts + if post['url'].endswith('/filesystem.Filesystem/Remove')] + assert removed == [askpass_path, posixpath.dirname(askpass_path)] + + +@pytest.mark.skipif(os.name == 'nt', reason='local askpass fixture requires POSIX shell') +def test_git_temporary_credentials_override_config_without_writing_to_store(): + try: + subprocess.check_output(['git', '--version']) + except OSError: + pytest.skip('git executable is required') + root = tempfile.mkdtemp(prefix='qiniu-git-credentials-') + git_home = os.path.join(root, 'home') + os.mkdir(git_home) + env = dict(os.environ) + for key in list(env): + if key.startswith('GIT_') or key == 'SSH_ASKPASS': + env.pop(key) + env.update({'HOME': git_home, 'XDG_CONFIG_HOME': git_home, + 'GIT_CONFIG_NOSYSTEM': '1', + 'GIT_CONFIG_GLOBAL': os.path.join(git_home, '.gitconfig'), + 'GIT_TERMINAL_PROMPT': '0', + 'NO_PROXY': '127.0.0.1,localhost', + 'no_proxy': '127.0.0.1,localhost'}) + remote = os.path.join(root, 'remote.git') + store = os.path.join(git_home, 'credentials') + helper = 'store --file=' + store + subprocess.check_call(['git', 'init', '--bare', '-q', remote], env=env) + subprocess.check_call(['git', '--git-dir=' + remote, 'update-server-info'], env=env) + subprocess.check_call(['git', 'config', '--global', 'credential.helper', helper], env=env) + + class Handler(SimpleHTTPRequestHandler): + def do_GET(self): + expected = 'Basic ' + base64.b64encode(b'demo:fake-token').decode('ascii') + if self.headers.get('Authorization') != expected: + self.send_response(401) + self.send_header('WWW-Authenticate', 'Basic realm="local"') + self.send_header('Content-Length', '0') + self.end_headers() + return + SimpleHTTPRequestHandler.do_GET(self) + + def translate_path(self, path): + return os.path.join(root, path.split('?', 1)[0].lstrip('/')) + + def log_message(self, *args): + pass + + def local_path(value): + return value.replace('/tmp/qiniu-git-auth', os.path.join(root, 'auth')) + + class Files(object): + def write(self, path, data, **opts): + with open(local_path(path), 'w') as output: + output.write(data) + + def remove(self, path, **opts): + path = local_path(path) + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) + + class Commands(object): + def __init__(self): + self.sandbox = type('SandboxContext', (object,), {'files': Files()})() + + def run(self, command, **opts): + process_env = dict(env) + process_env.update(opts.get('envs') or {}) + if 'GIT_ASKPASS' in process_env: + process_env['GIT_ASKPASS'] = local_path(process_env['GIT_ASKPASS']) + process = subprocess.Popen( + shlex.split(local_path(command)), stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=process_env, cwd=opts.get('cwd')) + expired = Event() + + def expire(): + expired.set() + try: + process.kill() + except OSError: + pass + + timer = Timer(10, expire) + timer.daemon = True + timer.start() + try: + stdout, stderr = process.communicate() + finally: + timer.cancel() + assert not expired.is_set(), 'local Git command timed out' + return CommandResult(process.pid, process.returncode, + stdout.decode('utf-8', 'replace'), + stderr.decode('utf-8', 'replace')) + + server = HTTPServer(('127.0.0.1', 0), Handler) + serving = Thread(target=server.serve_forever) + serving.daemon = True + serving.start() + try: + git = Git(Commands()) + url = 'http://127.0.0.1:{0}/remote.git'.format(server.server_port) + username_key = 'credential.{0}.username'.format(url) + subprocess.check_call( + ['git', 'config', '--global', 'credential.username', 'default-user'], env=env) + subprocess.check_call( + ['git', 'config', '--global', username_key, 'configured-user'], env=env) + explicit_path = os.path.join(root, 'explicit') + old_url = url.replace('http://', 'http://old-user:old-password@', 1) + result = git.clone(old_url, explicit_path, username='demo', password='fake-token') + assert result.exit_code == 0, result.stderr + assert not os.path.exists(store), 'temporary credentials reached the store helper' + assert not any(name.startswith('auth') for name in os.listdir(root)) + configured = subprocess.check_output( + ['git', 'config', '--global', 'credential.helper'], env=env) + assert configured.decode('utf-8').strip() == helper + configured = subprocess.check_output( + ['git', 'config', '--global', username_key], env=env) + assert configured.decode('utf-8').strip() == 'configured-user' + remote_url = subprocess.check_output( + ['git', '-C', explicit_path, 'remote', 'get-url', 'origin'], env=env) + assert remote_url.decode('utf-8').strip() == url + + subprocess.check_call( + ['git', 'config', '--global', '--unset', username_key], env=env) + subprocess.check_call( + ['git', 'config', '--global', '--unset', 'credential.username'], env=env) + with open(store, 'w') as output: + output.write('http://demo:fake-token@127.0.0.1:{0}\n'.format(server.server_port)) + result = git.clone(url, os.path.join(root, 'configured-helper')) + assert result.exit_code == 0, result.stderr + finally: + server.shutdown() + server.server_close() + serving.join(1) + shutil.rmtree(root) + + +@pytest.mark.parametrize('operation', ['clone', 'pull', 'push']) +@pytest.mark.parametrize('background', [False, True]) +def test_git_credentials_keep_request_user_and_private_directory( + operation, background): + class CredentialSession(EnvdSession): + def post(self, url, **opts): + response = super(CredentialSession, self).post(url, **opts) + process = self.posts[-1]['data'].get('process') or {} + args = process.get('args') or [''] + if args[-1] == 'git remote get-url origin': + return DummyResponse(raw=b''.join([ + encode_connect_envelope({'start': {'pid': 12}}), + encode_connect_envelope({'data': {'stdout': base64.b64encode( + b'https://example.test/private.git\n').decode('ascii')}}), + encode_connect_envelope({'end': {'exitCode': 0}}), + ]), content_type='application/connect+json') + return response + + def request_user(headers): + value = headers['Authorization'].split(' ', 1)[1] + return base64.b64decode(value).decode('ascii').split(':', 1)[0] + + session = CredentialSession() + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + directories = [] + for user in ('root', 'user'): + first_post = len(session.posts) + opts = {'user': user, 'username': 'demo', 'password': 'fake-token', + 'background': background} + if operation == 'clone': + result = sandbox.git.clone('https://example.test/private.git', **opts) + else: + result = getattr(sandbox.git, operation)( + '/workspace/repo', remote='origin', **opts) + upload = session.requests[-1] + assert request_user(upload['kwargs']['headers']) == user + assert parse_qs(urlparse(upload['url']).query)['username'] == [user] + starts = [post for post in session.posts[first_post:] + if post['url'].endswith('/process.Process/Start')] + assert all(request_user(post['headers']) == user for post in starts) + git_process = next(post['data']['process'] for post in starts + if (post['data']['process'].get('envs') or {}).get('GIT_ASKPASS')) + askpass = git_process['envs']['GIT_ASKPASS'] + directory = posixpath.dirname(askpass) + assert directory not in directories + directories.append(directory) + assert any('700' in post['data']['process']['args'][-1] and + directory in post['data']['process']['args'][-1] + for post in starts) + if background: + assert not any(post['url'].endswith('/filesystem.Filesystem/Remove') + for post in session.posts[first_post:]) + result = result.wait() + assert result.exit_code == 0 + cleanup = [post for post in session.posts[first_post:] + if post['url'].endswith('/filesystem.Filesystem/Remove')] + assert directory in [post['data']['path'] for post in cleanup] + assert all(request_user(post['headers']) == user for post in cleanup) + + +def test_git_background_handle_discard_cleans_credentials_as_command_user(): + sandbox, session = sandbox_with_envd_session() + handle = sandbox.git.clone('https://example.test/private.git', + username='demo', password='fake-token', + user='root', background=True) + assert not any(post['url'].endswith('/filesystem.Filesystem/Remove') + for post in session.posts) + del handle + gc.collect() + + cleanup = [post for post in session.posts + if post['url'].endswith('/filesystem.Filesystem/Remove')] + assert len(cleanup) == 2 + assert cleanup[1]['data']['path'] == posixpath.dirname(cleanup[0]['data']['path']) + assert all(post['headers']['Authorization'] == 'Basic cm9vdDo=' + for post in cleanup) + + +@pytest.mark.parametrize('url_method,operation,http_method', [ + ('download_url', 'read', 'GET'), ('upload_url', 'write', 'POST'), +]) +def test_public_file_urls_pass_envd_signature_expiration_and_tamper_checks( + url_method, operation, http_method): + token = 'envd-test-token' + + class Handler(BaseHTTPRequestHandler): + def authorize(self, expected_operation): + query = parse_qs(urlparse(self.path).query) + components = [query.get('path', [''])[0], expected_operation, + query.get('username', [''])[0], token] + expiration = query.get('signature_expiration', [None])[0] + if expiration is not None: + components.append(expiration) + payload = ':'.join(components).encode('utf-8') + # Envd auth.go and keys/sha256.go use standard base64 without padding. + expected = 'v1_' + base64.b64encode( + hashlib.sha256(payload).digest()).decode('ascii').rstrip('=') + valid = query.get('signature', [''])[0] == expected + if expiration is not None: + valid = valid and int(expiration) >= int(time.time()) + self.send_response(200 if valid else 401) + self.send_header('Content-Length', '0') + self.end_headers() + + def do_GET(self): + self.authorize('read') + + def do_POST(self): + self.rfile.read(int(self.headers.get('Content-Length', '0'))) + self.authorize('write') + + def log_message(self, *args): + pass + + server = HTTPServer(('127.0.0.1', 0), Handler) + serving = Thread(target=server.serve_forever) + serving.daemon = True + serving.start() + sandbox = Sandbox(sandbox_id='sbx123', envdAccessToken=token, + envd_url='http://127.0.0.1:{0}'.format(server.server_port)) + client = requests.Session() + client.trust_env = False + + def status(url, method=http_method): + response = client.request(method, url, data=b'', timeout=2) + return response.status_code + + try: + url = getattr(sandbox, url_method)( + '/tmp/hello world.txt', user='root', + signature_expiration=int(time.time()) + 300) + assert status(url) == 200 + assert status(url, method='POST' if operation == 'read' else 'GET') == 401 + + query = dict((key, values[0]) for key, values in + parse_qs(urlparse(url).query).items()) + for key, value in [('signature', 'v1_tampered'), + ('path', '/tmp/different.txt'), + ('username', 'different-user'), + ('signature_expiration', str(int(time.time()) + 600))]: + changed = dict(query) + changed[key] = value + tampered = urlparse(url)._replace(query=urlencode(changed)).geturl() + assert status(tampered) == 401 + + expired = getattr(sandbox, url_method)( + '/tmp/hello world.txt', user='root', + signature_expiration=int(time.time()) - 1) + assert status(expired) == 401 + no_expiration = getattr(sandbox, url_method)( + '/tmp/hello world.txt', user='root', signature_expiration=None) + assert status(no_expiration) == 200 + finally: + client.close() + server.shutdown() + server.server_close() + serving.join(1) + + +def test_git_clone_auth_failure_removes_temporary_credentials(): + class AuthFailureSession(EnvdSession): + def post(self, url, **opts): + response = super(AuthFailureSession, self).post(url, **opts) + process = self.posts[-1]['data'].get('process') or {} + if url.endswith('/process.Process/Start') and ( + process.get('envs') or {}).get('GIT_ASKPASS'): + return DummyResponse( + raw=b''.join([ + encode_connect_envelope({'start': {'pid': 12}}), + encode_connect_envelope({'data': { + 'stderr': base64.b64encode( + b'fatal: Authentication failed').decode('ascii'), + }}), + encode_connect_envelope({'end': {'exitCode': 128}}), + ]), content_type='application/connect+json') + return response + + session = AuthFailureSession() + sandbox = Sandbox(client=SandboxClient(api_key='api-key', session=session), + info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + + with pytest.raises(GitAuthException): + sandbox.git.clone('https://github.com/qiniu/private.git', + username='demo-user', password='private-token', user='root') + + removed = [post['data']['path'] for post in session.posts + if post['url'].endswith('/filesystem.Filesystem/Remove')] + assert len(removed) == 2 + assert removed[1] == posixpath.dirname(removed[0]) + cleanup = [post for post in session.posts + if post['url'].endswith('/filesystem.Filesystem/Remove')] + assert all(post['headers']['Authorization'] == 'Basic cm9vdDo=' + for post in cleanup) + + +@pytest.mark.parametrize('url,credentials', [ + ('https://github.com/qiniu/private.git', {'username': 'demo-user'}), + ('https://github.com/qiniu/private.git', {'password': 'private-token'}), + ('git@github.com:qiniu/private.git', + {'username': 'demo-user', 'password': 'private-token'}), +]) +def test_git_clone_rejects_incomplete_or_unsupported_credentials( + url, credentials): + sandbox, session = sandbox_with_envd_session() + with pytest.raises(InvalidArgumentException): + sandbox.git.clone(url, **credentials) + assert session.posts == [] + + def test_filesystem_uses_envd_rpc_and_signed_file_urls(): sandbox, session = sandbox_with_envd_session() diff --git a/tests/cases/test_services/test_sandbox/test_example_config.py b/tests/cases/test_services/test_sandbox/test_example_config.py index 2d3c0f13..d917165b 100644 --- a/tests/cases/test_services/test_sandbox/test_example_config.py +++ b/tests/cases/test_services/test_sandbox/test_example_config.py @@ -147,7 +147,7 @@ def test_examples_cover_primary_sandbox_surfaces(): 'update_injections', 'update_github_token', "os.getenv('QINIU_SANDBOX_GITHUB_TOKEN')", - 'list_sandboxes_v2(', + 'list_sandboxes(', 'template=[sandbox.template_id]', ], 'sandbox_templates.py': [ diff --git a/tests/cases/test_services/test_sandbox/test_integration.py b/tests/cases/test_services/test_sandbox/test_integration.py index 2bc6db08..b75e12ed 100644 --- a/tests/cases/test_services/test_sandbox/test_integration.py +++ b/tests/cases/test_services/test_sandbox/test_integration.py @@ -264,26 +264,45 @@ def test_filesystem_watch_dir_when_envd_supports_it(): sandbox.kill() -def test_pty_when_envd_supports_it(): +def test_commands_connect_stdin_close_and_kill(): sandbox = None try: - sandbox = create_integration_sandbox('pty') - try: - handle = sandbox.pty.create(PtySize(rows=24, cols=80), timeout=30) - except SandboxError as err: - if is_unsupported_runtime_error(err): - pytest.skip('envd does not support pty') - raise + sandbox = create_integration_sandbox('process_selectors') + handle = sandbox.commands.start('cat', stdin=True, tag='stdin-test', + request_timeout=30) + connected = sandbox.commands.connect(handle.pid, timeout=30) + tagged = sandbox.commands.connect(None, tag='stdin-test', timeout=30) + sandbox.commands.send_stdin(handle.pid, 'qiniu-stdin-ready\n') + sandbox.commands.close_stdin(handle.pid) + for stream in (handle, connected, tagged): + result = stream.wait() + assert result.exit_code == 0 + assert 'qiniu-stdin-ready' in result.stdout + + handle = sandbox.commands.start('exec sleep 60', request_timeout=30) + assert sandbox.commands.kill(handle.pid) is True + assert handle.wait().exit_code != 0 + finally: + if sandbox is not None: + sandbox.kill() - try: - sandbox.pty.send_stdin(handle.pid, 'echo qiniu-pty-ready\n') - sandbox.pty.send_stdin(handle.pid, 'exit\n') - except SandboxError as err: - if is_unsupported_runtime_error(err): - pytest.skip('envd can create pty but does not support input') - raise + +def test_pty_input_resize_reconnect_and_kill(): + sandbox = None + try: + sandbox = create_integration_sandbox('pty') + handle = sandbox.pty.create(PtySize(rows=24, cols=80), timeout=30) + handle.disconnect() + handle = sandbox.pty.connect(handle.pid, timeout=30) + sandbox.pty.resize(handle.pid, PtySize(rows=30, cols=100)) + sandbox.pty.send_stdin(handle.pid, 'echo qiniu-pty-ready\n') + sandbox.pty.send_stdin(handle.pid, 'exit\n') result = handle.wait() assert 'qiniu-pty-ready' in result.stdout + + handle = sandbox.pty.create(PtySize(rows=24, cols=80), timeout=30) + assert sandbox.pty.kill(handle.pid) is True + assert handle.wait().exit_code != 0 finally: if sandbox is not None: sandbox.kill()