|
| 1 | +# ADR-020 · 열린 과제 13-21 — 코드는 tree-sitter AST 로 함수·클래스 단위로 자른다 |
| 2 | +from langchain_core.documents import Document |
| 3 | + |
| 4 | +from app.parsing.splitter import split_documents |
| 5 | + |
| 6 | + |
| 7 | +def code_doc(text, language, extension): |
| 8 | + return Document(page_content=text, metadata={"language": language, "extension": extension, "project_id": "p1", "source": f"f{extension}"}) |
| 9 | + |
| 10 | + |
| 11 | +def assert_covers_and_lines_match(src, chunks): |
| 12 | + lines = src.split("\n") |
| 13 | + for line in lines: |
| 14 | + if line.strip(): |
| 15 | + assert any(line.rstrip() in c.page_content for c in chunks), f"사라진 줄: {line!r}" |
| 16 | + for c in chunks: |
| 17 | + start, end = c.metadata["start_line"], c.metadata["end_line"] |
| 18 | + first, last = c.page_content.split("\n")[0], c.page_content.split("\n")[-1] |
| 19 | + assert lines[start - 1].strip() == first.strip(), (start, first) |
| 20 | + assert lines[end - 1].strip() == last.strip(), (end, last) |
| 21 | + |
| 22 | + |
| 23 | +def whole_in_one_chunk(chunks, text): |
| 24 | + owners = [c for c in chunks if text in c.page_content] |
| 25 | + assert len(owners) == 1, f"{text[:30]!r} 가 {len(owners)}개 청크에 있다" |
| 26 | + return owners[0] |
| 27 | + |
| 28 | + |
| 29 | +BIG_BODY = "\n".join(f" value_{i} = compute_something(value_{i - 1}, factor={i})" for i in range(1, 12)) |
| 30 | +METHOD_BODY = BIG_BODY.replace("\n ", "\n ").replace(" value_1 ", " value_1 ", 1) |
| 31 | +PY_SRC = f'''import os |
| 32 | +from typing import Any |
| 33 | +
|
| 34 | +DEFAULT_TIMEOUT = 5.0 |
| 35 | +
|
| 36 | +
|
| 37 | +def tiny(x): |
| 38 | + return x |
| 39 | +
|
| 40 | +
|
| 41 | +def big_function(arg: int) -> int: |
| 42 | + """긴 함수 — 문자 기준이면 한가운데가 잘렸다""" |
| 43 | + value_0 = arg |
| 44 | +{BIG_BODY} |
| 45 | + return value_11 |
| 46 | +
|
| 47 | +
|
| 48 | +class Client: |
| 49 | + """클래스는 chunk_size 를 넘으므로 메서드 단위로 내려간다""" |
| 50 | +
|
| 51 | + def __init__(self, base_url: str): |
| 52 | + self.base_url = base_url |
| 53 | +
|
| 54 | + def request(self, method: str, path: str) -> dict: |
| 55 | + url = self.base_url + path |
| 56 | +{METHOD_BODY} |
| 57 | + return {{"method": method, "url": url}} |
| 58 | +
|
| 59 | + def close(self): |
| 60 | + pass |
| 61 | +''' |
| 62 | + |
| 63 | + |
| 64 | +def test_python_functions_are_not_cut_in_the_middle(): |
| 65 | + chunks = split_documents([code_doc(PY_SRC, "python", ".py")], chunk_size=900) |
| 66 | + |
| 67 | + assert_covers_and_lines_match(PY_SRC, chunks) |
| 68 | + for c in chunks: |
| 69 | + assert len(c.page_content) <= 900 |
| 70 | + |
| 71 | + big = whole_in_one_chunk(chunks, "def big_function(arg: int) -> int:\n" + ' """긴 함수') |
| 72 | + assert big.metadata["symbol_name"] == "big_function" |
| 73 | + assert big.metadata["symbol_kind"] == "function_definition" |
| 74 | + assert big.metadata["start_line"] == PY_SRC.split("\n").index("def big_function(arg: int) -> int:") + 1 |
| 75 | + assert "return value_11" in big.page_content |
| 76 | + |
| 77 | + request = whole_in_one_chunk(chunks, "def request(self, method: str, path: str) -> dict:") |
| 78 | + assert request.metadata["symbol_name"] == "request" |
| 79 | + assert 'return {"method": method, "url": url}' in request.page_content |
| 80 | + |
| 81 | + # 작은 것(import·상수·한 줄 함수)은 홀로 서지 않고 이웃과 합쳐져 50자 하한에 안 걸린다 |
| 82 | + consts = whole_in_one_chunk(chunks, "DEFAULT_TIMEOUT = 5.0") |
| 83 | + assert "import os" in consts.page_content |
| 84 | + assert consts.metadata["symbol_kind"] != "module", "파이썬 루트 노드(module)를 정의로 보면 안 된다" |
| 85 | + assert all(len(c.page_content) >= 50 for c in chunks) |
| 86 | + |
| 87 | + |
| 88 | +def test_oversized_function_splits_on_lines_and_keeps_symbol(): |
| 89 | + body = "\n".join(f" step_{i} = run_step_number({i}, previous=step_{i - 1})" for i in range(1, 60)) |
| 90 | + src = f"def huge(step_0):\n{body}\n return step_59\n" |
| 91 | + chunks = split_documents([code_doc(src, "python", ".py")], chunk_size=500) |
| 92 | + |
| 93 | + assert len(chunks) > 1 |
| 94 | + assert_covers_and_lines_match(src, chunks) |
| 95 | + assert {c.metadata["symbol_name"] for c in chunks} == {"huge"} |
| 96 | + src_lines = src.split("\n") |
| 97 | + for c in chunks: |
| 98 | + for line in c.page_content.split("\n"): |
| 99 | + assert line in src_lines, f"줄 중간에서 잘렸다: {line!r}" |
| 100 | + |
| 101 | + |
| 102 | +TS_SRC = '''import { useEffect, useState } from "react"; |
| 103 | +
|
| 104 | +export const CONFIG = { retries: 3, timeoutMs: 5000 }; |
| 105 | +
|
| 106 | +export const useThing = (id: string) => { |
| 107 | + const [state, setState] = useState<string | null>(null); |
| 108 | + useEffect(() => { |
| 109 | + let cancelled = false; |
| 110 | + fetchThing(id).then((value) => { |
| 111 | + if (!cancelled) setState(value); |
| 112 | + }); |
| 113 | + return () => { |
| 114 | + cancelled = true; |
| 115 | + }; |
| 116 | + }, [id]); |
| 117 | + return state; |
| 118 | +}; |
| 119 | +
|
| 120 | +export default function Page({ id }: { id: string }) { |
| 121 | + const thing = useThing(id); |
| 122 | + return <section className="page"><h1>{thing ?? "loading"}</h1></section>; |
| 123 | +} |
| 124 | +''' |
| 125 | + |
| 126 | + |
| 127 | +def test_tsx_arrow_components_get_symbol_names(): |
| 128 | + chunks = split_documents([code_doc(TS_SRC, "typescript", ".tsx")], chunk_size=400) |
| 129 | + |
| 130 | + assert_covers_and_lines_match(TS_SRC, chunks) |
| 131 | + hook = whole_in_one_chunk(chunks, "export const useThing = (id: string) => {") |
| 132 | + assert hook.metadata["symbol_name"] == "useThing" |
| 133 | + page = whole_in_one_chunk(chunks, "export default function Page") |
| 134 | + assert page.metadata["symbol_name"] == "Page" |
| 135 | + # 상수는 정의로 세우지 않는다 — 홀로 서면 50자 하한에 걸려 사라진다 |
| 136 | + assert whole_in_one_chunk(chunks, "export const CONFIG").metadata["symbol_name"] != "CONFIG" |
| 137 | + |
| 138 | + |
| 139 | +def test_unknown_language_falls_back_to_character_splitter(): |
| 140 | + src = "\n".join(f"key_{i}: value_{i}" for i in range(80)) |
| 141 | + chunks = split_documents([code_doc(src, "yaml", ".yml")], chunk_size=200, chunk_overlap=0) |
| 142 | + assert len(chunks) > 1 |
| 143 | + assert "symbol_name" not in chunks[0].metadata |
| 144 | + assert chunks[0].metadata["start_line"] == 1 |
0 commit comments