Skip to content

Commit dfaf8f7

Browse files
authored
Merge pull request #21 from Qode-4/develop
release: 2차 배포 — AST 청킹·펜스 수정·top_k 10·리랭커 top_n
2 parents 5048716 + a8918e0 commit dfaf8f7

10 files changed

Lines changed: 496 additions & 57 deletions

File tree

‎app/api/main.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ async def lifespan(app: FastAPI):
2121
class SearchRequest(BaseModel):
2222
query: str
2323
project_id: str
24-
top_k: int = 5
24+
top_k: int = 10 # 열린 과제 13-19. Node rag.service.ts RAG_TOP_K 와 같이 움직인다
2525

2626

2727
class IndexRequest(BaseModel):

‎app/parsing/loader.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ def collect_documents(repo_path: str, project_id: str, max_files: int | None = N
3636

3737
# 파일 읽기 (바이너리 파일이나 읽기 실패 시 건너뜀)
3838
try:
39-
with open(file_path, "r", encoding="utf-8") as f:
39+
with open(file_path, "r", encoding="utf-8-sig") as f: # BOM 이 있으면 벗긴다 — 첫 줄이 \ufeff 로 시작하면 청크·줄 번호가 어긋난다
4040
content = f.read()
4141
except (UnicodeDecodeError, OSError):
4242
logger.warning("파일 읽기 실패, 건너뜀: %s", relative_path)

‎app/parsing/splitter.py‎

Lines changed: 247 additions & 48 deletions
Large diffs are not rendered by default.

‎app/retrieval/pipeline.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ async def search_pipeline(query: str, project_id: str, top_k: int) -> dict:
1313
raw_results = hybrid_search(query, project_id, top_k)
1414
filtered = filter_by_threshold(raw_results)
1515
deduped = dedup(filtered)
16-
reranked = rerank(query, deduped)
16+
reranked = rerank(query, deduped, top_k)
1717

1818
return {
1919
"chunks": reranked,

‎app/retrieval/reranker.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@ def rerank(query: str, docs: list[Document], top_k: int | None = None) -> list[D
3131

3232
try:
3333
reranker = _get_reranker()
34-
34+
# CohereRerank 의 top_n 기본값은 3 이다 — 그대로 두면 top_k 를 몇으로 올려도 LLM 에는 3개만 간다
35+
# (2026-09-16 실측: top_k 10 → final 3). 전부 돌려받고 아래에서 top_k 로 자른다.
36+
reranker.top_n = len(docs)
3537
reranked_docs = reranker.compress_documents(documents=docs, query=query)
3638

3739
for d in reranked_docs:

‎requirements.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,6 @@ psycopg[binary]>=3.0.0
1212
psycopg2-binary
1313
fastapi>=0.110.0
1414
uvicorn>=0.29.0
15-
rank-bm25>=0.2.2
15+
rank-bm25>=0.2.2
16+
tree-sitter-language-pack>=1.0.0
17+

‎tests/parsing/test_splitter_ast.py‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
# ADR-020 · 열린 과제 13-21 — 코드는 tree-sitter AST 로 함수·클래스 단위로 자른다
2+
from langchain_core.documents import Document
3+
4+
from app.parsing.splitter import split_documents
5+
6+
7+
def code_doc(text, language, extension):
8+
return Document(page_content=text, metadata={"language": language, "extension": extension, "project_id": "p1", "source": f"f{extension}"})
9+
10+
11+
def assert_covers_and_lines_match(src, chunks):
12+
lines = src.split("\n")
13+
for line in lines:
14+
if line.strip():
15+
assert any(line.rstrip() in c.page_content for c in chunks), f"사라진 줄: {line!r}"
16+
for c in chunks:
17+
start, end = c.metadata["start_line"], c.metadata["end_line"]
18+
first, last = c.page_content.split("\n")[0], c.page_content.split("\n")[-1]
19+
assert lines[start - 1].strip() == first.strip(), (start, first)
20+
assert lines[end - 1].strip() == last.strip(), (end, last)
21+
22+
23+
def whole_in_one_chunk(chunks, text):
24+
owners = [c for c in chunks if text in c.page_content]
25+
assert len(owners) == 1, f"{text[:30]!r} 가 {len(owners)}개 청크에 있다"
26+
return owners[0]
27+
28+
29+
BIG_BODY = "\n".join(f" value_{i} = compute_something(value_{i - 1}, factor={i})" for i in range(1, 12))
30+
METHOD_BODY = BIG_BODY.replace("\n ", "\n ").replace(" value_1 ", " value_1 ", 1)
31+
PY_SRC = f'''import os
32+
from typing import Any
33+
34+
DEFAULT_TIMEOUT = 5.0
35+
36+
37+
def tiny(x):
38+
return x
39+
40+
41+
def big_function(arg: int) -> int:
42+
"""긴 함수 — 문자 기준이면 한가운데가 잘렸다"""
43+
value_0 = arg
44+
{BIG_BODY}
45+
return value_11
46+
47+
48+
class Client:
49+
"""클래스는 chunk_size 를 넘으므로 메서드 단위로 내려간다"""
50+
51+
def __init__(self, base_url: str):
52+
self.base_url = base_url
53+
54+
def request(self, method: str, path: str) -> dict:
55+
url = self.base_url + path
56+
{METHOD_BODY}
57+
return {{"method": method, "url": url}}
58+
59+
def close(self):
60+
pass
61+
'''
62+
63+
64+
def test_python_functions_are_not_cut_in_the_middle():
65+
chunks = split_documents([code_doc(PY_SRC, "python", ".py")], chunk_size=900)
66+
67+
assert_covers_and_lines_match(PY_SRC, chunks)
68+
for c in chunks:
69+
assert len(c.page_content) <= 900
70+
71+
big = whole_in_one_chunk(chunks, "def big_function(arg: int) -> int:\n" + ' """긴 함수')
72+
assert big.metadata["symbol_name"] == "big_function"
73+
assert big.metadata["symbol_kind"] == "function_definition"
74+
assert big.metadata["start_line"] == PY_SRC.split("\n").index("def big_function(arg: int) -> int:") + 1
75+
assert "return value_11" in big.page_content
76+
77+
request = whole_in_one_chunk(chunks, "def request(self, method: str, path: str) -> dict:")
78+
assert request.metadata["symbol_name"] == "request"
79+
assert 'return {"method": method, "url": url}' in request.page_content
80+
81+
# 작은 것(import·상수·한 줄 함수)은 홀로 서지 않고 이웃과 합쳐져 50자 하한에 안 걸린다
82+
consts = whole_in_one_chunk(chunks, "DEFAULT_TIMEOUT = 5.0")
83+
assert "import os" in consts.page_content
84+
assert consts.metadata["symbol_kind"] != "module", "파이썬 루트 노드(module)를 정의로 보면 안 된다"
85+
assert all(len(c.page_content) >= 50 for c in chunks)
86+
87+
88+
def test_oversized_function_splits_on_lines_and_keeps_symbol():
89+
body = "\n".join(f" step_{i} = run_step_number({i}, previous=step_{i - 1})" for i in range(1, 60))
90+
src = f"def huge(step_0):\n{body}\n return step_59\n"
91+
chunks = split_documents([code_doc(src, "python", ".py")], chunk_size=500)
92+
93+
assert len(chunks) > 1
94+
assert_covers_and_lines_match(src, chunks)
95+
assert {c.metadata["symbol_name"] for c in chunks} == {"huge"}
96+
src_lines = src.split("\n")
97+
for c in chunks:
98+
for line in c.page_content.split("\n"):
99+
assert line in src_lines, f"줄 중간에서 잘렸다: {line!r}"
100+
101+
102+
TS_SRC = '''import { useEffect, useState } from "react";
103+
104+
export const CONFIG = { retries: 3, timeoutMs: 5000 };
105+
106+
export const useThing = (id: string) => {
107+
const [state, setState] = useState<string | null>(null);
108+
useEffect(() => {
109+
let cancelled = false;
110+
fetchThing(id).then((value) => {
111+
if (!cancelled) setState(value);
112+
});
113+
return () => {
114+
cancelled = true;
115+
};
116+
}, [id]);
117+
return state;
118+
};
119+
120+
export default function Page({ id }: { id: string }) {
121+
const thing = useThing(id);
122+
return <section className="page"><h1>{thing ?? "loading"}</h1></section>;
123+
}
124+
'''
125+
126+
127+
def test_tsx_arrow_components_get_symbol_names():
128+
chunks = split_documents([code_doc(TS_SRC, "typescript", ".tsx")], chunk_size=400)
129+
130+
assert_covers_and_lines_match(TS_SRC, chunks)
131+
hook = whole_in_one_chunk(chunks, "export const useThing = (id: string) => {")
132+
assert hook.metadata["symbol_name"] == "useThing"
133+
page = whole_in_one_chunk(chunks, "export default function Page")
134+
assert page.metadata["symbol_name"] == "Page"
135+
# 상수는 정의로 세우지 않는다 — 홀로 서면 50자 하한에 걸려 사라진다
136+
assert whole_in_one_chunk(chunks, "export const CONFIG").metadata["symbol_name"] != "CONFIG"
137+
138+
139+
def test_unknown_language_falls_back_to_character_splitter():
140+
src = "\n".join(f"key_{i}: value_{i}" for i in range(80))
141+
chunks = split_documents([code_doc(src, "yaml", ".yml")], chunk_size=200, chunk_overlap=0)
142+
assert len(chunks) > 1
143+
assert "symbol_name" not in chunks[0].metadata
144+
assert chunks[0].metadata["start_line"] == 1

‎tests/parsing/test_splitter_lines.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
from app.parsing.splitter import split_documents
55

66

7-
def make_doc(text, language="python"):
7+
def make_doc(text, language="unknown"):
8+
# 문자 기준 분할기가 도는 언어로 고정한다 — 코드는 AST 경로(test_splitter_ast.py)로 간다
89
return Document(page_content=text, metadata={"language": language, "project_id": "p1"})
910

1011

@@ -48,9 +49,7 @@ def split_documents(self, docs):
4849
def test_unlocatable_chunk_has_no_line_range(monkeypatch):
4950
# 열린 과제 13-13 — 원문에서 못 찾으면 옛 코드는 start_line을 정하지 않았다.
5051
# 첫 청크면 NameError, 이후 청크면 이전 청크의 줄 번호를 그대로 인용한다.
51-
monkeypatch.setattr(
52-
splitter_module, "_build_language_splitter", lambda *_: {"python": _UnlocatableSplitter()}
53-
)
52+
monkeypatch.setattr(splitter_module, "_build_default_splitter", lambda *_: _UnlocatableSplitter())
5453
chunks = split_documents([make_doc("a = 1\nb = 2\nc = 3\n")], chunk_size=100, chunk_overlap=20)
5554

5655
# 못 찾은 청크는 줄 범위를 비운다 — 틀린 줄을 붙이지 않는다
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import re
2+
3+
from langchain_core.documents import Document
4+
5+
from app.parsing.splitter import split_documents
6+
7+
FENCE_LINE = re.compile(r"^```", re.MULTILINE)
8+
9+
10+
def md_doc(text):
11+
return Document(page_content=text, metadata={"language": "markdown", "project_id": "p1", "source": "README.md"})
12+
13+
14+
def test_code_fence_stays_in_one_chunk():
15+
# 열린 과제 13-1 — 기본 마크다운 분할 규칙은 닫는 ``` 앞에서 자른다.
16+
# 코드블록은 A 청크에, 닫는 ``` 는 B 청크 첫 줄에 놓여 B를 마크다운으로
17+
# 읽으면 그 아래가 전부 코드블록으로 먹힌다("백틱 뒤가 잘린다").
18+
# 또 '\n# ' 헤딩 규칙이 코드블록 안의 파이썬 주석에도 걸려 블록 한가운데를 자른다.
19+
sections = []
20+
for i in range(12):
21+
sections.append(
22+
f"## Step {i}\n\n설명 문장 {i}.\n\n"
23+
f"```python\n# 주석 {i}\nvalue_{i} = compute({i})\nprint(value_{i})\n```\n\n"
24+
f"펜스 뒤 문장 {i}은 살아야 한다.\n"
25+
)
26+
src = "\n".join(sections)
27+
28+
chunks = split_documents([md_doc(src)], chunk_size=300, chunk_overlap=0)
29+
30+
for c in chunks:
31+
text = c.page_content
32+
assert not text.startswith("```\n"), f"닫는 펜스가 청크 첫 줄에 떨어졌다: {text[:40]!r}"
33+
assert len(FENCE_LINE.findall(text)) % 2 == 0, f"펜스가 짝이 안 맞는다: {text[:60]!r}"
34+
assert not text.startswith("# 주석"), "코드블록 안 주석을 헤딩으로 보고 잘랐다"
35+
36+
# 텍스트 손실 없음 + 줄 번호 정확
37+
lines = src.split("\n")
38+
for line in lines:
39+
if line.strip():
40+
assert any(line in c.page_content for c in chunks), f"사라진 줄: {line!r}"
41+
for c in chunks:
42+
start = c.metadata["start_line"]
43+
assert start is not None
44+
assert lines[start - 1].startswith(c.page_content.split("\n")[0])
45+
46+
47+
def test_oversized_code_block_is_split_on_lines_only():
48+
# chunk_size를 넘는 코드블록은 어쩔 수 없이 잘리지만, 줄 중간(공백)에서 자르지는 않는다
49+
body = "\n".join(f"item_{i} = load_something_reasonably_long({i})" for i in range(40))
50+
src = f"# Big\n\n```python\n{body}\n```\n"
51+
chunks = split_documents([md_doc(src)], chunk_size=300, chunk_overlap=0)
52+
assert len(chunks) > 1
53+
for c in chunks:
54+
for line in c.page_content.split("\n"):
55+
assert line in src.split("\n"), f"줄 중간에서 잘렸다: {line!r}"
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# 열린 과제 13-19 후속 — 리랭커가 top_k 만큼 돌려줘야 top_k 상향이 LLM 에 닿는다
2+
from langchain_core.documents import Document
3+
4+
import app.retrieval.reranker as reranker_module
5+
from app.retrieval.reranker import rerank
6+
7+
8+
class _CohereLike:
9+
"""langchain_cohere.CohereRerank 흉내 — top_n(기본 3)개만 돌려준다."""
10+
11+
top_n = 3
12+
13+
def compress_documents(self, documents, query):
14+
ranked = sorted(documents, key=lambda d: d.metadata["score"]) # 벡터 점수 역순으로 뒤집어 재정렬을 흉내
15+
for d in ranked:
16+
d.metadata["relevance_score"] = 0.9
17+
return ranked[: self.top_n]
18+
19+
20+
def docs(n):
21+
return [Document(page_content=f"chunk {i}", metadata={"score": i / 10}) for i in range(n)]
22+
23+
24+
def test_rerank_returns_top_k_not_cohere_default_three(monkeypatch):
25+
monkeypatch.setattr(reranker_module, "RERANKER_ENABLED", True)
26+
monkeypatch.setattr(reranker_module, "_get_reranker", lambda: _CohereLike())
27+
28+
result = rerank("q", docs(9), top_k=10)
29+
30+
assert len(result) == 9, "Cohere 기본 top_n=3 에 잘리면 top_k 상향이 무의미하다"
31+
assert result[0].page_content == "chunk 0", "리랭커의 순서를 따라야 한다"
32+
assert all("rerank_score" in d.metadata for d in result)
33+
34+
35+
def test_rerank_slices_to_top_k(monkeypatch):
36+
monkeypatch.setattr(reranker_module, "RERANKER_ENABLED", True)
37+
monkeypatch.setattr(reranker_module, "_get_reranker", lambda: _CohereLike())
38+
assert len(rerank("q", docs(9), top_k=5)) == 5

0 commit comments

Comments
 (0)