Skip to content

Commit abecbe5

Browse files
lucic71lucic71
committed
Create a single index
1 parent 3dfb5dd commit abecbe5

3 files changed

Lines changed: 139 additions & 90 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -228,9 +228,8 @@ foreach(_header IN LISTS rule_common_headers)
228228
list(APPEND _common_headers --common-header=${_header})
229229
endforeach()
230230
set(cpp_rules_ir_outputs)
231-
set(cpp_rules_index_outputs)
232231
set(cpp_rules_index_inputs)
233-
set(_index_clean_stamp ${CMAKE_BINARY_DIR}/rules_index/clean.stamp)
232+
set(cpp_rules_index_outputs ${CMAKE_BINARY_DIR}/rules_index/c.json ${CMAKE_BINARY_DIR}/rules_index/cpp.json)
234233
set(rust_rules_ir_outputs)
235234
foreach(_rule_dir IN LISTS rule_subdirs)
236235
file(GLOB _cpp_srcs ${_rule_dir}/src.c ${_rule_dir}/src.cpp)
@@ -259,16 +258,6 @@ foreach(_rule_dir IN LISTS rule_subdirs)
259258
list(APPEND cpp_rules_ir_outputs ${_cpp_out})
260259

261260
if(NOT _rule_name STREQUAL "builtin_types")
262-
set(_index_stamp ${_out_dir}/index.stamp)
263-
add_custom_command(
264-
OUTPUT ${_index_stamp}
265-
COMMAND ${CMAKE_COMMAND} -E make_directory ${_out_dir}
266-
COMMAND $<TARGET_FILE:cpp-rule-indexer> --dir ${_rule_dir} --path ${_rule_name} --index ${CMAKE_BINARY_DIR}/rules_index ${_cxxflags} ${_common_headers}
267-
COMMAND ${CMAKE_COMMAND} -E touch ${_index_stamp}
268-
DEPENDS ${_index_clean_stamp}
269-
VERBATIM
270-
)
271-
list(APPEND cpp_rules_index_outputs ${_index_stamp})
272261
list(APPEND cpp_rules_index_inputs ${_cpp_srcs} ${_cxxflags_file})
273262
endif()
274263

@@ -303,10 +292,8 @@ endforeach()
303292
add_custom_target("preprocess-cpp-rules" ALL DEPENDS ${cpp_rules_ir_outputs})
304293

305294
add_custom_command(
306-
OUTPUT ${_index_clean_stamp}
307-
COMMAND ${CMAKE_COMMAND} -E rm -rf ${CMAKE_BINARY_DIR}/rules_index
308-
COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/rules_index
309-
COMMAND ${CMAKE_COMMAND} -E touch ${_index_clean_stamp}
295+
OUTPUT ${cpp_rules_index_outputs}
296+
COMMAND $<TARGET_FILE:cpp-rule-indexer> --rules ${PROJECT_SOURCE_DIR}/rules --index ${CMAKE_BINARY_DIR}/rules_index --exclude builtin_types ${_common_headers}
310297
DEPENDS ${cpp_rules_index_inputs} ${rule_common_headers} ${rule_indexer_sources} ${RUSTLS_HEADER} ${BROTLI_HEADERS}
311298
VERBATIM
312299
)

‎cpp2rust/converter/rules/rules_loader.cpp‎

Lines changed: 40 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,14 @@
99
#include <clang/Lex/Preprocessor.h>
1010
#include <clang/Sema/Sema.h>
1111
#include <llvm/ADT/DenseSet.h>
12+
#include <llvm/Support/Error.h>
13+
#include <llvm/Support/ErrorHandling.h>
14+
#include <llvm/Support/JSON.h>
1215
#include <llvm/Support/MemoryBuffer.h>
1316

1417
#include <algorithm>
1518
#include <filesystem>
16-
#include <format>
19+
#include <map>
1720
#include <ranges>
1821
#include <set>
1922
#include <vector>
@@ -227,17 +230,28 @@ class KeyCollector : public clang::RecursiveASTVisitor<KeyCollector> {
227230
llvm::DenseSet<const void *> seen_;
228231
};
229232

230-
std::vector<fs::path> ListFiles(const fs::path &dir) {
231-
std::vector<fs::path> files;
232-
std::error_code ec;
233-
for (fs::directory_iterator it(dir, ec), end; !ec && it != end;
234-
it.increment(ec)) {
235-
if (it->is_regular_file()) {
236-
files.push_back(it->path());
237-
}
233+
const llvm::json::Object &LoadIndex(const fs::path &path) {
234+
static std::map<fs::path, llvm::json::Object> indexes;
235+
if (auto it = indexes.find(path); it != indexes.end()) {
236+
return it->second;
237+
}
238+
auto buf = llvm::MemoryBuffer::getFile(path.string());
239+
if (!buf) {
240+
llvm::errs() << "Missing " << path.string() << ", run cpp-rule-indexer\n";
241+
llvm::report_fatal_error("cannot read the rule index");
242+
}
243+
auto parsed = llvm::json::parse((*buf)->getBuffer());
244+
if (!parsed) {
245+
llvm::errs() << "Failed to parse rule index: " << path.string() << ": "
246+
<< llvm::toString(parsed.takeError()) << '\n';
247+
llvm::report_fatal_error("cannot parse the rule index");
238248
}
239-
std::ranges::sort(files);
240-
return files;
249+
auto index = parsed->getAsObject();
250+
if (!index) {
251+
llvm::errs() << "Rule index is not an object: " << path.string() << '\n';
252+
llvm::report_fatal_error("cannot parse the rule index");
253+
}
254+
return indexes.emplace(path, std::move(*index)).first->second;
241255
}
242256

243257
std::string BuildRulesBuffer(const fs::path &index_dir,
@@ -247,9 +261,22 @@ std::string BuildRulesBuffer(const fs::path &index_dir,
247261
if (!is_cxx && lang == std::string("cpp")) {
248262
continue;
249263
}
264+
const auto &index = LoadIndex(index_dir / (std::string(lang) + ".json"));
265+
auto common = index.getString("common");
266+
auto rules = index.getObject("rules");
267+
std::set<std::string> namespaces;
250268
for (const auto &path : paths) {
251-
for (const auto &file : ListFiles(index_dir / lang / path)) {
252-
out += std::format("#include \"{}\"\n", file.string());
269+
auto entries = rules->getArray(path.generic_string());
270+
if (!entries) {
271+
continue;
272+
}
273+
for (const auto &entry : *entries) {
274+
auto rule = entry.getAsObject();
275+
if (auto ns = rule->getString("namespace");
276+
ns && namespaces.insert(ns->str()).second) {
277+
out += "namespace " + ns->str() + " {\n" + common->str() + "}\n";
278+
}
279+
out += *rule->getString("text");
253280
}
254281
}
255282
}

‎cpp2rust/rule_preprocessing/cpp_rule_indexer.cpp‎

Lines changed: 96 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@
1010
#include <clang/Tooling/Tooling.h>
1111
#include <clang/Tooling/Transformer/SourceCode.h>
1212
#include <llvm/ADT/STLExtras.h>
13-
#include <llvm/ADT/SmallVector.h>
1413
#include <llvm/ADT/StringExtras.h>
1514
#include <llvm/ADT/StringRef.h>
1615
#include <llvm/Support/CommandLine.h>
16+
#include <llvm/Support/JSON.h>
1717
#include <llvm/Support/MemoryBuffer.h>
1818
#include <llvm/Support/raw_ostream.h>
1919

@@ -36,8 +36,6 @@ namespace cpp2rust {
3636
struct RuleCtx {
3737
fs::path path;
3838
std::string name;
39-
fs::path index_dir;
40-
std::vector<std::string> common_headers;
4139
};
4240

4341
fs::path ClassOf(clang::QualType type) {
@@ -162,24 +160,21 @@ std::string CreateIncFile(const RuleCtx &dir, bool is_c,
162160
return std::format("#define {0} cpp2rust_rules_{1}_{0}\n{2}#undef {0}\n",
163161
name, dir.name, text);
164162
}
165-
std::string includes;
166-
if (!dir.common_headers.empty()) {
167-
includes = std::format("#ifndef CPP2RUST_RULES_{0}_INCLUDES\n"
168-
"#define CPP2RUST_RULES_{0}_INCLUDES\n",
169-
dir.name);
170-
for (const auto &header : dir.common_headers) {
171-
includes += std::format("#include \"{}\"\n", header);
172-
}
173-
includes += "#endif\n";
163+
return std::format("namespace cpp2rust_rules_{} {{\n{}}}\n", dir.name, text);
164+
}
165+
166+
void AddRule(llvm::json::Object &rules, std::string key,
167+
llvm::json::Value rule) {
168+
auto &slot = rules[llvm::json::ObjectKey(std::move(key))];
169+
if (!slot.getAsArray()) {
170+
slot = llvm::json::Array();
174171
}
175-
return std::format("namespace cpp2rust_rules_{} {{\n{}{}}}\n", dir.name,
176-
includes, text);
172+
slot.getAsArray()->push_back(std::move(rule));
177173
}
178174

179-
void IndexRuleFile(clang::ASTContext &ctx, const RuleCtx &dir) {
175+
void IndexRuleFile(clang::ASTContext &ctx, const RuleCtx &dir,
176+
const fs::path &src_path, llvm::json::Object &rules) {
180177
bool is_c = !ctx.getLangOpts().CPlusPlus;
181-
auto index_dir = dir.index_dir / (is_c ? "c" : "cpp");
182-
auto file_name = dir.name + ".inc";
183178
auto &sm = ctx.getSourceManager();
184179
for (auto *decl : ctx.getTranslationUnitDecl()->decls()) {
185180
if (decl->isImplicit() ||
@@ -210,17 +205,21 @@ void IndexRuleFile(clang::ASTContext &ctx, const RuleCtx &dir) {
210205
<< "' in rule dir " << dir.path.string() << '\n';
211206
std::exit(EXIT_FAILURE);
212207
}
213-
auto text =
214-
clang::tooling::getExtendedText(*decl, clang::tok::semi, ctx).str() +
215-
'\n';
216-
auto path = index_dir / key / file_name;
217-
fs::create_directories(path.parent_path());
218-
std::ofstream(path, std::ios::app) << CreateIncFile(dir, is_c, name, text);
208+
auto range = clang::tooling::getExtendedRange(*decl, clang::tok::semi, ctx);
209+
auto text = std::format(
210+
"#line {} \"{}\"\n{}\n", sm.getSpellingLineNumber(range.getBegin()),
211+
src_path.string(), clang::tooling::getText(range, ctx).str());
212+
llvm::json::Object rule{{"text", CreateIncFile(dir, is_c, name, text)}};
213+
if (!is_c) {
214+
rule["namespace"] = "cpp2rust_rules_" + dir.name;
215+
}
216+
AddRule(rules, key.generic_string(), std::move(rule));
219217
}
220218
}
221219

222220
void Index(const fs::path &src_path, const RuleCtx &dir,
223-
llvm::ArrayRef<llvm::StringRef> cxx_flags) {
221+
const std::vector<std::string> &cxx_flags,
222+
llvm::json::Object &rules) {
224223
bool is_c = src_path.extension() == ".c";
225224
auto flags = getPlatformClangBeginFlags();
226225
flags.push_back("-isystem" + src_path.parent_path().string());
@@ -241,7 +240,66 @@ void Index(const fs::path &src_path, const RuleCtx &dir,
241240
llvm::errs() << "ERROR: cannot parse " << src_path.string() << '\n';
242241
std::exit(EXIT_FAILURE);
243242
}
244-
IndexRuleFile(ast->getASTContext(), dir);
243+
IndexRuleFile(ast->getASTContext(), dir, src_path, rules);
244+
}
245+
246+
void WriteJson(const fs::path &path, llvm::json::Object object) {
247+
std::error_code ec;
248+
llvm::raw_fd_ostream os(path.string(), ec);
249+
if (ec) {
250+
llvm::errs() << "ERROR: cannot write " << path.string() << ": "
251+
<< ec.message() << '\n';
252+
std::exit(EXIT_FAILURE);
253+
}
254+
os << llvm::json::Value(std::move(object));
255+
}
256+
257+
std::vector<std::string> ReadCXXFlags(const fs::path &rule_dir) {
258+
std::vector<std::string> flags;
259+
std::ifstream file(rule_dir / "cxxflags");
260+
for (std::string flag; std::getline(file, flag);) {
261+
if (!flag.empty()) {
262+
flags.push_back(flag);
263+
}
264+
}
265+
return flags;
266+
}
267+
268+
void IndexRules(const fs::path &rules_dir, const fs::path &index_dir,
269+
const std::vector<std::string> &excluded,
270+
const std::vector<std::string> &common_headers) {
271+
std::vector<fs::path> rule_dirs;
272+
for (const auto &entry : fs::directory_iterator(rules_dir)) {
273+
if (entry.is_directory() &&
274+
!llvm::is_contained(excluded, entry.path().filename().string())) {
275+
rule_dirs.push_back(entry.path());
276+
}
277+
}
278+
std::ranges::sort(rule_dirs);
279+
fs::create_directories(index_dir);
280+
for (const char *lang : {"c", "cpp"}) {
281+
std::string common;
282+
if (lang == std::string("cpp")) {
283+
for (const auto &header : common_headers) {
284+
common += std::format("#include \"{}\"\n", header);
285+
}
286+
}
287+
llvm::json::Object rules;
288+
for (const auto &rule_dir : rule_dirs) {
289+
auto src_path = rule_dir / (std::string("src.") + lang);
290+
if (!fs::exists(src_path)) {
291+
continue;
292+
}
293+
llvm::errs() << "Indexing " << src_path.string() << '\n';
294+
RuleCtx dir;
295+
dir.path = rule_dir.filename();
296+
dir.name = rule_dir.filename().string();
297+
Index(src_path, dir, ReadCXXFlags(rule_dir), rules);
298+
}
299+
WriteJson(index_dir / (std::string(lang) + ".json"),
300+
llvm::json::Object{{"common", std::move(common)},
301+
{"rules", std::move(rules)}});
302+
}
245303
}
246304

247305
} // namespace cpp2rust
@@ -251,17 +309,10 @@ namespace {
251309
llvm::cl::OptionCategory cat("cpp-rule-indexer options");
252310

253311
llvm::cl::opt<std::string>
254-
SrcDir("dir",
255-
llvm::cl::desc("Path to a rule directory containing src.c and/or "
256-
"src.cpp."),
257-
llvm::cl::value_desc("rule-dir"), llvm::cl::Required,
258-
llvm::cl::cat(cat));
259-
260-
llvm::cl::opt<std::string>
261-
RulePath("path",
262-
llvm::cl::desc("Path of the rule directory relative to the "
263-
"rules root, e.g. std/vector."),
264-
llvm::cl::value_desc("rule-path"), llvm::cl::Required,
312+
RulesDir("rules",
313+
llvm::cl::desc("Path to the rules directory, whose subdirectories "
314+
"contain src.c and/or src.cpp."),
315+
llvm::cl::value_desc("rules-dir"), llvm::cl::Required,
265316
llvm::cl::cat(cat));
266317

267318
llvm::cl::opt<std::string>
@@ -270,38 +321,22 @@ llvm::cl::opt<std::string>
270321
llvm::cl::cat(cat));
271322

272323
llvm::cl::list<std::string>
273-
CommonHeaders("common-header",
274-
llvm::cl::desc("Header included by every indexed C++ rule"),
275-
llvm::cl::value_desc("header"), llvm::cl::ZeroOrMore,
276-
llvm::cl::cat(cat));
324+
Excluded("exclude", llvm::cl::desc("Rule directory that is not indexed"),
325+
llvm::cl::value_desc("rule-dir"), llvm::cl::ZeroOrMore,
326+
llvm::cl::cat(cat));
277327

278-
llvm::cl::list<std::string> CXXFlags("cxxflags",
279-
llvm::cl::desc("Additional CXXFLAGS"),
280-
llvm::cl::value_desc("cxxflags"),
281-
llvm::cl::ZeroOrMore, llvm::cl::cat(cat));
328+
llvm::cl::list<std::string> CommonHeaders(
329+
"common-header", llvm::cl::desc("Header the C++ rules depend on"),
330+
llvm::cl::value_desc("header"), llvm::cl::ZeroOrMore, llvm::cl::cat(cat));
282331

283332
} // namespace
284333

285334
int main(int argc, char *argv[]) {
286335
llvm::cl::HideUnrelatedOptions(cat);
287336
llvm::cl::ParseCommandLineOptions(argc, argv);
288337

289-
cpp2rust::RuleCtx dir;
290-
dir.path = RulePath.getValue();
291-
dir.name = RulePath.getValue();
292-
std::ranges::replace(dir.name, '/', '_');
293-
dir.index_dir = IndexDir.getValue();
294-
dir.common_headers.assign(CommonHeaders.begin(), CommonHeaders.end());
295-
296-
llvm::SmallVector<llvm::StringRef, 4> cxx_flags(CXXFlags.begin(),
297-
CXXFlags.end());
298-
for (const char *name : {"src.c", "src.cpp"}) {
299-
auto path = fs::path(SrcDir.getValue()) / name;
300-
if (!fs::exists(path)) {
301-
continue;
302-
}
303-
llvm::errs() << "Indexing " << path.string() << '\n';
304-
cpp2rust::Index(path, dir, cxx_flags);
305-
}
338+
cpp2rust::IndexRules(RulesDir.getValue(), IndexDir.getValue(),
339+
{Excluded.begin(), Excluded.end()},
340+
{CommonHeaders.begin(), CommonHeaders.end()});
306341
return EXIT_SUCCESS;
307342
}

0 commit comments

Comments
 (0)