postprocess: Preprocess the C source code passed to transforms

c2rust-ast-exporter/src/AstExporter.cpp

@@ -15,6 +15,8 @@
 #include "llvm/Support/Path.h"
 // Declares clang::SyntaxOnlyAction.
 #include "clang/Frontend/FrontendActions.h"
+// Declares clang::DoPrintPreprocessedInput.
+#include "clang/Frontend/Utils.h"
 #include "clang/Tooling/CommonOptionsParser.h"
 
 #include "clang/AST/DeclVisitor.h"
@@ -3030,6 +3032,41 @@ class TranslateAction : public clang::ASTFrontendAction {
     }
 };
 
+// Prints preprocessed source to a string, like `clang -E -fdirectives-only -C`:
+// conditional directives are resolved (dropping inactive regions and their
+// comments) but macro invocations are not expanded, comments are kept, and
+// line markers are kept so output lines can be mapped back to source lines.
+class PrintPreprocessedToStringAction
+    : public clang::PreprocessorFrontendAction {
+    PreprocessedOutputs *outputs;
+
+  public:
+    explicit PrintPreprocessedToStringAction(PreprocessedOutputs *outputs)
+        : outputs(outputs) {}
+
+  protected:
+    void ExecuteAction() override {
+        auto &CI = getCompilerInstance();
+
+        clang::PreprocessorOutputOptions Opts;
+        Opts.ShowCPP = 1;
+        Opts.ShowComments = 1;
+        Opts.ShowLineMarkers = 1;
+#if CLANG_VERSION_MAJOR >= 11
+        // Without directives-only support, fall back to full macro expansion;
+        // the output is still usable, just further from the original source.
+        Opts.DirectivesOnly = 1;
+        CI.getPreprocessor().SetMacroExpansionOnlyInDirectives();
+#endif // CLANG_VERSION_MAJOR >= 11
+
+        std::string out;
+        llvm::raw_string_ostream OS(out);
+        clang::DoPrintPreprocessedInput(CI.getPreprocessor(), &OS, Opts);
+        OS.flush();
+        (*outputs)[make_realpath(getCurrentFile().str())] = std::move(out);
+    }
+};
+
 // Apply a custom category to all command-line options so that they are the
 // only ones displayed.
 static llvm::cl::OptionCategory MyToolCategory("my-tool options");
@@ -3098,8 +3135,26 @@ class MyFrontendActionFactory : public FrontendActionFactory {
 #endif // CLANG_VERSION_MAJOR
 };
 
+class PreprocessActionFactory : public FrontendActionFactory {
+    PreprocessedOutputs *outputs;
+
+  public:
+    PreprocessActionFactory(PreprocessedOutputs *outputs) : outputs(outputs) {}
+
+#if CLANG_VERSION_MAJOR < 10
+    clang::FrontendAction *create() override {
+        return new PrintPreprocessedToStringAction(outputs);
+    }
+#else
+    std::unique_ptr<FrontendAction> create() override {
+        return std::make_unique<PrintPreprocessedToStringAction>(outputs);
+    }
+#endif // CLANG_VERSION_MAJOR
+};
+
 // Marshal the output map into something easy to manipulate in Rust
-ExportResult *make_export_result(const Outputs &outputs) {
+ExportResult *make_export_result(const Outputs &outputs,
+                                 const PreprocessedOutputs &preprocessed) {
     auto result = new ExportResult;
     auto n = outputs.size();
     result->resize(n);
@@ -3117,6 +3172,14 @@ ExportResult *make_export_result(const Outputs &outputs) {
         std::copy(std::begin(bytes), std::end(bytes), byte_array);
         result->bytes[i] = byte_array;
         result->sizes[i] = bytes.size();
+
+        auto pp = preprocessed.find(name);
+        if (pp != preprocessed.end()) {
+            auto const &text = pp->second;
+            auto pp_array = new char[text.size() + 1];
+            memcpy(pp_array, text.c_str(), text.size() + 1);
+            result->preprocessed[i] = pp_array;
+        }
         i++;
     }
 
@@ -3125,7 +3188,8 @@ ExportResult *make_export_result(const Outputs &outputs) {
 
 // Extract clang AST for the source file specified in the argument vector.
 // Note: The arguments should only reference one source file at a time.
-Outputs process(int argc, const char *argv[], int *result) {
+Outputs process(int argc, const char *argv[], int *result,
+                PreprocessedOutputs *preprocessed) {
     auto argv_ = augment_argv(argc, argv);
     int argc_ = argv_.size() - 1; // ignore the extra nullptr
 
@@ -3163,12 +3227,24 @@ Outputs process(int argc, const char *argv[], int *result) {
 
     *result = Tool.run(&myFrontendActionFactory);
     assert(outputs.size() == 1 && "Expected exactly one output.");
+
+    if (preprocessed) {
+        // The AST run above already reported diagnostics; don't repeat them.
+        IgnoringDiagConsumer diagConsumer;
+        Tool.setDiagnosticConsumer(&diagConsumer);
+        PreprocessActionFactory preprocessActionFactory(preprocessed);
+        // Failure here is not fatal: the preprocessed text is optional
+        // metadata, and its absence shows up as a missing map entry.
+        Tool.run(&preprocessActionFactory);
+    }
+
     return outputs;
 }
 
 // AST exporter library interface.
 extern "C" {
-ExportResult *ast_exporter(int argc, const char *argv[], int debug) {
+ExportResult *ast_exporter(int argc, const char *argv[], int debug,
+                           int emit_preprocessed) {
 #ifndef NDEBUG
     if (debug) {
         llvm::DebugFlag = true;
@@ -3177,8 +3253,10 @@ ExportResult *ast_exporter(int argc, const char *argv[], int debug) {
 #endif // NDEBUG
 
     int result;
-    auto outputs = process(argc, argv, &result);
-    return make_export_result(outputs);
+    PreprocessedOutputs preprocessed;
+    auto outputs = process(argc, argv, &result,
+                           emit_preprocessed ? &preprocessed : nullptr);
+    return make_export_result(outputs, preprocessed);
 }
 
 void drop_export_result(ExportResult *result) { delete result; }

c2rust-ast-exporter/src/AstExporter.hpp

@@ -12,7 +12,9 @@
 #include <vector>
 
 using Outputs = std::unordered_map<std::string, std::vector<uint8_t>>;
+using PreprocessedOutputs = std::unordered_map<std::string, std::string>;
 
-Outputs process(int argc, const char *argv[], int *result);
+Outputs process(int argc, const char *argv[], int *result,
+                PreprocessedOutputs *preprocessed = nullptr);
 
 #endif /* AstExporter_hpp */

c2rust-ast-exporter/src/ExportResult.cpp

@@ -7,7 +7,8 @@
 
 #include "ExportResult.hpp"
 
-ExportResult::ExportResult() : entries(0), names(), bytes(), sizes() {}
+ExportResult::ExportResult()
+    : entries(0), names(), bytes(), sizes(), preprocessed() {}
 
 ExportResult::~ExportResult() { deallocate(); }
 
@@ -16,20 +17,24 @@ void ExportResult::resize(std::size_t n) {
     names = new char *[n];
     bytes = new std::uint8_t *[n];
     sizes = new std::size_t[n];
+    preprocessed = new char *[n];
     std::fill_n(names, n, nullptr);
     std::fill_n(bytes, n, nullptr);
     std::fill_n(sizes, n, 0);
+    std::fill_n(preprocessed, n, nullptr);
     entries = n;
 }
 
 void ExportResult::deallocate() {
     for (std::size_t i = 0; i < entries; i++) {
         delete[] names[i];
         delete[] bytes[i];
+        delete[] preprocessed[i];
     }
     delete[] names;
     delete[] bytes;
     delete[] sizes;
+    delete[] preprocessed;
 
     entries = 0;
 }

c2rust-ast-exporter/src/ExportResult.hpp

@@ -17,6 +17,9 @@ struct ExportResult {
     char **names;
     std::uint8_t **bytes;
     std::size_t *sizes;
+    // Preprocessed source text per entry; entries may be null when
+    // preprocessed output was not requested or could not be produced.
+    char **preprocessed;
 
     ExportResult();
     ExportResult(ExportResult const &) = delete;

c2rust-ast-exporter/src/lib.rs

@@ -27,15 +27,19 @@ pub fn get_clang_major_version() -> Option<u32> {
         .ok()
 }
 
+/// Returns the untyped AST and, if `emit_preprocessed` was set and
+/// preprocessing succeeded, the preprocessed source text of the translation
+/// unit (directives-only, with comments and line markers preserved).
 pub fn get_untyped_ast(
     file_path: &Path,
     cc_db: &Path,
     extra_args: &[&str],
     debug: bool,
-) -> Result<clang_ast::AstContext, Error> {
-    let cbors = get_ast_cbors(file_path, cc_db, extra_args, debug);
-    let buffer = cbors
-        .values()
+    emit_preprocessed: bool,
+) -> Result<(clang_ast::AstContext, Option<String>), Error> {
+    let cbors = get_ast_cbors(file_path, cc_db, extra_args, debug, emit_preprocessed);
+    let (buffer, preprocessed) = cbors
+        .into_values()
         .next()
         .ok_or_else(|| Error::new(ErrorKind::InvalidData, "Could not parse input file"))?;
 
@@ -46,7 +50,9 @@ pub fn get_untyped_ast(
 
     let items: Value = from_slice(&buffer[..]).unwrap();
 
-    clang_ast::process(items).map_err(|e| Error::new(ErrorKind::InvalidData, format!("{}", e)))
+    let ast = clang_ast::process(items)
+        .map_err(|e| Error::new(ErrorKind::InvalidData, format!("{}", e)))?;
+    Ok((ast, preprocessed))
 }
 
 /// libClangTooling is not thread-safe, so we must not allow concurrent calls to `ast_exporter`.
@@ -57,9 +63,8 @@ fn get_ast_cbors(
     cc_db: &Path,
     extra_args: &[&str],
     debug: bool,
-) -> HashMap<String, Vec<u8>> {
-    let mut res = 0;
-
+    emit_preprocessed: bool,
+) -> HashMap<String, (Vec<u8>, Option<String>)> {
     let mut args_owned = vec![CString::new("ast_exporter").unwrap()];
     args_owned.push(CString::new(file_path.to_str().unwrap()).unwrap());
     args_owned.push(CString::new("-p").unwrap());
@@ -78,7 +83,7 @@ fn get_ast_cbors(
             args_ptrs.len() as c_int,
             args_ptrs.as_ptr(),
             debug.into(),
-            &mut res,
+            emit_preprocessed.into(),
         );
         drop(lock);
         hashmap = marshal_result(ptr);
@@ -98,12 +103,13 @@ extern "C" {
     /// # Safety
     ///
     /// Not thread-safe; must not be called multiple times concurrently.
-    // ExportResult *ast_exporter(int argc, char *argv[]);
+    // ExportResult *ast_exporter(int argc, char *argv[],
+    //                            int debug, int emit_preprocessed);
     fn ast_exporter(
         argc: c_int,
         argv: *const *const c_char,
         debug: c_int,
-        res: *mut c_int,
+        emit_preprocessed: c_int,
     ) -> *mut ffi::ExportResult;
 
     // void drop_export_result(ExportResult *result);
@@ -112,7 +118,9 @@ extern "C" {
     fn clang_version() -> *const c_char;
 }
 
-unsafe fn marshal_result(result: *const ffi::ExportResult) -> HashMap<String, Vec<u8>> {
+unsafe fn marshal_result(
+    result: *const ffi::ExportResult,
+) -> HashMap<String, (Vec<u8>, Option<String>)> {
     let mut output = HashMap::new();
 
     let n = (*result).entries as isize;
@@ -131,7 +139,15 @@ unsafe fn marshal_result(result: *const ffi::ExportResult) -> HashMap<String, Ve
         let mut v = Vec::new();
         v.extend_from_slice(bytes);
 
-        output.insert(name, v);
+        // Convert optional preprocessed source text
+        let cpreprocessed = *res.preprocessed.offset(i);
+        let preprocessed = if cpreprocessed.is_null() {
+            None
+        } else {
+            Some(CStr::from_ptr(cpreprocessed).to_str().unwrap().to_owned())
+        };
+
+        output.insert(name, (v, preprocessed));
     }
     output
 }

c2rust-postprocess/tests/examples/qsort.rs

@@ -1,4 +1,5 @@
 #![allow(
+    clippy::missing_safety_doc,
     dead_code,
     non_camel_case_types,
     non_snake_case,

c2rust-postprocess/tests/test_definitions.py

@@ -35,24 +35,24 @@ def test_get_rust_function_spans(transpile_qsort, pytestconfig):
     expected_fn_spans = [
         {
             "name": "swap",
-            "start_line": 10,
-            "end_line": 14,
-            "start_byte": 154,
-            "end_byte": 316,
+            "start_line": 11,
+            "end_line": 15,
+            "start_byte": 186,
+            "end_byte": 348,
         },
         {
             "name": "partition",
-            "start_line": 16,
-            "end_line": 39,
-            "start_byte": 330,
-            "end_byte": 1193,
+            "start_line": 17,
+            "end_line": 40,
+            "start_byte": 362,
+            "end_byte": 1225,
         },
         {
             "name": "quickSort",
-            "start_line": 41,
-            "end_line": 51,
-            "start_byte": 1207,
-            "end_byte": 1566,
+            "start_line": 42,
+            "end_line": 52,
+            "start_byte": 1239,
+            "end_byte": 1598,
         },
     ]
 

c2rust-transpile/src/lib.rs

@@ -677,11 +677,12 @@ fn transpile_single(
     }
 
     // Extract the untyped AST from the CBOR file
-    let untyped_context = match ast_exporter::get_untyped_ast(
+    let (untyped_context, preprocessed_source) = match ast_exporter::get_untyped_ast(
         input_path,
         cc_db,
         extra_clang_args,
         tcfg.debug_ast_exporter,
+        tcfg.emit_c_decl_map,
     ) {
         Err(e) => {
             warn!(
@@ -720,9 +721,17 @@ fn transpile_single(
         println!("{:#?}", Printer::new(io::stdout()).print(&typed_context));
     }
 
+    // Extract preprocessed text for each function definition so the C decl
+    // map can carry it alongside the original source snippet.
+    let preprocessed_definitions = preprocessed_source
+        .map(|source| {
+            translator::collect_preprocessed_definitions(&typed_context, input_path, &source)
+        })
+        .unwrap_or_default();
+
     // Perform the translation
     let (translated_string, maybe_decl_map, pragmas, crates) =
-        translator::translate(typed_context, tcfg, input_path);
+        translator::translate(typed_context, tcfg, input_path, &preprocessed_definitions);
 
     if let Some(decl_map) = maybe_decl_map {
         let decl_map_path = output_path.with_extension("c_decls.json");