TechOverflow Logo
TechOverflow

How to parse C++ structs & inline field comments to JSON using clang

In our prev

In our previous post How to parse C++ ‘struct’ definition from source code using clang we learned how to parse a C++ struct definition using the clang C++ parser. In this post, we will extend this to parse inline field comments and convert the struct definition to JSON.

Our test struct will look like this:

  typedef struct {
      double a; /* That weird parameter */
      double b; /* Another weird parameter */
      double c;
      double d; // This is documentation for d
      double e[8]; /* Array of doubles */

  } myParameters;

and our goal is to not only parse the individual fields but also their inline comments, and convert the whole struct definition to JSON.

Source code

#include <clang-c/Index.h>
#include <iostream>
#include <vector>
#include <string>
#include <cstring>
#include <rapidjson/document.h>
#include <rapidjson/writer.h>
#include <rapidjson/stringbuffer.h>
#include <rapidjson/prettywriter.h>

std::string extractInlineComment(CXCursor cursor, const char* sourceCode) {
    if (sourceCode == nullptr) {
        return "";
    }
    
    CXSourceRange range = clang_getCursorExtent(cursor);
    CXSourceLocation start = clang_getRangeStart(range);
    CXSourceLocation end = clang_getRangeEnd(range);
    
    CXFile file;
    unsigned startLine, startColumn, startOffset;
    unsigned endLine, endColumn, endOffset;
    
    clang_getExpansionLocation(start, &file, &startLine, &startColumn, &startOffset);
    clang_getExpansionLocation(end, &file, &endLine, &endColumn, &endOffset);
    
    // Find the line containing this field declaration
    const char* lineStart = sourceCode;
    unsigned currentLine = 1;
    
    // Navigate to the line containing the field
    while (currentLine < startLine && *lineStart) {
        if (*lineStart == '\n') {
            currentLine++;
        }
        lineStart++;
    }
    
    // Find the end of the line
    const char* lineEnd = lineStart;
    while (*lineEnd && *lineEnd != '\n') {
        lineEnd++;
    }
    
    // Extract the line as a string
    std::string line(lineStart, lineEnd - lineStart);
    
    // Look for comments in this line
    std::string comment = "";
    
    // Look for /* */ style comments
    size_t blockStart = line.find("/*");
    if (blockStart != std::string::npos) {
        size_t blockEnd = line.find("*/", blockStart);
        if (blockEnd != std::string::npos) {
            comment = line.substr(blockStart, blockEnd - blockStart + 2);
        }
    }
    
    // Look for // style comments if no block comment found
    if (comment.empty()) {
        size_t lineCommentStart = line.find("//");
        if (lineCommentStart != std::string::npos) {
            comment = line.substr(lineCommentStart);
        }
    }
    
    return comment;
}

struct VisitorData {
    const char* sourceCode;
    rapidjson::Value* fieldsArray;
    rapidjson::Document::AllocatorType* allocator;
};

void extractStructFields(CXCursor cursor, const char* sourceCode, rapidjson::Document& doc) {
    CXCursorKind kind = clang_getCursorKind(cursor);
    if (kind == CXCursor_StructDecl) {
        CXString structName = clang_getCursorSpelling(cursor);
        std::string structNameStr = clang_getCString(structName);
        
        // Create struct object in JSON
        rapidjson::Value structObj(rapidjson::kObjectType);
        rapidjson::Value fieldsArray(rapidjson::kArrayType);
        
        clang_disposeString(structName);
        
        VisitorData data = {sourceCode, &fieldsArray, &doc.GetAllocator()};
        
        clang_visitChildren(cursor, [](CXCursor c, CXCursor parent, CXClientData client_data) {
            VisitorData* data = static_cast<VisitorData*>(client_data);
            const char* sourceCode = data->sourceCode;
            rapidjson::Value* fieldsArray = data->fieldsArray;
            rapidjson::Document::AllocatorType* allocator = data->allocator;
            
            CXCursorKind kind = clang_getCursorKind(c);
            if (kind == CXCursor_FieldDecl) {
                CXString fieldName = clang_getCursorSpelling(c);
                CXType fieldType = clang_getCursorType(c);
                CXString typeName = clang_getTypeSpelling(fieldType);
                
                std::string comment = extractInlineComment(c, sourceCode);
                std::string fieldNameStr = clang_getCString(fieldName);
                std::string typeNameStr = clang_getCString(typeName);
                
                // Create field object
                rapidjson::Value fieldObj(rapidjson::kObjectType);
                rapidjson::Value nameVal(fieldNameStr.c_str(), *allocator);
                rapidjson::Value typeVal(typeNameStr.c_str(), *allocator);
                rapidjson::Value commentVal(comment.c_str(), *allocator);
                
                fieldObj.AddMember("name", nameVal, *allocator);
                fieldObj.AddMember("type", typeVal, *allocator);
                fieldObj.AddMember("comment", commentVal, *allocator);
                
                fieldsArray->PushBack(fieldObj, *allocator);
                
                clang_disposeString(fieldName);
                clang_disposeString(typeName);
            }
            return CXChildVisit_Continue;
        }, &data);
        
        // Add struct to document
        structObj.AddMember("fields", fieldsArray, doc.GetAllocator());
        rapidjson::Value structNameVal(structNameStr.c_str(), doc.GetAllocator());
        doc.AddMember(structNameVal, structObj, doc.GetAllocator());
    }
}

int main() {
    CXIndex index = clang_createIndex(0, 0);
    const char *code = R"(
        typedef struct {
            double a; /* That weird parameter */
            double b; /* Another weird parameter */
            double c;
            double d; // This is documentation for d
            double e[8]; /* Array of doubles */
        } myParameters;
    )";

    CXUnsavedFile unsavedFile = {"test.cpp", code, (unsigned long)strlen(code)};
    CXTranslationUnit unit = clang_parseTranslationUnit(index, "test.cpp", nullptr, 0, &unsavedFile, 1, 
                                                       CXTranslationUnit_DetailedPreprocessingRecord | 
                                                       CXTranslationUnit_SkipFunctionBodies);
    if (unit == nullptr) {
        std::cerr << "Failed to parse translation unit." << std::endl;
        return 1;
    }

    // Create JSON document
    rapidjson::Document doc;
    doc.SetObject();

    CXCursor cursor = clang_getTranslationUnitCursor(unit);
    auto mainData = std::make_pair(code, &doc);
    clang_visitChildren(cursor, [](CXCursor c, CXCursor parent, CXClientData client_data) {
        auto* data = static_cast<std::pair<const char*, rapidjson::Document*>*>(client_data);
        const char* sourceCode = data->first;
        rapidjson::Document* doc = data->second;
        extractStructFields(c, sourceCode, *doc);
        return CXChildVisit_Continue;
    }, &mainData);

    // Convert JSON to string and output
    rapidjson::StringBuffer buffer;
    rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
    doc.Accept(writer);
    
    std::cout << buffer.GetString() << std::endl;

    clang_disposeTranslationUnit(unit);
    clang_disposeIndex(index);
    return 0;
}

How to compile and run

On Ubuntu, install the required libraries using

sudo apt -y install libclang-19-dev

and compile the code using

g++ parse_struct_with_comments.cpp -o parse_struct_with_comments -std=c++17 -I/usr/lib/llvm-19/include -L/usr/lib/llvm-19/lib -lclang

Then run the program:

./parse_struct_with_comments

It will output the following JSON:

{
    "myParameters": {
        "fields": [
            {
                "name": "a",
                "type": "double",
                "comment": "/* That weird parameter */"
            },
            {
                "name": "b",
                "type": "double",
                "comment": "/* Another weird parameter */"
            },
            {
                "name": "c",
                "type": "double",
                "comment": ""
            },
            {
                "name": "d",
                "type": "double",
                "comment": "// This is documentation for d"
            },
            {
                "name": "e",
                "type": "double[8]",
                "comment": "/* Array of doubles */"
            }

        ]
    }
}

Source code variant with file I/O

#include <clang-c/Index.h>
#include <iostream>
#include <vector>
#include <string>
#include <cstring>
#include <fstream>
#include <rapidjson/document.h>
#include <rapidjson/writer.h>
#include <rapidjson/stringbuffer.h>
#include <rapidjson/prettywriter.h>

std::string extractInlineComment(CXCursor cursor, const char* sourceCode) {
    if (sourceCode == nullptr) {
        return "";
    }
    
    CXSourceRange range = clang_getCursorExtent(cursor);
    CXSourceLocation start = clang_getRangeStart(range);
    CXSourceLocation end = clang_getRangeEnd(range);
    
    CXFile file;
    unsigned startLine, startColumn, startOffset;
    unsigned endLine, endColumn, endOffset;
    
    clang_getExpansionLocation(start, &file, &startLine, &startColumn, &startOffset);
    clang_getExpansionLocation(end, &file, &endLine, &endColumn, &endOffset);
    
    // Find the line containing this field declaration
    const char* lineStart = sourceCode;
    unsigned currentLine = 1;
    
    // Navigate to the line containing the field
    while (currentLine < startLine && *lineStart) {
        if (*lineStart == '\n') {
            currentLine++;
        }
        lineStart++;
    }
    
    // Find the end of the line
    const char* lineEnd = lineStart;
    while (*lineEnd && *lineEnd != '\n') {
        lineEnd++;
    }
    
    // Extract the line as a string
    std::string line(lineStart, lineEnd - lineStart);
    
    // Look for comments in this line
    std::string comment = "";
    
    // Look for /* */ style comments
    size_t blockStart = line.find("/*");
    if (blockStart != std::string::npos) {
        size_t blockEnd = line.find("*/", blockStart);
        if (blockEnd != std::string::npos) {
            comment = line.substr(blockStart, blockEnd - blockStart + 2);
        }
    }
    
    // Look for // style comments if no block comment found
    if (comment.empty()) {
        size_t lineCommentStart = line.find("//");
        if (lineCommentStart != std::string::npos) {
            comment = line.substr(lineCommentStart);
        }
    }
    
    return comment;
}

struct VisitorData {
    const char* sourceCode;
    rapidjson::Value* fieldsArray;
    rapidjson::Document::AllocatorType* allocator;
};

void extractStructFields(CXCursor cursor, const char* sourceCode, rapidjson::Document& doc) {
    CXCursorKind kind = clang_getCursorKind(cursor);
    if (kind == CXCursor_StructDecl) {
        CXString structName = clang_getCursorSpelling(cursor);
        std::string structNameStr = clang_getCString(structName);
        
        // Create struct object in JSON
        rapidjson::Value structObj(rapidjson::kObjectType);
        rapidjson::Value fieldsArray(rapidjson::kArrayType);
        
        clang_disposeString(structName);
        
        VisitorData data = {sourceCode, &fieldsArray, &doc.GetAllocator()};
        
        clang_visitChildren(cursor, [](CXCursor c, CXCursor parent, CXClientData client_data) {
            VisitorData* data = static_cast<VisitorData*>(client_data);
            const char* sourceCode = data->sourceCode;
            rapidjson::Value* fieldsArray = data->fieldsArray;
            rapidjson::Document::AllocatorType* allocator = data->allocator;
            
            CXCursorKind kind = clang_getCursorKind(c);
            if (kind == CXCursor_FieldDecl) {
                CXString fieldName = clang_getCursorSpelling(c);
                CXType fieldType = clang_getCursorType(c);
                CXString typeName = clang_getTypeSpelling(fieldType);
                
                std::string comment = extractInlineComment(c, sourceCode);
                std::string fieldNameStr = clang_getCString(fieldName);
                std::string typeNameStr = clang_getCString(typeName);
                
                // Create field object
                rapidjson::Value fieldObj(rapidjson::kObjectType);
                rapidjson::Value nameVal(fieldNameStr.c_str(), *allocator);
                rapidjson::Value typeVal(typeNameStr.c_str(), *allocator);
                rapidjson::Value commentVal(comment.c_str(), *allocator);
                
                fieldObj.AddMember("name", nameVal, *allocator);
                fieldObj.AddMember("type", typeVal, *allocator);
                fieldObj.AddMember("comment", commentVal, *allocator);
                
                fieldsArray->PushBack(fieldObj, *allocator);
                
                clang_disposeString(fieldName);
                clang_disposeString(typeName);
            }
            return CXChildVisit_Continue;
        }, &data);
        
        // Add struct to document
        structObj.AddMember("fields", fieldsArray, doc.GetAllocator());
        rapidjson::Value structNameVal(structNameStr.c_str(), doc.GetAllocator());
        doc.AddMember(structNameVal, structObj, doc.GetAllocator());
    }
}

int main(int argc, char* argv[]) {
    if (argc != 2) {
        std::cerr << "Usage: " << argv[0] << " <filename>" << std::endl;
        return 1;
    }

    const char* filename = argv[1];
    CXIndex index = clang_createIndex(0, 0);
    
    CXTranslationUnit unit = clang_parseTranslationUnit(index, filename, nullptr, 0, nullptr, 0, 
                                                       CXTranslationUnit_DetailedPreprocessingRecord | 
                                                       CXTranslationUnit_SkipFunctionBodies);
    if (unit == nullptr) {
        std::cerr << "Failed to parse translation unit." << std::endl;
        return 1;
    }

    // Create JSON document
    rapidjson::Document doc;
    doc.SetObject();

    // Read the source file content
    std::ifstream file(filename, std::ios::binary);
    if (!file) {
        std::cerr << "Failed to open file: " << filename << std::endl;
        return 1;
    }
    
    file.seekg(0, std::ios::end);
    size_t fileSize = file.tellg();
    file.seekg(0, std::ios::beg);
    
    std::string sourceCode(fileSize, '\0');
    file.read(&sourceCode[0], fileSize);
    file.close();

    struct CallbackData {
        rapidjson::Document* doc;
        const char* sourceCode;
    };
    
    CallbackData callbackData = {&doc, sourceCode.c_str()};

    CXCursor cursor = clang_getTranslationUnitCursor(unit);
    clang_visitChildren(cursor, [](CXCursor c, CXCursor parent, CXClientData client_data) {
        CallbackData* data = static_cast<CallbackData*>(client_data);
        extractStructFields(c, data->sourceCode, *(data->doc));
        return CXChildVisit_Continue;
    }, &callbackData);

    // Convert JSON to string and output
    rapidjson::StringBuffer buffer;
    rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
    doc.Accept(writer);
    
    std::cout << buffer.GetString() << std::endl;

    clang_disposeTranslationUnit(unit);
    clang_disposeIndex(index);
    return 0;
}