tokenizer.cpp

extern "C" {
#include "clang-c/Index.h"
}
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/raw_ostream.h"

using namespace llvm;

static cl::opt<std::string> FileName(cl::Positional, cl::desc("Input File"), cl::Required);

int
main(int argc, char** argv)
{
    cl::ParseCommandLineOptions(argc, argv, "Tokenizer example\n");
    CXIndex index = clang_createIndex(0,0);
    const char *args[] = {
        "-I/usr/include",
        "-I."
    };

    CXTranslationUnit translationUnit =
        clang_parseTranslationUnit(index, FileName.c_str(), args, 2, NULL, 0, CXTranslationUnit_None);

    CXFile file = clang_getFile(translationUnit, FileName.c_str());
    CXSourceLocation loc_start = clang_getLocationForOffset(translationUnit, file, 0);
    CXSourceLocation loc_end = clang_getLocationForOffset(translationUnit, file, 60);
    CXSourceRange range = clang_getRange(loc_start, loc_end);
    unsigned numTokens = 0;
    CXToken *tokens = nullptr;
    clang_tokenize(translationUnit, range, &tokens, &numTokens);

    for (unsigned i = 0; i < numTokens; ++i)
    {
        enum CXTokenKind kind = clang_getTokenKind(tokens[i]);
        CXString name = clang_getTokenSpelling(translationUnit, tokens[i]);
        CXSourceLocation loc = clang_getTokenLocation(translationUnit, tokens[i]);

        CXString fName;
        unsigned line = 0, col = 0;
        clang_getPresumedLocation(loc, &fName, &line, &col);

        switch (kind)
        {
        case CXToken_Punctuation:
            outs() << "PUNCTUATION(" << clang_getCString(name) << ") ";
            break;
        case CXToken_Keyword:
            outs() << "KEYWORD(" << clang_getCString(name) << ") ";
            break;
        case CXToken_Identifier:
            outs() << "IDENTIFIER(" << clang_getCString(name) << ") ";
            break;
        case CXToken_Literal:
            outs() << "COMMENT(" << clang_getCString(name) << ") ";
            break;
        default:
            outs() << "UNKNOWN(" << clang_getCString(name) << ") ";
            break;
        }

        outs() << "\tLoc=<" << clang_getCString(fName) << "," << line << "," << col << ">\n";

        clang_disposeString(name);
        clang_disposeString(fName);
    }

    outs() << "\n";
    clang_disposeTokens(translationUnit, tokens, numTokens);
    clang_disposeTranslationUnit(translationUnit);
    return 0;
}