Extract source code tokenizer to a separate file.

server/TracySourceTokenizer.hpp

@@ -0,0 +1,53 @@
+#ifndef __TRACYSOURCETOKENIZER_HPP__
+#define __TRACYSOURCETOKENIZER_HPP__
+
+#include <stdint.h>
+#include <vector>
+
+namespace tracy
+{
+
+class Tokenizer
+{
+public:
+    enum class TokenColor : uint8_t
+    {
+        Default,
+        Comment,
+        Preprocessor,
+        String,
+        CharacterLiteral,
+        Keyword,
+        Number,
+        Punctuation,
+        Type,
+        Special
+    };
+
+    struct Token
+    {
+        const char* begin;
+        const char* end;
+        TokenColor color;
+    };
+
+    struct Line
+    {
+        const char* begin;
+        const char* end;
+        std::vector<Token> tokens;
+    };
+
+    void Reset();
+    std::vector<Token> Tokenize( const char* begin, const char* end );
+
+private:
+    TokenColor IdentifyToken( const char*& begin, const char* end );
+
+    bool m_isInComment;
+    bool m_isInPreprocessor;
+};
+
+}
+
+#endif