2017-05-10 14 views
1

我正在升級以前使用的Ecmascript引擎 Quex 0.64.8Quex 0.67.5。我使用了詞法分析器,並且 正在運行,它似乎現在只能檢測到ANSI令牌 ,而不像以前那樣檢測到UTF-8令牌。Quex:定義標識符的UTF8正則表達式

從本質上講,我做的是供應--codec utf8標誌,同時 運行Quex和使用下面的代碼來識別標識:

PATTERN_IDSTART [^0-9+\-<>*()\[\]?=&|~\\/\^%!{}\n\t\r"':;,. ] 

PATTERN_IDPART {PATTERN_IDSTART}|{PATTERN_DIGIT} 

PATTERN_ID  {PATTERN_IDSTART}{PATTERN_IDPART}* 

的想法是,而不是指定所有允許的令牌,我定義 這是不可接受,並排除這些。新的詞法分析器 檢測到諸如「test1」或「safari」等標識符就好了,但似乎 有「日本語」和「Örjan」的問題。我也只使用utf-8 並且不使用ICU或Iconv。

感覺就像我在這裏誤解了一些東西。任何幫助 解決這個問題將不勝感激。

編輯:

這可能是知道我跑Quex與以下參數有用:

-i ${CMAKE_CURRENT_SOURCE_DIR}/ecmascript.qx 
--analyzer-class ecmascript_lexer 
--foreign-token-id-file ${BISON_ECMASCRIPT_PARSER_OUTPUT_HEADER} 
--token-id-prefix TOK_ 
--template-compression 
--codec utf8 //--encoding utf8 since Quex 0.67.5 
--buffer-element-size 1 
--buffer-element-type char 
--odir ${CMAKE_CURRENT_BINARY_DIR}/generated 
--language c++ 
--warning-on-outrun 

編輯2:

我失敗了,因爲UTF重新創建一個小例子在這個例子中工作的-8解析 。因此,我創建了一個獨立的 版本的ecmascript引擎的詞法分析器部分,而不是 ,希望能夠更容易地看到發生了什麼問題。

現在不再清楚我的問題實際上是否與 有關,用於解析utf8令牌。很可能在我的.qx文件中出現了一些錯誤 ......不管怎樣,這裏是我的ecmascript詞法分析器的獨立版本 。

的CMakeLists.txt

cmake_minimum_required(VERSION 2.8) 

project(ecmascript CXX) 

if(MSVC) 
    add_definitions(-D_CRT_SECURE_NO_WARNINGS) 
endif() 

set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) 
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) 

set(QUEX_NAMES "quex") 

if(CMAKE_HOST_WIN32) 
    set(QUEX_NAMES "quex.bat" ${QUEX_NAMES}) 
else() 
    set(QUEX_NAMES "quex-exe.py" ${QUEX_NAMES}) 
endif() 

find_program(QUEX NAMES ${QUEX_NAMES} REQUIRED 
      HINTS ENV QUEX_PATH DOC "Path to Quex's executable." 
      NO_DEFAULT_PATH) 

find_path(QUEX_INCLUDE_DIR quex/core.py REQUIRED 
      HINTS ENV QUEX_PATH DOC "Path to Quex's include directory" 
      NO_DEFAULT_PATH) 

file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/generated) 

add_definitions(-DQUEX_OPTION_LINE_NUMBER_COUNTING 
       -DQUEX_OPTION_ASSERTS_DISABLED) 


add_definitions(-DQUEX_SETTING_BUFFER_SIZE=1024) # Sätter bufferstorleken på lexern 

set(ECMASCRIPT_LEXER ${CMAKE_CURRENT_BINARY_DIR}/generated/ecmascript_lexer) 

add_custom_command(OUTPUT ${ECMASCRIPT_LEXER} 
          ${ECMASCRIPT_LEXER}.cpp 
          ${ECMASCRIPT_LEXER}-token 
          ${ECMASCRIPT_LEXER}-configuration 
          ${ECMASCRIPT_LEXER}-token_ids 
          COMMAND ${QUEX} -i ${CMAKE_CURRENT_SOURCE_DIR}/ecmascript.qx 
             --analyzer-class ecmascript_lexer # Namnet på lexern 
             --foreign-token-id-file ${CMAKE_CURRENT_SOURCE_DIR}/ecmascript_yacc.hpp # token-id genereras av bison 
             --token-id-prefix TOK_ # Custom prefix för tokens (se ecmascript.y för detaljer) 
             --template-compression # Optimera lägesövergångar om möjligt 
             --encoding utf8   # Basera lexern på teckentabell UTF8 
             --buffer-element-size 1 # Använd en datatyp som är 1 byte stor 
             --buffer-element-type uint8_t 
             --odir ${CMAKE_CURRENT_BINARY_DIR}/generated # Berätta var alla genererade filer ska ligga 
             --language c++ 
        DEPENDS ecmascript.qx VERBATIM 
        COMMENT "Generating ecmascript lexer..." 
        MAIN_DEPENDENCY ${CMAKE_CURRENT_SOURCE_DIR}/ecmascript.qx) # Detta styr i vilken ordning Quex och Bison körs 

include_directories(${QUEX_INCLUDE_DIR} 
        ${CMAKE_CURRENT_SOURCE_DIR} 
        ${CMAKE_CURRENT_BINARY_DIR}/generated) 

set(es_lexer ${ECMASCRIPT_LEXER} 
      ${ECMASCRIPT_LEXER}.cpp 
      _main.cpp) 

set(es_generated ${es_lexer} ecmascript_yacc.hpp) 

add_executable(es_lexer ${es_generated})    

ecmascript.qx

header { 
    #include <quex/code_base/extra/accumulator/Accumulator> 

    #include "ecmascript_yacc.hpp" 
    #include <cstdlib> 
    #include <cstdio> 

    #define BACKSPACE   '\x08' 
    #define TAB     '\x09' 
    #define NEWLINE    '\x0A' 
    #define VERTICALTAB   '\x0B' 
    #define FORMFEED   '\x0C' 
    #define CARRIAGERETURN  '\x0D' 
    #define DOUBLEQUOTE   '\x22' 
    #define SINGLEQUOTE   '\x27' 
    #define DOUBLEBACKSLASH  '\x5C' 
    #define NULLTERM   '\x00' 
} 

footer { 
    #include <quex/code_base/extra/accumulator/Accumulator.i> 
} 

define { 
    PATTERN_NEWLINE [\n\r] 

    PATTERN_DIGIT  [0-9] 
    PATTERN_NOZDIGIT [1-9] 
    PATTERN_DECINTLIT "0"|{PATTERN_NOZDIGIT}{PATTERN_DIGIT}* 
    PATTERN_EXPIND  "e"|"E" 
    PATTERN_SIGNEDINT {PATTERN_DIGIT}+|"+"{PATTERN_DIGIT}+|"-"{PATTERN_DIGIT}+ 
    PATTERN_EXPPART  {PATTERN_EXPIND}{PATTERN_SIGNEDINT} 

    PATTERN_DECNUMBER {PATTERN_DECINTLIT}"."{PATTERN_DIGIT}*{PATTERN_EXPPART}?|"."{PATTERN_DIGIT}+{PATTERN_EXPPART}?|{PATTERN_DECINTLIT}{PATTERN_EXPPART}? 

    PATTERN_HEXDIGIT [0-9a-fA-F] 
    PATTERN_HEXNUMBER "0x"{PATTERN_HEXDIGIT}+|"0X"{PATTERN_HEXDIGIT}+ 

    PATTERN_UNIESCSEQ \\u{PATTERN_HEXDIGIT}{4} 

    PATTERN_STRING  "\""(\\"\""|[^"])*"\"" 

    PATTERN_DOUBLE_QUOTE_STRING_DELIMITER "\"" 

    PATTERN_SINGLE_QUOTE_STRING_DELIMITER "'" 

    PATTERN_SINGLELINE_COMMENT "//"[^\n\r]* 

    PATTERN_IDSTART [^0-9+\-<>*()\[\]?=&|~\\/\^%!{}\n\t\r"':;,. ] 

    PATTERN_IDPART {PATTERN_IDSTART}|{PATTERN_DIGIT} 

    PATTERN_ID  {PATTERN_IDSTART}{PATTERN_IDPART}*  
} 

mode EOF : <inheritable: only> {  
    on_end_of_stream { 
     self_send(TOK_LINETERM); 

     self_send(TOK_TERMINATION); 
    } 
} 

mode RestrictedProduction : EOF 
          <skip: [ \t]> 
{ 
    {PATTERN_NEWLINE}{ 
     self_send(';'); 
     self << Program; 
    } 

    on_failure { 
     self.undo(); 
     self << Program; 
    } 
} 

mode StringHelper : EOF 
        <inheritable: only> 
{ 
    on_entry { 
     self_send(TOK_QUOTE); 
    } 

    on_exit { 
     if(self.accumulator.text.begin != self.accumulator.text.end) 
      self_send(TOK_STRLITPART); 

     self_accumulator_flush(TOK_QUOTE); 
    } 

    {PATTERN_NEWLINE} => '\n'; 

    "\\b" { self_accumulator_add_character(BACKSPACE); } 

    "\\t" { self_accumulator_add_character(TAB); } 



    "\\n" { self_accumulator_add_character(NEWLINE); } 

    "\\v" { self_accumulator_add_character(VERTICALTAB); } 

    "\\f" { self_accumulator_add_character(FORMFEED); } 

    "\\r" { self_accumulator_add_character(CARRIAGERETURN); } 

    "\\\"" { self_accumulator_add_character(DOUBLEQUOTE); } 

    "\\'" { self_accumulator_add_character(SINGLEQUOTE); } 

    "\\\\" { self_accumulator_add_character(DOUBLEBACKSLASH); } 

    "\\0" { self_accumulator_add_character(NULLTERM); } 

    "\\x"{PATTERN_HEXDIGIT}{2} 
    { 
     { 
      unsigned long ulResult = strtoul(reinterpret_cast<char*>(Lexeme+2),0,16); 
      uint8_t *const pBuffer = reinterpret_cast<uint8_t*>(&ulResult); 
      self_accumulator_add(pBuffer,pBuffer+2);    
     } 
    } 

    on_failure { 
     self_accumulator_add(Lexeme, LexemeEnd); 
    } 
} 

mode SingleQuoteString : StringHelper 
{ 
    {PATTERN_SINGLE_QUOTE_STRING_DELIMITER} 
    { 
     // Om vi hittade slutet på strängen så växlar vi tillbaka till Program-läget 
     self << Program; 
    } 
} 

mode DoubleQuoteString : StringHelper 
{ 
    {PATTERN_DOUBLE_QUOTE_STRING_DELIMITER} 
    { 
     // Om vi hittade slutet på strängen så växlar vi tillbaka till Program-läget 
     self << Program; 
    } 
} 

mode PrefixHelper : EOF 
        <skip: [ \t]> // Ignorera whitespace 
{ 
    on_entry { 
     self.seek_backward(3); 
    } 

    {PATTERN_NEWLINE} 
    { 
     if(self.iParaCount == 0) 
      self_send(';'); 
    } 

    "++" 
    { 
     self_send(TOK_PLUSPLUS); 
     self << Program; 
    } 

    "--" 
    { 
     self_send(TOK_MINUSMINUS); 
     self << Program; 
    } 

    on_failure { 
     (void)Lexeme; 
    } 
} 

mode Operators : <inheritable: only> 
{ 
    "||"  => TOK_OR; 
    "&&"  => TOK_AND; 
    "++"  { self << PrefixHelper; } 
    "--"  { self << PrefixHelper; } 
    "==="  => TOK_EQEQEQ; 
    "=="  => TOK_EQEQ; 
    "!=="  => TOK_NEQEQ; 
    "!="  => TOK_NEQ; 
    "*="  => TOK_MULTEQ; 
    "/="  => TOK_DIVEQ; 
    "%="  => TOK_MODEQ; 
    "+="  => TOK_PLUSEQ; 
    "\-="  => TOK_MINUSEQ; 
    ">>>="  => TOK_GTGTGTEQ; 
    ">>>"  => TOK_GTGTGT; 
    "<<="  => TOK_LTLTEQ; 
    ">>="  => TOK_GTGTEQ; 
    "<<"  => TOK_LTLT; 
    ">>"  => TOK_GTGT; 
    "<="  => TOK_LTE; 
    ">="  => TOK_GTE; 
    "&="  => TOK_AMPEQ; 
    "^="  => TOK_CIRCEQ; 
    "|="  => TOK_PIPEEQ; 

    ['=']  => '='; 
    ['!']  => '!'; 
    ['(']  { self_send('('); ++self.iParaCount; } 
    ['+']  => '+'; 
    ['\-']  => '-'; 
    ['*']  => '*'; 
    ['/']  => '/'; 
    ['%']  => '%'; 
    ['<']  => '<'; 
    ['>']  => '>'; 
    ['\[']  => '['; 
    ['\]']  => ']'; 
    ['.']  => '.'; 
    [',']  => ','; 
    ['?']  => '?'; 
    [':']  => ':'; 
    ['~']  => '~'; 
    ['&']  => '&'; 
    ['^']  => '^'; 
    ['|']  => '|'; 
    ['{']  => '{'; 
    [';']  => ';'; 
    [')']  { self_send(')'); --self.iParaCount; } 
    ['}']  { self_send(TOK_LINETERM); self_send('}'); } 
} 

mode Keywords : <inheritable: only> 
{ 
    function  => TOK_FUNCTION; 
    return   { self_send(TOK_RETURN); self << RestrictedProduction; } 
    var    => TOK_VAR; 
    null   => TOK_NULL; 
    true   => TOK_TRUE; 
    false   => TOK_FALSE; 
    instanceof  => TOK_INSTANCEOF; 
    in    => TOK_IN; 
    delete   => TOK_DELETE; 
    void   => TOK_VOID; 
    typeof   => TOK_TYPEOF; 
    this   => TOK_THIS; 
    if    => TOK_IF; 
    else   => TOK_ELSE; 
    with   => TOK_WITH; 
    throw   { self_send(TOK_THROW); self << RestrictedProduction; } 
    try    => TOK_TRY; 
    catch   => TOK_CATCH; 
    finally   => TOK_FINALLY; 
    for    => TOK_FOR; 
    break   { self_send(TOK_BREAK); self << RestrictedProduction; } 
    continue  { self_send(TOK_CONTINUE); self << RestrictedProduction; } 
    while   => TOK_WHILE; 
    do    => TOK_DO; 
    switch   => TOK_SWITCH; 
    case   => TOK_CASE; 
    default   => TOK_DEFAULT; 
    new    => TOK_NEW; 
    synchronized => TOK_SYNCHRONIZED; 
} 

mode Values : <inheritable: only> 
{ 
    {PATTERN_DECNUMBER} => TOK_DECLIT(Lexeme); 

    {PATTERN_HEXNUMBER} => TOK_HEXINTLIT(Lexeme); 

    {PATTERN_DOUBLE_QUOTE_STRING_DELIMITER} { self << DoubleQuoteString; } 

    {PATTERN_SINGLE_QUOTE_STRING_DELIMITER} { self << SingleQuoteString; } 
} 

mode Identifiers : <inheritable: only> 
{ 
    {PATTERN_ID} => TOK_ID(Lexeme); 
} 

mode Program : Keywords, 
       Identifiers, 
       Values, 
       Operators, 
       EOF    
       <skip: [ \t]> 
       <skip_range: "/*" "*/"> 
{ 
    {PATTERN_NEWLINE} 
    { 
     if(self.iParaCount == 0) 
      self_send(TOK_LINETERM); 
    } 

    {PATTERN_SINGLELINE_COMMENT} 
    {} 
} 

body { 

    void push_token(const unsigned int uiToken) 
    { 
     self.uiLastToken = self.uiCurrentToken; 
     self.uiCurrentToken = uiToken; 
    } 

    bool use_auto_semi() const 
    { return uiLastToken == TOK_LINETERM; } 

    unsigned int uiLastToken, 
       uiCurrentToken; 

    int iParaCount; 

    quex::Token* pLastID; 

    QUEX_NAME(Accumulator) accumulator; 
} 

constructor { 
    self.uiLastToken = 0; 
    self.uiCurrentToken = 0; 
    self.iParaCount = 0; 
    self.pLastID = 0; 

    if(!QUEX_NAME(Accumulator_construct)(&me->accumulator, me)) { 
     return false; 
    } 
} 

destructor { 
    QUEX_NAME(Accumulator_destruct)(&me->accumulator); 
} 

start = Program; 

ecmascript_yacc.hpp

#ifndef YY_ECMASCRIPT_YY_C_USERS_PATRIKJ_WORK_GIT_ECMASCRIPT_BUILD_VC14_X64_GENERATED_ECMASCRIPT_YACC_HPP_INCLUDED 
# define YY_ECMASCRIPT_YY_C_USERS_PATRIKJ_WORK_GIT_ECMASCRIPT_BUILD_VC14_X64_GENERATED_ECMASCRIPT_YACC_HPP_INCLUDED 

/* Token type. */ 
#ifndef YYTOKENTYPE 
# define YYTOKENTYPE 
    enum yytokentype 
    { 
    TOK_TERMINATION = 0, 
    TOK_UNINITIALIZED = 1, 
    TOK_ID = 258, 
    TOK_NULL = 259, 
    TOK_TRUE = 260, 
    TOK_FALSE = 261, 
    TOK_DECLIT = 262, 
    TOK_HEXINTLIT = 263, 
    TOK_OR = 264, 
    TOK_AND = 265, 
    TOK_PLUSPLUS = 266, 
    TOK_MINUSMINUS = 267, 
    TOK_EQEQ = 268, 
    TOK_NEQ = 269, 
    TOK_EQEQEQ = 270, 
    TOK_NEQEQ = 271, 
    TOK_LTE = 272, 
    TOK_GTE = 273, 
    TOK_INSTANCEOF = 274, 
    TOK_IN = 275, 
    TOK_STRLITPART = 276, 
    TOK_QUOTE = 277, 
    TOK_VOID = 278, 
    TOK_TYPEOF = 279, 
    TOK_DELETE = 280, 
    TOK_THIS = 281, 
    TOK_LTLT = 282, 
    TOK_GTGT = 283, 
    TOK_GTGTGT = 284, 
    TOK_MULTEQ = 285, 
    TOK_DIVEQ = 286, 
    TOK_MODEQ = 287, 
    TOK_PLUSEQ = 288, 
    TOK_MINUSEQ = 289, 
    TOK_LTLTEQ = 290, 
    TOK_GTGTEQ = 291, 
    TOK_GTGTGTEQ = 292, 
    TOK_AMPEQ = 293, 
    TOK_CIRCEQ = 294, 
    TOK_PIPEEQ = 295, 
    TOK_IF = 296, 
    TOK_ELSE = 297, 
    TOK_RETURN = 298, 
    TOK_VAR = 299, 
    TOK_WITH = 300, 
    TOK_THROW = 301, 
    TOK_TRY = 302, 
    TOK_CATCH = 303, 
    TOK_FINALLY = 304, 
    TOK_FOR = 305, 
    TOK_BREAK = 306, 
    TOK_CONTINUE = 307, 
    TOK_WHILE = 308, 
    TOK_DO = 309, 
    TOK_SWITCH = 310, 
    TOK_CASE = 311, 
    TOK_DEFAULT = 312, 
    TOK_NEW = 313, 
    TOK_FUNCTION = 314, 
    TOK_SYNCHRONIZED = 315, 
    TOK_LINETERM = 316 
    }; 
#endif 

/* Value type. */ 
#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED 
typedef int YYSTYPE; 
# define YYSTYPE_IS_TRIVIAL 1 
# define YYSTYPE_IS_DECLARED 1 
#endif 

#endif /* !YY_ECMASCRIPT_YY_C_USERS_PATRIKJ_WORK_GIT_ECMASCRIPT_BUILD_VC14_X64_GENERATED_ECMASCRIPT_YACC_HPP_INCLUDED */ 

_main.cpp

#include <iostream> 

#include "ecmascript_lexer" 

/****************************************************************************************/ 
void print_token(quex::Token* token) 
{ 
    std::cout << token->get_string() << std::endl; 
} 

/****************************************************************************************/ 
int main(int argc, char** argv) 
{ 
    quex::Token*  token = 0; 
    quex::ecmascript_lexer qlex; 

    quex::ecmascript_lexer *lexer = quex::ecmascript_lexer::from_file_name("id_test.js", 0); 

    while(lexer->error_code == E_Error_None) 
    { 
     get_token: 

     lexer->receive(&token); 

     if(!token) 
      break; 

     print_token(token); 

     lexer->push_token(token->type_id()); 

     if(token->type_id() == TOK_LINETERM) 
      goto get_token; 

     if(token->type_id() == TOK_ID) 
      lexer->pLastID = token; 

     if(token->type_id() == TOK_TERMINATION) 
      break; 
    } 

    delete lexer; 
    return 0; 
} 

id_test。JS //用於測試的詞法分析器

test1 = safari; 
myFunc() 

function t(){} 

if(test1 < 23) 
    return myFunc(45); 

myFunc(); 

svenskaåäö(); 

var kalleö = 34; 

var _us=kalleö; 

_us = 678 

日本語 = "Nihongo" // Japanska språkets namn 

$myCar = _us 

var new1 = kalleö ? t(); 

"kalleÖ, _us and $myCar should be ignored here" 

الفصحى = "Arabiska" 

/* 
    var new1 = kalleÖ ? t(); 

    "kalleÖ, _us and $myCar should be ignored here" 
*/ 

// var new1 = kalleÖ ? t(); 
대한민국 = 45; 
대한민국X45 = "Value of: 대한민국" + 대한민국; 

ärta="ärta + 2" 

mix帝With대한민국 = "success?"; 

Örjan; 

önes; 
cake; 

Россия; 
РоссияX; 
РоссияX 
XРоссия; 
XРоссия; 

始皇帝 = "The First emperor" 
始皇帝x2 = "The First emperor, twice?" 

最好的問候,

帕特里克Ĵ

+1

我會問題'Quex:爲標識符定義UTF8正則表達式'。 –

+0

它與C++有什麼關係? –

+0

它生成一個用C++編碼的詞法分析器。我更願意將這個 問題標記爲「Quex」而不是C++,但缺乏這樣做的權限。 – PatrikJ

回答

1

我建議你依靠的Unicode屬性,特別是 ID_StartID_Continue,這樣你.qx文件包含

define { 
    ID_START \P{ID_Start} 
    ID_CONTINUE \P{ID_Continue} 
    ID   {ID_START}{ID_CONTINUE}* 
} 

Quex然後採樣UCS數據庫,你將不會我們擔心 特定的代碼點。

另外,如果你只是想支持的一個子集,使用intersection來 切出所期望的UCS範圍,如下面的例子:

... 
    ID_START [: intersection([\X900-\X970], \P{ID_Start}) :] 
    ID_CONTINUE [: intersection([\X900-\X970], \P{ID_Continue}) :] 
... 

PS,你的解決方案不是完全錯誤的。給定一個文件example.qx:

define { 
    PATTERN_IDSTART [^0-9+\-<>*()\[\]?=&|~\\/\^%!{}\n\t\r"':;,. ] 
    PATTERN_IDPART {PATTERN_IDSTART}|[0-9] 
    PATTERN_ID  {PATTERN_IDSTART}{PATTERN_IDPART}* 
} 

token { ID; WS; } 

mode X { 
    {PATTERN_ID} => QUEX_TKN_ID(Lexeme); 
    [ \n\t]  => QUEX_TKN_WS(Lexeme); 
} 

而且一些用戶文件 'example.c':

> quex -i tmp.qx --encoding utf8 --language C -o EasyLexer 
> gcc -I$QUEX_PATH example.c EasyLexer.c -o example 
> ./example example.txt 

提供

ID 'Örjan' 
WS '\n' 
ID '日本語' 
WS '\n' 

#include <stdio.h> 

#include "EasyLexer.h" 

void 
print_token(quex_Token* token_p) 
{ 
    const size_t BufferSize = 1024; 
    char   buffer[1024]; 
    printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize)); 
} 

int 
main(int argc, char** argv) 
{ 
    quex_Token*  token_p = NULL; 
    quex_EasyLexer qlex; 

    quex_EasyLexer_from_file_name(&qlex, "example.txt", NULL); 

    while(qlex.error_code == E_Error_None) { 
     quex_EasyLexer_receive(&qlex, &token_p); 
     if(! token_p) break; 

     print_token(token_p); 
     if(token_p->_id == QUEX_TKN_TERMINATION) break; 
    } 

    quex_EasyLexer_destruct(&qlex); 
    return 0; 
} 

然後在命令行上執行

假設文件'example.txt'是UT F-8編碼幷包含

Örjan 
日本語 

我不知道該說些什麼。我是否理解錯誤?

+0

當然,我可以使用屬性來代替。但是「我的方式」有什麼問題?你能詳細說明一下嗎?任何想法爲什麼它以前工作,但現在不工作? – PatrikJ

+0

如果我調用get_string函數,我得到的輸出結果如下: ID'<.text轉換爲pretty char disabled>',也許這與我的問題有關? – PatrikJ

+0

如果您使用版本庫中的版本,則不會發布此消息,而不是正式版本。無論如何,詞法轉換器的重構與你的問題無關。但可能是,你正在使用'--bet char'?所以,如果你使用UTF8,堅持'--bet uint8_t'。 –