使用Boost.Spirit.Lex和流迭代器

Question

我想使用Boost.Spirit.Lex來解析二進制文件;爲此，我寫了下面的程序（這裏是摘錄）：

#include <boost/spirit/include/lex_lexertl.hpp> 
#include <boost/spirit/include/support_multi_pass.hpp> 
#include <boost/bind.hpp> 
#include <boost/ref.hpp> 
#include <fstream> 
#include <iterator> 
#include <string> 

namespace spirit = boost::spirit; 
namespace lex = spirit::lex; 

#define X 1 
#define Y 2 
#define Z 3 

template<typename L> 
class word_count_tokens : public lex::lexer<L> 
{ 
    public: 
     word_count_tokens() { 
      this->self.add 
       ("[^ \t\n]+", X) 
       ("\n", Y) 
       (".", Z); 
     } 
}; 

class counter 
{ 
    public: 
     typedef bool result_type; 

     template<typename T> 
     bool operator() (const T &t, size_t &c, size_t &w, size_t &l) const { 
      switch (t.id()) { 
       case X: 
        ++w; c += t.value().size(); 
        break; 
       case Y: 
        ++l; ++c; 
        break; 
       case Z: 
        ++c; 
        break; 
      } 

      return true; 
     } 
}; 

int main (int argc, char **argv) 
{ 
    std::ifstream ifs (argv[1], std::ios::in | std::ios::binary); 
    auto first = spirit::make_default_multi_pass (std::istream_iterator<char> (ifs)); 
    auto last = spirit::make_default_multi_pass (std::istream_iterator<char>()); 
    size_t w, c, l; 
    word_count_tokens<lex::lexertl::lexer<>> word_count_functor; 

    w = c = l = 0; 

    bool r = lex::tokenize (first, last, word_count_functor, boost::bind (counter(), _1, boost::ref (c), boost::ref (w), boost::ref (l))); 

    ifs.close(); 

    if (r) { 
     std::cout << l << ", " << w << ", " << c << std::endl; 
    } 

    return 0; 
} 

構建返回以下錯誤：

lexer.hpp:390:46: error: non-const lvalue reference to type 'const char *' cannot bind to a value of unrelated type 

現在，錯誤是由於定義具體詞法分析器，lex::lexer<>;實際上它的第一個參數默認爲const char *。如果我使用spirit::istream_iterator或spirit::make_default_multi_pass (.....)，我也會得到同樣的錯誤。
但是，如果我指定了正確的模板參數lex::lexer<>，我會獲得大量的錯誤！

解決方案？

更新

我已經推杆所有源文件;這是word_counter網站的例子。

Answer 1

好了，因爲這個問題被改變，這裏有一個新的答案，解決一些點與完整的代碼示例。

1. 首先，您需要使用自定義令牌類型。即

   word_count_tokens<lex::lexertl::lexer<lex::lexertl::token<boost::spirit::istream_iterator>>> word_count_functor; 
   // instead of: 
   // word_count_tokens<lex::lexertl::lexer<>> word_count_functor; 
   

   顯然，這是習慣的typedef lex::lexertl::token<boost::spirit::istream_iterator>

2. 您需要使用min_token_id代替令牌標識1,2,3。此外，使其易於維護的枚舉：

   enum token_ids { 
       X = lex::min_token_id + 1, 
       Y, 
       Z, 
   }; 
   

3. 可以不再僅僅使用.size()上的默認令牌value()，因爲迭代器範圍不再RandomAccessRange。相反，採用boost::distance()這是專門爲iterator_range：

    ++w; c += boost::distance(t.value()); // t.value().size(); 
   

結合這些修正：Live On Coliru

#include <boost/spirit/include/lex_lexertl.hpp> 
#include <boost/spirit/include/support_istream_iterator.hpp> 
#include <boost/bind.hpp> 
#include <fstream> 

namespace spirit = boost::spirit; 
namespace lex = spirit::lex; 

enum token_ids { 
    X = lex::min_token_id + 1, 
    Y, 
    Z, 
}; 

template<typename L> 
class word_count_tokens : public lex::lexer<L> 
{ 
    public: 
     word_count_tokens() { 
      this->self.add 
       ("[^ \t\n]+", X) 
       ("\n"  , Y) 
       ("."  , Z); 
     } 
}; 

struct counter 
{ 
    typedef bool result_type; 

    template<typename T> 
    bool operator() (const T &t, size_t &c, size_t &w, size_t &l) const { 
     switch (t.id()) { 
      case X: 
       ++w; c += boost::distance(t.value()); // t.value().size(); 
       break; 
      case Y: 
       ++l; ++c; 
       break; 
      case Z: 
       ++c; 
       break; 
     } 

     return true; 
    } 
}; 

int main (int argc, char **argv) 
{ 
    std::ifstream ifs (argv[1], std::ios::in | std::ios::binary); 
    ifs >> std::noskipws; 
    boost::spirit::istream_iterator first(ifs), last; 
    word_count_tokens<lex::lexertl::lexer<lex::lexertl::token<boost::spirit::istream_iterator>>> word_count_functor; 

    size_t w = 0, c = 0, l = 0; 
    bool r = lex::tokenize (first, last, word_count_functor, 
      boost::bind (counter(), _1, boost::ref (c), boost::ref (w), boost::ref (l))); 

    ifs.close(); 

    if (r) { 
     std::cout << l << ", " << w << ", " << c << std::endl; 
    } 
} 

當上本身運行，版畫

65, 183, 1665 

Answer 2

我認爲真正的問題沒有顯示。你不會顯示first或last，我有一種感覺，你可能會有臨時的。

這是我想出了驗證樣本，或許你可以看到它是你在做什麼---錯---不同:)

• Live on Coliru（內存映射的字節 - 矢量，經由const char*）
• And this alternative（使用spirit::istream_iterator）

#include <boost/spirit/include/lex_lexertl.hpp> 
#include <boost/spirit/include/qi.hpp> 
#include <fstream> 
#ifdef MEMORY_MAPPED 
# include <boost/iostreams/device/mapped_file.hpp> 
#endif 

namespace /*anon*/ 
{ 
    namespace qi =boost::spirit::qi; 
    namespace lex=boost::spirit::lex; 

    template <typename Lexer> 
     struct mylexer_t : lex::lexer<Lexer> 
    { 
     mylexer_t() 
     { 
      fileheader = "hello"; 

      this->self = fileheader 
       | space [ lex::_pass = lex::pass_flags::pass_ignore ]; 
     } 

     lex::token_def<lex::omit> 
      fileheader, space; 
    }; 

    template <typename Iterator> struct my_grammar_t 
     : public qi::grammar<Iterator> 
    { 
     template <typename TokenDef> 
      my_grammar_t(TokenDef const& tok) 
       : my_grammar_t::base_type(header) 
     { 
      header = tok.fileheader; 
      BOOST_SPIRIT_DEBUG_NODE(header); 
     } 

     private: 
     qi::rule<Iterator> header; 
    }; 
} 

namespace /* */ { 

    std::string safechar(char ch) { 
     switch (ch) { 
      case '\t': return "\\t"; break; 
      case '\0': return "\\0"; break; 
      case '\r': return "\\r"; break; 
      case '\n': return "\\n"; break; 
     } 
     return std::string(1, ch); 
    } 

    template <typename It> 
     std::string showtoken(const boost::iterator_range<It>& range) 
     { 
      std::ostringstream oss; 
      oss << '['; 
      std::transform(range.begin(), range.end(), std::ostream_iterator<std::string>(oss), safechar); 
      oss << ']'; 
      return oss.str(); 
     } 
} 

bool parsefile(const std::string& spec) 
{ 
#ifdef MEMORY_MAPPED 
    typedef char const* It; 
    boost::iostreams::mapped_file mmap(spec.c_str(), boost::iostreams::mapped_file::readonly); 
    char const *first = mmap.const_data(); 
    char const *last = first + mmap.size(); 
#else 
    typedef char const* It; 
    std::ifstream in(spec.c_str()); 
    in.unsetf(std::ios::skipws); 

    std::string v(std::istreambuf_iterator<char>(in.rdbuf()), std::istreambuf_iterator<char>()); 
    It first = &v[0]; 
    It last = first+v.size(); 
#endif 

    typedef lex::lexertl::token<It /*, boost::mpl::vector<char, unsigned int, std::string> */> token_type; 
    typedef lex::lexertl::actor_lexer<token_type> lexer_type; 

    typedef mylexer_t<lexer_type>::iterator_type iterator_type; 
    try 
    { 
     static mylexer_t<lexer_type> mylexer; 
     static my_grammar_t<iterator_type> parser(mylexer); 

     auto iter = mylexer.begin(first, last); 
     auto end = mylexer.end(); 

     bool r = qi::parse(iter, end, parser); 

     r = r && (iter == end); 

     if (!r) 
      std::cerr << spec << ": parsing failed at: \"" << std::string(first, last) << "\"\n"; 
     return r; 
    } 
    catch (const qi::expectation_failure<iterator_type>& e) 
    { 
     std::cerr << "FIXME: expected " << e.what_ << ", got '"; 
     for (auto it=e.first; it!=e.last; it++) 
      std::cerr << showtoken(it->value()); 
     std::cerr << "'" << std::endl; 
     return false; 
    } 
} 

int main() 
{ 
    if (parsefile("input.bin")) 
     return 0; 
    return 1; 
} 

對於變種：

typedef boost::spirit::istream_iterator It; 
std::ifstream in(spec.c_str()); 
in.unsetf(std::ios::skipws); 

It first(in), last;