2012-09-19 106 views
5

我有一個原始字節數組,我需要將其標記爲java中的一個字節數組列表。以下方法聲明更好地解釋了這一點。Tokenize字節數組

public static List<byte[]> splitMessage(byte[] rawByte, String tokenDelimiter) 

示例運行。

示例執行1:

原始字節

byte[] rawBytes = new byte[]{72,118,121,49,85,118,97,113,111,124,44,124,49,48,43,57,48,36,63,49,66,70,22,18,124,44,124,23,27,25,54,24,24,34,44,57,69,66,49,47,66,16,39,35,32,36,30,50,63,124,44,124,16,18,24,64,4,94,124,44,124,19,31,42,55,66,46,34,62,34,37}; 

tokenDelimiter|,|(即124,44,124

所以列表返回是:

Token 1: 72,118,121,49,85,118,97,113,111 
Token 2: 49,48,43,57,48,36,63,49,66,70,22,18 
Token 3: 23,27,25,54,24,24,34,44,57,69,66,49,47,66,16,39,35,32,36,30,50,63, 
Token 4: 16,18,24,64,4,94 
Token 5: 19,31,42,55,66,46,34,62,34,37 

示例執行2:

byte[] rawBytes = new byte[]{72,118,121,49,85,118,97,113,111,124,44,124,49,48,43,57,48,36,63,49,66,70,22,18,124,44,124,124,44,124,23,27,25,54,24,24,34,44,57,69,66,49,47,66,16,39,35,32,36,30,50,63,124,44,124,16,18,24,64,4,94,124,44,124,19,31,42,55,66,46,34,62,34,37,124,44,124,124,44,124}; 

tokenDelimiter|,|(即124,44,124

Token 1: 72,118,121,49,85,118,97,113,111 
Token 2: 49,48,43,57,48,36,63,49,66,70,22,18 
Token 3: <Empty> 
Token 3: 23,27,25,54,24,24,34,44,57,69,66,49,47,66,16,39,35,32,36,30,50,63, 
Token 4: 16,18,24,64,4,94 
Token 5: 19,31,42,55,66,46,34,62,34,37 
Token 6: <Empty> 
Token 7: <Empty> 

我能夠才達到例如從下面的代碼段中運行。但是在第二個標籤中卡住了。

public static List<byte[]> splitMessageSept19(byte[] rawByte, String tokenDelimiter) throws UnsupportedEncodingException 
{ 
    List<byte[]> tokens = new ArrayList<byte[]>(); 

    final byte[] byteArray = tokenDelimiter.getBytes("UTF-8"); 
    final byte byteDelimitorFirstByte = byteArray[0]; 

    int bytenum =0 ; 
    int lastIndex = 0; 
    int storIterator =0; 
    for (int iterator = 0 ; iterator <= rawByte.length ; iterator++) 
    { 
     if (iterator == rawByte.length || rawByte[iterator] == byteDelimitorFirstByte) 
     { 
      storIterator = iterator; 
      if (iterator != rawByte.length) 
      { 
       for (int i=0 ; i < byteArray.length ; i++) 
       { 
        if (rawByte[iterator] == byteArray[i]) 
        { 
         iterator++ ; 
         continue; 
        } 
        else 
        { 
         break; 
        } 
       } 
      } 
      byte[] byteArrayExtracted = new byte[storIterator - lastIndex]; 
      System.arraycopy(rawByte, lastIndex, byteArrayExtracted, 0, 
          storIterator - lastIndex); 
      lastIndex = iterator ; 
      tokens.add(byteArrayExtracted); 
      byteArrayExtracted = null; 
     } 
    } 
    for (byte[] bytetoken : tokens) 
    { 
     System.out.println("Token received is: " + new String(bytetoken, "UTF-8")); 
    } 
    return tokens; 
} 

有沒有人遇到類似的令牌化數組問題?請建議是否有其他方法來標記數組。

請注意:我不想將字節流轉換爲String,以字符串格式進行標記並轉換回字節。它可能存在編碼問題。

+0

爲什麼你不只是跳過你的代碼中的空令牌? – mishadoff

回答

1

如果您使用ISO-8859-1,那麼字節會保留原樣。

private static final Charset ISO_8859_1 = Charset.forName("ISO-8859-1"); 

public static List<byte[]> splitMessageSept19(byte[] rawByte, String tokenDelimiter) { 
    Pattern pattern = Pattern.compile(tokenDelimiter, Pattern.LITERAL); 
    String[] parts = pattern.split(new String(rawByte, ISO_8859_1), -1); 
    List<byte[]> ret = new ArrayList<byte[]>(); 
    for (String part : parts) 
     ret.add(part.getBytes(ISO_8859_1)); 
    return ret; 
} 

public static void main(String... args) { 
    StringBuilder sb = new StringBuilder(); 
    for(int i=0;i<256;i++) 
     sb.append((char) i); 
    byte[] bytes = sb.toString().getBytes(ISO_8859_1); 
    List<byte[]> list = splitMessageSept19(bytes, ","); 
    for (byte[] b : list) 
     System.out.println(Arrays.toString(b)); 
} 

打印

[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18 ,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43號決議] [45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68 ,69,70,71,72,73,74,75,66,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93 ,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118 ,119,120,121,122,123,124,125,126,127,-128,-127,-126,-125,-124,-123,-122,-121,-120,-119, - 118,-117,-116,-115,-114,-113,-112, -111,-110,-109,-108,-107,-106,-105,-104,-103,-102,-101,-100,-99,-98,-97,-96,-95 ,-94,-93,-92,-91,-90,-89,-88,-87,-86,-85,-84,-83,-82,-81,-80,-79, - -78,-77,-76,-75,-74,-73,-72,-71,-70,-69,-68,-67,-66,-65,-64,-63,-62, -61,-60,-59,-58,-57,-56,-55,-54,-53,-52,-51,-50,-49,-48,-47,-46,-45 ,-44,-43,-42,-41,-40,-39,-38,-37,-36,-35,-34,-33,-32,-31,-30,-29, - 28,-27,-26,-25,-24,-23,-22,-21,-20,-19,-18,-17,-16,-15,-14,-13,-12, -11,-10,-9,-8,-7,-6,-5,-4,-3,-2,-1]

調用

byte[] rawBytes = new byte[]{72,118,121,49,85,118,97,113,111,124,44,124,49,48,43,57,48,36,63,49,66,70,22,18,124,44,124,124,44,124,23,27,25,54,24,24,34,44,57,69,66,49,47,66,16,39,35,32,36,30,50,63,124,44,124,16,18,24,64,4,94,124,44,124,19,31,42,55,66,46,34,62,34,37,124,44,124,124,44,124}; 
List<byte[]> list = splitMessageSept19(rawBytes, "|,|"); 

產生

[72, 118, 121, 49, 85, 118, 97, 113, 111] 
[49, 48, 43, 57, 48, 36, 63, 49, 66, 70, 22, 18] 
[] 
[23, 27, 25, 54, 24, 24, 34, 44, 57, 69, 66, 49, 47, 66, 16, 39, 35, 32, 36, 30, 50, 63] 
[16, 18, 24, 64, 4, 94] 
[19, 31, 42, 55, 66, 46, 34, 62, 34, 37] 
[] 
[] 
+0

我正在尋找一種算法,它不會將字節流轉換爲字符串,以字符串格式進行標記並轉換回字節。因爲我擔心會有編碼問題。我試過你的解決方案,這不會給編碼問題。感謝您的快速回復。 – user813063

+0

訣竅是使用'ISO-8859-1'編碼,只要提供'0 <= n <= 255',映射'n => n'。 –

0

您應該看看KMP算法:KMP on wikipedia以及其他字符串搜索算法。

作爲一個快速解決您的代碼試試這個:

public static List<byte[]> splitMessageSept19(byte[] rawByte, String tokenDelimiter) throws UnsupportedEncodingException 
{ 
    List<byte[]> tokens = new ArrayList<byte[]>(); 

    final byte[] byteArray = tokenDelimiter.getBytes("UTF-8"); 
    int lastIndex = 0; 

    for (int iterator = 0; iterator < rawByte.length - byteArray.length + 1;) 
    { 
     boolean patternFound = true; 
     for (int i = 0; i < byteArray.length; i++) 
     { 
      if (rawByte[iterator + i] != byteArray[i]) 
      { 
       patternFound = false; 
       break; 
      } 
     } 
     if (patternFound) 
     { 
      byte[] byteArrayExtracted = new byte[iterator - lastIndex]; 
      System.arraycopy(rawByte, lastIndex, byteArrayExtracted, 0, iterator - lastIndex); 
      iterator += byteArray.length; 
      lastIndex = iterator; 
      tokens.add(byteArrayExtracted); 
     } 
     else 
      iterator++; 

    } 
    for (byte[] bytetoken : tokens) 
    { 
     System.out.println("Token received is: " + new String(bytetoken, "UTF-8")); 
    } 
    return tokens; 
} 

我還沒有編譯這個代碼,所以它可能被打破,但即使我希望你的想法。

這是一個天真的算法,它非常慢,特別是如果分隔符很長。如果你想要更好的東西,去看看其他一些字符串搜索算法。