在應用程序引擎在1M左右，造成UrlFetch問題

Question

在應用程序引擎超過1M取（），我使用範圍頭，然後結合這些pieces.and我的代碼：

int startpos=0; 
int endpos; 
int seg=1; 
int len=1; 
while(len>0){ 
endpos=startpos+seg; 
httpConn = (HttpURLConnection) u.openConnection(); 
httpConn.setRequestMethod("GET"); 
con.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.1.14) Gecko/20080404 Firefox/2.0.0.14"); 

con.setRequestProperty("Range", "bytes=" + startpos + "-" + endpos); 
con.connect(); 
InputStream in=con.getInputStream(); 

len=con.getContentLength(); 
byte[] b=new byte[len]; 
in.read(b, 0, len); 

startpos+=len; 

} 但是當它到「InputStream in = con.getInputStream（）;」，它的調試是「URL抓取響應太大的問題」 所以我不知道這些代碼有什麼問題。 還有其他的方法來獲取（）超過1M？

Answer 1

並非所有的HTTP服務器都支持範圍請求，尤其是當它涉及到動態內容服務的框架 - 他們會簡單地忽略Range頭和送你整個響應。

最近的1.4.0版本增加的URLFetch響應限制32MB，雖然如此，你不再需要做到這一點。

Answer 2

我有同樣的問題和入侵了一點類來使用HTTP範圍參數AppEngine上模擬的輸入流。它允許您以面向行的方式閱讀大於極限的文件。我在下面附上吧，雖然你可能需要適應它爲您的目的：

package com.theodorebook.AEStreamer; 

import java.io.InputStream; 
import java.net.URL; 
import java.net.URLConnection; 
import java.util.Arrays; 
import java.util.logging.Logger; 

/** 
* A class to simulate a stream in appengine, which insists on downloading 
* an entire URL before letting you do anything with it. This enables one 
* to read files larger than the size limits. 
* 
* @author Theodore Book (theodorebook at gmail dot com) 
* 
*/ 
public class AEStreamer { 
    private static final int BITE_SIZE = 0x10000; //How big a chunk to grab at a time 
    private static final byte TERMINATOR = '\n'; //String terminator 

    private int mCurrentPosition = 0; //The current position in the file 
    private int mOffset = -1; //The offset of the current block 
    private long mValidBytes = 0; //The number of valid bytes in the chunk 
    private byte[] mChunk = new byte[BITE_SIZE]; 
    private boolean mComplete = false; 
    private String mURL; 

    private static final Logger log = Logger.getLogger(AEStreamer.class.getName()); 

    public AEStreamer(String url) { 
     mURL = url; 
    } 

    /** 
    * Returns the next line from the source, or null on empty 
    * @return 
    */ 
    public String readLine() { 
     String line = ""; 

     //See if we have something to read 
     if (mCurrentPosition >= mOffset + mValidBytes) { 
      if (mComplete) 
       return null; 
      readChunk(); 
     } 
     if (mValidBytes == 0) 
      return null; 

     //Read until we reach a terminator 
     int endPtr = mCurrentPosition - mOffset; 
     while (mChunk[endPtr] != TERMINATOR) { 
      endPtr++; 

      //If we reach the end of the block 
      if (endPtr == mValidBytes) { 
       line += new String(Arrays.copyOfRange(mChunk, mCurrentPosition - mOffset, endPtr)); 
       mCurrentPosition += (endPtr - mCurrentPosition + mOffset); 
       if (mComplete) { 
        return line; 
       } else { 
        readChunk(); 
        endPtr = mCurrentPosition - mOffset; 
       } 
      } 
     } 
     line += new String(Arrays.copyOfRange(mChunk, mCurrentPosition - mOffset, endPtr)); 
     mCurrentPosition += (endPtr - mCurrentPosition + mOffset); 
     mCurrentPosition++; 
     return line; 
    } 

    /** 
    * Reads the next chunk from the server 
    */ 
    private void readChunk() { 
     if (mOffset < 0) 
      mOffset = 0; 
     else 
      mOffset += BITE_SIZE; 

     try { 
      URL url = new URL(mURL); 
      URLConnection request = url.openConnection(); 
      request.setRequestProperty("Range", "bytes=" + (mOffset + 1) + "-" + (mOffset + BITE_SIZE)); 
      InputStream inStream = request.getInputStream(); 
      mValidBytes = inStream.read(mChunk); 
      inStream.close(); 
     } catch (Exception e) { 
      log.severe("Unable to read " + mURL + ": " + e.getLocalizedMessage()); 
      mComplete = true; 
      mValidBytes = 0; 
      return; 
     } 

     if (mValidBytes < BITE_SIZE) 
      mComplete = true; 

     //log.info("Read " + mValidBytes + " bytes"); 
    } 
}