Changed so I you can pass a per-request user-agent string.
This commit is contained in:
kebernet
parent
9b5acb1585
commit
9a8c0fd5ac
4 changed files with 352 additions and 317 deletions
4
pom.xml
4
pom.xml
|
|
@ -3,9 +3,9 @@
|
|||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.rometools</groupId>
|
||||
<artifactId>rome-fetcher</artifactId>
|
||||
<artifactId>fetcher</artifactId>
|
||||
|
||||
<name>Rome HTTP Fetcher</name>
|
||||
<name>rome-fetcher</name>
|
||||
<version>1.0.1-SNAPSHOT</version>
|
||||
<inceptionYear>2004</inceptionYear>
|
||||
<organization>
|
||||
|
|
|
|||
|
|
@ -16,46 +16,53 @@
|
|||
*/
|
||||
package com.sun.syndication.fetcher;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
|
||||
import com.sun.syndication.feed.synd.SyndFeed;
|
||||
import com.sun.syndication.io.FeedException;
|
||||
|
||||
public interface FeedFetcher {
|
||||
import java.io.IOException;
|
||||
|
||||
import java.net.URL;
|
||||
|
||||
|
||||
public interface FeedFetcher {
|
||||
/**
|
||||
* <p>The default user agent. It is not marked final so
|
||||
* buggy java compiler will not write this string
|
||||
* into all classes that reference it.</p>
|
||||
*
|
||||
* <p>http://tinyurl.com/64t5n points to https://rome.dev.java.net/
|
||||
* <p>http://tinyurl.com/64t5n points to https://rome.dev.java.net
|
||||
* Some servers ban user agents with "Java" in the name.</p>
|
||||
*
|
||||
*/
|
||||
public static String DEFAULT_USER_AGENT = "Rome Client (http://tinyurl.com/64t5n)";
|
||||
|
||||
/**
|
||||
* @return the User-Agent currently being sent to servers
|
||||
*/
|
||||
public abstract String getUserAgent();
|
||||
|
||||
/**
|
||||
* @param string The User-Agent to sent to servers
|
||||
*/
|
||||
public abstract void setUserAgent(String string);
|
||||
|
||||
/**
|
||||
* Retrieve a feed over HTTP
|
||||
*
|
||||
* @param feedUrl A non-null URL of a RSS/Atom feed to retrieve
|
||||
* @return A {@link com.sun.syndication.feed.synd.SyndFeed} object
|
||||
* @throws IllegalArgumentException if the URL is null;
|
||||
* @throws IOException if a TCP error occurs
|
||||
* @throws FeedException if the feed is not valid
|
||||
* @throws FetcherException if a HTTP error occurred
|
||||
* @return the User-Agent currently being sent to servers
|
||||
*/
|
||||
public abstract SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException;
|
||||
public abstract String getUserAgent();
|
||||
|
||||
/**
|
||||
* <p>Turn on or off rfc3229 delta encoding</p>
|
||||
*
|
||||
* <p>See http://www.ietf.org/rfc/rfc3229.txt and http://bobwyman.pubsub.com/main/2004/09/using_rfc3229_w.html</p>
|
||||
*
|
||||
* <p>NOTE: This is experimental and feedback is welcome!</p>
|
||||
*
|
||||
* @param useDeltaEncoding
|
||||
*/
|
||||
public abstract void setUsingDeltaEncoding(boolean useDeltaEncoding);
|
||||
|
||||
/**
|
||||
* <p>Is this fetcher using rfc3229 delta encoding?</p>
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public abstract boolean isUsingDeltaEncoding();
|
||||
|
||||
/**
|
||||
* <p>Add a FetcherListener.</p>
|
||||
|
|
@ -75,22 +82,20 @@ public interface FeedFetcher {
|
|||
public abstract void removeFetcherEventListener(FetcherListener listener);
|
||||
|
||||
/**
|
||||
* <p>Is this fetcher using rfc3229 delta encoding?</p>
|
||||
* Retrieve a feed over HTTP
|
||||
*
|
||||
* @return
|
||||
* @param feedUrl A non-null URL of a RSS/Atom feed to retrieve
|
||||
* @return A {@link com.sun.syndication.feed.synd.SyndFeed} object
|
||||
* @throws IllegalArgumentException if the URL is null;
|
||||
* @throws IOException if a TCP error occurs
|
||||
* @throws FeedException if the feed is not valid
|
||||
* @throws FetcherException if a HTTP error occurred
|
||||
*/
|
||||
public abstract boolean isUsingDeltaEncoding();
|
||||
public abstract SyndFeed retrieveFeed(URL feedUrl)
|
||||
throws IllegalArgumentException, IOException, FeedException, FetcherException;
|
||||
|
||||
/**
|
||||
* <p>Turn on or off rfc3229 delta encoding</p>
|
||||
*
|
||||
* <p>See http://www.ietf.org/rfc/rfc3229.txt and http://bobwyman.pubsub.com/main/2004/09/using_rfc3229_w.html</p>
|
||||
*
|
||||
* <p>NOTE: This is experimental and feedback is welcome!</p>
|
||||
*
|
||||
* @param useDeltaEncoding
|
||||
*/
|
||||
public abstract void setUsingDeltaEncoding(boolean useDeltaEncoding);
|
||||
public SyndFeed retrieveFeed(String userAgent, URL url)
|
||||
throws IllegalArgumentException, IOException, FeedException, FetcherException;
|
||||
|
||||
/**
|
||||
* If set to true, the WireFeed will be made accessible from the SyndFeed object returned from the Fetcher
|
||||
|
|
|
|||
|
|
@ -14,15 +14,14 @@
|
|||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package com.sun.syndication.fetcher.impl;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
import com.sun.syndication.feed.synd.SyndFeed;
|
||||
import com.sun.syndication.fetcher.FetcherEvent;
|
||||
import com.sun.syndication.fetcher.FetcherException;
|
||||
import com.sun.syndication.io.FeedException;
|
||||
import com.sun.syndication.io.SyndFeedInput;
|
||||
import com.sun.syndication.io.XmlReader;
|
||||
|
||||
import org.apache.commons.httpclient.Credentials;
|
||||
import org.apache.commons.httpclient.Header;
|
||||
|
|
@ -32,20 +31,22 @@ import org.apache.commons.httpclient.HttpMethod;
|
|||
import org.apache.commons.httpclient.methods.GetMethod;
|
||||
import org.apache.commons.httpclient.params.HttpClientParams;
|
||||
|
||||
import com.sun.syndication.feed.synd.SyndFeed;
|
||||
import com.sun.syndication.fetcher.FetcherEvent;
|
||||
import com.sun.syndication.fetcher.FetcherException;
|
||||
import com.sun.syndication.io.FeedException;
|
||||
import com.sun.syndication.io.SyndFeedInput;
|
||||
import com.sun.syndication.io.XmlReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
|
||||
/**
|
||||
* @author Nick Lothian
|
||||
*/
|
||||
public class HttpClientFeedFetcher extends AbstractFeedFetcher {
|
||||
|
||||
private FeedFetcherCache feedInfoCache;
|
||||
private CredentialSupplier credentialSupplier;
|
||||
private FeedFetcherCache feedInfoCache;
|
||||
private volatile HttpClientMethodCallbackIntf httpClientMethodCallback;
|
||||
private volatile HttpClientParams httpClientParams;
|
||||
|
||||
|
|
@ -62,26 +63,11 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
|
|||
setFeedInfoCache(cache);
|
||||
}
|
||||
|
||||
|
||||
public HttpClientFeedFetcher(FeedFetcherCache cache, CredentialSupplier credentialSupplier) {
|
||||
this(cache);
|
||||
setCredentialSupplier(credentialSupplier);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return Returns the httpClientParams.
|
||||
*/
|
||||
public synchronized HttpClientParams getHttpClientParams() {
|
||||
return this.httpClientParams;
|
||||
}
|
||||
/**
|
||||
* @param httpClientParams The httpClientParams to set.
|
||||
*/
|
||||
public synchronized void setHttpClientParams(HttpClientParams httpClientParams) {
|
||||
this.httpClientParams = httpClientParams;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param timeout Sets the connect timeout for the HttpClient but using the URLConnection method name.
|
||||
* Uses the HttpClientParams method setConnectionManagerTimeout instead of setConnectTimeout
|
||||
|
|
@ -97,7 +83,58 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
|
|||
*
|
||||
*/
|
||||
public int getConnectTimeout() {
|
||||
return (int) this.getHttpClientParams().getConnectionManagerTimeout();
|
||||
return (int) this.getHttpClientParams()
|
||||
.getConnectionManagerTimeout();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param credentialSupplier The credentialSupplier to set.
|
||||
*/
|
||||
public synchronized void setCredentialSupplier(CredentialSupplier credentialSupplier) {
|
||||
this.credentialSupplier = credentialSupplier;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Returns the credentialSupplier.
|
||||
*/
|
||||
public synchronized CredentialSupplier getCredentialSupplier() {
|
||||
return credentialSupplier;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param feedInfoCache the feedInfoCache to set
|
||||
*/
|
||||
public synchronized void setFeedInfoCache(FeedFetcherCache feedInfoCache) {
|
||||
this.feedInfoCache = feedInfoCache;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the feedInfoCache.
|
||||
*/
|
||||
public synchronized FeedFetcherCache getFeedInfoCache() {
|
||||
return feedInfoCache;
|
||||
}
|
||||
|
||||
public synchronized void setHttpClientMethodCallback(HttpClientMethodCallbackIntf httpClientMethodCallback) {
|
||||
this.httpClientMethodCallback = httpClientMethodCallback;
|
||||
}
|
||||
|
||||
public HttpClientMethodCallbackIntf getHttpClientMethodCallback() {
|
||||
return httpClientMethodCallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param httpClientParams The httpClientParams to set.
|
||||
*/
|
||||
public synchronized void setHttpClientParams(HttpClientParams httpClientParams) {
|
||||
this.httpClientParams = httpClientParams;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Returns the httpClientParams.
|
||||
*/
|
||||
public synchronized HttpClientParams getHttpClientParams() {
|
||||
return this.httpClientParams;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -111,71 +148,48 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
|
|||
* @param timeout Sets the read timeout for the URLConnection to a specified timeout, in milliseconds.
|
||||
*/
|
||||
public int getReadTimeout() {
|
||||
return (int) this.getHttpClientParams().getSoTimeout();
|
||||
return (int) this.getHttpClientParams()
|
||||
.getSoTimeout();
|
||||
}
|
||||
|
||||
public HttpClientMethodCallbackIntf getHttpClientMethodCallback() {
|
||||
return httpClientMethodCallback;
|
||||
}
|
||||
|
||||
public synchronized void setHttpClientMethodCallback(HttpClientMethodCallbackIntf httpClientMethodCallback) {
|
||||
this.httpClientMethodCallback = httpClientMethodCallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the feedInfoCache.
|
||||
*/
|
||||
public synchronized FeedFetcherCache getFeedInfoCache() {
|
||||
return feedInfoCache;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param feedInfoCache the feedInfoCache to set
|
||||
*/
|
||||
public synchronized void setFeedInfoCache(FeedFetcherCache feedInfoCache) {
|
||||
this.feedInfoCache = feedInfoCache;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return Returns the credentialSupplier.
|
||||
*/
|
||||
public synchronized CredentialSupplier getCredentialSupplier() {
|
||||
return credentialSupplier;
|
||||
}
|
||||
/**
|
||||
* @param credentialSupplier The credentialSupplier to set.
|
||||
*/
|
||||
public synchronized void setCredentialSupplier(CredentialSupplier credentialSupplier) {
|
||||
this.credentialSupplier = credentialSupplier;
|
||||
public SyndFeed retrieveFeed(URL url) throws IllegalArgumentException, IOException, FeedException, FetcherException {
|
||||
return this.retrieveFeed(this.getUserAgent(), url);
|
||||
}
|
||||
|
||||
/**
|
||||
* @see com.sun.syndication.fetcher.FeedFetcher#retrieveFeed(java.net.URL)
|
||||
*/
|
||||
public SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
|
||||
public SyndFeed retrieveFeed(String userAgent, URL feedUrl)
|
||||
throws IllegalArgumentException, IOException, FeedException, FetcherException {
|
||||
if (feedUrl == null) {
|
||||
throw new IllegalArgumentException("null is not a valid URL");
|
||||
}
|
||||
|
||||
// TODO Fix this
|
||||
//System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.SimpleLog");
|
||||
HttpClient client = new HttpClient(httpClientParams);
|
||||
|
||||
if (getCredentialSupplier() != null) {
|
||||
client.getState().setAuthenticationPreemptive(true);
|
||||
// TODO what should realm be here?
|
||||
Credentials credentials = getCredentialSupplier().getCredentials(null, feedUrl.getHost());
|
||||
if (credentials != null) {
|
||||
client.getState().setCredentials(null, feedUrl.getHost(), credentials);
|
||||
}
|
||||
}
|
||||
client.getState()
|
||||
.setAuthenticationPreemptive(true);
|
||||
|
||||
// TODO what should realm be here?
|
||||
Credentials credentials = getCredentialSupplier()
|
||||
.getCredentials(null, feedUrl.getHost());
|
||||
|
||||
if (credentials != null) {
|
||||
client.getState()
|
||||
.setCredentials(null, feedUrl.getHost(), credentials);
|
||||
}
|
||||
}
|
||||
|
||||
System.setProperty("httpclient.useragent", getUserAgent());
|
||||
|
||||
String urlStr = feedUrl.toString();
|
||||
|
||||
HttpMethod method = new GetMethod(urlStr);
|
||||
method.addRequestHeader("Accept-Encoding", "gzip");
|
||||
method.addRequestHeader("User-Agent", getUserAgent());
|
||||
method.addRequestHeader("User-Agent", userAgent);
|
||||
method.setFollowRedirects(true);
|
||||
|
||||
if (httpClientMethodCallback != null) {
|
||||
|
|
@ -185,9 +199,9 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
|
|||
}
|
||||
|
||||
FeedFetcherCache cache = getFeedInfoCache();
|
||||
|
||||
if (cache != null) {
|
||||
// retrieve feed
|
||||
|
||||
try {
|
||||
if (isUsingDeltaEncoding()) {
|
||||
method.setRequestHeader("A-IM", "feed");
|
||||
|
|
@ -196,6 +210,7 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
|
|||
// get the feed info from the cache
|
||||
// Note that syndFeedInfo will be null if it is not in the cache
|
||||
SyndFeedInfo syndFeedInfo = cache.getFeedInfo(feedUrl);
|
||||
|
||||
if (syndFeedInfo != null) {
|
||||
method.setRequestHeader("If-None-Match", syndFeedInfo.getETag());
|
||||
|
||||
|
|
@ -223,7 +238,6 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
|
|||
method.releaseConnection();
|
||||
method.recycle();
|
||||
}
|
||||
|
||||
} else {
|
||||
// cache is not in use
|
||||
try {
|
||||
|
|
@ -239,6 +253,19 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
|
|||
}
|
||||
}
|
||||
|
||||
private SyndFeed getFeed(SyndFeedInfo syndFeedInfo, String urlStr, HttpMethod method, int statusCode)
|
||||
throws IOException, HttpException, FetcherException, FeedException {
|
||||
if ((statusCode == HttpURLConnection.HTTP_NOT_MODIFIED) && (syndFeedInfo != null)) {
|
||||
fireEvent(FetcherEvent.EVENT_TYPE_FEED_UNCHANGED, urlStr);
|
||||
|
||||
return syndFeedInfo.getSyndFeed();
|
||||
}
|
||||
|
||||
SyndFeed feed = retrieveFeed(urlStr, method);
|
||||
fireEvent(FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, urlStr, feed);
|
||||
|
||||
return feed;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param feedUrl
|
||||
|
|
@ -248,7 +275,8 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
|
|||
* @return
|
||||
* @throws MalformedURLException
|
||||
*/
|
||||
private SyndFeedInfo buildSyndFeedInfo(URL feedUrl, String urlStr, HttpMethod method, SyndFeed feed, int statusCode) throws MalformedURLException {
|
||||
private SyndFeedInfo buildSyndFeedInfo(URL feedUrl, String urlStr, HttpMethod method, SyndFeed feed, int statusCode)
|
||||
throws MalformedURLException {
|
||||
SyndFeedInfo syndFeedInfo;
|
||||
syndFeedInfo = new SyndFeedInfo();
|
||||
|
||||
|
|
@ -257,12 +285,16 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
|
|||
syndFeedInfo.setId(feedUrl.toString());
|
||||
|
||||
Header imHeader = method.getResponseHeader("IM");
|
||||
if (imHeader != null && imHeader.getValue().indexOf("feed") >= 0 && isUsingDeltaEncoding()) {
|
||||
|
||||
if ((imHeader != null) && (imHeader.getValue()
|
||||
.indexOf("feed") >= 0) && isUsingDeltaEncoding()) {
|
||||
FeedFetcherCache cache = getFeedInfoCache();
|
||||
if (cache != null && statusCode == 226) {
|
||||
|
||||
if ((cache != null) && (statusCode == 226)) {
|
||||
// client is setup to use http delta encoding and the server supports it and has returned a delta encoded response
|
||||
// This response only includes new items
|
||||
SyndFeedInfo cachedInfo = cache.getFeedInfo(feedUrl);
|
||||
|
||||
if (cachedInfo != null) {
|
||||
SyndFeed cachedFeed = cachedInfo.getSyndFeed();
|
||||
|
||||
|
|
@ -273,11 +305,13 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
|
|||
}
|
||||
|
||||
Header lastModifiedHeader = method.getResponseHeader("Last-Modified");
|
||||
|
||||
if (lastModifiedHeader != null) {
|
||||
syndFeedInfo.setLastModified(lastModifiedHeader.getValue());
|
||||
}
|
||||
|
||||
Header eTagHeader = method.getResponseHeader("ETag");
|
||||
|
||||
if (eTagHeader != null) {
|
||||
syndFeedInfo.setETag(eTagHeader.getValue());
|
||||
}
|
||||
|
|
@ -297,21 +331,26 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
|
|||
* @throws FetcherException
|
||||
* @throws FeedException
|
||||
*/
|
||||
private SyndFeed retrieveFeed(String urlStr, HttpMethod method) throws IOException, HttpException, FetcherException, FeedException {
|
||||
|
||||
private SyndFeed retrieveFeed(String urlStr, HttpMethod method)
|
||||
throws IOException, HttpException, FetcherException, FeedException {
|
||||
InputStream stream = null;
|
||||
if ((method.getResponseHeader("Content-Encoding") != null) && ("gzip".equalsIgnoreCase(method.getResponseHeader("Content-Encoding").getValue()))) {
|
||||
|
||||
if ((method.getResponseHeader("Content-Encoding") != null) &&
|
||||
("gzip".equalsIgnoreCase(method.getResponseHeader("Content-Encoding").getValue()))) {
|
||||
stream = new GZIPInputStream(method.getResponseBodyAsStream());
|
||||
} else {
|
||||
stream = method.getResponseBodyAsStream();
|
||||
}
|
||||
|
||||
try {
|
||||
XmlReader reader = null;
|
||||
|
||||
if (method.getResponseHeader("Content-Type") != null) {
|
||||
reader = new XmlReader(stream, method.getResponseHeader("Content-Type").getValue(), true);
|
||||
} else {
|
||||
reader = new XmlReader(stream, true);
|
||||
}
|
||||
|
||||
SyndFeedInput syndFeedInput = new SyndFeedInput();
|
||||
syndFeedInput.setPreserveWireFeed(isPreserveWireFeed());
|
||||
|
||||
|
|
@ -323,18 +362,6 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
|
|||
}
|
||||
}
|
||||
|
||||
private SyndFeed getFeed(SyndFeedInfo syndFeedInfo, String urlStr, HttpMethod method, int statusCode) throws IOException, HttpException, FetcherException, FeedException {
|
||||
|
||||
if (statusCode == HttpURLConnection.HTTP_NOT_MODIFIED && syndFeedInfo != null) {
|
||||
fireEvent(FetcherEvent.EVENT_TYPE_FEED_UNCHANGED, urlStr);
|
||||
return syndFeedInfo.getSyndFeed();
|
||||
}
|
||||
|
||||
SyndFeed feed = retrieveFeed(urlStr, method);
|
||||
fireEvent(FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, urlStr, feed);
|
||||
return feed;
|
||||
}
|
||||
|
||||
public interface CredentialSupplier {
|
||||
public Credentials getCredentials(String realm, String host);
|
||||
}
|
||||
|
|
@ -349,5 +376,4 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
|
|||
*/
|
||||
public void afterHttpClientMethodCreate(HttpMethod method);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -83,6 +83,10 @@ public class HttpURLFeedFetcher extends AbstractFeedFetcher {
|
|||
setFeedInfoCache(feedInfoCache);
|
||||
}
|
||||
|
||||
public SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
|
||||
return this.retrieveFeed(this.getUserAgent(), feedUrl);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a feed over HTTP
|
||||
*
|
||||
|
|
@ -93,7 +97,7 @@ public class HttpURLFeedFetcher extends AbstractFeedFetcher {
|
|||
* @throws FeedException if the feed is not valid
|
||||
* @throws FetcherException if a HTTP error occurred
|
||||
*/
|
||||
public SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
|
||||
public SyndFeed retrieveFeed(String userAgent, URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
|
||||
if (feedUrl == null) {
|
||||
throw new IllegalArgumentException("null is not a valid URL");
|
||||
}
|
||||
|
|
@ -140,6 +144,9 @@ public class HttpURLFeedFetcher extends AbstractFeedFetcher {
|
|||
fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, connection);
|
||||
InputStream inputStream = null;
|
||||
setRequestHeaders(connection, null);
|
||||
|
||||
connection.addRequestProperty("User-Agent", userAgent);
|
||||
|
||||
httpConnection.connect();
|
||||
try {
|
||||
inputStream = httpConnection.getInputStream();
|
||||
|
|
@ -238,9 +245,6 @@ public class HttpURLFeedFetcher extends AbstractFeedFetcher {
|
|||
// header to retrieve feed gzipped
|
||||
connection.setRequestProperty("Accept-Encoding", "gzip");
|
||||
|
||||
// set the user agent
|
||||
connection.addRequestProperty("User-Agent", getUserAgent());
|
||||
|
||||
if (isUsingDeltaEncoding()) {
|
||||
connection.addRequestProperty("A-IM", "feed");
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue