Changed so I you can pass a per-request user-agent string.

pom.xml

@@ -3,9 +3,9 @@
     <modelVersion>4.0.0</modelVersion>
 
     <groupId>org.rometools</groupId>
-    <artifactId>rome-fetcher</artifactId>
+    <artifactId>fetcher</artifactId>
 
-    <name>Rome HTTP Fetcher</name>
+    <name>rome-fetcher</name>
     <version>1.0.1-SNAPSHOT</version>
     <inceptionYear>2004</inceptionYear>
     <organization>

src/java/com/sun/syndication/fetcher/FeedFetcher.java

@@ -16,46 +16,53 @@
  */
 package com.sun.syndication.fetcher;
 
-import java.io.IOException;
-import java.net.URL;
-
 import com.sun.syndication.feed.synd.SyndFeed;
 import com.sun.syndication.io.FeedException;
 
-public interface FeedFetcher {
+import java.io.IOException;
 
+import java.net.URL;
+
+
+public interface FeedFetcher {
     /**
      * <p>The default user agent. It is not marked final so
      * buggy java compiler will not write this string
      * into all classes that reference it.</p>
      *
-     * <p>http://tinyurl.com/64t5n points to https://rome.dev.java.net/
+     * <p>http://tinyurl.com/64t5n points to https://rome.dev.java.net
      * Some servers ban user agents with "Java" in the name.</p>
      *
      */
     public static String DEFAULT_USER_AGENT = "Rome Client (http://tinyurl.com/64t5n)";
 
-    /**
-     * @return the User-Agent currently being sent to servers
-     */
-    public abstract String getUserAgent();
-
     /**
      * @param string The User-Agent to sent to servers
      */
     public abstract void setUserAgent(String string);
 
     /**
-     * Retrieve a feed over HTTP
+     * @return the User-Agent currently being sent to servers
-     *
-     * @param feedUrl A non-null URL of a RSS/Atom feed to retrieve
-     * @return A {@link com.sun.syndication.feed.synd.SyndFeed} object
-     * @throws IllegalArgumentException if the URL is null;
-     * @throws IOException if a TCP error occurs
-     * @throws FeedException if the feed is not valid
-     * @throws FetcherException if a HTTP error occurred
      */
-    public abstract SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException;
+    public abstract String getUserAgent();
+
+    /**
+     * <p>Turn on or off rfc3229 delta encoding</p>
+     *
+     * <p>See http://www.ietf.org/rfc/rfc3229.txt and http://bobwyman.pubsub.com/main/2004/09/using_rfc3229_w.html</p>
+     *
+     * <p>NOTE: This is experimental and feedback is welcome!</p>
+     *
+     * @param useDeltaEncoding
+     */
+    public abstract void setUsingDeltaEncoding(boolean useDeltaEncoding);
+
+    /**
+     * <p>Is this fetcher using rfc3229 delta encoding?</p>
+     *
+     * @return
+     */
+    public abstract boolean isUsingDeltaEncoding();
 
     /**
      * <p>Add a FetcherListener.</p>
@@ -75,22 +82,20 @@ public interface FeedFetcher {
     public abstract void removeFetcherEventListener(FetcherListener listener);
 
     /**
-     * <p>Is this fetcher using rfc3229 delta encoding?</p>
+     * Retrieve a feed over HTTP
      *
-     * @return
+     * @param feedUrl A non-null URL of a RSS/Atom feed to retrieve
+     * @return A {@link com.sun.syndication.feed.synd.SyndFeed} object
+     * @throws IllegalArgumentException if the URL is null;
+     * @throws IOException if a TCP error occurs
+     * @throws FeedException if the feed is not valid
+     * @throws FetcherException if a HTTP error occurred
      */
-    public abstract boolean isUsingDeltaEncoding();
+    public abstract SyndFeed retrieveFeed(URL feedUrl)
+        throws IllegalArgumentException, IOException, FeedException, FetcherException;
 
-    /**
+    public SyndFeed retrieveFeed(String userAgent, URL url)
-     * <p>Turn on or off rfc3229 delta encoding</p>
+        throws IllegalArgumentException, IOException, FeedException, FetcherException;
-     * 
-     * <p>See http://www.ietf.org/rfc/rfc3229.txt and http://bobwyman.pubsub.com/main/2004/09/using_rfc3229_w.html</p>
-     * 
-     * <p>NOTE: This is experimental and feedback is welcome!</p>
-     * 
-     * @param useDeltaEncoding
-     */
-    public abstract void setUsingDeltaEncoding(boolean useDeltaEncoding);
 
     /**
      * If set to true, the WireFeed will be made accessible from the SyndFeed object returned from the Fetcher

src/java/com/sun/syndication/fetcher/impl/HttpClientFeedFetcher.java

@@ -14,15 +14,14 @@
  * limitations under the License.
  *
  */
-
 package com.sun.syndication.fetcher.impl;
 
-import java.io.IOException;
+import com.sun.syndication.feed.synd.SyndFeed;
-import java.io.InputStream;
+import com.sun.syndication.fetcher.FetcherEvent;
-import java.net.HttpURLConnection;
+import com.sun.syndication.fetcher.FetcherException;
-import java.net.MalformedURLException;
+import com.sun.syndication.io.FeedException;
-import java.net.URL;
+import com.sun.syndication.io.SyndFeedInput;
-import java.util.zip.GZIPInputStream;
+import com.sun.syndication.io.XmlReader;
 
 import org.apache.commons.httpclient.Credentials;
 import org.apache.commons.httpclient.Header;
@@ -32,20 +31,22 @@ import org.apache.commons.httpclient.HttpMethod;
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.apache.commons.httpclient.params.HttpClientParams;
 
-import com.sun.syndication.feed.synd.SyndFeed;
+import java.io.IOException;
-import com.sun.syndication.fetcher.FetcherEvent;
+import java.io.InputStream;
-import com.sun.syndication.fetcher.FetcherException;
+
-import com.sun.syndication.io.FeedException;
+import java.net.HttpURLConnection;
-import com.sun.syndication.io.SyndFeedInput;
+import java.net.MalformedURLException;
-import com.sun.syndication.io.XmlReader;
+import java.net.URL;
+
+import java.util.zip.GZIPInputStream;
+
 
 /**
  * @author Nick Lothian
  */
 public class HttpClientFeedFetcher extends AbstractFeedFetcher {
-
-	private FeedFetcherCache feedInfoCache;
     private CredentialSupplier credentialSupplier;
+    private FeedFetcherCache feedInfoCache;
     private volatile HttpClientMethodCallbackIntf httpClientMethodCallback;
     private volatile HttpClientParams httpClientParams;
 
@@ -62,26 +63,11 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
         setFeedInfoCache(cache);
     }
 
-	
     public HttpClientFeedFetcher(FeedFetcherCache cache, CredentialSupplier credentialSupplier) {
         this(cache);
         setCredentialSupplier(credentialSupplier);
     }
 
-
-	/**
-     * @return Returns the httpClientParams.
-     */
-    public synchronized HttpClientParams getHttpClientParams() {
-        return this.httpClientParams;
-    }
-    /**
-     * @param httpClientParams The httpClientParams to set.
-     */
-    public synchronized void setHttpClientParams(HttpClientParams httpClientParams) {
-        this.httpClientParams = httpClientParams;
-    }	
-
     /**
      * @param timeout Sets the connect timeout for the HttpClient but using the URLConnection method name.
          *                   Uses the HttpClientParams method setConnectionManagerTimeout instead of setConnectTimeout
@@ -97,7 +83,58 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
      *
      */
     public int getConnectTimeout() {
-		return (int) this.getHttpClientParams().getConnectionManagerTimeout();
+        return (int) this.getHttpClientParams()
+                         .getConnectionManagerTimeout();
+    }
+
+    /**
+     * @param credentialSupplier The credentialSupplier to set.
+     */
+    public synchronized void setCredentialSupplier(CredentialSupplier credentialSupplier) {
+        this.credentialSupplier = credentialSupplier;
+    }
+
+    /**
+    * @return Returns the credentialSupplier.
+    */
+    public synchronized CredentialSupplier getCredentialSupplier() {
+        return credentialSupplier;
+    }
+
+    /**
+         * @param feedInfoCache the feedInfoCache to set
+         */
+    public synchronized void setFeedInfoCache(FeedFetcherCache feedInfoCache) {
+        this.feedInfoCache = feedInfoCache;
+    }
+
+    /**
+     * @return the feedInfoCache.
+     */
+    public synchronized FeedFetcherCache getFeedInfoCache() {
+        return feedInfoCache;
+    }
+
+    public synchronized void setHttpClientMethodCallback(HttpClientMethodCallbackIntf httpClientMethodCallback) {
+        this.httpClientMethodCallback = httpClientMethodCallback;
+    }
+
+    public HttpClientMethodCallbackIntf getHttpClientMethodCallback() {
+        return httpClientMethodCallback;
+    }
+
+    /**
+     * @param httpClientParams The httpClientParams to set.
+     */
+    public synchronized void setHttpClientParams(HttpClientParams httpClientParams) {
+        this.httpClientParams = httpClientParams;
+    }
+
+    /**
+    * @return Returns the httpClientParams.
+    */
+    public synchronized HttpClientParams getHttpClientParams() {
+        return this.httpClientParams;
     }
 
     /**
@@ -111,71 +148,48 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
      * @param timeout Sets the read timeout for the URLConnection to a specified timeout, in milliseconds.
      */
     public int getReadTimeout() {
-		return (int) this.getHttpClientParams().getSoTimeout();
+        return (int) this.getHttpClientParams()
+                         .getSoTimeout();
     }
 
-	public HttpClientMethodCallbackIntf getHttpClientMethodCallback() {
+    public SyndFeed retrieveFeed(URL url) throws IllegalArgumentException, IOException, FeedException, FetcherException {
-		return httpClientMethodCallback;
+        return this.retrieveFeed(this.getUserAgent(), url);
-	}
-
-	public synchronized void setHttpClientMethodCallback(HttpClientMethodCallbackIntf httpClientMethodCallback) {
-		this.httpClientMethodCallback = httpClientMethodCallback;
-	}	
-	
-	/**
-	 * @return the feedInfoCache.
-	 */
-	public synchronized FeedFetcherCache getFeedInfoCache() {
-		return feedInfoCache;
-	}
-	
-    /**
-	 * @param feedInfoCache the feedInfoCache to set
-	 */
-	public synchronized void setFeedInfoCache(FeedFetcherCache feedInfoCache) {
-		this.feedInfoCache = feedInfoCache;
-	}
-
-	/**
-     * @return Returns the credentialSupplier.
-     */
-    public synchronized CredentialSupplier getCredentialSupplier() {
-        return credentialSupplier;
-    }
-    /**
-     * @param credentialSupplier The credentialSupplier to set.
-     */
-    public synchronized void setCredentialSupplier(CredentialSupplier credentialSupplier) {
-        this.credentialSupplier = credentialSupplier;
     }
 
     /**
      * @see com.sun.syndication.fetcher.FeedFetcher#retrieveFeed(java.net.URL)
      */
-	public SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
+    public SyndFeed retrieveFeed(String userAgent, URL feedUrl)
+        throws IllegalArgumentException, IOException, FeedException, FetcherException {
         if (feedUrl == null) {
             throw new IllegalArgumentException("null is not a valid URL");
         }
+
         // TODO Fix this
         //System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.SimpleLog");
         HttpClient client = new HttpClient(httpClientParams);
 
         if (getCredentialSupplier() != null) {
-			client.getState().setAuthenticationPreemptive(true);
+            client.getState()
-			// TODO what should realm be here?
+                  .setAuthenticationPreemptive(true);
-			Credentials credentials = getCredentialSupplier().getCredentials(null, feedUrl.getHost()); 
-			if (credentials != null) {
-			    client.getState().setCredentials(null, feedUrl.getHost(), credentials);
-			}			
-		}
 
+            // TODO what should realm be here?
+            Credentials credentials = getCredentialSupplier()
+                                          .getCredentials(null, feedUrl.getHost());
+
+            if (credentials != null) {
+                client.getState()
+                      .setCredentials(null, feedUrl.getHost(), credentials);
+            }
+        }
 
         System.setProperty("httpclient.useragent", getUserAgent());
+
         String urlStr = feedUrl.toString();
 
         HttpMethod method = new GetMethod(urlStr);
         method.addRequestHeader("Accept-Encoding", "gzip");
-		method.addRequestHeader("User-Agent", getUserAgent());		
+        method.addRequestHeader("User-Agent", userAgent);
         method.setFollowRedirects(true);
 
         if (httpClientMethodCallback != null) {
@@ -185,9 +199,9 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
         }
 
         FeedFetcherCache cache = getFeedInfoCache();
+
         if (cache != null) {
             // retrieve feed
-
             try {
                 if (isUsingDeltaEncoding()) {
                     method.setRequestHeader("A-IM", "feed");
@@ -196,11 +210,12 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
                 // get the feed info from the cache
                 // Note that syndFeedInfo will be null if it is not in the cache
                 SyndFeedInfo syndFeedInfo = cache.getFeedInfo(feedUrl);
+
                 if (syndFeedInfo != null) {
                     method.setRequestHeader("If-None-Match", syndFeedInfo.getETag());
 
                     if (syndFeedInfo.getLastModified() instanceof String) {
-				        method.setRequestHeader("If-Modified-Since", (String)syndFeedInfo.getLastModified());
+                        method.setRequestHeader("If-Modified-Since", (String) syndFeedInfo.getLastModified());
                     }
                 }
 
@@ -223,7 +238,6 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
                 method.releaseConnection();
                 method.recycle();
             }
-				
         } else {
             // cache is not in use		    
             try {
@@ -239,6 +253,19 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
         }
     }
 
+    private SyndFeed getFeed(SyndFeedInfo syndFeedInfo, String urlStr, HttpMethod method, int statusCode)
+        throws IOException, HttpException, FetcherException, FeedException {
+        if ((statusCode == HttpURLConnection.HTTP_NOT_MODIFIED) && (syndFeedInfo != null)) {
+            fireEvent(FetcherEvent.EVENT_TYPE_FEED_UNCHANGED, urlStr);
+
+            return syndFeedInfo.getSyndFeed();
+        }
+
+        SyndFeed feed = retrieveFeed(urlStr, method);
+        fireEvent(FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, urlStr, feed);
+
+        return feed;
+    }
 
     /**
     * @param feedUrl
@@ -248,7 +275,8 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
     * @return
     * @throws MalformedURLException
     */
-    private SyndFeedInfo buildSyndFeedInfo(URL feedUrl, String urlStr, HttpMethod method, SyndFeed feed, int statusCode) throws MalformedURLException {
+    private SyndFeedInfo buildSyndFeedInfo(URL feedUrl, String urlStr, HttpMethod method, SyndFeed feed, int statusCode)
+        throws MalformedURLException {
         SyndFeedInfo syndFeedInfo;
         syndFeedInfo = new SyndFeedInfo();
 
@@ -257,12 +285,16 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
         syndFeedInfo.setId(feedUrl.toString());
 
         Header imHeader = method.getResponseHeader("IM");
-        if (imHeader != null && imHeader.getValue().indexOf("feed") >= 0 && isUsingDeltaEncoding()) {
+
+        if ((imHeader != null) && (imHeader.getValue()
+                                               .indexOf("feed") >= 0) && isUsingDeltaEncoding()) {
             FeedFetcherCache cache = getFeedInfoCache();
-			if (cache != null && statusCode == 226) {
+
+            if ((cache != null) && (statusCode == 226)) {
                 // client is setup to use http delta encoding and the server supports it and has returned a delta encoded response
                 // This response only includes new items
                 SyndFeedInfo cachedInfo = cache.getFeedInfo(feedUrl);
+
                 if (cachedInfo != null) {
                     SyndFeed cachedFeed = cachedInfo.getSyndFeed();
 
@@ -273,11 +305,13 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
         }
 
         Header lastModifiedHeader = method.getResponseHeader("Last-Modified");
+
         if (lastModifiedHeader != null) {
             syndFeedInfo.setLastModified(lastModifiedHeader.getValue());
         }
 
         Header eTagHeader = method.getResponseHeader("ETag");
+
         if (eTagHeader != null) {
             syndFeedInfo.setETag(eTagHeader.getValue());
         }
@@ -297,21 +331,26 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
          * @throws FetcherException
          * @throws FeedException
          */
-	private SyndFeed retrieveFeed(String urlStr, HttpMethod method) throws IOException, HttpException, FetcherException, FeedException {
+    private SyndFeed retrieveFeed(String urlStr, HttpMethod method)
-		
+        throws IOException, HttpException, FetcherException, FeedException {
         InputStream stream = null;
-		if ((method.getResponseHeader("Content-Encoding") != null) && ("gzip".equalsIgnoreCase(method.getResponseHeader("Content-Encoding").getValue()))) {		
+
+        if ((method.getResponseHeader("Content-Encoding") != null) &&
+                ("gzip".equalsIgnoreCase(method.getResponseHeader("Content-Encoding").getValue()))) {
             stream = new GZIPInputStream(method.getResponseBodyAsStream());
         } else {
             stream = method.getResponseBodyAsStream();
         }
+
         try {
             XmlReader reader = null;
+
             if (method.getResponseHeader("Content-Type") != null) {
                 reader = new XmlReader(stream, method.getResponseHeader("Content-Type").getValue(), true);
             } else {
                 reader = new XmlReader(stream, true);
             }
+
             SyndFeedInput syndFeedInput = new SyndFeedInput();
             syndFeedInput.setPreserveWireFeed(isPreserveWireFeed());
 
@@ -323,18 +362,6 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
         }
     }
 
-	private SyndFeed getFeed(SyndFeedInfo syndFeedInfo, String urlStr, HttpMethod method, int statusCode) throws IOException, HttpException, FetcherException, FeedException {
-
-		if (statusCode == HttpURLConnection.HTTP_NOT_MODIFIED && syndFeedInfo != null) {
-		    fireEvent(FetcherEvent.EVENT_TYPE_FEED_UNCHANGED, urlStr);
-		    return syndFeedInfo.getSyndFeed();
-		}
-		
-		SyndFeed feed = retrieveFeed(urlStr, method);
-		fireEvent(FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, urlStr, feed);			
-		return feed;
-	}
-	
     public interface CredentialSupplier {
         public Credentials getCredentials(String realm, String host);
     }
@@ -349,5 +376,4 @@ public class HttpClientFeedFetcher extends AbstractFeedFetcher {
              */
         public void afterHttpClientMethodCreate(HttpMethod method);
     }
-
 }

src/java/com/sun/syndication/fetcher/impl/HttpURLFeedFetcher.java

@@ -83,6 +83,10 @@ public class HttpURLFeedFetcher extends AbstractFeedFetcher {
 		setFeedInfoCache(feedInfoCache);
 	}
 
+        public SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
+            return this.retrieveFeed(this.getUserAgent(), feedUrl);
+        }
+
 	/**
 	 * Retrieve a feed over HTTP
 	 *
@@ -93,7 +97,7 @@ public class HttpURLFeedFetcher extends AbstractFeedFetcher {
 	 * @throws FeedException if the feed is not valid
 	 * @throws FetcherException if a HTTP error occurred
 	 */
-	public SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
+	public SyndFeed retrieveFeed(String userAgent, URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
 		if (feedUrl == null) {
 			throw new IllegalArgumentException("null is not a valid URL");
 		}
@@ -140,6 +144,9 @@ public class HttpURLFeedFetcher extends AbstractFeedFetcher {
 			fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, connection);
 			InputStream inputStream = null;
 			setRequestHeaders(connection, null);
+
+                        connection.addRequestProperty("User-Agent", userAgent);
+
 			httpConnection.connect();
 			try {
 				inputStream = httpConnection.getInputStream();						
@@ -238,9 +245,6 @@ public class HttpURLFeedFetcher extends AbstractFeedFetcher {
 		// header to retrieve feed gzipped
 		connection.setRequestProperty("Accept-Encoding", "gzip");
 
-		// set the user agent
-		connection.addRequestProperty("User-Agent", getUserAgent());	
-		
 		if (isUsingDeltaEncoding()) {
 		    connection.addRequestProperty("A-IM", "feed");
 		}