Initial refactoring.

pom.xml

@@ -1,21 +1,23 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
     <modelVersion>4.0.0</modelVersion>
-
     <groupId>org.rometools</groupId>
-    <artifactId>fetcher</artifactId>
+    <artifactId>rome-fetcher</artifactId>
-
     <name>rome-fetcher</name>
-    <version>1.0.1-SNAPSHOT</version>
+    <version>1.5-SNAPSHOT</version>
     <inceptionYear>2004</inceptionYear>
     <organization>
         <name>ROME Project</name>
         <url>http://rometools.jira.com</url>
     </organization>
-    <!--scm>
+    <issueManagement>
-        <connection>scm:cvs:pserver:snoopdave@cvs.dev.java.net:/cvs:rome/subprojects/fetcher</connection>
+        <url>https://rometools.jira.com/secure/IssueNavigator.jspa</url>
-        <url>https://rome.dev.java.net/source/browse/rome/</url>
+    </issueManagement>
-  </scm-->
+    <scm>
+        <connection>scm:svn:https://rometools.jira.com/svn/FETCHER/trunk</connection>
+        <developerConnection>scm:svn:https://rometools.jira.com/svn/FETCHER/trunk</developerConnection>
+        <url>https://rometools.jira.com/source/browse/FETCHER</url>
+    </scm>
     <licenses>
         <license>
             <name>Apache 2</name>

src/java/org/rometools/fetcher/FeedFetcher.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.rometools.fetcher;
+
+import com.sun.syndication.feed.synd.SyndFeed;
+import com.sun.syndication.io.FeedException;
+
+import java.io.IOException;
+
+import java.net.URL;
+
+
+public interface FeedFetcher {
+    /**
+     * <p>The default user agent. It is not marked final so
+     * buggy java compiler will not write this string
+     * into all classes that reference it.</p>
+     *
+     * <p>http://tinyurl.com/64t5n points to https://rome.dev.java.net
+     * Some servers ban user agents with "Java" in the name.</p>
+     *
+     */
+    public static String DEFAULT_USER_AGENT = "Rome Client (http://tinyurl.com/64t5n)";
+
+    /**
+     * @param string The User-Agent to sent to servers
+     */
+    public abstract void setUserAgent(String string);
+
+    /**
+     * @return the User-Agent currently being sent to servers
+     */
+    public abstract String getUserAgent();
+
+    /**
+     * <p>Turn on or off rfc3229 delta encoding</p>
+     *
+     * <p>See http://www.ietf.org/rfc/rfc3229.txt and http://bobwyman.pubsub.com/main/2004/09/using_rfc3229_w.html</p>
+     *
+     * <p>NOTE: This is experimental and feedback is welcome!</p>
+     *
+     * @param useDeltaEncoding
+     */
+    public abstract void setUsingDeltaEncoding(boolean useDeltaEncoding);
+
+    /**
+     * <p>Is this fetcher using rfc3229 delta encoding?</p>
+     *
+     * @return
+     */
+    public abstract boolean isUsingDeltaEncoding();
+
+    /**
+     * <p>Add a FetcherListener.</p>
+     *
+     * <p>The FetcherListener will receive an FetcherEvent when
+     * a Fetcher event (feed polled, retrieved, etc) occurs</p>
+     *
+     * @param listener The FetcherListener to recieve the event
+     */
+    public abstract void addFetcherEventListener(FetcherListener listener);
+
+    /**
+     * <p>Remove a FetcherListener</p>
+     *
+     * @param listener The FetcherListener to remove
+     */
+    public abstract void removeFetcherEventListener(FetcherListener listener);
+
+    /**
+     * Retrieve a feed over HTTP
+     *
+     * @param feedUrl A non-null URL of a RSS/Atom feed to retrieve
+     * @return A {@link com.sun.syndication.feed.synd.SyndFeed} object
+     * @throws IllegalArgumentException if the URL is null;
+     * @throws IOException if a TCP error occurs
+     * @throws FeedException if the feed is not valid
+     * @throws FetcherException if a HTTP error occurred
+     */
+    public abstract SyndFeed retrieveFeed(URL feedUrl)
+        throws IllegalArgumentException, IOException, FeedException, FetcherException;
+
+    public SyndFeed retrieveFeed(String userAgent, URL url)
+        throws IllegalArgumentException, IOException, FeedException, FetcherException;
+
+    /**
+     * If set to true, the WireFeed will be made accessible from the SyndFeed object returned from the Fetcher
+     * via the originalWireFeed() method. Each Entry in the feed will have the corresponding wireEntry property set.
+     */
+    void setPreserveWireFeed(boolean preserveWireFeed);
+}

src/java/org/rometools/fetcher/FetcherEvent.java

@@ -0,0 +1,85 @@
+package org.rometools.fetcher;
+
+import java.util.EventObject;
+
+import com.sun.syndication.feed.synd.SyndFeed;
+
+/**
+ * Implementation note: FetcherEvent is not thread safe. Make sure that
+ * they are only ever accessed by one thread. If necessary, make all getters
+ * and setters synchronized, or alternatively make all fields final.
+ * 
+ * @author nl
+ */
+public class FetcherEvent extends EventObject {
+
+	private static final long serialVersionUID = 3985600601904140103L;
+
+	public static final String EVENT_TYPE_FEED_POLLED = "FEED_POLLED";
+	public static final String EVENT_TYPE_FEED_RETRIEVED = "FEED_RETRIEVED";
+	public static final String EVENT_TYPE_FEED_UNCHANGED = "FEED_UNCHANGED";
+	
+	private String eventType;
+	private String urlString;
+	private SyndFeed feed;
+    
+	public FetcherEvent(Object source) {
+		super(source);
+	}
+
+
+	public FetcherEvent(Object source, String urlStr, String eventType) {
+		this(source);
+		setUrlString(urlStr);
+		setEventType(eventType);
+	}	
+
+	public FetcherEvent(Object source, String urlStr, String eventType, SyndFeed feed) {
+		this(source, urlStr, eventType);
+		setFeed(feed);
+	}	
+	
+	
+    /**
+     * @return Returns the feed.
+     * 
+     * <p>The feed will only be set if the eventType is EVENT_TYPE_FEED_RETRIEVED</p>
+     */
+    public SyndFeed getFeed() {
+        return feed;
+    }
+    
+    /**
+     * @param feed The feed to set.
+     * 
+     * <p>The feed will only be set if the eventType is EVENT_TYPE_FEED_RETRIEVED</p>
+     */
+    public void setFeed(SyndFeed feed) {
+        this.feed = feed;
+    }
+	
+	/**
+	 * @return Returns the eventType.
+	 */
+	public String getEventType() {
+		return eventType;
+	}
+	/**
+	 * @param eventType The eventType to set.
+	 */
+	public void setEventType(String eventType) {
+		this.eventType = eventType;
+	}
+	/**
+	 * @return Returns the urlString.
+	 */
+	public String getUrlString() {
+		return urlString;
+	}
+	/**
+	 * @param urlString The urlString to set.
+	 */
+	public void setUrlString(String urlString) {
+		this.urlString = urlString;
+	}
+}

src/java/org/rometools/fetcher/FetcherException.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.rometools.fetcher;
+
+/**
+ * @author Nick Lothian
+ *
+ */
+public class FetcherException extends Exception {
+	private static final long serialVersionUID = -7479645796948092380L;
+
+	int responseCode;
+	
+	public FetcherException(Throwable cause) {
+		super();
+		initCause(cause);
+	}
+	
+	public FetcherException(String message, Throwable cause) {
+		super(message);
+		initCause(cause);
+	}
+	
+	public FetcherException(String message) {
+		super(message);
+	}
+	
+	public FetcherException(int responseCode, String message) {
+		this(message);		
+		this.responseCode = responseCode;
+	}
+
+	public int getResponseCode() {
+		return responseCode;
+	}
+
+}

src/java/org/rometools/fetcher/FetcherListener.java

@@ -0,0 +1,15 @@
+package org.rometools.fetcher;
+
+import java.util.EventListener;
+
+
+public interface FetcherListener extends EventListener {
+
+	/**
+	 * <p>Called when a fetcher event occurs</p>
+	 * 
+	 * @param event the event that fired
+	 */
+	public void fetcherEvent(FetcherEvent event);
+	
+}

src/java/org/rometools/fetcher/impl/AbstractFeedFetcher.java

@@ -0,0 +1,219 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.rometools.fetcher.impl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URLConnection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Properties;
+import java.util.Set;
+
+import com.sun.syndication.feed.synd.SyndFeed;
+import org.rometools.fetcher.FeedFetcher;
+import org.rometools.fetcher.FetcherEvent;
+import org.rometools.fetcher.FetcherException;
+import org.rometools.fetcher.FetcherListener;
+
+
+public abstract class AbstractFeedFetcher implements FeedFetcher {
+	private final Set fetcherEventListeners;
+	private String userAgent;
+	private boolean usingDeltaEncoding;	
+	private boolean preserveWireFeed;
+	
+	
+    
+	public AbstractFeedFetcher() {
+		fetcherEventListeners = Collections.synchronizedSet(new HashSet());
+		
+		Properties props = new Properties(System.getProperties());
+		String resourceName = "fetcher.properties";
+		
+		try {
+			InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(resourceName);
+			if (inputStream == null) {
+			    inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName);
+			}			
+			if (inputStream != null) {
+				props.load(inputStream);
+				System.getProperties().putAll(props);
+				inputStream.close();
+			} else {
+				System.err.println("Could not find " + resourceName + " on classpath");
+			}
+		} catch (IOException e) {
+			// do nothing - we don't want to fail just because we could not find the version
+			System.err.println("Error reading " + resourceName + " from classpath: " + e.getMessage());
+		}		
+		
+		
+		setUserAgent(DEFAULT_USER_AGENT + " Ver: " + System.getProperty("rome.fetcher.version", "UNKNOWN"));
+	}
+
+	/**
+	 * @return the User-Agent currently being sent to servers
+	 */
+	public synchronized String getUserAgent() {
+		return userAgent;
+	}
+
+	/**
+	 * @param string The User-Agent to sent to servers
+	 */
+	public synchronized void setUserAgent(String string) {
+		userAgent = string;
+	}
+
+	/**
+	 * @param eventType The event type to fire
+	 * @param connection the current connection
+	 */
+	protected void fireEvent(String eventType, URLConnection connection) {	    
+		fireEvent(eventType, connection.getURL().toExternalForm(), null);
+	}
+	
+	
+	/**
+	 * @param eventType The event type to fire
+	 * @param connection the current connection
+	 * @param feed The feed to pass to the event
+	 */
+	protected void fireEvent(String eventType, URLConnection connection, SyndFeed feed) {	    
+		fireEvent(eventType, connection.getURL().toExternalForm(), feed);
+	}
+
+	/**
+	 * @param eventType The event type to fire
+	 * @param urlStr the current url as a string
+	 */
+	protected void fireEvent(String eventType, String urlStr) {
+	    fireEvent(eventType, urlStr, null);
+	}	
+	
+	/**
+	 * @param eventType The event type to fire
+	 * @param urlStr the current url as a string
+	 * @param feed The feed to pass to the event
+	 */
+	protected void fireEvent(String eventType, String urlStr, SyndFeed feed) {
+		FetcherEvent fetcherEvent = new FetcherEvent(this, urlStr, eventType, feed);
+		synchronized(fetcherEventListeners) {
+			Iterator iter = fetcherEventListeners.iterator();
+			while ( iter.hasNext()) {
+				FetcherListener fetcherEventListener = (FetcherListener) iter.next();
+				fetcherEventListener.fetcherEvent(fetcherEvent);							
+			}					
+		}
+	}	
+	
+	/**
+	 * @see com.sun.syndication.fetcher.FeedFetcher#addFetcherEventListener(com.sun.syndication.fetcher.FetcherListener)
+	 */
+	public void addFetcherEventListener(FetcherListener listener) {
+		if (listener != null) {
+			fetcherEventListeners.add(listener);		
+		}	
+		
+	}
+
+	/**
+	 * @see com.sun.syndication.fetcher.FeedFetcher#removeFetcherEventListener(com.sun.syndication.fetcher.FetcherListener)
+	 */
+	public void removeFetcherEventListener(FetcherListener listener) {
+		if (listener != null) {
+			fetcherEventListeners.remove(listener);		
+		}		
+	}
+
+    /**
+     * @return Returns the useDeltaEncoding.
+     */
+    public synchronized boolean isUsingDeltaEncoding() {
+        return usingDeltaEncoding;
+    }
+    /**
+     * @param useDeltaEncoding The useDeltaEncoding to set.
+     */
+    public synchronized void setUsingDeltaEncoding(boolean useDeltaEncoding) {
+        this.usingDeltaEncoding = useDeltaEncoding;
+    }		
+	
+	/**
+	 * <p>Handles HTTP error codes.</p>
+	 *
+	 * @param responseCode the HTTP response code
+	 * @throws FetcherException if response code is in the range 400 to 599 inclusive
+	 */
+	protected void handleErrorCodes(int responseCode) throws FetcherException {
+		// Handle 2xx codes as OK, so ignore them here
+		// 3xx codes are handled by the HttpURLConnection class
+	    if (responseCode == 403) {
+	        // Authentication is required
+	        throwAuthenticationError(responseCode);
+	    } else if (responseCode >= 400 && responseCode < 500) {
+			throw4XXError(responseCode);
+		} else if (responseCode >= 500 && responseCode < 600) {
+			throw new FetcherException(responseCode, "The server encounted an error. HTTP Response code was:" + responseCode);
+		}
+	}
+	
+	protected void throw4XXError(int responseCode) throws FetcherException {
+		throw new FetcherException(responseCode, "The requested resource could not be found. HTTP Response code was:" + responseCode);
+	}
+
+	protected void throwAuthenticationError(int responseCode) throws FetcherException {
+		throw new FetcherException(responseCode, "Authentication required for that resource. HTTP Response code was:" + responseCode);
+	}
+	
+	/**
+	 * <p>Combine the entries in two feeds into a single feed.</p>
+	 * 
+	 * <p>The returned feed will have the same data as the newFeed parameter, with 
+	 * the entries from originalFeed appended to the end of its entries.</p>
+	 * 
+	 * @param originalFeed
+	 * @param newFeed
+	 * @return
+	 */
+	public static SyndFeed combineFeeds(SyndFeed originalFeed, SyndFeed newFeed) {
+	    SyndFeed result;
+        try {
+            result = (SyndFeed) newFeed.clone();
+            
+            result.getEntries().addAll(result.getEntries().size(), originalFeed.getEntries());
+            
+            return result;
+        } catch (CloneNotSupportedException e) {
+            IllegalArgumentException iae = new IllegalArgumentException("Cannot clone feed");
+            iae.initCause(e);
+            throw iae;
+        }        
+	}
+
+	public boolean isPreserveWireFeed() {
+		return preserveWireFeed;
+	}
+
+	public void setPreserveWireFeed(boolean preserveWireFeed) {
+		this.preserveWireFeed = preserveWireFeed;
+	}
+	
+}

src/java/org/rometools/fetcher/impl/AbstractFeedFetcherBeanInfo.java

@@ -0,0 +1,29 @@
+package org.rometools.fetcher.impl;
+
+import java.beans.EventSetDescriptor;
+import java.beans.SimpleBeanInfo;
+import java.lang.reflect.Method;
+
+import org.rometools.fetcher.FetcherEvent;
+import org.rometools.fetcher.FetcherListener;
+
+public class AbstractFeedFetcherBeanInfo extends SimpleBeanInfo {
+
+	public EventSetDescriptor[] getEventSetDescriptors() {
+		try {
+			Class clz = AbstractFeedFetcher.class; // get the class object which we'll describe
+			Method addMethod = clz.getMethod("addFetcherEventListener", new Class[] { FetcherListener.class });
+			Method removeMethod = clz.getMethod("removeFetcherEventListener", new Class[] { FetcherListener.class });
+			Method listenerMethod = FetcherListener.class.getMethod("fetcherEvent", new Class[] { FetcherEvent.class });
+
+			EventSetDescriptor est = new EventSetDescriptor("fetcherEvent", clz, new Method[] { listenerMethod }, addMethod, removeMethod);
+			EventSetDescriptor[] results = new EventSetDescriptor[] { est };
+
+			return results;
+		} catch (Exception e) {
+			// IntrospectionException, SecurityException and/or NoSuchMethodException can be thrown here
+			// the best we can do is to convert them to runtime exceptions
+			throw new RuntimeException(e);
+		}
+	}
+}

src/java/org/rometools/fetcher/impl/DiskFeedInfoCache.java

@@ -0,0 +1,134 @@
+/*
+ * Copyright 2005 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.rometools.fetcher.impl;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.net.URL;
+import javax.swing.text.Utilities;
+
+
+/**
+ * Disk based cache.
+ */
+public class DiskFeedInfoCache implements FeedFetcherCache {
+    
+    protected String cachePath = null;
+    public DiskFeedInfoCache(String cachePath) {
+        this.cachePath = cachePath;
+    }
+    public SyndFeedInfo getFeedInfo(URL url) {
+        SyndFeedInfo info = null;
+        String fileName = cachePath + File.separator + "feed_"
+                + replaceNonAlphanumeric(url.toString(),'_').trim();
+        FileInputStream fis;
+        try {
+            fis = new FileInputStream(fileName);
+            ObjectInputStream ois = new ObjectInputStream(fis);
+            info = (SyndFeedInfo)ois.readObject();
+            fis.close();
+        } catch (FileNotFoundException fnfe) {
+            // That's OK, we'l return null
+        } catch (ClassNotFoundException cnfe) {
+            // Error writing to cache is fatal
+            throw new RuntimeException("Attempting to read from cache", cnfe);
+        } catch (IOException fnfe) {
+            // Error writing to cache is fatal
+            throw new RuntimeException("Attempting to read from cache", fnfe);
+        }
+        return info;
+    }
+    
+    public void setFeedInfo(URL url, SyndFeedInfo feedInfo) {
+        String fileName = cachePath + File.separator + "feed_"
+                + replaceNonAlphanumeric(url.toString(),'_').trim();
+        FileOutputStream fos;
+        try {
+            fos = new FileOutputStream(fileName);
+            ObjectOutputStream oos = new ObjectOutputStream(fos);
+            oos.writeObject(feedInfo);
+            fos.flush();
+            fos.close();
+        } catch (Exception e) {
+            // Error writing to cache is fatal
+            throw new RuntimeException("Attempting to write to cache", e);
+        }
+    }
+    
+    public static String replaceNonAlphanumeric(String str, char subst) {
+        StringBuffer ret = new StringBuffer(str.length());
+        char[] testChars = str.toCharArray();
+        for (int i = 0; i < testChars.length; i++) {
+            if (Character.isLetterOrDigit(testChars[i])) {
+                ret.append(testChars[i]);
+            } else {
+                ret.append( subst );
+            }
+        }
+        return ret.toString();
+    }
+    
+    /**
+     * Clear the cache. 
+     */
+    public synchronized void clear() {
+    	final File file = new File(this.cachePath);
+    	//only do the delete if the directory exists
+    	if( file.exists() && file.canWrite() ) {
+    		//make the directory empty
+    		final String[] files = file.list();
+    		final int len = files.length;
+    		for( int i=0; i<len; i++ ) {
+    			File deleteMe = new File(this.cachePath + File.separator + files[i]);
+    			deleteMe.delete();
+    		}
+    		
+    		//don't delete the cache directory
+    	}
+    }
+    
+    public SyndFeedInfo remove( URL url ) {
+        SyndFeedInfo info = null;
+        String fileName = cachePath + File.separator + "feed_"
+                + replaceNonAlphanumeric(url.toString(),'_').trim();
+        FileInputStream fis;
+        try {
+            fis = new FileInputStream(fileName);
+            ObjectInputStream ois = new ObjectInputStream(fis);
+            info = (SyndFeedInfo)ois.readObject();
+            fis.close();
+            
+            File file = new File( fileName );
+            if( file.exists() ) { 
+            	file.delete();
+            }
+        } catch (FileNotFoundException fnfe) {
+            // That's OK, we'l return null
+        } catch (ClassNotFoundException cnfe) {
+            // Error writing to cahce is fatal
+            throw new RuntimeException("Attempting to read from cache", cnfe);
+        } catch (IOException fnfe) {
+            // Error writing to cahce is fatal
+            throw new RuntimeException("Attempting to read from cache", fnfe);
+        }
+        return info;   	
+    }
+}

src/java/org/rometools/fetcher/impl/FeedFetcherCache.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.rometools.fetcher.impl;
+
+import java.net.URL;
+
+/**
+ * <p>An interface to allow caching of feed details. Implementing this allows the
+ * {@link com.sun.syndication.fetcher.io.HttpURLFeedFetcher} class to 
+ * enable conditional gets</p> 
+ * 
+ * @author Nick Lothian
+ *
+ */
+public interface FeedFetcherCache {
+	/**
+	 * Get a SyndFeedInfo object from the cache.
+	 * 
+	 * @param feedUrl The url of the feed
+	 * @return A SyndFeedInfo or null if it is not in the cache
+	 */
+	public SyndFeedInfo getFeedInfo(URL feedUrl);
+	
+	/**
+	 * Add a SyndFeedInfo object to the cache
+	 * 
+	 * @param feedUrl  The url of the feed
+	 * @param syndFeedInfo A SyndFeedInfo for the feed
+	 */
+	public void setFeedInfo(URL feedUrl, SyndFeedInfo syndFeedInfo);
+	
+	/**
+	 * Removes all items from the cache.
+	 */
+	public void clear();
+	
+	/**
+	 * Removes the SyndFeedInfo identified by the url from the cache.
+	 * @return The removed SyndFeedInfo
+	 */
+	public SyndFeedInfo remove( URL feedUrl );
+}

src/java/org/rometools/fetcher/impl/HashMapFeedInfoCache.java

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.rometools.fetcher.impl;
+
+import java.io.Serializable;
+import java.net.URL;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+
+/**
+ * <p>A very simple implementation of the {@link com.sun.syndication.fetcher.impl.FeedFetcherCache} interface.</p>
+ * 
+ * <p>This implementation uses a HashMap to cache retrieved feeds. This implementation is
+ * most suitible for sort term (client aggregator?) use, as the memory usage will increase
+ * over time as the number of feeds in the cache increases.</p>
+ * 
+ * @author Nick Lothian
+ *
+ */
+public class HashMapFeedInfoCache implements FeedFetcherCache, Serializable {
+	private static final long serialVersionUID = -1594665619950916222L;
+
+	static HashMapFeedInfoCache _instance;
+	
+	private Map infoCache;
+	
+	/**
+	 * <p>Constructor for HashMapFeedInfoCache</p>
+	 * 
+	 * <p>Only use this if you want multiple instances of the cache. 
+	 * Usually getInstance() is more appropriate.</p>
+	 *
+	 */
+	public HashMapFeedInfoCache() {
+		setInfoCache(createInfoCache());
+	}
+
+	/**
+	 * Get the global instance of the cache
+	 * @return an implementation of FeedFetcherCache
+	 */
+	public static synchronized FeedFetcherCache getInstance() {
+		if (_instance == null) {
+			_instance = new HashMapFeedInfoCache();			
+		}
+		return _instance;
+	}
+
+	protected Map createInfoCache() {
+ 		return (Collections.synchronizedMap(new HashMap()));
+ 	}
+
+	
+	protected Object get(Object key) {
+		return getInfoCache().get(key);
+	}
+
+	/**
+	 * @see extensions.io.FeedFetcherCache#getFeedInfo(java.net.URL)
+	 */
+	public SyndFeedInfo getFeedInfo(URL feedUrl) {
+		return (SyndFeedInfo) get(feedUrl.toString());
+	}
+
+	protected void put(Object key, Object value) {
+		getInfoCache().put(key, value);
+	}
+
+	/**
+	 * @see extensions.io.FeedFetcherCache#setFeedInfo(java.net.URL, extensions.io.SyndFeedInfo)
+	 */
+	public void setFeedInfo(URL feedUrl, SyndFeedInfo syndFeedInfo) {
+		put(feedUrl.toString(), syndFeedInfo);		
+	}
+
+	protected synchronized final Map getInfoCache() {
+		return infoCache;
+	}
+
+	/**
+	 * The API of this class indicates that map must thread safe. In other
+	 * words, be sure to wrap it in a synchronized map unless you know
+	 * what you are doing.
+	 * 
+	 * @param map the map to use as the info cache.
+	 */
+	protected synchronized final void setInfoCache(Map map) {
+		infoCache = map;
+	}
+	
+	/**
+	 * @see com.sun.syndication.fetcher.impl.FeedFetcherCache#clear()
+	 */
+	public void clear() {
+		synchronized( infoCache ) {
+			infoCache.clear();
+		}
+	}
+	
+	/**
+	 * @see com.sun.syndication.fetcher.impl.FeedFetcherCache#remove(java.net.URL)
+	 */
+	public SyndFeedInfo remove( final URL url ) {
+		if( url == null ) return null;
+		
+		return (SyndFeedInfo) infoCache.remove( url.toString() );
+	}
+
+}

src/java/org/rometools/fetcher/impl/HttpClientFeedFetcher.java

@@ -0,0 +1,379 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.rometools.fetcher.impl;
+
+import com.sun.syndication.feed.synd.SyndFeed;
+import org.rometools.fetcher.FetcherEvent;
+import org.rometools.fetcher.FetcherException;
+import com.sun.syndication.io.FeedException;
+import com.sun.syndication.io.SyndFeedInput;
+import com.sun.syndication.io.XmlReader;
+
+import org.apache.commons.httpclient.Credentials;
+import org.apache.commons.httpclient.Header;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpException;
+import org.apache.commons.httpclient.HttpMethod;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.params.HttpClientParams;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+import java.util.zip.GZIPInputStream;
+
+
+/**
+ * @author Nick Lothian
+ */
+public class HttpClientFeedFetcher extends AbstractFeedFetcher {
+    private CredentialSupplier credentialSupplier;
+    private FeedFetcherCache feedInfoCache;
+    private volatile HttpClientMethodCallbackIntf httpClientMethodCallback;
+    private volatile HttpClientParams httpClientParams;
+
+    public HttpClientFeedFetcher() {
+        super();
+        setHttpClientParams(new HttpClientParams());
+    }
+
+    /**
+     * @param cache
+     */
+    public HttpClientFeedFetcher(FeedFetcherCache cache) {
+        this();
+        setFeedInfoCache(cache);
+    }
+
+    public HttpClientFeedFetcher(FeedFetcherCache cache, CredentialSupplier credentialSupplier) {
+        this(cache);
+        setCredentialSupplier(credentialSupplier);
+    }
+
+    /**
+     * @param timeout Sets the connect timeout for the HttpClient but using the URLConnection method name.
+         *                   Uses the HttpClientParams method setConnectionManagerTimeout instead of setConnectTimeout
+         *
+         */
+    public synchronized void setConnectTimeout(int timeout) {
+        httpClientParams.setConnectionManagerTimeout(timeout);
+    }
+
+    /**
+     * @return The currently used connect timeout for the HttpClient but using the URLConnection method name.
+     *                    Uses the HttpClientParams method getConnectionManagerTimeout instead of getConnectTimeout
+     *
+     */
+    public int getConnectTimeout() {
+        return (int) this.getHttpClientParams()
+                         .getConnectionManagerTimeout();
+    }
+
+    /**
+     * @param credentialSupplier The credentialSupplier to set.
+     */
+    public synchronized void setCredentialSupplier(CredentialSupplier credentialSupplier) {
+        this.credentialSupplier = credentialSupplier;
+    }
+
+    /**
+    * @return Returns the credentialSupplier.
+    */
+    public synchronized CredentialSupplier getCredentialSupplier() {
+        return credentialSupplier;
+    }
+
+    /**
+         * @param feedInfoCache the feedInfoCache to set
+         */
+    public synchronized void setFeedInfoCache(FeedFetcherCache feedInfoCache) {
+        this.feedInfoCache = feedInfoCache;
+    }
+
+    /**
+     * @return the feedInfoCache.
+     */
+    public synchronized FeedFetcherCache getFeedInfoCache() {
+        return feedInfoCache;
+    }
+
+    public synchronized void setHttpClientMethodCallback(HttpClientMethodCallbackIntf httpClientMethodCallback) {
+        this.httpClientMethodCallback = httpClientMethodCallback;
+    }
+
+    public HttpClientMethodCallbackIntf getHttpClientMethodCallback() {
+        return httpClientMethodCallback;
+    }
+
+    /**
+     * @param httpClientParams The httpClientParams to set.
+     */
+    public synchronized void setHttpClientParams(HttpClientParams httpClientParams) {
+        this.httpClientParams = httpClientParams;
+    }
+
+    /**
+    * @return Returns the httpClientParams.
+    */
+    public synchronized HttpClientParams getHttpClientParams() {
+        return this.httpClientParams;
+    }
+
+    /**
+     * @return The currently used read timeout for the URLConnection, 0 is unlimited, i.e. no timeout
+     */
+    public synchronized void setReadTimeout(int timeout) {
+        httpClientParams.setSoTimeout(timeout);
+    }
+
+    /**
+     * @param timeout Sets the read timeout for the URLConnection to a specified timeout, in milliseconds.
+     */
+    public int getReadTimeout() {
+        return (int) this.getHttpClientParams()
+                         .getSoTimeout();
+    }
+
+    public SyndFeed retrieveFeed(URL url) throws IllegalArgumentException, IOException, FeedException, FetcherException {
+        return this.retrieveFeed(this.getUserAgent(), url);
+    }
+
+    /**
+     * @see com.sun.syndication.fetcher.FeedFetcher#retrieveFeed(java.net.URL)
+     */
+    public SyndFeed retrieveFeed(String userAgent, URL feedUrl)
+        throws IllegalArgumentException, IOException, FeedException, FetcherException {
+        if (feedUrl == null) {
+            throw new IllegalArgumentException("null is not a valid URL");
+        }
+
+        // TODO Fix this
+        //System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.SimpleLog");
+        HttpClient client = new HttpClient(httpClientParams);
+
+        if (getCredentialSupplier() != null) {
+            client.getState()
+                  .setAuthenticationPreemptive(true);
+
+            // TODO what should realm be here?
+            Credentials credentials = getCredentialSupplier()
+                                          .getCredentials(null, feedUrl.getHost());
+
+            if (credentials != null) {
+                client.getState()
+                      .setCredentials(null, feedUrl.getHost(), credentials);
+            }
+        }
+
+        System.setProperty("httpclient.useragent", userAgent);
+
+        String urlStr = feedUrl.toString();
+
+        HttpMethod method = new GetMethod(urlStr);
+        method.addRequestHeader("Accept-Encoding", "gzip");
+        method.addRequestHeader("User-Agent", userAgent);
+        method.setFollowRedirects(true);
+
+        if (httpClientMethodCallback != null) {
+            synchronized (httpClientMethodCallback) {
+                httpClientMethodCallback.afterHttpClientMethodCreate(method);
+            }
+        }
+
+        FeedFetcherCache cache = getFeedInfoCache();
+
+        if (cache != null) {
+            // retrieve feed
+            try {
+                if (isUsingDeltaEncoding()) {
+                    method.setRequestHeader("A-IM", "feed");
+                }
+
+                // get the feed info from the cache
+                // Note that syndFeedInfo will be null if it is not in the cache
+                SyndFeedInfo syndFeedInfo = cache.getFeedInfo(feedUrl);
+
+                if (syndFeedInfo != null) {
+                    method.setRequestHeader("If-None-Match", syndFeedInfo.getETag());
+
+                    if (syndFeedInfo.getLastModified() instanceof String) {
+                        method.setRequestHeader("If-Modified-Since", (String) syndFeedInfo.getLastModified());
+                    }
+                }
+
+                int statusCode = client.executeMethod(method);
+                fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);
+                handleErrorCodes(statusCode);
+
+                SyndFeed feed = getFeed(syndFeedInfo, urlStr, method, statusCode);
+
+                syndFeedInfo = buildSyndFeedInfo(feedUrl, urlStr, method, feed, statusCode);
+
+                cache.setFeedInfo(new URL(urlStr), syndFeedInfo);
+
+                // the feed may have been modified to pick up cached values
+                // (eg - for delta encoding)
+                feed = syndFeedInfo.getSyndFeed();
+
+                return feed;
+            } finally {
+                method.releaseConnection();
+                method.recycle();
+            }
+        } else {
+            // cache is not in use		    
+            try {
+                int statusCode = client.executeMethod(method);
+                fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);
+                handleErrorCodes(statusCode);
+
+                return getFeed(null, urlStr, method, statusCode);
+            } finally {
+                method.releaseConnection();
+                method.recycle();
+            }
+        }
+    }
+
+    private SyndFeed getFeed(SyndFeedInfo syndFeedInfo, String urlStr, HttpMethod method, int statusCode)
+        throws IOException, HttpException, FetcherException, FeedException {
+        if ((statusCode == HttpURLConnection.HTTP_NOT_MODIFIED) && (syndFeedInfo != null)) {
+            fireEvent(FetcherEvent.EVENT_TYPE_FEED_UNCHANGED, urlStr);
+
+            return syndFeedInfo.getSyndFeed();
+        }
+
+        SyndFeed feed = retrieveFeed(urlStr, method);
+        fireEvent(FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, urlStr, feed);
+
+        return feed;
+    }
+
+    /**
+    * @param feedUrl
+    * @param urlStr
+    * @param method
+    * @param feed
+    * @return
+    * @throws MalformedURLException
+    */
+    private SyndFeedInfo buildSyndFeedInfo(URL feedUrl, String urlStr, HttpMethod method, SyndFeed feed, int statusCode)
+        throws MalformedURLException {
+        SyndFeedInfo syndFeedInfo;
+        syndFeedInfo = new SyndFeedInfo();
+
+        // this may be different to feedURL because of 3XX redirects
+        syndFeedInfo.setUrl(new URL(urlStr));
+        syndFeedInfo.setId(feedUrl.toString());
+
+        Header imHeader = method.getResponseHeader("IM");
+
+        if ((imHeader != null) && (imHeader.getValue()
+                                               .indexOf("feed") >= 0) && isUsingDeltaEncoding()) {
+            FeedFetcherCache cache = getFeedInfoCache();
+
+            if ((cache != null) && (statusCode == 226)) {
+                // client is setup to use http delta encoding and the server supports it and has returned a delta encoded response
+                // This response only includes new items
+                SyndFeedInfo cachedInfo = cache.getFeedInfo(feedUrl);
+
+                if (cachedInfo != null) {
+                    SyndFeed cachedFeed = cachedInfo.getSyndFeed();
+
+                    // set the new feed to be the orginal feed plus the new items
+                    feed = combineFeeds(cachedFeed, feed);
+                }
+            }
+        }
+
+        Header lastModifiedHeader = method.getResponseHeader("Last-Modified");
+
+        if (lastModifiedHeader != null) {
+            syndFeedInfo.setLastModified(lastModifiedHeader.getValue());
+        }
+
+        Header eTagHeader = method.getResponseHeader("ETag");
+
+        if (eTagHeader != null) {
+            syndFeedInfo.setETag(eTagHeader.getValue());
+        }
+
+        syndFeedInfo.setSyndFeed(feed);
+
+        return syndFeedInfo;
+    }
+
+    /**
+         * @param client
+         * @param urlStr
+         * @param method
+         * @return
+         * @throws IOException
+         * @throws HttpException
+         * @throws FetcherException
+         * @throws FeedException
+         */
+    private SyndFeed retrieveFeed(String urlStr, HttpMethod method)
+        throws IOException, HttpException, FetcherException, FeedException {
+        InputStream stream = null;
+
+        if ((method.getResponseHeader("Content-Encoding") != null) &&
+                ("gzip".equalsIgnoreCase(method.getResponseHeader("Content-Encoding").getValue()))) {
+            stream = new GZIPInputStream(method.getResponseBodyAsStream());
+        } else {
+            stream = method.getResponseBodyAsStream();
+        }
+
+        try {
+            XmlReader reader = null;
+
+            if (method.getResponseHeader("Content-Type") != null) {
+                reader = new XmlReader(stream, method.getResponseHeader("Content-Type").getValue(), true);
+            } else {
+                reader = new XmlReader(stream, true);
+            }
+
+            SyndFeedInput syndFeedInput = new SyndFeedInput();
+            syndFeedInput.setPreserveWireFeed(isPreserveWireFeed());
+
+            return syndFeedInput.build(reader);
+        } finally {
+            if (stream != null) {
+                stream.close();
+            }
+        }
+    }
+
+    public interface CredentialSupplier {
+        public Credentials getCredentials(String realm, String host);
+    }
+
+    public interface HttpClientMethodCallbackIntf {
+        /**
+             * Allows access to the underlying HttpClient HttpMethod object.
+             * Note that in most cases, method.setRequestHeader(String, String)
+             * is what you want to do (rather than method.addRequestHeader(String, String))
+             *
+             * @param method
+             */
+        public void afterHttpClientMethodCreate(HttpMethod method);
+    }
+}

src/java/org/rometools/fetcher/impl/HttpURLFeedFetcher.java

@@ -0,0 +1,299 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.rometools.fetcher.impl;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.zip.GZIPInputStream;
+
+import com.sun.syndication.feed.synd.SyndFeed;
+import org.rometools.fetcher.FetcherEvent;
+import org.rometools.fetcher.FetcherException;
+import com.sun.syndication.io.FeedException;
+import com.sun.syndication.io.SyndFeedInput;
+import com.sun.syndication.io.XmlReader;
+
+/**
+ * <p>Class to retrieve syndication files via HTTP.</p>
+ *
+ * <p>If passed a {@link com.sun.syndication.fetcher.impl.FeedFetcherCache} in the
+ * constructor it will use conditional gets to only retrieve modified content.</p>
+ *
+ * <p>The class uses the Accept-Encoding: gzip header to retrieve gzipped feeds where
+ * supported by the server.</p>
+ *
+ * <p>Simple usage:
+ * <pre>
+ * 	// create the cache
+ *	FeedFetcherCache feedInfoCache = HashMapFeedInfoCache.getFeedInfoCache();
+ *	// retrieve the feed the first time
+ *	// any subsequent request will use conditional gets and only
+ *	// retrieve the resource if it has changed
+ *	SyndFeed feed = new HttpURLFeedFetcher(feedInfoCache).retrieveFeed(feedUrl);
+ *</pre>
+ *
+ * </p>
+ *
+ * @see <a href="http://fishbowl.pastiche.org/2002/10/21/http_conditional_get_for_rss_hackers">http://fishbowl.pastiche.org/2002/10/21/http_conditional_get_for_rss_hackers</a>
+ * @see <a href="http://diveintomark.org/archives/2003/07/21/atom_aggregator_behavior_http_level">http://diveintomark.org/archives/2003/07/21/atom_aggregator_behavior_http_level</a>
+ * @see <a href="http://bobwyman.pubsub.com/main/2004/09/using_rfc3229_w.html">http://bobwyman.pubsub.com/main/2004/09/using_rfc3229_w.html</a>
+ * @author Nick Lothian
+ */
+public class HttpURLFeedFetcher extends AbstractFeedFetcher {
+	static final int POLL_EVENT = 1;
+	static final int RETRIEVE_EVENT = 2;
+	static final int UNCHANGED_EVENT = 3;
+
+	private FeedFetcherCache feedInfoCache;
+
+
+	/**
+	 * Constructor to use HttpURLFeedFetcher without caching of feeds
+	 *
+	 */
+	public HttpURLFeedFetcher() {
+		super();
+	}
+
+	/**
+	 * Constructor to enable HttpURLFeedFetcher to cache feeds
+	 *
+	 * @param feedCache - an instance of the FeedFetcherCache interface
+	 */
+	public HttpURLFeedFetcher(FeedFetcherCache feedInfoCache) {
+		this();
+		setFeedInfoCache(feedInfoCache);
+	}
+
+        public SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
+            return this.retrieveFeed(this.getUserAgent(), feedUrl);
+        }
+
+	/**
+	 * Retrieve a feed over HTTP
+	 *
+	 * @param feedUrl A non-null URL of a RSS/Atom feed to retrieve
+	 * @return A {@link com.sun.syndication.feed.synd.SyndFeed} object
+	 * @throws IllegalArgumentException if the URL is null;
+	 * @throws IOException if a TCP error occurs
+	 * @throws FeedException if the feed is not valid
+	 * @throws FetcherException if a HTTP error occurred
+	 */
+	public SyndFeed retrieveFeed(String userAgent, URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
+		if (feedUrl == null) {
+			throw new IllegalArgumentException("null is not a valid URL");
+		}
+		
+		URLConnection connection = feedUrl.openConnection();
+		if (!(connection instanceof HttpURLConnection)) {		    
+		    throw new IllegalArgumentException(feedUrl.toExternalForm() + " is not a valid HTTP Url");
+		}
+		HttpURLConnection httpConnection = (HttpURLConnection)connection;		
+		// httpConnection.setInstanceFollowRedirects(true); // this is true by default, but can be changed on a claswide basis		
+		
+		FeedFetcherCache cache = getFeedInfoCache();
+		if (cache != null) {
+			SyndFeedInfo syndFeedInfo = cache.getFeedInfo(feedUrl);
+			setRequestHeaders(connection, syndFeedInfo);
+			httpConnection.connect();
+			try {
+				fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, connection);
+								
+				if (syndFeedInfo == null) {
+					// this is a feed that hasn't been retrieved
+					syndFeedInfo = new SyndFeedInfo();
+					retrieveAndCacheFeed(feedUrl, syndFeedInfo, httpConnection);
+				} else {
+					// check the response code
+					int responseCode = httpConnection.getResponseCode();
+					if (responseCode != HttpURLConnection.HTTP_NOT_MODIFIED) {
+						// the response code is not 304 NOT MODIFIED
+						// This is either because the feed server
+						// does not support condition gets
+						// or because the feed hasn't changed
+						retrieveAndCacheFeed(feedUrl, syndFeedInfo, httpConnection);
+					} else {
+						// the feed does not need retrieving
+						fireEvent(FetcherEvent.EVENT_TYPE_FEED_UNCHANGED, connection);
+					}
+				}
+	
+				return syndFeedInfo.getSyndFeed();
+			} finally {
+			    httpConnection.disconnect();
+			}
+		} else {			
+			fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, connection);
+			InputStream inputStream = null;
+			setRequestHeaders(connection, null);
+
+                        connection.addRequestProperty("User-Agent", userAgent);
+
+			httpConnection.connect();
+			try {
+				inputStream = httpConnection.getInputStream();						
+				return getSyndFeedFromStream(inputStream, connection);
+			} catch (java.io.IOException e) {
+				handleErrorCodes(((HttpURLConnection)connection).getResponseCode());
+			} finally {
+			    if (inputStream != null) {
+			        inputStream.close();
+			    }
+			    httpConnection.disconnect();
+			}
+			// we will never actually get to this line
+			return null;
+		}
+	}
+
+	protected void retrieveAndCacheFeed(URL feedUrl, SyndFeedInfo syndFeedInfo, HttpURLConnection connection) throws IllegalArgumentException, FeedException, FetcherException, IOException {
+		handleErrorCodes(connection.getResponseCode());		
+
+		resetFeedInfo(feedUrl, syndFeedInfo, connection);
+		FeedFetcherCache cache = getFeedInfoCache();
+		// resetting feed info in the cache
+		// could be needed for some implementations
+		// of FeedFetcherCache (eg, distributed HashTables)
+		if (cache != null) {
+			cache.setFeedInfo(feedUrl, syndFeedInfo);
+		}
+	}
+
+	protected void resetFeedInfo(URL orignalUrl, SyndFeedInfo syndFeedInfo, HttpURLConnection connection) throws IllegalArgumentException, IOException, FeedException {
+		// need to always set the URL because this may have changed due to 3xx redirects
+		syndFeedInfo.setUrl(connection.getURL());
+
+		// the ID is a persistant value that should stay the same even if the URL for the
+		// feed changes (eg, by 3xx redirects)
+		syndFeedInfo.setId(orignalUrl.toString());
+
+		// This will be 0 if the server doesn't support or isn't setting the last modified header
+		syndFeedInfo.setLastModified(new Long(connection.getLastModified()));
+
+		// This will be null if the server doesn't support or isn't setting the ETag header
+		syndFeedInfo.setETag(connection.getHeaderField("ETag"));
+
+		// get the contents
+		InputStream inputStream = null;
+		try {
+			inputStream = connection.getInputStream();
+			SyndFeed syndFeed = getSyndFeedFromStream(inputStream, connection);
+			
+			String imHeader = connection.getHeaderField("IM");			
+			if (isUsingDeltaEncoding() && (imHeader!= null && imHeader.indexOf("feed") >= 0)) {
+				FeedFetcherCache cache = getFeedInfoCache();
+				if (cache != null && connection.getResponseCode() == 226) {
+				    // client is setup to use http delta encoding and the server supports it and has returned a delta encoded response
+				    // This response only includes new items
+				    SyndFeedInfo cachedInfo = cache.getFeedInfo(orignalUrl);
+				    if (cachedInfo != null) {
+					    SyndFeed cachedFeed = cachedInfo.getSyndFeed();
+					    
+					    // set the new feed to be the orginal feed plus the new items
+					    syndFeed = combineFeeds(cachedFeed, syndFeed);			        
+				    }
+				}
+			}
+			
+			syndFeedInfo.setSyndFeed(syndFeed);
+		} finally {
+			if (inputStream != null) {
+				inputStream.close();
+			}
+		}
+	}
+
+	/**
+	 * <p>Set appropriate HTTP headers, including conditional get and gzip encoding headers</p>
+	 *
+	 * @param connection A URLConnection
+	 * @param syndFeedInfo The SyndFeedInfo for the feed to be retrieved. May be null
+	 */
+	protected void setRequestHeaders(URLConnection connection, SyndFeedInfo syndFeedInfo) {
+		if (syndFeedInfo != null) {
+			// set the headers to get feed only if modified
+			// we support the use of both last modified and eTag headers
+			if (syndFeedInfo.getLastModified() != null) {			    
+			    Object lastModified = syndFeedInfo.getLastModified();
+			    if (lastModified instanceof Long) {
+			        connection.setIfModifiedSince(((Long)syndFeedInfo.getLastModified()).longValue());
+			    }				
+			}
+			if (syndFeedInfo.getETag() != null) {
+				connection.setRequestProperty("If-None-Match", syndFeedInfo.getETag());
+			}
+
+		}
+		// header to retrieve feed gzipped
+		connection.setRequestProperty("Accept-Encoding", "gzip");
+
+		if (isUsingDeltaEncoding()) {
+		    connection.addRequestProperty("A-IM", "feed");
+		}		
+	}
+
+	private SyndFeed readSyndFeedFromStream(InputStream inputStream, URLConnection connection) throws IOException, IllegalArgumentException, FeedException {
+		BufferedInputStream is;
+		if ("gzip".equalsIgnoreCase(connection.getContentEncoding())) {
+			// handle gzip encoded content
+			is = new BufferedInputStream(new GZIPInputStream(inputStream));
+		} else {
+			is = new BufferedInputStream(inputStream);
+		}
+
+		//InputStreamReader reader = new InputStreamReader(is, ResponseHandler.getCharacterEncoding(connection));
+
+		//SyndFeedInput input = new SyndFeedInput();
+
+	    XmlReader reader = null;	    
+	    if (connection.getHeaderField("Content-Type") != null) {
+	        reader = new XmlReader(is, connection.getHeaderField("Content-Type"), true);
+	    } else {
+	        reader = new XmlReader(is, true);
+	    }
+	    
+	    SyndFeedInput syndFeedInput = new SyndFeedInput();
+	    syndFeedInput.setPreserveWireFeed(isPreserveWireFeed());
+	    
+		return syndFeedInput.build(reader);
+	    
+	}
+
+	private SyndFeed getSyndFeedFromStream(InputStream inputStream, URLConnection connection) throws IOException, IllegalArgumentException, FeedException {
+		SyndFeed feed = readSyndFeedFromStream(inputStream, connection);
+		fireEvent(FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, connection, feed);
+		return feed;
+	}
+
+	/**
+	 * @return The FeedFetcherCache used by this fetcher (Could be null)
+	 */
+	public synchronized FeedFetcherCache getFeedInfoCache() {
+		return feedInfoCache;
+	}
+
+	/**
+	 * @param cache The cache to be used by this fetcher (pass null to stop using a cache)
+	 */
+	public synchronized void setFeedInfoCache(FeedFetcherCache cache) {
+		feedInfoCache = cache;
+	}
+}

src/java/org/rometools/fetcher/impl/LinkedHashMapFeedInfoCache.java

@@ -0,0 +1,70 @@
+package org.rometools.fetcher.impl;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * <p>An implementation of the {@link com.sun.syndication.fetcher.impl.FeedFetcherCache} interface.</p>
+ * 
+ * <p>Unlike the HashMapFeedInfoCache this implementation will not grow unbound</p>
+ * 
+ * @author Javier Kohen
+ * @author Nick Lothian
+ *
+ */
+public class LinkedHashMapFeedInfoCache extends HashMapFeedInfoCache {
+	private final class CacheImpl extends LinkedHashMap {
+		private static final long serialVersionUID = -6977191330127794920L;
+
+		public CacheImpl() {
+			super(16, 0.75F, true);
+		}
+		
+		protected boolean removeEldestEntry(Map.Entry eldest) {
+			return size() > getMaxEntries();
+		}
+	}
+	
+	private static final int DEFAULT_MAX_ENTRIES = 20;
+
+	private static final long serialVersionUID = 1694228973357997417L;
+
+	private int maxEntries = DEFAULT_MAX_ENTRIES;
+
+	private final static LinkedHashMapFeedInfoCache _instance = new LinkedHashMapFeedInfoCache();	
+	
+	
+	/**
+	 * Get the global instance of the cache
+	 * @return an implementation of FeedFetcherCache
+	 */
+	public static final FeedFetcherCache getInstance() {
+		return _instance;
+	}
+
+	/**
+	 * <p>Constructor for HashMapFeedInfoCache</p>
+	 * 
+	 * <p>Only use this if you want multiple instances of the cache. 
+	 * Usually {@link #getInstance()} is more appropriate.</p>
+	 *
+	 * @see #getInstance()
+	 */
+	public LinkedHashMapFeedInfoCache() {
+		super();
+	}
+
+	protected Map createInfoCache() {
+		return Collections.synchronizedMap(new CacheImpl());
+	}
+
+	public synchronized final int getMaxEntries() {
+		return maxEntries;
+	}
+
+	public synchronized final void setMaxEntries(int maxEntries) {
+		this.maxEntries = maxEntries;
+	}	
+	
+}

src/java/org/rometools/fetcher/impl/ResponseHandler.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.rometools.fetcher.impl;
+
+import java.net.URLConnection;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Utility class to help deal with HTTP responses
+ *
+ */
+public class ResponseHandler {
+	public static final String defaultCharacterEncoding = "ISO-8859-1";
+	
+	private final static Pattern characterEncodingPattern = Pattern.compile("charset=([.[^; ]]*)");
+	
+	public static String getCharacterEncoding(URLConnection connection) {
+		return getCharacterEncoding(connection.getContentType());
+	}
+	
+	/**
+	 * 
+	 * <p>Gets the character encoding of a response. (Note that this is different to
+	 * the content-encoding)</p>
+	 * 
+	 * @param contentTypeHeader the value of the content-type HTTP header eg: text/html; charset=ISO-8859-4
+	 * @return the character encoding, eg: ISO-8859-4
+	 */
+	public static String getCharacterEncoding(String contentTypeHeader) {
+		if (contentTypeHeader == null) {
+			return defaultCharacterEncoding;
+		}
+		
+		Matcher m = characterEncodingPattern.matcher(contentTypeHeader);
+		//if (!m.matches()) {
+		if (!m.find()) {
+			return defaultCharacterEncoding;
+		} else {
+			return m.group(1);
+		}
+	}
+}

src/java/org/rometools/fetcher/impl/SyndFeedInfo.java

@@ -0,0 +1,149 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.rometools.fetcher.impl;
+
+import java.io.Serializable;
+import java.net.URL;
+
+import com.sun.syndication.feed.impl.ObjectBean;
+import com.sun.syndication.feed.synd.SyndFeed;
+
+/**
+ * <p>A class to represent a {@link com.sun.syndication.feed.synd.SyndFeed}
+ * and some useful information about it.</p>
+ * 
+ * <p>This class is thread safe, as expected by the different feed fetcher
+ * implementations.</p>
+ *
+ * @author Nick Lothian
+ */
+public class SyndFeedInfo implements Cloneable, Serializable {
+	private static final long serialVersionUID = -1874786860901426015L;
+	
+	private final ObjectBean _objBean;
+	private String id;
+	private URL url;
+	private Object lastModified;
+	private String eTag;
+	private SyndFeed syndFeed;
+
+    public SyndFeedInfo() {
+        _objBean = new ObjectBean(this.getClass(),this);
+    }
+
+    /**
+     * Creates a deep 'bean' clone of the object.
+     * <p>
+     * @return a clone of the object.
+     * @throws CloneNotSupportedException thrown if an element of the object cannot be cloned.
+     *
+     */
+    public Object clone() throws CloneNotSupportedException {
+        return _objBean.clone();
+    }
+
+    /**
+     * Indicates whether some other object is "equal to" this one as defined by the Object equals() method.
+     * <p>
+     * @param other he reference object with which to compare.
+     * @return <b>true</b> if 'this' object is equal to the 'other' object.
+     *
+     */
+    public boolean equals(Object other) {
+        return _objBean.equals(other);
+    }
+
+    /**
+     * Returns a hashcode value for the object.
+     * <p>
+     * It follows the contract defined by the Object hashCode() method.
+     * <p>
+     * @return the hashcode of the bean object.
+     *
+     */
+    public int hashCode() {
+        return _objBean.hashCode();
+    }
+
+    /**
+     * Returns the String representation for the object.
+     * <p>
+     * @return String representation for the object.
+     *
+     */
+    public String toString() {
+        return _objBean.toString();
+    }
+
+
+	/**
+	 * @return the ETag the feed was last retrieved with
+	 */
+	public synchronized String getETag() {
+		return eTag;
+	}
+
+	/**
+	 * @return the last modified date for the feed
+	 */
+	public synchronized Object getLastModified() {
+		return lastModified;
+	}
+
+	/**
+	 * @return the URL the feed was served from
+	 */
+	public synchronized URL getUrl() {
+		return url;
+	}
+
+	public synchronized void setETag(String string) {
+		eTag = string;
+	}
+
+	public synchronized void setLastModified(Object o) {
+		lastModified = o;
+	}
+
+	public synchronized void setUrl(URL url) {
+		this.url = url;
+	}
+
+	public synchronized SyndFeed getSyndFeed() {
+		return syndFeed;
+	}
+
+	public synchronized void setSyndFeed(SyndFeed feed) {
+		syndFeed = feed;
+	}
+
+	/**
+	 * @return A unique ID to identify the feed
+	 */
+	public synchronized String getId() {
+		return id;
+	}
+
+	/**
+	 * @param string A unique ID to identify the feed. Note that if the URL of the feed
+	 * changes this will remain the same
+	 */
+	public synchronized void setId(String string) {
+		id = string;
+	}
+
+}

src/java/org/rometools/fetcher/samples/FeedAggregator.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.rometools.fetcher.samples;
+
+import java.io.PrintWriter;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.sun.syndication.feed.synd.SyndFeedImpl;
+import com.sun.syndication.feed.synd.SyndFeed;
+import org.rometools.fetcher.FeedFetcher;
+import org.rometools.fetcher.impl.FeedFetcherCache;
+import org.rometools.fetcher.impl.HashMapFeedInfoCache;
+import org.rometools.fetcher.impl.HttpURLFeedFetcher;
+import com.sun.syndication.io.SyndFeedOutput;
+
+/**
+ * <p>It aggregates a list of RSS/Atom feeds (they can be of different types)
+ * into a single feed of the specified type.</p>
+ *
+ * <p>Converted from the original FeedAggregator sample</p>
+ *
+ * @author Alejandro Abdelnur
+ * @author Nick Lothian
+ *
+ */
+public class FeedAggregator {
+
+    public static void main(String[] args) {
+        boolean ok = false;
+        if (args.length>=2) {
+            try {
+                String outputType = args[0];
+
+                SyndFeed feed = new SyndFeedImpl();
+                feed.setFeedType(outputType);
+
+                feed.setTitle("Aggregated Feed");
+                feed.setDescription("Anonymous Aggregated Feed");
+                feed.setAuthor("anonymous");
+                feed.setLink("http://www.anonymous.com");
+
+                List entries = new ArrayList();
+                feed.setEntries(entries);
+
+				FeedFetcherCache feedInfoCache = HashMapFeedInfoCache.getInstance();
+				FeedFetcher feedFetcher = new HttpURLFeedFetcher(feedInfoCache);
+
+                for (int i=1;i<args.length;i++) {
+                    URL inputUrl = new URL(args[i]);
+					SyndFeed inFeed = feedFetcher.retrieveFeed(inputUrl);
+                    entries.addAll(inFeed.getEntries());
+                }
+
+                SyndFeedOutput output = new SyndFeedOutput();
+				output.output(feed, new PrintWriter(System.out));
+
+                ok = true;
+            }
+            catch (Exception ex) {
+                System.out.println("ERROR: "+ex.getMessage());
+                ex.printStackTrace();
+            }
+        }
+
+        if (!ok) {
+            System.out.println();
+            System.out.println("FeedAggregator aggregates different feeds into a single one.");
+            System.out.println("The first parameter must be the feed type for the aggregated feed.");
+            System.out.println(" [valid values are: rss_0.9, rss_0.91, rss_0.92, rss_0.93, ]");
+            System.out.println(" [                  rss_0.94, rss_1.0, rss_2.0 & atom_0.3  ]");
+            System.out.println("The second to last parameters are the URLs of feeds to aggregate.");
+            System.out.println();
+        }
+    }
+
+}

src/java/org/rometools/fetcher/samples/FeedReader.java

@@ -0,0 +1,96 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.rometools.fetcher.samples;
+import java.net.URL;
+
+import com.sun.syndication.feed.synd.SyndFeed;
+import org.rometools.fetcher.FeedFetcher;
+import org.rometools.fetcher.FetcherEvent;
+import org.rometools.fetcher.FetcherListener;
+import org.rometools.fetcher.impl.FeedFetcherCache;
+import org.rometools.fetcher.impl.HashMapFeedInfoCache;
+import org.rometools.fetcher.impl.HttpURLFeedFetcher;
+/**
+ * Reads and prints any RSS/Atom feed type. Converted from the
+ * original Rome sample FeedReader
+ * <p>
+ * @author Alejandro Abdelnur
+ * @author Nick Lothian
+ *
+ */
+public class FeedReader {
+	public static void main(String[] args) {
+		boolean ok = false;
+		if (args.length==1) {
+			try {
+				URL feedUrl = new URL(args[0]);
+				FeedFetcherCache feedInfoCache = HashMapFeedInfoCache.getInstance();
+				FeedFetcher fetcher = new HttpURLFeedFetcher(feedInfoCache);
+
+				FetcherEventListenerImpl listener = new FetcherEventListenerImpl();
+
+				fetcher.addFetcherEventListener(listener);
+
+				System.err.println("Retrieving feed " + feedUrl);
+				// Retrieve the feed.
+				// We will get a Feed Polled Event and then a
+				// Feed Retrieved event (assuming the feed is valid)
+				SyndFeed feed = fetcher.retrieveFeed(feedUrl);
+
+				System.err.println(feedUrl + " retrieved");
+				System.err.println(feedUrl + " has a title: " + feed.getTitle() + " and contains " + feed.getEntries().size() + " entries.");
+				// We will now retrieve the feed again. If the feed is unmodified
+				// and the server supports conditional gets, we will get a "Feed
+				// Unchanged" event after the Feed Polled event
+				System.err.println("Polling " + feedUrl + " again to test conditional get support.");
+				SyndFeed feed2 = fetcher.retrieveFeed(feedUrl);
+				System.err.println("If a \"Feed Unchanged\" event fired then the server supports conditional gets.");
+
+				ok = true;
+			}
+			catch (Exception ex) {
+				System.out.println("ERROR: "+ex.getMessage());
+				ex.printStackTrace();
+			}
+		}
+
+		if (!ok) {
+			System.out.println();
+			System.out.println("FeedReader reads and prints any RSS/Atom feed type.");
+			System.out.println("The first parameter must be the URL of the feed to read.");
+			System.out.println();
+		}
+
+	}
+
+	static class FetcherEventListenerImpl implements FetcherListener {
+		/**
+		 * @see com.sun.syndication.fetcher.FetcherListener#fetcherEvent(com.sun.syndication.fetcher.FetcherEvent)
+		 */
+		public void fetcherEvent(FetcherEvent event) {
+			String eventType = event.getEventType();
+			if (FetcherEvent.EVENT_TYPE_FEED_POLLED.equals(eventType)) {
+				System.err.println("\tEVENT: Feed Polled. URL = " + event.getUrlString());
+			} else if (FetcherEvent.EVENT_TYPE_FEED_RETRIEVED.equals(eventType)) {
+				System.err.println("\tEVENT: Feed Retrieved. URL = " + event.getUrlString());
+			} else if (FetcherEvent.EVENT_TYPE_FEED_UNCHANGED.equals(eventType)) {
+				System.err.println("\tEVENT: Feed Unchanged. URL = " + event.getUrlString());
+			}
+		}
+	}
+}

src/test/org/rometools/test/AbstractJettyTest.java

@@ -0,0 +1,421 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.rometools.test;
+
+import org.rometools.fetcher.impl.HashMapFeedInfoCache;
+import org.rometools.fetcher.impl.FeedFetcherCache;
+import java.net.URL;
+
+import junit.framework.TestCase;
+
+import org.mortbay.http.BasicAuthenticator;
+import org.mortbay.http.HashUserRealm;
+import org.mortbay.http.HttpContext;
+import org.mortbay.http.HttpServer;
+import org.mortbay.http.SecurityConstraint;
+import org.mortbay.http.SocketListener;
+import org.mortbay.http.UserRealm;
+import org.mortbay.http.handler.SecurityHandler;
+import org.mortbay.jetty.servlet.ServletHandler;
+
+import com.sun.syndication.feed.atom.Entry;
+import com.sun.syndication.feed.synd.SyndEntry;
+import com.sun.syndication.feed.synd.SyndFeed;
+import org.rometools.fetcher.FeedFetcher;
+import org.rometools.fetcher.FetcherEvent;
+import org.rometools.fetcher.FetcherException;
+import org.rometools.fetcher.FetcherListener;
+
+/**
+ * @author nl
+ */
+public abstract class AbstractJettyTest extends TestCase {
+
+	private HttpServer server;
+    private int testPort = 8283;
+	
+	/**
+	 * @param s
+	 */
+	public AbstractJettyTest(String s) {
+		super(s);
+	}
+
+	protected HttpServer getServer() {
+		return server;
+	}
+
+	protected abstract FeedFetcher getFeedFetcher();
+	
+	protected abstract FeedFetcher getFeedFetcher(FeedFetcherCache cache);
+	
+	/**
+	 * @see junit.framework.TestCase#setUp()
+	 */
+	protected void setUp() throws Exception {
+		setupServer();
+	
+		HttpContext context = createContext();
+		
+		ServletHandler servlets = createServletHandler();
+		context.addHandler(servlets);
+		
+		server.addContext(context);		
+		
+		server.start();
+	}
+
+	/**
+     * @throws InterruptedException
+     */
+    private void setupServer() throws InterruptedException {
+        // Create the server
+		if (server != null) {
+			server.stop();
+			server = null;
+		}
+		server = new HttpServer();
+	
+		// Create a port listener
+		SocketListener listener=new SocketListener();
+		listener.setPort(testPort);
+		server.addListener(listener);
+    }
+
+    /**
+     * @return
+     */
+    private ServletHandler createServletHandler() {
+        ServletHandler servlets = new ServletHandler();
+		servlets.addServlet("FetcherTestServlet",FetcherTestServlet.SERVLET_MAPPING,"org.rometools.test.FetcherTestServlet");
+		servlets.addServlet("FetcherTestServlet",FetcherTestServlet.SERVLET_MAPPING2,"org.rometools.test.FetcherTestServlet");
+        return servlets;
+    }
+
+    /**
+     * @return
+     */
+    private HttpContext createContext() {
+        HttpContext context = new HttpContext();
+		context.setContextPath("/rome/*");
+        return context;
+    }
+
+    /**
+	 * @see junit.framework.TestCase#tearDown()
+	 */
+	protected void tearDown() throws Exception {
+		if (server != null) {
+			server.stop();
+			server.destroy();
+			server = null;
+		}
+	}
+
+	class FetcherEventListenerImpl implements FetcherListener {
+		boolean polled = false;
+		boolean retrieved = false;
+		boolean unchanged = false;
+
+		public void reset() {
+			polled = false;
+			retrieved = false;
+			unchanged = false;
+		}
+
+		/**
+		 * @see com.sun.syndication.fetcher.FetcherListener#fetcherEvent(com.sun.syndication.fetcher.FetcherEvent)
+		 */
+		public void fetcherEvent(FetcherEvent event) {
+			String eventType = event.getEventType();
+			if (FetcherEvent.EVENT_TYPE_FEED_POLLED.equals(eventType)) {
+				System.err.println("\tEVENT: Feed Polled. URL = " + event.getUrlString());
+				polled = true;
+			} else if (FetcherEvent.EVENT_TYPE_FEED_RETRIEVED.equals(eventType)) {
+				System.err.println("\tEVENT: Feed Retrieved. URL = " + event.getUrlString());
+				retrieved = true;
+			} else if (FetcherEvent.EVENT_TYPE_FEED_UNCHANGED.equals(eventType)) {
+				System.err.println("\tEVENT: Feed Unchanged. URL = " + event.getUrlString());
+				unchanged = true;
+			}
+		}
+	}
+
+	public void testRetrieveFeed() {
+		FeedFetcher feedFetcher = getFeedFetcher();
+		try {
+			SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
+			assertNotNull(feed);
+			assertEquals("atom_1.0.feed.title", feed.getTitle());
+		} catch (Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}
+	}
+
+	public void testBasicAuthentication() {	    
+		try {
+            setupServer();
+            
+            HttpContext context = createContext();
+
+            URL url = this.getClass().getResource("/testuser.properties");            
+            UserRealm ur = new HashUserRealm("test", url.getFile());						
+            context.setRealm(ur);
+
+            BasicAuthenticator ba = new BasicAuthenticator();
+            context.setAuthenticator(ba);		
+            
+            SecurityHandler sh =  new SecurityHandler();					
+            context.addHandler(sh);
+
+            SecurityConstraint sc = new SecurityConstraint();
+            sc.setName("test");
+            sc.addRole("*");
+            sc.setAuthenticate(true);		
+            context.addSecurityConstraint("/", sc);			
+            
+            ServletHandler servlets = createServletHandler();
+            context.addHandler(servlets);
+            
+            server.addContext(context);		
+            
+            server.start();            
+            
+            FeedFetcher feedFetcher = getAuthenticatedFeedFetcher();
+			SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
+			assertNotNull(feed);
+			assertEquals("atom_1.0.feed.title", feed.getTitle());
+            
+            
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }	    
+	    
+		
+	}
+	
+	public abstract FeedFetcher getAuthenticatedFeedFetcher();
+	
+	/**
+	 * Test getting a feed via a http 301 redirect
+	 *
+	 */
+	public void testRetrieveRedirectedFeed() {
+		FeedFetcher feedFetcher = getFeedFetcher();
+		try {			
+			SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet?redirect=TRUE"));
+			assertNotNull(feed);
+			assertEquals("atom_1.0.feed.title", feed.getTitle());
+		} catch (Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}
+	}
+
+	/**
+	 * Test error handling
+	 *
+	 */
+	public void testErrorHandling() {
+		FeedFetcher feedFetcher = getFeedFetcher();
+		try {
+			SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet?error=404"));
+			fail("4xx error handling did not work correctly");
+		} catch (FetcherException e) {
+			// expect this exception
+			assertEquals(404, e.getResponseCode());
+		} catch (Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}
+	
+		try {
+			SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet?error=500"));
+			fail("5xx error handling did not work correctly");
+		} catch (FetcherException e) {
+			// expect this exception
+			assertEquals(500, e.getResponseCode());
+		} catch (Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}
+	}
+
+	public void testUserAgent() {
+		FeedFetcher feedFetcher = getFeedFetcher();
+		//System.out.println(feedFetcher.getUserAgent());
+		//System.out.println(System.getProperty("rome.fetcher.version", "UNKNOWN"));
+		assertEquals("Rome Client (http://tinyurl.com/64t5n) Ver: " + System.getProperty("rome.fetcher.version", "UNKNOWN"), feedFetcher.getUserAgent());
+	}
+
+	/**
+	 * Test events fired when there is no cache in use
+	 *
+	 */
+	public void testFetchEvents() {
+		FeedFetcher feedFetcher = getFeedFetcher();
+		FetcherEventListenerImpl listener = new FetcherEventListenerImpl();
+		feedFetcher.addFetcherEventListener(listener);
+		try {
+			SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
+			assertNotNull(feed);
+			assertTrue(listener.polled);
+			assertTrue(listener.retrieved);
+			assertFalse(listener.unchanged);
+			listener.reset();
+	
+			// since there is no cache, the events fired should be exactly the same if
+			// we re-retrieve the feed
+			feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
+			assertNotNull(feed);
+			assertTrue(listener.polled);
+			assertTrue(listener.retrieved);
+			assertFalse(listener.unchanged);
+			listener.reset();
+		} catch (Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}
+	}
+
+	/**
+	 * Test events fired when there is a cache in use
+	 *
+	 */
+	public void testFetchEventsWithCache() {
+		FeedFetcherCache feedInfoCache = new HashMapFeedInfoCache();
+		FeedFetcher feedFetcher = getFeedFetcher(feedInfoCache);
+		FetcherEventListenerImpl listener = new FetcherEventListenerImpl();
+		feedFetcher.addFetcherEventListener(listener);
+		try {
+			SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
+			assertNotNull(feed);
+			assertTrue(listener.polled);
+			assertTrue(listener.retrieved);
+			assertFalse(listener.unchanged);
+			listener.reset();
+	
+			// Since the feed is cached, the second request should not
+			// actually retrieve the feed
+			feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
+			assertNotNull(feed);
+			assertTrue(listener.polled);
+			assertFalse(listener.retrieved);
+			assertTrue(listener.unchanged);
+			listener.reset();
+	
+			// now simulate getting the feed after it has changed
+			feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet?refreshfeed=TRUE"));
+			assertNotNull(feed);
+			assertTrue(listener.polled);
+			assertTrue(listener.retrieved);
+			assertFalse(listener.unchanged);
+			listener.reset();
+		} catch (Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}
+	}
+	
+	/**
+	 * Test handling of GZipped feed
+	 *
+	 */
+	public void testGZippedFeed() {
+	    FeedFetcher feedFetcher = getFeedFetcher();
+		try {
+			SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet?gzipfeed=TRUE"));
+			assertNotNull(feed);
+			assertEquals("atom_1.0.feed.title", feed.getTitle());
+		} catch (Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}	    
+	}
+	
+	public void testPreserveWireFeed() throws Exception {
+		FeedFetcher feedFetcher = getFeedFetcher();
+
+		// first check we the WireFeed is not preserved by default
+		SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
+		assertNotNull(feed);
+		assertEquals("atom_1.0.feed.title", feed.getTitle());
+		assertNull(feed.originalWireFeed());
+		
+		SyndEntry syndEntry = (SyndEntry)feed.getEntries().get(0);
+		assertNotNull(syndEntry);
+		assertNull(syndEntry.getWireEntry());
+		
+		// now turn on WireFeed preservation	
+		feedFetcher.setPreserveWireFeed(true);
+		try {
+			feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
+			assertNotNull(feed);
+			assertEquals("atom_1.0.feed.title", feed.getTitle());
+			assertNotNull(feed.originalWireFeed());
+
+			syndEntry = (SyndEntry)feed.getEntries().get(0);
+			assertNotNull(syndEntry);
+			assertNotNull(syndEntry.getWireEntry());
+			
+			Entry entry = (Entry) syndEntry.getWireEntry();
+			assertEquals("atom_1.0.feed.entry[0].rights", entry.getRights());
+			
+		} finally {
+			feedFetcher.setPreserveWireFeed(false); //reset 
+		}
+		
+	}
+	
+	public void testDeltaEncoding() {
+	    FeedFetcherCache feedInfoCache = new HashMapFeedInfoCache();
+		FeedFetcher feedFetcher = getFeedFetcher(feedInfoCache);	    		
+		try {
+		    feedFetcher.setUsingDeltaEncoding(true);
+		    
+		    // first retrieval should just grab the default feed
+			SyndFeed feed1 = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet?deltaencode=TRUE&refreshfeed=TRUE"));
+			assertNotNull(feed1);
+			assertEquals("atom_1.0.feed.title", feed1.getTitle());
+			assertEquals(2, feed1.getEntries().size());
+			SyndEntry entry1 = (SyndEntry) feed1.getEntries().get(0);
+			assertEquals("atom_1.0.feed.entry[0].title", entry1.getTitle());
+			
+			// second retrieval should get only the new item
+			/*
+			 * This is breaking with Rome 0.5 ??
+			 */ 
+			SyndFeed feed2 = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet?deltaencode=TRUE&refreshfeed=TRUE"));					
+			assertNotNull(feed2);
+			assertEquals(FetcherTestServlet.DELTA_FEED_TITLE, feed2.getTitle());
+			assertEquals(3, feed2.getEntries().size());
+			entry1 = (SyndEntry) feed2.getEntries().get(0);
+			assertEquals(FetcherTestServlet.DELTA_FEED_ENTRY_TITLE, entry1.getTitle());
+			
+			SyndEntry entry2 = (SyndEntry) feed2.getEntries().get(1);
+			assertEquals("atom_1.0.feed.entry[0].title", entry2.getTitle());			
+			
+		} catch (Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}	    	    
+	}
+		
+	
+}

src/test/org/rometools/test/DiskFeedInfoCacheTest.java

@@ -0,0 +1,42 @@
+package org.rometools.test;
+
+import org.rometools.fetcher.impl.DiskFeedInfoCache;
+import org.rometools.fetcher.impl.SyndFeedInfo;
+import java.net.*;
+import java.io.File;
+import junit.framework.TestCase;
+
+public class DiskFeedInfoCacheTest extends TestCase {
+
+	public void testClear() throws Exception {
+		File cacheDir = new File("test-cache");
+		cacheDir.mkdir();
+		cacheDir.deleteOnExit();
+		
+		final DiskFeedInfoCache cache = new DiskFeedInfoCache(cacheDir.getCanonicalPath());
+		SyndFeedInfo info = new SyndFeedInfo();
+		URL url = new URL("http://nowhere.com");
+		cache.setFeedInfo(url, info);
+		
+		cache.clear();
+		final Object returned = cache.getFeedInfo(url);
+		assertTrue( returned == null );
+	}
+	
+	public void testRemove() throws Exception {
+		File cacheDir = new File("test-cache");
+		cacheDir.mkdir();
+		cacheDir.deleteOnExit();
+		
+		final DiskFeedInfoCache cache = new DiskFeedInfoCache( cacheDir.getCanonicalPath() );
+		SyndFeedInfo info = new SyndFeedInfo();
+		URL url = new URL("http://nowhere.com");
+		cache.setFeedInfo( url, info );
+		
+		SyndFeedInfo removedInfo = cache.remove( url );
+		assertTrue( removedInfo.equals(info) );
+		SyndFeedInfo shouldBeNull = cache.remove( url );
+		assertTrue( null == shouldBeNull );	
+	}
+	
+}

src/test/org/rometools/test/FetcherTestServlet.java

@@ -0,0 +1,244 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.rometools.test;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.zip.GZIPOutputStream;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import com.sun.syndication.feed.synd.SyndContent;
+import com.sun.syndication.feed.synd.SyndContentImpl;
+import com.sun.syndication.feed.synd.SyndEntry;
+import com.sun.syndication.feed.synd.SyndEntryImpl;
+import com.sun.syndication.feed.synd.SyndFeed;
+import com.sun.syndication.feed.synd.SyndFeedImpl;
+import com.sun.syndication.io.FeedException;
+import com.sun.syndication.io.SyndFeedOutput;
+
+
+public class FetcherTestServlet extends javax.servlet.http.HttpServlet {
+    public static final String ETAG_1 = "ETAG-1";
+    public static final String ETAG_2 = "ETAG-2";
+    
+    public static final String DELTA_FEED_TITLE = "Delta Encoded Feed";
+    public static final String DELTA_FEED_ENTRY_TITLE = "Delta Encoded Feed Entry";
+    
+    public static final String SERVLET_MAPPING = "/FetcherTestServlet/*";
+    public static final String SERVLET_MAPPING2 = "/FetcherTestServlet2/*";    
+    
+	/**
+	 * @throws IOException
+	 * @throws 
+	 * @see javax.servlet.http.HttpServlet#doGet(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse)
+	 */
+	protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+		
+		if ("TRUE".equalsIgnoreCase(request.getParameter("redirect"))) {
+			//	testing redirection support			
+			response.sendRedirect("/rome/FetcherTestServlet2/");
+			return;
+		} else if (request.getParameter("error") != null) {
+			//response.sendError(HttpServletResponse.SC_NOT_FOUND);	
+			int errorToThrow = Integer.parseInt(request.getParameter("error"));
+			response.sendError(errorToThrow);
+			return;
+		} else {
+			
+			// We manually set the date headers using strings
+			// instead of the get/setDateHeader methods because
+			// those methods return longs, which has too much
+			// precision for the real date headers
+			// this is just a random date
+			String lastModifiedDate = "Thu, 08 Jan 2009 23:06:39 GMT"; 
+			String eTag = ETAG_1;
+			
+			if ("TRUE".equalsIgnoreCase(request.getParameter("refreshfeed"))) {
+				lastModifiedDate = "Fri, 09 Jan 2009 12:06:39 GMT";
+				eTag = ETAG_2; 
+			}
+			
+			boolean serveFeed = checkModified(request, lastModifiedDate, eTag) || ("TRUE".equalsIgnoreCase(request.getParameter("deltaencode")));
+			boolean gzip = "TRUE".equalsIgnoreCase(request.getParameter("gzipfeed"));
+			
+			if (serveFeed) {
+			    String aimHeader = request.getHeader("A-IM");
+			    boolean serveDeltaEncodedFeed = ((aimHeader != null) && (aimHeader.indexOf("feed") >=0) && "TRUE".equalsIgnoreCase(request.getParameter("deltaencode")));
+			    if (serveDeltaEncodedFeed) {
+			        try {
+                        sendDeltaEncodedData(response, lastModifiedDate, request.getHeader("If-None-Match"), eTag, gzip);
+                    } catch (FeedException e) {
+                        throw new ServletException(e);
+                    }
+			    } else {
+			        sendFeedData(response, lastModifiedDate, eTag, gzip);
+			    }
+				return;					
+			} else {
+				response.sendError(HttpServletResponse.SC_NOT_MODIFIED);
+				return;							
+			}
+		}							
+	}
+
+
+
+    private boolean checkModified(HttpServletRequest request, String lastModifiedDate, String eTag) {
+		
+		String requestedETag = request.getHeader("If-None-Match");			
+		String requestedLastModified = request.getHeader("If-Modified-Since");
+		boolean modified = true;
+		boolean mustServer = false;			
+		if (requestedETag != null) {
+			if (eTag.equals(requestedETag)) {
+				modified = false;
+			} else {
+				modified = true;
+				mustServer = true;				
+			}
+		}
+		if (requestedLastModified != null) {
+			if (lastModifiedDate.equals(requestedLastModified)) {
+				modified = false;
+			} else {
+				modified = true;
+				mustServer = true;					
+			}
+		}
+		boolean serveFeed = (modified || mustServer);
+		return serveFeed;
+	}
+
+	/**
+     * @param request
+     * @param lastModifiedDate
+     * @param tag
+     * @param gzip
+	 * @throws IOException
+	 * @throws FeedException
+     */
+    private void sendDeltaEncodedData(HttpServletResponse response, String lastModifiedDate, String requestedETag, String responseETag, boolean gzip) throws IOException, FeedException {
+        if (ETAG_1.equals(requestedETag) || ETAG_2.equals(requestedETag)) {               
+		    OutputStream out = null;
+		    if (gzip) {
+		        response.setHeader("Content-Encoding", "gzip");
+		        out = new GZIPOutputStream(response.getOutputStream());	        
+		    } else {
+		        out = response.getOutputStream();	        
+		    }
+		    
+			response.setContentType("text/xml");
+			response.setStatus(226);
+			if (gzip) {
+			    response.setHeader("IM", "feed, gzip");
+			} else {
+			    response.setHeader("IM", "feed");
+			}
+			
+			if (responseETag != null) {
+				response.setHeader("ETag", responseETag);
+			}		
+			if (lastModifiedDate != null) {
+				response.setHeader("Last-Modified", lastModifiedDate);
+			}		    
+		    
+	        SyndFeed feed = new SyndFeedImpl();
+	        feed.setFeedType("atom_1.0");
+	
+	        feed.setTitle(DELTA_FEED_TITLE);        
+	        feed.setLink("http://rome.dev.java.net");
+	        feed.setDescription("This tests using rfc3229 delta encoding.");
+	
+	        List entries = new ArrayList();
+	        SyndEntry entry;
+	        SyndContent description;
+	
+	        entry = new SyndEntryImpl();
+	        entry.setTitle(DELTA_FEED_ENTRY_TITLE);
+	        entry.setLink("http://bobwyman.pubsub.com/main/2004/09/using_rfc3229_w.html");
+	        try {
+	            DateFormat dateParser = new SimpleDateFormat("yyyy-MM-dd");
+	            entry.setPublishedDate(dateParser.parse("2004-11-25"));
+	        }
+	        catch (ParseException ex) {
+	            // 
+	        }
+	        description = new SyndContentImpl();
+	        description.setType("text/plain");
+	        description.setValue("Test for RFC3229 Delta Encoding");	        
+	        entry.setDescription(description);	        
+	        entries.add(entry);
+	        
+	        feed.setEntries(entries);
+	        
+	        SyndFeedOutput output = new SyndFeedOutput();        
+	        output.output(feed, new OutputStreamWriter(out));
+        } else {
+            sendFeedData(response, lastModifiedDate, responseETag, gzip);
+        }
+    }    
+    
+	private void sendFeedData(HttpServletResponse response, String lastModifiedDate, String eTag, boolean gzip) throws IOException {
+	    OutputStream out = null;
+	    if (gzip) {
+	        response.setHeader("Content-Encoding", "gzip");
+	        out = new GZIPOutputStream(response.getOutputStream());	        
+	    } else {
+	        out = response.getOutputStream();
+	    }	    
+	    
+		response.setContentType("text/xml");
+		if (eTag != null) {
+			response.setHeader("ETag", eTag);
+		}		
+		if (lastModifiedDate != null) {
+			response.setHeader("Last-Modified", lastModifiedDate);
+		}
+		
+		InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream("/atom_1.0.xml");
+		if (inputStream == null) {
+		    inputStream = this.getClass().getResourceAsStream("/atom_1.0.xml");
+		}
+		
+		BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
+		try {
+			String line;
+			while ((line = reader.readLine()) != null) {									
+			    out.write(line.getBytes());
+				line = null;
+			}
+		} finally {
+			if (reader != null) {
+			    reader.close();
+			}
+		}
+		
+		out.close();
+	}
+}

src/test/org/rometools/test/HashMapFeedInfoCacheTest.java

@@ -0,0 +1,41 @@
+package org.rometools.test;
+
+import org.rometools.fetcher.impl.HashMapFeedInfoCache;
+import org.rometools.fetcher.impl.SyndFeedInfo;
+import java.net.URL;
+
+import junit.framework.TestCase;
+
+public class HashMapFeedInfoCacheTest extends TestCase {
+
+	public void testRemove() throws Exception {
+		final HashMapFeedInfoCache cache = new HashMapFeedInfoCache();
+		assertNotNull( cache );
+		
+		final URL url = new URL("http://foo.com");
+		final SyndFeedInfo syndFeedInfo = new SyndFeedInfo();
+		syndFeedInfo.setUrl(url);
+		cache.setFeedInfo(url, syndFeedInfo);
+		
+		final SyndFeedInfo returned = cache.remove(url);
+		assertTrue( returned.equals(syndFeedInfo) );
+		assertTrue( url.equals( returned.getUrl() ));
+	}
+	
+	public void testClear() throws Exception {
+		final HashMapFeedInfoCache cache = new HashMapFeedInfoCache();
+		assertNotNull( cache );
+		
+		final URL url = new URL("http://foo.com");
+		final SyndFeedInfo syndFeedInfo = new SyndFeedInfo();
+		syndFeedInfo.setUrl(url);
+		cache.setFeedInfo(url, syndFeedInfo);
+		
+		//clear it
+		cache.clear();
+		
+		//we should not get a result back
+		final Object returned = cache.getFeedInfo(url);
+		assertTrue( returned == null );
+	}
+}

src/test/org/rometools/test/HttpClientFeedFetcherTest.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.rometools.test;
+
+import org.rometools.fetcher.impl.HttpClientFeedFetcher;
+import org.rometools.fetcher.impl.FeedFetcherCache;
+import org.apache.commons.httpclient.Credentials;
+import org.apache.commons.httpclient.UsernamePasswordCredentials;
+
+import org.rometools.fetcher.FeedFetcher;
+
+/**
+ * @author Nick Lothian
+ */
+public class HttpClientFeedFetcherTest extends AbstractJettyTest {
+
+	public HttpClientFeedFetcherTest(String s) {
+		super(s);
+	}
+	
+	/**
+	 * @see com.sun.syndication.fetcher.impl.AbstractJettyTest#getFeedFetcher()
+	 */
+	protected FeedFetcher getFeedFetcher() {		
+		return new HttpClientFeedFetcher();
+	}
+	
+	protected FeedFetcher getFeedFetcher(FeedFetcherCache cache) {
+		return new HttpClientFeedFetcher(cache);
+	}
+
+    /**
+     * @see com.sun.syndication.fetcher.impl.AbstractJettyTest#getAuthenticatedFeedFetcher()
+     */
+    public FeedFetcher getAuthenticatedFeedFetcher() {
+        return new HttpClientFeedFetcher(null, new HttpClientFeedFetcher.CredentialSupplier() {
+            public Credentials getCredentials(String realm, String host) {
+                if ("localhost".equals(host)) {
+                	return new UsernamePasswordCredentials("username", "password");
+                } else {
+                    return null;
+                }                
+            }            
+        }); 
+    }
+}

src/test/org/rometools/test/HttpURLFeedFetcherTest.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.rometools.test;
+
+import org.rometools.fetcher.impl.HttpURLFeedFetcher;
+import org.rometools.fetcher.impl.FeedFetcherCache;
+import org.rometools.fetcher.FeedFetcher;
+
+
+public class HttpURLFeedFetcherTest extends AbstractJettyTest {
+	
+	public HttpURLFeedFetcherTest(String s) {
+		super(s);
+	}
+
+	/**
+	 * @see com.sun.syndication.fetcher.impl.AbstractJettyTest#getFeedFetcher()
+	 */
+	protected FeedFetcher getFeedFetcher() {
+		return new HttpURLFeedFetcher();
+	}		
+	
+	protected FeedFetcher getFeedFetcher(FeedFetcherCache cache) {
+		return new HttpURLFeedFetcher(cache);
+	}
+
+    /**
+     * @see com.sun.syndication.fetcher.impl.AbstractJettyTest#getAuthenticatedFeedFetcher()
+     */
+    public FeedFetcher getAuthenticatedFeedFetcher() {
+        // setup the authenticator
+        java.net.Authenticator.setDefault(new TestBasicAuthenticator());
+        
+        FeedFetcher feedFetcher = getFeedFetcher();	
+        
+        return feedFetcher;
+    }
+	
+
+
+}

src/test/org/rometools/test/ResponseHandlerTest.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.rometools.test;
+
+import org.rometools.fetcher.impl.ResponseHandler;
+
+import junit.framework.TestCase;
+
+public class ResponseHandlerTest extends TestCase {
+
+	/**
+	 * Constructor for ResponseHandlerTest.
+	 */
+	public ResponseHandlerTest(String arg0) {
+		super(arg0);
+	}
+
+	public void testGetCharacterEncodingString() {
+		assertEquals(ResponseHandler.defaultCharacterEncoding, ResponseHandler.getCharacterEncoding((String)null));
+		assertEquals(ResponseHandler.defaultCharacterEncoding, ResponseHandler.getCharacterEncoding("text/xml"));
+		assertEquals(ResponseHandler.defaultCharacterEncoding, ResponseHandler.getCharacterEncoding("text/xml;"));
+		assertEquals("ISO-8859-4", ResponseHandler.getCharacterEncoding("text/xml; charset=ISO-8859-4"));
+		assertEquals("ISO-8859-4", ResponseHandler.getCharacterEncoding("text/xml;charset=ISO-8859-4"));
+		assertEquals("ISO-8859-4", ResponseHandler.getCharacterEncoding("text/xml;charset=ISO-8859-4;something"));
+		assertEquals(ResponseHandler.defaultCharacterEncoding, ResponseHandler.getCharacterEncoding("text/xml;something"));
+	}
+
+}

src/test/org/rometools/test/TestBasicAuthenticator.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2004 Sun Microsystems, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.rometools.test;
+
+import java.net.Authenticator;
+import java.net.PasswordAuthentication;
+
+/**
+ * @author nl
+ */
+public class TestBasicAuthenticator extends Authenticator {
+
+    /**
+     * @see java.net.Authenticator#getPasswordAuthentication()
+     */
+    protected PasswordAuthentication getPasswordAuthentication() {
+        if ("localhost".equals(getRequestingHost())) {
+            return new PasswordAuthentication("username", "password".toCharArray());
+        } else {
+            return null;
+        }
+    }
+}