PatchesAndForks/rome
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Initial refactoring.

Browse source
This commit is contained in:
kebernet 2011-03-14 23:52:57 +00:00
parent 422c6baf73
commit 54b4e421a3
25 changed files with 2913 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
pom.xml
View file

@ -1,21 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.rometools</groupId>
<artifactId>fetcher</artifactId>
<artifactId>rome-fetcher</artifactId>
<name>rome-fetcher</name>
<version>1.0.1-SNAPSHOT</version>
<version>1.5-SNAPSHOT</version>
<inceptionYear>2004</inceptionYear>
<organization>
<name>ROME Project</name>
<url>http://rometools.jira.com</url>
</organization>
<!--scm>
<connection>scm:cvs:pserver:snoopdave@cvs.dev.java.net:/cvs:rome/subprojects/fetcher</connection>
<url>https://rome.dev.java.net/source/browse/rome/</url>
</scm-->
<issueManagement>
<url>https://rometools.jira.com/secure/IssueNavigator.jspa</url>
</issueManagement>
<scm>
<connection>scm:svn:https://rometools.jira.com/svn/FETCHER/trunk</connection>
<developerConnection>scm:svn:https://rometools.jira.com/svn/FETCHER/trunk</developerConnection>
<url>https://rometools.jira.com/source/browse/FETCHER</url>
</scm>
<licenses>
<license>
<name>Apache 2</name>

105
src/java/org/rometools/fetcher/FeedFetcher.java Normal file
View file

@ -0,0 +1,105 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.fetcher;
import com.sun.syndication.feed.synd.SyndFeed;
import com.sun.syndication.io.FeedException;
import java.io.IOException;
import java.net.URL;
public interface FeedFetcher {
/**
* <p>The default user agent. It is not marked final so
* buggy java compiler will not write this string
* into all classes that reference it.</p>
*
* <p>http://tinyurl.com/64t5n points to https://rome.dev.java.net
* Some servers ban user agents with "Java" in the name.</p>
*
*/
public static String DEFAULT_USER_AGENT = "Rome Client (http://tinyurl.com/64t5n)";
/**
* @param string The User-Agent to sent to servers
*/
public abstract void setUserAgent(String string);
/**
* @return the User-Agent currently being sent to servers
*/
public abstract String getUserAgent();
/**
* <p>Turn on or off rfc3229 delta encoding</p>
*
* <p>See http://www.ietf.org/rfc/rfc3229.txt and http://bobwyman.pubsub.com/main/2004/09/using_rfc3229_w.html</p>
*
* <p>NOTE: This is experimental and feedback is welcome!</p>
*
* @param useDeltaEncoding
*/
public abstract void setUsingDeltaEncoding(boolean useDeltaEncoding);
/**
* <p>Is this fetcher using rfc3229 delta encoding?</p>
*
* @return
*/
public abstract boolean isUsingDeltaEncoding();
/**
* <p>Add a FetcherListener.</p>
*
* <p>The FetcherListener will receive an FetcherEvent when
* a Fetcher event (feed polled, retrieved, etc) occurs</p>
*
* @param listener The FetcherListener to recieve the event
*/
public abstract void addFetcherEventListener(FetcherListener listener);
/**
* <p>Remove a FetcherListener</p>
*
* @param listener The FetcherListener to remove
*/
public abstract void removeFetcherEventListener(FetcherListener listener);
/**
* Retrieve a feed over HTTP
*
* @param feedUrl A non-null URL of a RSS/Atom feed to retrieve
* @return A {@link com.sun.syndication.feed.synd.SyndFeed} object
* @throws IllegalArgumentException if the URL is null;
* @throws IOException if a TCP error occurs
* @throws FeedException if the feed is not valid
* @throws FetcherException if a HTTP error occurred
*/
public abstract SyndFeed retrieveFeed(URL feedUrl)
throws IllegalArgumentException, IOException, FeedException, FetcherException;
public SyndFeed retrieveFeed(String userAgent, URL url)
throws IllegalArgumentException, IOException, FeedException, FetcherException;
/**
* If set to true, the WireFeed will be made accessible from the SyndFeed object returned from the Fetcher
* via the originalWireFeed() method. Each Entry in the feed will have the corresponding wireEntry property set.
*/
void setPreserveWireFeed(boolean preserveWireFeed);
}

85
src/java/org/rometools/fetcher/FetcherEvent.java Normal file
View file

@ -0,0 +1,85 @@
package org.rometools.fetcher;
import java.util.EventObject;
import com.sun.syndication.feed.synd.SyndFeed;
/**
* Implementation note: FetcherEvent is not thread safe. Make sure that
* they are only ever accessed by one thread. If necessary, make all getters
* and setters synchronized, or alternatively make all fields final.
*
* @author nl
*/
public class FetcherEvent extends EventObject {
private static final long serialVersionUID = 3985600601904140103L;
public static final String EVENT_TYPE_FEED_POLLED = "FEED_POLLED";
public static final String EVENT_TYPE_FEED_RETRIEVED = "FEED_RETRIEVED";
public static final String EVENT_TYPE_FEED_UNCHANGED = "FEED_UNCHANGED";
private String eventType;
private String urlString;
private SyndFeed feed;
public FetcherEvent(Object source) {
super(source);
}
public FetcherEvent(Object source, String urlStr, String eventType) {
this(source);
setUrlString(urlStr);
setEventType(eventType);
}
public FetcherEvent(Object source, String urlStr, String eventType, SyndFeed feed) {
this(source, urlStr, eventType);
setFeed(feed);
}
/**
* @return Returns the feed.
*
* <p>The feed will only be set if the eventType is EVENT_TYPE_FEED_RETRIEVED</p>
*/
public SyndFeed getFeed() {
return feed;
}
/**
* @param feed The feed to set.
*
* <p>The feed will only be set if the eventType is EVENT_TYPE_FEED_RETRIEVED</p>
*/
public void setFeed(SyndFeed feed) {
this.feed = feed;
}
/**
* @return Returns the eventType.
*/
public String getEventType() {
return eventType;
}
/**
* @param eventType The eventType to set.
*/
public void setEventType(String eventType) {
this.eventType = eventType;
}
/**
* @return Returns the urlString.
*/
public String getUrlString() {
return urlString;
}
/**
* @param urlString The urlString to set.
*/
public void setUrlString(String urlString) {
this.urlString = urlString;
}
}

51
src/java/org/rometools/fetcher/FetcherException.java Normal file
View file

@ -0,0 +1,51 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.fetcher;
/**
* @author Nick Lothian
*
*/
public class FetcherException extends Exception {
private static final long serialVersionUID = -7479645796948092380L;
int responseCode;
public FetcherException(Throwable cause) {
super();
initCause(cause);
}
public FetcherException(String message, Throwable cause) {
super(message);
initCause(cause);
}
public FetcherException(String message) {
super(message);
}
public FetcherException(int responseCode, String message) {
this(message);
this.responseCode = responseCode;
}
public int getResponseCode() {
return responseCode;
}
}

15
src/java/org/rometools/fetcher/FetcherListener.java Normal file
View file

@ -0,0 +1,15 @@
package org.rometools.fetcher;
import java.util.EventListener;
public interface FetcherListener extends EventListener {
/**
* <p>Called when a fetcher event occurs</p>
*
* @param event the event that fired
*/
public void fetcherEvent(FetcherEvent event);
}

219
src/java/org/rometools/fetcher/impl/AbstractFeedFetcher.java Normal file
View file

@ -0,0 +1,219 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.fetcher.impl;
import java.io.IOException;
import java.io.InputStream;
import java.net.URLConnection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Properties;
import java.util.Set;
import com.sun.syndication.feed.synd.SyndFeed;
import org.rometools.fetcher.FeedFetcher;
import org.rometools.fetcher.FetcherEvent;
import org.rometools.fetcher.FetcherException;
import org.rometools.fetcher.FetcherListener;
public abstract class AbstractFeedFetcher implements FeedFetcher {
private final Set fetcherEventListeners;
private String userAgent;
private boolean usingDeltaEncoding;
private boolean preserveWireFeed;
public AbstractFeedFetcher() {
fetcherEventListeners = Collections.synchronizedSet(new HashSet());
Properties props = new Properties(System.getProperties());
String resourceName = "fetcher.properties";
try {
InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(resourceName);
if (inputStream == null) {
inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName);
}
if (inputStream != null) {
props.load(inputStream);
System.getProperties().putAll(props);
inputStream.close();
} else {
System.err.println("Could not find " + resourceName + " on classpath");
}
} catch (IOException e) {
// do nothing - we don't want to fail just because we could not find the version
System.err.println("Error reading " + resourceName + " from classpath: " + e.getMessage());
}
setUserAgent(DEFAULT_USER_AGENT + " Ver: " + System.getProperty("rome.fetcher.version", "UNKNOWN"));
}
/**
* @return the User-Agent currently being sent to servers
*/
public synchronized String getUserAgent() {
return userAgent;
}
/**
* @param string The User-Agent to sent to servers
*/
public synchronized void setUserAgent(String string) {
userAgent = string;
}
/**
* @param eventType The event type to fire
* @param connection the current connection
*/
protected void fireEvent(String eventType, URLConnection connection) {
fireEvent(eventType, connection.getURL().toExternalForm(), null);
}
/**
* @param eventType The event type to fire
* @param connection the current connection
* @param feed The feed to pass to the event
*/
protected void fireEvent(String eventType, URLConnection connection, SyndFeed feed) {
fireEvent(eventType, connection.getURL().toExternalForm(), feed);
}
/**
* @param eventType The event type to fire
* @param urlStr the current url as a string
*/
protected void fireEvent(String eventType, String urlStr) {
fireEvent(eventType, urlStr, null);
}
/**
* @param eventType The event type to fire
* @param urlStr the current url as a string
* @param feed The feed to pass to the event
*/
protected void fireEvent(String eventType, String urlStr, SyndFeed feed) {
FetcherEvent fetcherEvent = new FetcherEvent(this, urlStr, eventType, feed);
synchronized(fetcherEventListeners) {
Iterator iter = fetcherEventListeners.iterator();
while ( iter.hasNext()) {
FetcherListener fetcherEventListener = (FetcherListener) iter.next();
fetcherEventListener.fetcherEvent(fetcherEvent);
}
}
}
/**
* @see com.sun.syndication.fetcher.FeedFetcher#addFetcherEventListener(com.sun.syndication.fetcher.FetcherListener)
*/
public void addFetcherEventListener(FetcherListener listener) {
if (listener != null) {
fetcherEventListeners.add(listener);
}
}
/**
* @see com.sun.syndication.fetcher.FeedFetcher#removeFetcherEventListener(com.sun.syndication.fetcher.FetcherListener)
*/
public void removeFetcherEventListener(FetcherListener listener) {
if (listener != null) {
fetcherEventListeners.remove(listener);
}
}
/**
* @return Returns the useDeltaEncoding.
*/
public synchronized boolean isUsingDeltaEncoding() {
return usingDeltaEncoding;
}
/**
* @param useDeltaEncoding The useDeltaEncoding to set.
*/
public synchronized void setUsingDeltaEncoding(boolean useDeltaEncoding) {
this.usingDeltaEncoding = useDeltaEncoding;
}
/**
* <p>Handles HTTP error codes.</p>
*
* @param responseCode the HTTP response code
* @throws FetcherException if response code is in the range 400 to 599 inclusive
*/
protected void handleErrorCodes(int responseCode) throws FetcherException {
// Handle 2xx codes as OK, so ignore them here
// 3xx codes are handled by the HttpURLConnection class
if (responseCode == 403) {
// Authentication is required
throwAuthenticationError(responseCode);
} else if (responseCode >= 400 && responseCode < 500) {
throw4XXError(responseCode);
} else if (responseCode >= 500 && responseCode < 600) {
throw new FetcherException(responseCode, "The server encounted an error. HTTP Response code was:" + responseCode);
}
}
protected void throw4XXError(int responseCode) throws FetcherException {
throw new FetcherException(responseCode, "The requested resource could not be found. HTTP Response code was:" + responseCode);
}
protected void throwAuthenticationError(int responseCode) throws FetcherException {
throw new FetcherException(responseCode, "Authentication required for that resource. HTTP Response code was:" + responseCode);
}
/**
* <p>Combine the entries in two feeds into a single feed.</p>
*
* <p>The returned feed will have the same data as the newFeed parameter, with
* the entries from originalFeed appended to the end of its entries.</p>
*
* @param originalFeed
* @param newFeed
* @return
*/
public static SyndFeed combineFeeds(SyndFeed originalFeed, SyndFeed newFeed) {
SyndFeed result;
try {
result = (SyndFeed) newFeed.clone();
result.getEntries().addAll(result.getEntries().size(), originalFeed.getEntries());
return result;
} catch (CloneNotSupportedException e) {
IllegalArgumentException iae = new IllegalArgumentException("Cannot clone feed");
iae.initCause(e);
throw iae;
}
}
public boolean isPreserveWireFeed() {
return preserveWireFeed;
}
public void setPreserveWireFeed(boolean preserveWireFeed) {
this.preserveWireFeed = preserveWireFeed;
}
}

29
src/java/org/rometools/fetcher/impl/AbstractFeedFetcherBeanInfo.java Normal file
View file

@ -0,0 +1,29 @@
package org.rometools.fetcher.impl;
import java.beans.EventSetDescriptor;
import java.beans.SimpleBeanInfo;
import java.lang.reflect.Method;
import org.rometools.fetcher.FetcherEvent;
import org.rometools.fetcher.FetcherListener;
public class AbstractFeedFetcherBeanInfo extends SimpleBeanInfo {
public EventSetDescriptor[] getEventSetDescriptors() {
try {
Class clz = AbstractFeedFetcher.class; // get the class object which we'll describe
Method addMethod = clz.getMethod("addFetcherEventListener", new Class[] { FetcherListener.class });
Method removeMethod = clz.getMethod("removeFetcherEventListener", new Class[] { FetcherListener.class });
Method listenerMethod = FetcherListener.class.getMethod("fetcherEvent", new Class[] { FetcherEvent.class });
EventSetDescriptor est = new EventSetDescriptor("fetcherEvent", clz, new Method[] { listenerMethod }, addMethod, removeMethod);
EventSetDescriptor[] results = new EventSetDescriptor[] { est };
return results;
} catch (Exception e) {
// IntrospectionException, SecurityException and/or NoSuchMethodException can be thrown here
// the best we can do is to convert them to runtime exceptions
throw new RuntimeException(e);
}
}
}

134
src/java/org/rometools/fetcher/impl/DiskFeedInfoCache.java Normal file
View file

@ -0,0 +1,134 @@
/*
* Copyright 2005 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.rometools.fetcher.impl;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.net.URL;
import javax.swing.text.Utilities;
/**
* Disk based cache.
*/
public class DiskFeedInfoCache implements FeedFetcherCache {
protected String cachePath = null;
public DiskFeedInfoCache(String cachePath) {
this.cachePath = cachePath;
}
public SyndFeedInfo getFeedInfo(URL url) {
SyndFeedInfo info = null;
String fileName = cachePath + File.separator + "feed_"
+ replaceNonAlphanumeric(url.toString(),'_').trim();
FileInputStream fis;
try {
fis = new FileInputStream(fileName);
ObjectInputStream ois = new ObjectInputStream(fis);
info = (SyndFeedInfo)ois.readObject();
fis.close();
} catch (FileNotFoundException fnfe) {
// That's OK, we'l return null
} catch (ClassNotFoundException cnfe) {
// Error writing to cache is fatal
throw new RuntimeException("Attempting to read from cache", cnfe);
} catch (IOException fnfe) {
// Error writing to cache is fatal
throw new RuntimeException("Attempting to read from cache", fnfe);
}
return info;
}
public void setFeedInfo(URL url, SyndFeedInfo feedInfo) {
String fileName = cachePath + File.separator + "feed_"
+ replaceNonAlphanumeric(url.toString(),'_').trim();
FileOutputStream fos;
try {
fos = new FileOutputStream(fileName);
ObjectOutputStream oos = new ObjectOutputStream(fos);
oos.writeObject(feedInfo);
fos.flush();
fos.close();
} catch (Exception e) {
// Error writing to cache is fatal
throw new RuntimeException("Attempting to write to cache", e);
}
}
public static String replaceNonAlphanumeric(String str, char subst) {
StringBuffer ret = new StringBuffer(str.length());
char[] testChars = str.toCharArray();
for (int i = 0; i < testChars.length; i++) {
if (Character.isLetterOrDigit(testChars[i])) {
ret.append(testChars[i]);
} else {
ret.append( subst );
}
}
return ret.toString();
}
/**
* Clear the cache.
*/
public synchronized void clear() {
final File file = new File(this.cachePath);
//only do the delete if the directory exists
if( file.exists() && file.canWrite() ) {
//make the directory empty
final String[] files = file.list();
final int len = files.length;
for( int i=0; i<len; i++ ) {
File deleteMe = new File(this.cachePath + File.separator + files[i]);
deleteMe.delete();
}
//don't delete the cache directory
}
}
public SyndFeedInfo remove( URL url ) {
SyndFeedInfo info = null;
String fileName = cachePath + File.separator + "feed_"
+ replaceNonAlphanumeric(url.toString(),'_').trim();
FileInputStream fis;
try {
fis = new FileInputStream(fileName);
ObjectInputStream ois = new ObjectInputStream(fis);
info = (SyndFeedInfo)ois.readObject();
fis.close();
File file = new File( fileName );
if( file.exists() ) {
file.delete();
}
} catch (FileNotFoundException fnfe) {
// That's OK, we'l return null
} catch (ClassNotFoundException cnfe) {
// Error writing to cahce is fatal
throw new RuntimeException("Attempting to read from cache", cnfe);
} catch (IOException fnfe) {
// Error writing to cahce is fatal
throw new RuntimeException("Attempting to read from cache", fnfe);
}
return info;
}
}

56
src/java/org/rometools/fetcher/impl/FeedFetcherCache.java Normal file
View file

@ -0,0 +1,56 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.fetcher.impl;
import java.net.URL;
/**
* <p>An interface to allow caching of feed details. Implementing this allows the
* {@link com.sun.syndication.fetcher.io.HttpURLFeedFetcher} class to
* enable conditional gets</p>
*
* @author Nick Lothian
*
*/
public interface FeedFetcherCache {
/**
* Get a SyndFeedInfo object from the cache.
*
* @param feedUrl The url of the feed
* @return A SyndFeedInfo or null if it is not in the cache
*/
public SyndFeedInfo getFeedInfo(URL feedUrl);
/**
* Add a SyndFeedInfo object to the cache
*
* @param feedUrl The url of the feed
* @param syndFeedInfo A SyndFeedInfo for the feed
*/
public void setFeedInfo(URL feedUrl, SyndFeedInfo syndFeedInfo);
/**
* Removes all items from the cache.
*/
public void clear();
/**
* Removes the SyndFeedInfo identified by the url from the cache.
* @return The removed SyndFeedInfo
*/
public SyndFeedInfo remove( URL feedUrl );
}

125
src/java/org/rometools/fetcher/impl/HashMapFeedInfoCache.java Normal file
View file

@ -0,0 +1,125 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.fetcher.impl;
import java.io.Serializable;
import java.net.URL;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
/**
* <p>A very simple implementation of the {@link com.sun.syndication.fetcher.impl.FeedFetcherCache} interface.</p>
*
* <p>This implementation uses a HashMap to cache retrieved feeds. This implementation is
* most suitible for sort term (client aggregator?) use, as the memory usage will increase
* over time as the number of feeds in the cache increases.</p>
*
* @author Nick Lothian
*
*/
public class HashMapFeedInfoCache implements FeedFetcherCache, Serializable {
private static final long serialVersionUID = -1594665619950916222L;
static HashMapFeedInfoCache _instance;
private Map infoCache;
/**
* <p>Constructor for HashMapFeedInfoCache</p>
*
* <p>Only use this if you want multiple instances of the cache.
* Usually getInstance() is more appropriate.</p>
*
*/
public HashMapFeedInfoCache() {
setInfoCache(createInfoCache());
}
/**
* Get the global instance of the cache
* @return an implementation of FeedFetcherCache
*/
public static synchronized FeedFetcherCache getInstance() {
if (_instance == null) {
_instance = new HashMapFeedInfoCache();
}
return _instance;
}
protected Map createInfoCache() {
return (Collections.synchronizedMap(new HashMap()));
}
protected Object get(Object key) {
return getInfoCache().get(key);
}
/**
* @see extensions.io.FeedFetcherCache#getFeedInfo(java.net.URL)
*/
public SyndFeedInfo getFeedInfo(URL feedUrl) {
return (SyndFeedInfo) get(feedUrl.toString());
}
protected void put(Object key, Object value) {
getInfoCache().put(key, value);
}
/**
* @see extensions.io.FeedFetcherCache#setFeedInfo(java.net.URL, extensions.io.SyndFeedInfo)
*/
public void setFeedInfo(URL feedUrl, SyndFeedInfo syndFeedInfo) {
put(feedUrl.toString(), syndFeedInfo);
}
protected synchronized final Map getInfoCache() {
return infoCache;
}
/**
* The API of this class indicates that map must thread safe. In other
* words, be sure to wrap it in a synchronized map unless you know
* what you are doing.
*
* @param map the map to use as the info cache.
*/
protected synchronized final void setInfoCache(Map map) {
infoCache = map;
}
/**
* @see com.sun.syndication.fetcher.impl.FeedFetcherCache#clear()
*/
public void clear() {
synchronized( infoCache ) {
infoCache.clear();
}
}
/**
* @see com.sun.syndication.fetcher.impl.FeedFetcherCache#remove(java.net.URL)
*/
public SyndFeedInfo remove( final URL url ) {
if( url == null ) return null;
return (SyndFeedInfo) infoCache.remove( url.toString() );
}
}

379
src/java/org/rometools/fetcher/impl/HttpClientFeedFetcher.java Normal file
View file

@ -0,0 +1,379 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.fetcher.impl;
import com.sun.syndication.feed.synd.SyndFeed;
import org.rometools.fetcher.FetcherEvent;
import org.rometools.fetcher.FetcherException;
import com.sun.syndication.io.FeedException;
import com.sun.syndication.io.SyndFeedInput;
import com.sun.syndication.io.XmlReader;
import org.apache.commons.httpclient.Credentials;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpClientParams;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.zip.GZIPInputStream;
/**
* @author Nick Lothian
*/
public class HttpClientFeedFetcher extends AbstractFeedFetcher {
private CredentialSupplier credentialSupplier;
private FeedFetcherCache feedInfoCache;
private volatile HttpClientMethodCallbackIntf httpClientMethodCallback;
private volatile HttpClientParams httpClientParams;
public HttpClientFeedFetcher() {
super();
setHttpClientParams(new HttpClientParams());
}
/**
* @param cache
*/
public HttpClientFeedFetcher(FeedFetcherCache cache) {
this();
setFeedInfoCache(cache);
}
public HttpClientFeedFetcher(FeedFetcherCache cache, CredentialSupplier credentialSupplier) {
this(cache);
setCredentialSupplier(credentialSupplier);
}
/**
* @param timeout Sets the connect timeout for the HttpClient but using the URLConnection method name.
* Uses the HttpClientParams method setConnectionManagerTimeout instead of setConnectTimeout
*
*/
public synchronized void setConnectTimeout(int timeout) {
httpClientParams.setConnectionManagerTimeout(timeout);
}
/**
* @return The currently used connect timeout for the HttpClient but using the URLConnection method name.
* Uses the HttpClientParams method getConnectionManagerTimeout instead of getConnectTimeout
*
*/
public int getConnectTimeout() {
return (int) this.getHttpClientParams()
.getConnectionManagerTimeout();
}
/**
* @param credentialSupplier The credentialSupplier to set.
*/
public synchronized void setCredentialSupplier(CredentialSupplier credentialSupplier) {
this.credentialSupplier = credentialSupplier;
}
/**
* @return Returns the credentialSupplier.
*/
public synchronized CredentialSupplier getCredentialSupplier() {
return credentialSupplier;
}
/**
* @param feedInfoCache the feedInfoCache to set
*/
public synchronized void setFeedInfoCache(FeedFetcherCache feedInfoCache) {
this.feedInfoCache = feedInfoCache;
}
/**
* @return the feedInfoCache.
*/
public synchronized FeedFetcherCache getFeedInfoCache() {
return feedInfoCache;
}
public synchronized void setHttpClientMethodCallback(HttpClientMethodCallbackIntf httpClientMethodCallback) {
this.httpClientMethodCallback = httpClientMethodCallback;
}
public HttpClientMethodCallbackIntf getHttpClientMethodCallback() {
return httpClientMethodCallback;
}
/**
* @param httpClientParams The httpClientParams to set.
*/
public synchronized void setHttpClientParams(HttpClientParams httpClientParams) {
this.httpClientParams = httpClientParams;
}
/**
* @return Returns the httpClientParams.
*/
public synchronized HttpClientParams getHttpClientParams() {
return this.httpClientParams;
}
/**
* @return The currently used read timeout for the URLConnection, 0 is unlimited, i.e. no timeout
*/
public synchronized void setReadTimeout(int timeout) {
httpClientParams.setSoTimeout(timeout);
}
/**
* @param timeout Sets the read timeout for the URLConnection to a specified timeout, in milliseconds.
*/
public int getReadTimeout() {
return (int) this.getHttpClientParams()
.getSoTimeout();
}
public SyndFeed retrieveFeed(URL url) throws IllegalArgumentException, IOException, FeedException, FetcherException {
return this.retrieveFeed(this.getUserAgent(), url);
}
/**
* @see com.sun.syndication.fetcher.FeedFetcher#retrieveFeed(java.net.URL)
*/
public SyndFeed retrieveFeed(String userAgent, URL feedUrl)
throws IllegalArgumentException, IOException, FeedException, FetcherException {
if (feedUrl == null) {
throw new IllegalArgumentException("null is not a valid URL");
}
// TODO Fix this
//System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.SimpleLog");
HttpClient client = new HttpClient(httpClientParams);
if (getCredentialSupplier() != null) {
client.getState()
.setAuthenticationPreemptive(true);
// TODO what should realm be here?
Credentials credentials = getCredentialSupplier()
.getCredentials(null, feedUrl.getHost());
if (credentials != null) {
client.getState()
.setCredentials(null, feedUrl.getHost(), credentials);
}
}
System.setProperty("httpclient.useragent", userAgent);
String urlStr = feedUrl.toString();
HttpMethod method = new GetMethod(urlStr);
method.addRequestHeader("Accept-Encoding", "gzip");
method.addRequestHeader("User-Agent", userAgent);
method.setFollowRedirects(true);
if (httpClientMethodCallback != null) {
synchronized (httpClientMethodCallback) {
httpClientMethodCallback.afterHttpClientMethodCreate(method);
}
}
FeedFetcherCache cache = getFeedInfoCache();
if (cache != null) {
// retrieve feed
try {
if (isUsingDeltaEncoding()) {
method.setRequestHeader("A-IM", "feed");
}
// get the feed info from the cache
// Note that syndFeedInfo will be null if it is not in the cache
SyndFeedInfo syndFeedInfo = cache.getFeedInfo(feedUrl);
if (syndFeedInfo != null) {
method.setRequestHeader("If-None-Match", syndFeedInfo.getETag());
if (syndFeedInfo.getLastModified() instanceof String) {
method.setRequestHeader("If-Modified-Since", (String) syndFeedInfo.getLastModified());
}
}
int statusCode = client.executeMethod(method);
fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);
handleErrorCodes(statusCode);
SyndFeed feed = getFeed(syndFeedInfo, urlStr, method, statusCode);
syndFeedInfo = buildSyndFeedInfo(feedUrl, urlStr, method, feed, statusCode);
cache.setFeedInfo(new URL(urlStr), syndFeedInfo);
// the feed may have been modified to pick up cached values
// (eg - for delta encoding)
feed = syndFeedInfo.getSyndFeed();
return feed;
} finally {
method.releaseConnection();
method.recycle();
}
} else {
// cache is not in use
try {
int statusCode = client.executeMethod(method);
fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);
handleErrorCodes(statusCode);
return getFeed(null, urlStr, method, statusCode);
} finally {
method.releaseConnection();
method.recycle();
}
}
}
private SyndFeed getFeed(SyndFeedInfo syndFeedInfo, String urlStr, HttpMethod method, int statusCode)
throws IOException, HttpException, FetcherException, FeedException {
if ((statusCode == HttpURLConnection.HTTP_NOT_MODIFIED) && (syndFeedInfo != null)) {
fireEvent(FetcherEvent.EVENT_TYPE_FEED_UNCHANGED, urlStr);
return syndFeedInfo.getSyndFeed();
}
SyndFeed feed = retrieveFeed(urlStr, method);
fireEvent(FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, urlStr, feed);
return feed;
}
/**
* @param feedUrl
* @param urlStr
* @param method
* @param feed
* @return
* @throws MalformedURLException
*/
private SyndFeedInfo buildSyndFeedInfo(URL feedUrl, String urlStr, HttpMethod method, SyndFeed feed, int statusCode)
throws MalformedURLException {
SyndFeedInfo syndFeedInfo;
syndFeedInfo = new SyndFeedInfo();
// this may be different to feedURL because of 3XX redirects
syndFeedInfo.setUrl(new URL(urlStr));
syndFeedInfo.setId(feedUrl.toString());
Header imHeader = method.getResponseHeader("IM");
if ((imHeader != null) && (imHeader.getValue()
.indexOf("feed") >= 0) && isUsingDeltaEncoding()) {
FeedFetcherCache cache = getFeedInfoCache();
if ((cache != null) && (statusCode == 226)) {
// client is setup to use http delta encoding and the server supports it and has returned a delta encoded response
// This response only includes new items
SyndFeedInfo cachedInfo = cache.getFeedInfo(feedUrl);
if (cachedInfo != null) {
SyndFeed cachedFeed = cachedInfo.getSyndFeed();
// set the new feed to be the orginal feed plus the new items
feed = combineFeeds(cachedFeed, feed);
}
}
}
Header lastModifiedHeader = method.getResponseHeader("Last-Modified");
if (lastModifiedHeader != null) {
syndFeedInfo.setLastModified(lastModifiedHeader.getValue());
}
Header eTagHeader = method.getResponseHeader("ETag");
if (eTagHeader != null) {
syndFeedInfo.setETag(eTagHeader.getValue());
}
syndFeedInfo.setSyndFeed(feed);
return syndFeedInfo;
}
/**
* @param client
* @param urlStr
* @param method
* @return
* @throws IOException
* @throws HttpException
* @throws FetcherException
* @throws FeedException
*/
private SyndFeed retrieveFeed(String urlStr, HttpMethod method)
throws IOException, HttpException, FetcherException, FeedException {
InputStream stream = null;
if ((method.getResponseHeader("Content-Encoding") != null) &&
("gzip".equalsIgnoreCase(method.getResponseHeader("Content-Encoding").getValue()))) {
stream = new GZIPInputStream(method.getResponseBodyAsStream());
} else {
stream = method.getResponseBodyAsStream();
}
try {
XmlReader reader = null;
if (method.getResponseHeader("Content-Type") != null) {
reader = new XmlReader(stream, method.getResponseHeader("Content-Type").getValue(), true);
} else {
reader = new XmlReader(stream, true);
}
SyndFeedInput syndFeedInput = new SyndFeedInput();
syndFeedInput.setPreserveWireFeed(isPreserveWireFeed());
return syndFeedInput.build(reader);
} finally {
if (stream != null) {
stream.close();
}
}
}
public interface CredentialSupplier {
public Credentials getCredentials(String realm, String host);
}
public interface HttpClientMethodCallbackIntf {
/**
* Allows access to the underlying HttpClient HttpMethod object.
* Note that in most cases, method.setRequestHeader(String, String)
* is what you want to do (rather than method.addRequestHeader(String, String))
*
* @param method
*/
public void afterHttpClientMethodCreate(HttpMethod method);
}
}

299
src/java/org/rometools/fetcher/impl/HttpURLFeedFetcher.java Normal file
View file

@ -0,0 +1,299 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.fetcher.impl;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.zip.GZIPInputStream;
import com.sun.syndication.feed.synd.SyndFeed;
import org.rometools.fetcher.FetcherEvent;
import org.rometools.fetcher.FetcherException;
import com.sun.syndication.io.FeedException;
import com.sun.syndication.io.SyndFeedInput;
import com.sun.syndication.io.XmlReader;
/**
* <p>Class to retrieve syndication files via HTTP.</p>
*
* <p>If passed a {@link com.sun.syndication.fetcher.impl.FeedFetcherCache} in the
* constructor it will use conditional gets to only retrieve modified content.</p>
*
* <p>The class uses the Accept-Encoding: gzip header to retrieve gzipped feeds where
* supported by the server.</p>
*
* <p>Simple usage:
* <pre>
* // create the cache
* FeedFetcherCache feedInfoCache = HashMapFeedInfoCache.getFeedInfoCache();
* // retrieve the feed the first time
* // any subsequent request will use conditional gets and only
* // retrieve the resource if it has changed
* SyndFeed feed = new HttpURLFeedFetcher(feedInfoCache).retrieveFeed(feedUrl);
*</pre>
*
* </p>
*
* @see <a href="http://fishbowl.pastiche.org/2002/10/21/http_conditional_get_for_rss_hackers">http://fishbowl.pastiche.org/2002/10/21/http_conditional_get_for_rss_hackers</a>
* @see <a href="http://diveintomark.org/archives/2003/07/21/atom_aggregator_behavior_http_level">http://diveintomark.org/archives/2003/07/21/atom_aggregator_behavior_http_level</a>
* @see <a href="http://bobwyman.pubsub.com/main/2004/09/using_rfc3229_w.html">http://bobwyman.pubsub.com/main/2004/09/using_rfc3229_w.html</a>
* @author Nick Lothian
*/
public class HttpURLFeedFetcher extends AbstractFeedFetcher {
static final int POLL_EVENT = 1;
static final int RETRIEVE_EVENT = 2;
static final int UNCHANGED_EVENT = 3;
private FeedFetcherCache feedInfoCache;
/**
* Constructor to use HttpURLFeedFetcher without caching of feeds
*
*/
public HttpURLFeedFetcher() {
super();
}
/**
* Constructor to enable HttpURLFeedFetcher to cache feeds
*
* @param feedCache - an instance of the FeedFetcherCache interface
*/
public HttpURLFeedFetcher(FeedFetcherCache feedInfoCache) {
this();
setFeedInfoCache(feedInfoCache);
}
public SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
return this.retrieveFeed(this.getUserAgent(), feedUrl);
}
/**
* Retrieve a feed over HTTP
*
* @param feedUrl A non-null URL of a RSS/Atom feed to retrieve
* @return A {@link com.sun.syndication.feed.synd.SyndFeed} object
* @throws IllegalArgumentException if the URL is null;
* @throws IOException if a TCP error occurs
* @throws FeedException if the feed is not valid
* @throws FetcherException if a HTTP error occurred
*/
public SyndFeed retrieveFeed(String userAgent, URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
if (feedUrl == null) {
throw new IllegalArgumentException("null is not a valid URL");
}
URLConnection connection = feedUrl.openConnection();
if (!(connection instanceof HttpURLConnection)) {
throw new IllegalArgumentException(feedUrl.toExternalForm() + " is not a valid HTTP Url");
}
HttpURLConnection httpConnection = (HttpURLConnection)connection;
// httpConnection.setInstanceFollowRedirects(true); // this is true by default, but can be changed on a claswide basis
FeedFetcherCache cache = getFeedInfoCache();
if (cache != null) {
SyndFeedInfo syndFeedInfo = cache.getFeedInfo(feedUrl);
setRequestHeaders(connection, syndFeedInfo);
httpConnection.connect();
try {
fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, connection);
if (syndFeedInfo == null) {
// this is a feed that hasn't been retrieved
syndFeedInfo = new SyndFeedInfo();
retrieveAndCacheFeed(feedUrl, syndFeedInfo, httpConnection);
} else {
// check the response code
int responseCode = httpConnection.getResponseCode();
if (responseCode != HttpURLConnection.HTTP_NOT_MODIFIED) {
// the response code is not 304 NOT MODIFIED
// This is either because the feed server
// does not support condition gets
// or because the feed hasn't changed
retrieveAndCacheFeed(feedUrl, syndFeedInfo, httpConnection);
} else {
// the feed does not need retrieving
fireEvent(FetcherEvent.EVENT_TYPE_FEED_UNCHANGED, connection);
}
}
return syndFeedInfo.getSyndFeed();
} finally {
httpConnection.disconnect();
}
} else {
fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, connection);
InputStream inputStream = null;
setRequestHeaders(connection, null);
connection.addRequestProperty("User-Agent", userAgent);
httpConnection.connect();
try {
inputStream = httpConnection.getInputStream();
return getSyndFeedFromStream(inputStream, connection);
} catch (java.io.IOException e) {
handleErrorCodes(((HttpURLConnection)connection).getResponseCode());
} finally {
if (inputStream != null) {
inputStream.close();
}
httpConnection.disconnect();
}
// we will never actually get to this line
return null;
}
}
protected void retrieveAndCacheFeed(URL feedUrl, SyndFeedInfo syndFeedInfo, HttpURLConnection connection) throws IllegalArgumentException, FeedException, FetcherException, IOException {
handleErrorCodes(connection.getResponseCode());
resetFeedInfo(feedUrl, syndFeedInfo, connection);
FeedFetcherCache cache = getFeedInfoCache();
// resetting feed info in the cache
// could be needed for some implementations
// of FeedFetcherCache (eg, distributed HashTables)
if (cache != null) {
cache.setFeedInfo(feedUrl, syndFeedInfo);
}
}
protected void resetFeedInfo(URL orignalUrl, SyndFeedInfo syndFeedInfo, HttpURLConnection connection) throws IllegalArgumentException, IOException, FeedException {
// need to always set the URL because this may have changed due to 3xx redirects
syndFeedInfo.setUrl(connection.getURL());
// the ID is a persistant value that should stay the same even if the URL for the
// feed changes (eg, by 3xx redirects)
syndFeedInfo.setId(orignalUrl.toString());
// This will be 0 if the server doesn't support or isn't setting the last modified header
syndFeedInfo.setLastModified(new Long(connection.getLastModified()));
// This will be null if the server doesn't support or isn't setting the ETag header
syndFeedInfo.setETag(connection.getHeaderField("ETag"));
// get the contents
InputStream inputStream = null;
try {
inputStream = connection.getInputStream();
SyndFeed syndFeed = getSyndFeedFromStream(inputStream, connection);
String imHeader = connection.getHeaderField("IM");
if (isUsingDeltaEncoding() && (imHeader!= null && imHeader.indexOf("feed") >= 0)) {
FeedFetcherCache cache = getFeedInfoCache();
if (cache != null && connection.getResponseCode() == 226) {
// client is setup to use http delta encoding and the server supports it and has returned a delta encoded response
// This response only includes new items
SyndFeedInfo cachedInfo = cache.getFeedInfo(orignalUrl);
if (cachedInfo != null) {
SyndFeed cachedFeed = cachedInfo.getSyndFeed();
// set the new feed to be the orginal feed plus the new items
syndFeed = combineFeeds(cachedFeed, syndFeed);
}
}
}
syndFeedInfo.setSyndFeed(syndFeed);
} finally {
if (inputStream != null) {
inputStream.close();
}
}
}
/**
* <p>Set appropriate HTTP headers, including conditional get and gzip encoding headers</p>
*
* @param connection A URLConnection
* @param syndFeedInfo The SyndFeedInfo for the feed to be retrieved. May be null
*/
protected void setRequestHeaders(URLConnection connection, SyndFeedInfo syndFeedInfo) {
if (syndFeedInfo != null) {
// set the headers to get feed only if modified
// we support the use of both last modified and eTag headers
if (syndFeedInfo.getLastModified() != null) {
Object lastModified = syndFeedInfo.getLastModified();
if (lastModified instanceof Long) {
connection.setIfModifiedSince(((Long)syndFeedInfo.getLastModified()).longValue());
}
}
if (syndFeedInfo.getETag() != null) {
connection.setRequestProperty("If-None-Match", syndFeedInfo.getETag());
}
}
// header to retrieve feed gzipped
connection.setRequestProperty("Accept-Encoding", "gzip");
if (isUsingDeltaEncoding()) {
connection.addRequestProperty("A-IM", "feed");
}
}
private SyndFeed readSyndFeedFromStream(InputStream inputStream, URLConnection connection) throws IOException, IllegalArgumentException, FeedException {
BufferedInputStream is;
if ("gzip".equalsIgnoreCase(connection.getContentEncoding())) {
// handle gzip encoded content
is = new BufferedInputStream(new GZIPInputStream(inputStream));
} else {
is = new BufferedInputStream(inputStream);
}
//InputStreamReader reader = new InputStreamReader(is, ResponseHandler.getCharacterEncoding(connection));
//SyndFeedInput input = new SyndFeedInput();
XmlReader reader = null;
if (connection.getHeaderField("Content-Type") != null) {
reader = new XmlReader(is, connection.getHeaderField("Content-Type"), true);
} else {
reader = new XmlReader(is, true);
}
SyndFeedInput syndFeedInput = new SyndFeedInput();
syndFeedInput.setPreserveWireFeed(isPreserveWireFeed());
return syndFeedInput.build(reader);
}
private SyndFeed getSyndFeedFromStream(InputStream inputStream, URLConnection connection) throws IOException, IllegalArgumentException, FeedException {
SyndFeed feed = readSyndFeedFromStream(inputStream, connection);
fireEvent(FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, connection, feed);
return feed;
}
/**
* @return The FeedFetcherCache used by this fetcher (Could be null)
*/
public synchronized FeedFetcherCache getFeedInfoCache() {
return feedInfoCache;
}
/**
* @param cache The cache to be used by this fetcher (pass null to stop using a cache)
*/
public synchronized void setFeedInfoCache(FeedFetcherCache cache) {
feedInfoCache = cache;
}
}

70
src/java/org/rometools/fetcher/impl/LinkedHashMapFeedInfoCache.java Normal file
View file

@ -0,0 +1,70 @@
package org.rometools.fetcher.impl;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
/**
* <p>An implementation of the {@link com.sun.syndication.fetcher.impl.FeedFetcherCache} interface.</p>
*
* <p>Unlike the HashMapFeedInfoCache this implementation will not grow unbound</p>
*
* @author Javier Kohen
* @author Nick Lothian
*
*/
public class LinkedHashMapFeedInfoCache extends HashMapFeedInfoCache {
private final class CacheImpl extends LinkedHashMap {
private static final long serialVersionUID = -6977191330127794920L;
public CacheImpl() {
super(16, 0.75F, true);
}
protected boolean removeEldestEntry(Map.Entry eldest) {
return size() > getMaxEntries();
}
}
private static final int DEFAULT_MAX_ENTRIES = 20;
private static final long serialVersionUID = 1694228973357997417L;
private int maxEntries = DEFAULT_MAX_ENTRIES;
private final static LinkedHashMapFeedInfoCache _instance = new LinkedHashMapFeedInfoCache();
/**
* Get the global instance of the cache
* @return an implementation of FeedFetcherCache
*/
public static final FeedFetcherCache getInstance() {
return _instance;
}
/**
* <p>Constructor for HashMapFeedInfoCache</p>
*
* <p>Only use this if you want multiple instances of the cache.
* Usually {@link #getInstance()} is more appropriate.</p>
*
* @see #getInstance()
*/
public LinkedHashMapFeedInfoCache() {
super();
}
protected Map createInfoCache() {
return Collections.synchronizedMap(new CacheImpl());
}
public synchronized final int getMaxEntries() {
return maxEntries;
}
public synchronized final void setMaxEntries(int maxEntries) {
this.maxEntries = maxEntries;
}
}

57
src/java/org/rometools/fetcher/impl/ResponseHandler.java Normal file
View file

@ -0,0 +1,57 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.fetcher.impl;
import java.net.URLConnection;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Utility class to help deal with HTTP responses
*
*/
public class ResponseHandler {
public static final String defaultCharacterEncoding = "ISO-8859-1";
private final static Pattern characterEncodingPattern = Pattern.compile("charset=([.[^; ]]*)");
public static String getCharacterEncoding(URLConnection connection) {
return getCharacterEncoding(connection.getContentType());
}
/**
*
* <p>Gets the character encoding of a response. (Note that this is different to
* the content-encoding)</p>
*
* @param contentTypeHeader the value of the content-type HTTP header eg: text/html; charset=ISO-8859-4
* @return the character encoding, eg: ISO-8859-4
*/
public static String getCharacterEncoding(String contentTypeHeader) {
if (contentTypeHeader == null) {
return defaultCharacterEncoding;
}
Matcher m = characterEncodingPattern.matcher(contentTypeHeader);
//if (!m.matches()) {
if (!m.find()) {
return defaultCharacterEncoding;
} else {
return m.group(1);
}
}
}

149
src/java/org/rometools/fetcher/impl/SyndFeedInfo.java Normal file
View file

@ -0,0 +1,149 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.fetcher.impl;
import java.io.Serializable;
import java.net.URL;
import com.sun.syndication.feed.impl.ObjectBean;
import com.sun.syndication.feed.synd.SyndFeed;
/**
* <p>A class to represent a {@link com.sun.syndication.feed.synd.SyndFeed}
* and some useful information about it.</p>
*
* <p>This class is thread safe, as expected by the different feed fetcher
* implementations.</p>
*
* @author Nick Lothian
*/
public class SyndFeedInfo implements Cloneable, Serializable {
private static final long serialVersionUID = -1874786860901426015L;
private final ObjectBean _objBean;
private String id;
private URL url;
private Object lastModified;
private String eTag;
private SyndFeed syndFeed;
public SyndFeedInfo() {
_objBean = new ObjectBean(this.getClass(),this);
}
/**
* Creates a deep 'bean' clone of the object.
* <p>
* @return a clone of the object.
* @throws CloneNotSupportedException thrown if an element of the object cannot be cloned.
*
*/
public Object clone() throws CloneNotSupportedException {
return _objBean.clone();
}
/**
* Indicates whether some other object is "equal to" this one as defined by the Object equals() method.
* <p>
* @param other he reference object with which to compare.
* @return <b>true</b> if 'this' object is equal to the 'other' object.
*
*/
public boolean equals(Object other) {
return _objBean.equals(other);
}
/**
* Returns a hashcode value for the object.
* <p>
* It follows the contract defined by the Object hashCode() method.
* <p>
* @return the hashcode of the bean object.
*
*/
public int hashCode() {
return _objBean.hashCode();
}
/**
* Returns the String representation for the object.
* <p>
* @return String representation for the object.
*
*/
public String toString() {
return _objBean.toString();
}
/**
* @return the ETag the feed was last retrieved with
*/
public synchronized String getETag() {
return eTag;
}
/**
* @return the last modified date for the feed
*/
public synchronized Object getLastModified() {
return lastModified;
}
/**
* @return the URL the feed was served from
*/
public synchronized URL getUrl() {
return url;
}
public synchronized void setETag(String string) {
eTag = string;
}
public synchronized void setLastModified(Object o) {
lastModified = o;
}
public synchronized void setUrl(URL url) {
this.url = url;
}
public synchronized SyndFeed getSyndFeed() {
return syndFeed;
}
public synchronized void setSyndFeed(SyndFeed feed) {
syndFeed = feed;
}
/**
* @return A unique ID to identify the feed
*/
public synchronized String getId() {
return id;
}
/**
* @param string A unique ID to identify the feed. Note that if the URL of the feed
* changes this will remain the same
*/
public synchronized void setId(String string) {
id = string;
}
}

92
src/java/org/rometools/fetcher/samples/FeedAggregator.java Normal file
View file

@ -0,0 +1,92 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.fetcher.samples;
import java.io.PrintWriter;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import com.sun.syndication.feed.synd.SyndFeedImpl;
import com.sun.syndication.feed.synd.SyndFeed;
import org.rometools.fetcher.FeedFetcher;
import org.rometools.fetcher.impl.FeedFetcherCache;
import org.rometools.fetcher.impl.HashMapFeedInfoCache;
import org.rometools.fetcher.impl.HttpURLFeedFetcher;
import com.sun.syndication.io.SyndFeedOutput;
/**
* <p>It aggregates a list of RSS/Atom feeds (they can be of different types)
* into a single feed of the specified type.</p>
*
* <p>Converted from the original FeedAggregator sample</p>
*
* @author Alejandro Abdelnur
* @author Nick Lothian
*
*/
public class FeedAggregator {
public static void main(String[] args) {
boolean ok = false;
if (args.length>=2) {
try {
String outputType = args[0];
SyndFeed feed = new SyndFeedImpl();
feed.setFeedType(outputType);
feed.setTitle("Aggregated Feed");
feed.setDescription("Anonymous Aggregated Feed");
feed.setAuthor("anonymous");
feed.setLink("http://www.anonymous.com");
List entries = new ArrayList();
feed.setEntries(entries);
FeedFetcherCache feedInfoCache = HashMapFeedInfoCache.getInstance();
FeedFetcher feedFetcher = new HttpURLFeedFetcher(feedInfoCache);
for (int i=1;i<args.length;i++) {
URL inputUrl = new URL(args[i]);
SyndFeed inFeed = feedFetcher.retrieveFeed(inputUrl);
entries.addAll(inFeed.getEntries());
}
SyndFeedOutput output = new SyndFeedOutput();
output.output(feed, new PrintWriter(System.out));
ok = true;
}
catch (Exception ex) {
System.out.println("ERROR: "+ex.getMessage());
ex.printStackTrace();
}
}
if (!ok) {
System.out.println();
System.out.println("FeedAggregator aggregates different feeds into a single one.");
System.out.println("The first parameter must be the feed type for the aggregated feed.");
System.out.println(" [valid values are: rss_0.9, rss_0.91, rss_0.92, rss_0.93, ]");
System.out.println(" [ rss_0.94, rss_1.0, rss_2.0 & atom_0.3 ]");
System.out.println("The second to last parameters are the URLs of feeds to aggregate.");
System.out.println();
}
}
}

96
src/java/org/rometools/fetcher/samples/FeedReader.java Normal file
View file

@ -0,0 +1,96 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.fetcher.samples;
import java.net.URL;
import com.sun.syndication.feed.synd.SyndFeed;
import org.rometools.fetcher.FeedFetcher;
import org.rometools.fetcher.FetcherEvent;
import org.rometools.fetcher.FetcherListener;
import org.rometools.fetcher.impl.FeedFetcherCache;
import org.rometools.fetcher.impl.HashMapFeedInfoCache;
import org.rometools.fetcher.impl.HttpURLFeedFetcher;
/**
* Reads and prints any RSS/Atom feed type. Converted from the
* original Rome sample FeedReader
* <p>
* @author Alejandro Abdelnur
* @author Nick Lothian
*
*/
public class FeedReader {
public static void main(String[] args) {
boolean ok = false;
if (args.length==1) {
try {
URL feedUrl = new URL(args[0]);
FeedFetcherCache feedInfoCache = HashMapFeedInfoCache.getInstance();
FeedFetcher fetcher = new HttpURLFeedFetcher(feedInfoCache);
FetcherEventListenerImpl listener = new FetcherEventListenerImpl();
fetcher.addFetcherEventListener(listener);
System.err.println("Retrieving feed " + feedUrl);
// Retrieve the feed.
// We will get a Feed Polled Event and then a
// Feed Retrieved event (assuming the feed is valid)
SyndFeed feed = fetcher.retrieveFeed(feedUrl);
System.err.println(feedUrl + " retrieved");
System.err.println(feedUrl + " has a title: " + feed.getTitle() + " and contains " + feed.getEntries().size() + " entries.");
// We will now retrieve the feed again. If the feed is unmodified
// and the server supports conditional gets, we will get a "Feed
// Unchanged" event after the Feed Polled event
System.err.println("Polling " + feedUrl + " again to test conditional get support.");
SyndFeed feed2 = fetcher.retrieveFeed(feedUrl);
System.err.println("If a \"Feed Unchanged\" event fired then the server supports conditional gets.");
ok = true;
}
catch (Exception ex) {
System.out.println("ERROR: "+ex.getMessage());
ex.printStackTrace();
}
}
if (!ok) {
System.out.println();
System.out.println("FeedReader reads and prints any RSS/Atom feed type.");
System.out.println("The first parameter must be the URL of the feed to read.");
System.out.println();
}
}
static class FetcherEventListenerImpl implements FetcherListener {
/**
* @see com.sun.syndication.fetcher.FetcherListener#fetcherEvent(com.sun.syndication.fetcher.FetcherEvent)
*/
public void fetcherEvent(FetcherEvent event) {
String eventType = event.getEventType();
if (FetcherEvent.EVENT_TYPE_FEED_POLLED.equals(eventType)) {
System.err.println("\tEVENT: Feed Polled. URL = " + event.getUrlString());
} else if (FetcherEvent.EVENT_TYPE_FEED_RETRIEVED.equals(eventType)) {
System.err.println("\tEVENT: Feed Retrieved. URL = " + event.getUrlString());
} else if (FetcherEvent.EVENT_TYPE_FEED_UNCHANGED.equals(eventType)) {
System.err.println("\tEVENT: Feed Unchanged. URL = " + event.getUrlString());
}
}
}
}

421
src/test/org/rometools/test/AbstractJettyTest.java Normal file
View file

@ -0,0 +1,421 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.test;
import org.rometools.fetcher.impl.HashMapFeedInfoCache;
import org.rometools.fetcher.impl.FeedFetcherCache;
import java.net.URL;
import junit.framework.TestCase;
import org.mortbay.http.BasicAuthenticator;
import org.mortbay.http.HashUserRealm;
import org.mortbay.http.HttpContext;
import org.mortbay.http.HttpServer;
import org.mortbay.http.SecurityConstraint;
import org.mortbay.http.SocketListener;
import org.mortbay.http.UserRealm;
import org.mortbay.http.handler.SecurityHandler;
import org.mortbay.jetty.servlet.ServletHandler;
import com.sun.syndication.feed.atom.Entry;
import com.sun.syndication.feed.synd.SyndEntry;
import com.sun.syndication.feed.synd.SyndFeed;
import org.rometools.fetcher.FeedFetcher;
import org.rometools.fetcher.FetcherEvent;
import org.rometools.fetcher.FetcherException;
import org.rometools.fetcher.FetcherListener;
/**
* @author nl
*/
public abstract class AbstractJettyTest extends TestCase {
private HttpServer server;
private int testPort = 8283;
/**
* @param s
*/
public AbstractJettyTest(String s) {
super(s);
}
protected HttpServer getServer() {
return server;
}
protected abstract FeedFetcher getFeedFetcher();
protected abstract FeedFetcher getFeedFetcher(FeedFetcherCache cache);
/**
* @see junit.framework.TestCase#setUp()
*/
protected void setUp() throws Exception {
setupServer();
HttpContext context = createContext();
ServletHandler servlets = createServletHandler();
context.addHandler(servlets);
server.addContext(context);
server.start();
}
/**
* @throws InterruptedException
*/
private void setupServer() throws InterruptedException {
// Create the server
if (server != null) {
server.stop();
server = null;
}
server = new HttpServer();
// Create a port listener
SocketListener listener=new SocketListener();
listener.setPort(testPort);
server.addListener(listener);
}
/**
* @return
*/
private ServletHandler createServletHandler() {
ServletHandler servlets = new ServletHandler();
servlets.addServlet("FetcherTestServlet",FetcherTestServlet.SERVLET_MAPPING,"org.rometools.test.FetcherTestServlet");
servlets.addServlet("FetcherTestServlet",FetcherTestServlet.SERVLET_MAPPING2,"org.rometools.test.FetcherTestServlet");
return servlets;
}
/**
* @return
*/
private HttpContext createContext() {
HttpContext context = new HttpContext();
context.setContextPath("/rome/*");
return context;
}
/**
* @see junit.framework.TestCase#tearDown()
*/
protected void tearDown() throws Exception {
if (server != null) {
server.stop();
server.destroy();
server = null;
}
}
class FetcherEventListenerImpl implements FetcherListener {
boolean polled = false;
boolean retrieved = false;
boolean unchanged = false;
public void reset() {
polled = false;
retrieved = false;
unchanged = false;
}
/**
* @see com.sun.syndication.fetcher.FetcherListener#fetcherEvent(com.sun.syndication.fetcher.FetcherEvent)
*/
public void fetcherEvent(FetcherEvent event) {
String eventType = event.getEventType();
if (FetcherEvent.EVENT_TYPE_FEED_POLLED.equals(eventType)) {
System.err.println("\tEVENT: Feed Polled. URL = " + event.getUrlString());
polled = true;
} else if (FetcherEvent.EVENT_TYPE_FEED_RETRIEVED.equals(eventType)) {
System.err.println("\tEVENT: Feed Retrieved. URL = " + event.getUrlString());
retrieved = true;
} else if (FetcherEvent.EVENT_TYPE_FEED_UNCHANGED.equals(eventType)) {
System.err.println("\tEVENT: Feed Unchanged. URL = " + event.getUrlString());
unchanged = true;
}
}
}
public void testRetrieveFeed() {
FeedFetcher feedFetcher = getFeedFetcher();
try {
SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
assertNotNull(feed);
assertEquals("atom_1.0.feed.title", feed.getTitle());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
public void testBasicAuthentication() {
try {
setupServer();
HttpContext context = createContext();
URL url = this.getClass().getResource("/testuser.properties");
UserRealm ur = new HashUserRealm("test", url.getFile());
context.setRealm(ur);
BasicAuthenticator ba = new BasicAuthenticator();
context.setAuthenticator(ba);
SecurityHandler sh = new SecurityHandler();
context.addHandler(sh);
SecurityConstraint sc = new SecurityConstraint();
sc.setName("test");
sc.addRole("*");
sc.setAuthenticate(true);
context.addSecurityConstraint("/", sc);
ServletHandler servlets = createServletHandler();
context.addHandler(servlets);
server.addContext(context);
server.start();
FeedFetcher feedFetcher = getAuthenticatedFeedFetcher();
SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
assertNotNull(feed);
assertEquals("atom_1.0.feed.title", feed.getTitle());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
public abstract FeedFetcher getAuthenticatedFeedFetcher();
/**
* Test getting a feed via a http 301 redirect
*
*/
public void testRetrieveRedirectedFeed() {
FeedFetcher feedFetcher = getFeedFetcher();
try {
SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet?redirect=TRUE"));
assertNotNull(feed);
assertEquals("atom_1.0.feed.title", feed.getTitle());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
/**
* Test error handling
*
*/
public void testErrorHandling() {
FeedFetcher feedFetcher = getFeedFetcher();
try {
SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet?error=404"));
fail("4xx error handling did not work correctly");
} catch (FetcherException e) {
// expect this exception
assertEquals(404, e.getResponseCode());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
try {
SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet?error=500"));
fail("5xx error handling did not work correctly");
} catch (FetcherException e) {
// expect this exception
assertEquals(500, e.getResponseCode());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
public void testUserAgent() {
FeedFetcher feedFetcher = getFeedFetcher();
//System.out.println(feedFetcher.getUserAgent());
//System.out.println(System.getProperty("rome.fetcher.version", "UNKNOWN"));
assertEquals("Rome Client (http://tinyurl.com/64t5n) Ver: " + System.getProperty("rome.fetcher.version", "UNKNOWN"), feedFetcher.getUserAgent());
}
/**
* Test events fired when there is no cache in use
*
*/
public void testFetchEvents() {
FeedFetcher feedFetcher = getFeedFetcher();
FetcherEventListenerImpl listener = new FetcherEventListenerImpl();
feedFetcher.addFetcherEventListener(listener);
try {
SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
assertNotNull(feed);
assertTrue(listener.polled);
assertTrue(listener.retrieved);
assertFalse(listener.unchanged);
listener.reset();
// since there is no cache, the events fired should be exactly the same if
// we re-retrieve the feed
feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
assertNotNull(feed);
assertTrue(listener.polled);
assertTrue(listener.retrieved);
assertFalse(listener.unchanged);
listener.reset();
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
/**
* Test events fired when there is a cache in use
*
*/
public void testFetchEventsWithCache() {
FeedFetcherCache feedInfoCache = new HashMapFeedInfoCache();
FeedFetcher feedFetcher = getFeedFetcher(feedInfoCache);
FetcherEventListenerImpl listener = new FetcherEventListenerImpl();
feedFetcher.addFetcherEventListener(listener);
try {
SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
assertNotNull(feed);
assertTrue(listener.polled);
assertTrue(listener.retrieved);
assertFalse(listener.unchanged);
listener.reset();
// Since the feed is cached, the second request should not
// actually retrieve the feed
feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
assertNotNull(feed);
assertTrue(listener.polled);
assertFalse(listener.retrieved);
assertTrue(listener.unchanged);
listener.reset();
// now simulate getting the feed after it has changed
feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet?refreshfeed=TRUE"));
assertNotNull(feed);
assertTrue(listener.polled);
assertTrue(listener.retrieved);
assertFalse(listener.unchanged);
listener.reset();
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
/**
* Test handling of GZipped feed
*
*/
public void testGZippedFeed() {
FeedFetcher feedFetcher = getFeedFetcher();
try {
SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet?gzipfeed=TRUE"));
assertNotNull(feed);
assertEquals("atom_1.0.feed.title", feed.getTitle());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
public void testPreserveWireFeed() throws Exception {
FeedFetcher feedFetcher = getFeedFetcher();
// first check we the WireFeed is not preserved by default
SyndFeed feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
assertNotNull(feed);
assertEquals("atom_1.0.feed.title", feed.getTitle());
assertNull(feed.originalWireFeed());
SyndEntry syndEntry = (SyndEntry)feed.getEntries().get(0);
assertNotNull(syndEntry);
assertNull(syndEntry.getWireEntry());
// now turn on WireFeed preservation
feedFetcher.setPreserveWireFeed(true);
try {
feed = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet/"));
assertNotNull(feed);
assertEquals("atom_1.0.feed.title", feed.getTitle());
assertNotNull(feed.originalWireFeed());
syndEntry = (SyndEntry)feed.getEntries().get(0);
assertNotNull(syndEntry);
assertNotNull(syndEntry.getWireEntry());
Entry entry = (Entry) syndEntry.getWireEntry();
assertEquals("atom_1.0.feed.entry[0].rights", entry.getRights());
} finally {
feedFetcher.setPreserveWireFeed(false); //reset
}
}
public void testDeltaEncoding() {
FeedFetcherCache feedInfoCache = new HashMapFeedInfoCache();
FeedFetcher feedFetcher = getFeedFetcher(feedInfoCache);
try {
feedFetcher.setUsingDeltaEncoding(true);
// first retrieval should just grab the default feed
SyndFeed feed1 = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet?deltaencode=TRUE&refreshfeed=TRUE"));
assertNotNull(feed1);
assertEquals("atom_1.0.feed.title", feed1.getTitle());
assertEquals(2, feed1.getEntries().size());
SyndEntry entry1 = (SyndEntry) feed1.getEntries().get(0);
assertEquals("atom_1.0.feed.entry[0].title", entry1.getTitle());
// second retrieval should get only the new item
/*
* This is breaking with Rome 0.5 ??
*/
SyndFeed feed2 = feedFetcher.retrieveFeed(new URL("http://localhost:"+testPort+"/rome/FetcherTestServlet?deltaencode=TRUE&refreshfeed=TRUE"));
assertNotNull(feed2);
assertEquals(FetcherTestServlet.DELTA_FEED_TITLE, feed2.getTitle());
assertEquals(3, feed2.getEntries().size());
entry1 = (SyndEntry) feed2.getEntries().get(0);
assertEquals(FetcherTestServlet.DELTA_FEED_ENTRY_TITLE, entry1.getTitle());
SyndEntry entry2 = (SyndEntry) feed2.getEntries().get(1);
assertEquals("atom_1.0.feed.entry[0].title", entry2.getTitle());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
}

42
src/test/org/rometools/test/DiskFeedInfoCacheTest.java Normal file
View file

@ -0,0 +1,42 @@
package org.rometools.test;
import org.rometools.fetcher.impl.DiskFeedInfoCache;
import org.rometools.fetcher.impl.SyndFeedInfo;
import java.net.*;
import java.io.File;
import junit.framework.TestCase;
public class DiskFeedInfoCacheTest extends TestCase {
public void testClear() throws Exception {
File cacheDir = new File("test-cache");
cacheDir.mkdir();
cacheDir.deleteOnExit();
final DiskFeedInfoCache cache = new DiskFeedInfoCache(cacheDir.getCanonicalPath());
SyndFeedInfo info = new SyndFeedInfo();
URL url = new URL("http://nowhere.com");
cache.setFeedInfo(url, info);
cache.clear();
final Object returned = cache.getFeedInfo(url);
assertTrue( returned == null );
}
public void testRemove() throws Exception {
File cacheDir = new File("test-cache");
cacheDir.mkdir();
cacheDir.deleteOnExit();
final DiskFeedInfoCache cache = new DiskFeedInfoCache( cacheDir.getCanonicalPath() );
SyndFeedInfo info = new SyndFeedInfo();
URL url = new URL("http://nowhere.com");
cache.setFeedInfo( url, info );
SyndFeedInfo removedInfo = cache.remove( url );
assertTrue( removedInfo.equals(info) );
SyndFeedInfo shouldBeNull = cache.remove( url );
assertTrue( null == shouldBeNull );
}
}

244
src/test/org/rometools/test/FetcherTestServlet.java Normal file
View file

@ -0,0 +1,244 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.test;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.GZIPOutputStream;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.sun.syndication.feed.synd.SyndContent;
import com.sun.syndication.feed.synd.SyndContentImpl;
import com.sun.syndication.feed.synd.SyndEntry;
import com.sun.syndication.feed.synd.SyndEntryImpl;
import com.sun.syndication.feed.synd.SyndFeed;
import com.sun.syndication.feed.synd.SyndFeedImpl;
import com.sun.syndication.io.FeedException;
import com.sun.syndication.io.SyndFeedOutput;
public class FetcherTestServlet extends javax.servlet.http.HttpServlet {
public static final String ETAG_1 = "ETAG-1";
public static final String ETAG_2 = "ETAG-2";
public static final String DELTA_FEED_TITLE = "Delta Encoded Feed";
public static final String DELTA_FEED_ENTRY_TITLE = "Delta Encoded Feed Entry";
public static final String SERVLET_MAPPING = "/FetcherTestServlet/*";
public static final String SERVLET_MAPPING2 = "/FetcherTestServlet2/*";
/**
* @throws IOException
* @throws
* @see javax.servlet.http.HttpServlet#doGet(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse)
*/
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
if ("TRUE".equalsIgnoreCase(request.getParameter("redirect"))) {
// testing redirection support
response.sendRedirect("/rome/FetcherTestServlet2/");
return;
} else if (request.getParameter("error") != null) {
//response.sendError(HttpServletResponse.SC_NOT_FOUND);
int errorToThrow = Integer.parseInt(request.getParameter("error"));
response.sendError(errorToThrow);
return;
} else {
// We manually set the date headers using strings
// instead of the get/setDateHeader methods because
// those methods return longs, which has too much
// precision for the real date headers
// this is just a random date
String lastModifiedDate = "Thu, 08 Jan 2009 23:06:39 GMT";
String eTag = ETAG_1;
if ("TRUE".equalsIgnoreCase(request.getParameter("refreshfeed"))) {
lastModifiedDate = "Fri, 09 Jan 2009 12:06:39 GMT";
eTag = ETAG_2;
}
boolean serveFeed = checkModified(request, lastModifiedDate, eTag) || ("TRUE".equalsIgnoreCase(request.getParameter("deltaencode")));
boolean gzip = "TRUE".equalsIgnoreCase(request.getParameter("gzipfeed"));
if (serveFeed) {
String aimHeader = request.getHeader("A-IM");
boolean serveDeltaEncodedFeed = ((aimHeader != null) && (aimHeader.indexOf("feed") >=0) && "TRUE".equalsIgnoreCase(request.getParameter("deltaencode")));
if (serveDeltaEncodedFeed) {
try {
sendDeltaEncodedData(response, lastModifiedDate, request.getHeader("If-None-Match"), eTag, gzip);
} catch (FeedException e) {
throw new ServletException(e);
}
} else {
sendFeedData(response, lastModifiedDate, eTag, gzip);
}
return;
} else {
response.sendError(HttpServletResponse.SC_NOT_MODIFIED);
return;
}
}
}
private boolean checkModified(HttpServletRequest request, String lastModifiedDate, String eTag) {
String requestedETag = request.getHeader("If-None-Match");
String requestedLastModified = request.getHeader("If-Modified-Since");
boolean modified = true;
boolean mustServer = false;
if (requestedETag != null) {
if (eTag.equals(requestedETag)) {
modified = false;
} else {
modified = true;
mustServer = true;
}
}
if (requestedLastModified != null) {
if (lastModifiedDate.equals(requestedLastModified)) {
modified = false;
} else {
modified = true;
mustServer = true;
}
}
boolean serveFeed = (modified || mustServer);
return serveFeed;
}
/**
* @param request
* @param lastModifiedDate
* @param tag
* @param gzip
* @throws IOException
* @throws FeedException
*/
private void sendDeltaEncodedData(HttpServletResponse response, String lastModifiedDate, String requestedETag, String responseETag, boolean gzip) throws IOException, FeedException {
if (ETAG_1.equals(requestedETag) || ETAG_2.equals(requestedETag)) {
OutputStream out = null;
if (gzip) {
response.setHeader("Content-Encoding", "gzip");
out = new GZIPOutputStream(response.getOutputStream());
} else {
out = response.getOutputStream();
}
response.setContentType("text/xml");
response.setStatus(226);
if (gzip) {
response.setHeader("IM", "feed, gzip");
} else {
response.setHeader("IM", "feed");
}
if (responseETag != null) {
response.setHeader("ETag", responseETag);
}
if (lastModifiedDate != null) {
response.setHeader("Last-Modified", lastModifiedDate);
}
SyndFeed feed = new SyndFeedImpl();
feed.setFeedType("atom_1.0");
feed.setTitle(DELTA_FEED_TITLE);
feed.setLink("http://rome.dev.java.net");
feed.setDescription("This tests using rfc3229 delta encoding.");
List entries = new ArrayList();
SyndEntry entry;
SyndContent description;
entry = new SyndEntryImpl();
entry.setTitle(DELTA_FEED_ENTRY_TITLE);
entry.setLink("http://bobwyman.pubsub.com/main/2004/09/using_rfc3229_w.html");
try {
DateFormat dateParser = new SimpleDateFormat("yyyy-MM-dd");
entry.setPublishedDate(dateParser.parse("2004-11-25"));
}
catch (ParseException ex) {
//
}
description = new SyndContentImpl();
description.setType("text/plain");
description.setValue("Test for RFC3229 Delta Encoding");
entry.setDescription(description);
entries.add(entry);
feed.setEntries(entries);
SyndFeedOutput output = new SyndFeedOutput();
output.output(feed, new OutputStreamWriter(out));
} else {
sendFeedData(response, lastModifiedDate, responseETag, gzip);
}
}
private void sendFeedData(HttpServletResponse response, String lastModifiedDate, String eTag, boolean gzip) throws IOException {
OutputStream out = null;
if (gzip) {
response.setHeader("Content-Encoding", "gzip");
out = new GZIPOutputStream(response.getOutputStream());
} else {
out = response.getOutputStream();
}
response.setContentType("text/xml");
if (eTag != null) {
response.setHeader("ETag", eTag);
}
if (lastModifiedDate != null) {
response.setHeader("Last-Modified", lastModifiedDate);
}
InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream("/atom_1.0.xml");
if (inputStream == null) {
inputStream = this.getClass().getResourceAsStream("/atom_1.0.xml");
}
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
try {
String line;
while ((line = reader.readLine()) != null) {
out.write(line.getBytes());
line = null;
}
} finally {
if (reader != null) {
reader.close();
}
}
out.close();
}
}

41
src/test/org/rometools/test/HashMapFeedInfoCacheTest.java Normal file
View file

@ -0,0 +1,41 @@
package org.rometools.test;
import org.rometools.fetcher.impl.HashMapFeedInfoCache;
import org.rometools.fetcher.impl.SyndFeedInfo;
import java.net.URL;
import junit.framework.TestCase;
public class HashMapFeedInfoCacheTest extends TestCase {
public void testRemove() throws Exception {
final HashMapFeedInfoCache cache = new HashMapFeedInfoCache();
assertNotNull( cache );
final URL url = new URL("http://foo.com");
final SyndFeedInfo syndFeedInfo = new SyndFeedInfo();
syndFeedInfo.setUrl(url);
cache.setFeedInfo(url, syndFeedInfo);
final SyndFeedInfo returned = cache.remove(url);
assertTrue( returned.equals(syndFeedInfo) );
assertTrue( url.equals( returned.getUrl() ));
}
public void testClear() throws Exception {
final HashMapFeedInfoCache cache = new HashMapFeedInfoCache();
assertNotNull( cache );
final URL url = new URL("http://foo.com");
final SyndFeedInfo syndFeedInfo = new SyndFeedInfo();
syndFeedInfo.setUrl(url);
cache.setFeedInfo(url, syndFeedInfo);
//clear it
cache.clear();
//we should not get a result back
final Object returned = cache.getFeedInfo(url);
assertTrue( returned == null );
}
}

60
src/test/org/rometools/test/HttpClientFeedFetcherTest.java Normal file
View file

@ -0,0 +1,60 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.test;
import org.rometools.fetcher.impl.HttpClientFeedFetcher;
import org.rometools.fetcher.impl.FeedFetcherCache;
import org.apache.commons.httpclient.Credentials;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.rometools.fetcher.FeedFetcher;
/**
* @author Nick Lothian
*/
public class HttpClientFeedFetcherTest extends AbstractJettyTest {
public HttpClientFeedFetcherTest(String s) {
super(s);
}
/**
* @see com.sun.syndication.fetcher.impl.AbstractJettyTest#getFeedFetcher()
*/
protected FeedFetcher getFeedFetcher() {
return new HttpClientFeedFetcher();
}
protected FeedFetcher getFeedFetcher(FeedFetcherCache cache) {
return new HttpClientFeedFetcher(cache);
}
/**
* @see com.sun.syndication.fetcher.impl.AbstractJettyTest#getAuthenticatedFeedFetcher()
*/
public FeedFetcher getAuthenticatedFeedFetcher() {
return new HttpClientFeedFetcher(null, new HttpClientFeedFetcher.CredentialSupplier() {
public Credentials getCredentials(String realm, String host) {
if ("localhost".equals(host)) {
return new UsernamePasswordCredentials("username", "password");
} else {
return null;
}
}
});
}
}

55
src/test/org/rometools/test/HttpURLFeedFetcherTest.java Normal file
View file

@ -0,0 +1,55 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.test;
import org.rometools.fetcher.impl.HttpURLFeedFetcher;
import org.rometools.fetcher.impl.FeedFetcherCache;
import org.rometools.fetcher.FeedFetcher;
public class HttpURLFeedFetcherTest extends AbstractJettyTest {
public HttpURLFeedFetcherTest(String s) {
super(s);
}
/**
* @see com.sun.syndication.fetcher.impl.AbstractJettyTest#getFeedFetcher()
*/
protected FeedFetcher getFeedFetcher() {
return new HttpURLFeedFetcher();
}
protected FeedFetcher getFeedFetcher(FeedFetcherCache cache) {
return new HttpURLFeedFetcher(cache);
}
/**
* @see com.sun.syndication.fetcher.impl.AbstractJettyTest#getAuthenticatedFeedFetcher()
*/
public FeedFetcher getAuthenticatedFeedFetcher() {
// setup the authenticator
java.net.Authenticator.setDefault(new TestBasicAuthenticator());
FeedFetcher feedFetcher = getFeedFetcher();
return feedFetcher;
}
}

42
src/test/org/rometools/test/ResponseHandlerTest.java Normal file
View file

@ -0,0 +1,42 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.test;
import org.rometools.fetcher.impl.ResponseHandler;
import junit.framework.TestCase;
public class ResponseHandlerTest extends TestCase {
/**
* Constructor for ResponseHandlerTest.
*/
public ResponseHandlerTest(String arg0) {
super(arg0);
}
public void testGetCharacterEncodingString() {
assertEquals(ResponseHandler.defaultCharacterEncoding, ResponseHandler.getCharacterEncoding((String)null));
assertEquals(ResponseHandler.defaultCharacterEncoding, ResponseHandler.getCharacterEncoding("text/xml"));
assertEquals(ResponseHandler.defaultCharacterEncoding, ResponseHandler.getCharacterEncoding("text/xml;"));
assertEquals("ISO-8859-4", ResponseHandler.getCharacterEncoding("text/xml; charset=ISO-8859-4"));
assertEquals("ISO-8859-4", ResponseHandler.getCharacterEncoding("text/xml;charset=ISO-8859-4"));
assertEquals("ISO-8859-4", ResponseHandler.getCharacterEncoding("text/xml;charset=ISO-8859-4;something"));
assertEquals(ResponseHandler.defaultCharacterEncoding, ResponseHandler.getCharacterEncoding("text/xml;something"));
}
}

37
src/test/org/rometools/test/TestBasicAuthenticator.java Normal file
View file

@ -0,0 +1,37 @@
/*
* Copyright 2004 Sun Microsystems, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.rometools.test;
import java.net.Authenticator;
import java.net.PasswordAuthentication;
/**
* @author nl
*/
public class TestBasicAuthenticator extends Authenticator {
/**
* @see java.net.Authenticator#getPasswordAuthentication()
*/
protected PasswordAuthentication getPasswordAuthentication() {
if ("localhost".equals(getRequestingHost())) {
return new PasswordAuthentication("username", "password".toCharArray());
} else {
return null;
}
}
}