View Javadoc

1   /*
2    * Copyright 2004 Sun Microsystems, Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   *
16   */
17  
18  package com.sun.syndication.fetcher.impl;
19  
20  import java.io.IOException;
21  import java.io.InputStream;
22  import java.net.HttpURLConnection;
23  import java.net.MalformedURLException;
24  import java.net.URL;
25  import java.util.zip.GZIPInputStream;
26  
27  import org.apache.commons.httpclient.Credentials;
28  import org.apache.commons.httpclient.Header;
29  import org.apache.commons.httpclient.HttpClient;
30  import org.apache.commons.httpclient.HttpException;
31  import org.apache.commons.httpclient.HttpMethod;
32  import org.apache.commons.httpclient.methods.GetMethod;
33  
34  import com.sun.syndication.feed.synd.SyndFeed;
35  import com.sun.syndication.fetcher.FetcherEvent;
36  import com.sun.syndication.fetcher.FetcherException;
37  import com.sun.syndication.io.FeedException;
38  import com.sun.syndication.io.SyndFeedInput;
39  import com.sun.syndication.io.XmlReader;
40  
41  /***
42   * @author Nick Lothian
43   */
44  public class HttpClientFeedFetcher extends AbstractFeedFetcher {
45  
46  	private FeedFetcherCache feedInfoCache;
47      private CredentialSupplier credentialSupplier;
48  		
49  	public HttpClientFeedFetcher() {
50  		super();
51  	}
52  	
53  	/***
54  	 * @param cache
55  	 */
56  	public HttpClientFeedFetcher(FeedFetcherCache cache) {
57  		this();
58  		this.feedInfoCache = cache;
59  	}
60  
61  	
62  	public HttpClientFeedFetcher(FeedFetcherCache cache, CredentialSupplier credentialSupplier) {
63  	    this(cache);
64  	    this.credentialSupplier = credentialSupplier;
65  	}
66  	
67      /***
68       * @return Returns the credentialSupplier.
69       */
70      public CredentialSupplier getCredentialSupplier() {
71          return credentialSupplier;
72      }
73      /***
74       * @param credentialSupplier The credentialSupplier to set.
75       */
76      public void setCredentialSupplier(CredentialSupplier credentialSupplier) {
77          this.credentialSupplier = credentialSupplier;
78      }	
79  	
80  	/***
81  	 * @see com.sun.syndication.fetcher.FeedFetcher#retrieveFeed(java.net.URL)
82  	 */
83  	public SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
84  		if (feedUrl == null) {
85  			throw new IllegalArgumentException("null is not a valid URL");
86  		}
87  		// TODO Fix this
88  		//System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.SimpleLog");
89  		HttpClient client = new HttpClient();
90  		
91  		if (getCredentialSupplier() != null) {
92  			client.getState().setAuthenticationPreemptive(true);
93  			// TODO what should realm be here?
94  			Credentials credentials = getCredentialSupplier().getCredentials(null, feedUrl.getHost()); 
95  			if (credentials != null) {
96  			    client.getState().setCredentials(null, feedUrl.getHost(), credentials);
97  			}			
98  		}
99  		
100 		
101 		System.setProperty("httpclient.useragent", getUserAgent());
102 		String urlStr = feedUrl.toString();
103 		if (feedInfoCache != null) {
104 			// get the feed info from the cache
105 		    // Note that syndFeedInfo will be null if it is not in the cache
106 			SyndFeedInfo syndFeedInfo = feedInfoCache.getFeedInfo(feedUrl);			
107 							
108 			// retrieve feed
109 			HttpMethod method = new GetMethod(urlStr);
110 			method.addRequestHeader("Accept-Encoding", "gzip");
111 			try {
112 				if (isUsingDeltaEncoding()) {
113 				    method.setRequestHeader("A-IM", "feed");
114 				}	    
115 			    
116 			    if (syndFeedInfo != null) {
117 				    method.setRequestHeader("If-None-Match", syndFeedInfo.getETag());
118 				    
119 				    if (syndFeedInfo.getLastModified() instanceof String) {
120 				        method.setRequestHeader("If-Modified-Since", (String)syndFeedInfo.getLastModified());
121 				    }
122 			    }
123 			    
124 			    method.setFollowRedirects(true);			
125 				
126 				int statusCode = client.executeMethod(method);
127 				fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);				
128 				handleErrorCodes(statusCode);			    
129 			    			    
130 			    SyndFeed feed = retrieveFeed(syndFeedInfo, urlStr, method, statusCode);
131 			    				    
132 				syndFeedInfo = buildSyndFeedInfo(feedUrl, urlStr, method, feed, statusCode);
133 				
134 				feedInfoCache.setFeedInfo(new URL(urlStr), syndFeedInfo);	
135 				
136 				// the feed may have been modified to pick up cached values
137 				// (eg - for delta encoding)
138 				feed = syndFeedInfo.getSyndFeed();
139 	
140 				return feed;
141 			} finally {
142 				method.releaseConnection();
143 			}
144 				
145 		} else {
146 		    // cache is not in use		    
147 			HttpMethod method = new GetMethod(urlStr);
148 			try {
149 			    method.setFollowRedirects(true);			
150 				
151 				int statusCode = client.executeMethod(method);
152 				fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);
153 				handleErrorCodes(statusCode);		
154 			    
155 				return retrieveFeed(null, urlStr, method, statusCode);
156 			} finally {
157 				method.releaseConnection();
158 			}
159 		}
160 	}
161 
162 
163 	/***
164      * @param feedUrl
165      * @param urlStr
166      * @param method
167      * @param feed
168      * @return
169      * @throws MalformedURLException
170      */
171     private SyndFeedInfo buildSyndFeedInfo(URL feedUrl, String urlStr, HttpMethod method, SyndFeed feed, int statusCode) throws MalformedURLException {
172         SyndFeedInfo syndFeedInfo;
173         syndFeedInfo = new SyndFeedInfo();
174         
175         // this may be different to feedURL because of 3XX redirects
176         syndFeedInfo.setUrl(new URL(urlStr));
177         syndFeedInfo.setId(feedUrl.toString());                					
178                 
179         Header imHeader = method.getResponseHeader("IM");
180         if (imHeader != null && imHeader.getValue().indexOf("feed") >= 0 && isUsingDeltaEncoding() && feedInfoCache != null && statusCode == 226) {
181             // client is setup to use http delta encoding and the server supports it and has returned a delta encoded response
182             // This response only includes new items
183             SyndFeedInfo cachedInfo = feedInfoCache.getFeedInfo(feedUrl);
184 		    if (cachedInfo != null) {
185 			    SyndFeed cachedFeed = cachedInfo.getSyndFeed();
186 			    
187 			    // set the new feed to be the orginal feed plus the new items
188 			    feed = combineFeeds(cachedFeed, feed);			        
189 		    }            
190         }
191         
192         Header lastModifiedHeader = method.getResponseHeader("Last-Modified");
193         if (lastModifiedHeader != null) {
194             syndFeedInfo.setLastModified(lastModifiedHeader.getValue());
195         }
196         
197         Header eTagHeader = method.getResponseHeader("ETag");
198         if (eTagHeader != null) {
199             syndFeedInfo.setETag(eTagHeader.getValue());
200         }
201         
202         syndFeedInfo.setSyndFeed(feed);
203         
204         return syndFeedInfo;
205     }
206 
207     /***
208 	 * @param client
209 	 * @param urlStr
210 	 * @param method
211 	 * @return
212 	 * @throws IOException
213 	 * @throws HttpException
214 	 * @throws FetcherException
215 	 * @throws FeedException
216 	 */
217 	private SyndFeed retrieveFeed(SyndFeedInfo syndFeedInfo, String urlStr, HttpMethod method, int statusCode) throws IOException, HttpException, FetcherException, FeedException {
218 
219 		if (statusCode == HttpURLConnection.HTTP_NOT_MODIFIED && syndFeedInfo != null) {
220 		    fireEvent(FetcherEvent.EVENT_TYPE_FEED_UNCHANGED, urlStr);
221 		    return syndFeedInfo.getSyndFeed();
222 		}
223 		
224 		
225 		InputStream stream = null;
226 		if ((method.getResponseHeader("Content-Encoding") != null) && ("gzip".equalsIgnoreCase(method.getResponseHeader("Content-Encoding").getValue()))) {		
227 		    stream = new GZIPInputStream(method.getResponseBodyAsStream());
228 		} else {
229 		    stream = method.getResponseBodyAsStream();
230 		}		
231 		try {		
232 		    XmlReader reader = null;
233 		    if (method.getResponseHeader("Content-Type") != null) {
234 		        reader = new XmlReader(stream, method.getResponseHeader("Content-Type").getValue(), true);
235 		    } else {
236 		        reader = new XmlReader(stream, true);
237 		    }
238 			SyndFeed feed = new SyndFeedInput().build(reader);
239 
240 			fireEvent(FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, urlStr, feed);			
241 			
242 			return feed;
243 			
244 		} finally {
245 		    if (stream != null) {
246 		        stream.close();
247 		    }
248 			
249 		}
250 	}
251 	
252     public interface CredentialSupplier {
253         public Credentials getCredentials(String realm, String host);
254     }
255 	
256 
257 }