View Javadoc

1   /*
2    * Copyright 2004 Sun Microsystems, Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   *
16   */
17  
18  package com.sun.syndication.fetcher.impl;
19  
20  import java.io.IOException;
21  import java.io.InputStream;
22  import java.io.InputStreamReader;
23  import java.net.HttpURLConnection;
24  import java.net.MalformedURLException;
25  import java.net.URL;
26  import java.util.zip.GZIPInputStream;
27  
28  import org.apache.commons.httpclient.Credentials;
29  import org.apache.commons.httpclient.Header;
30  import org.apache.commons.httpclient.HttpClient;
31  import org.apache.commons.httpclient.HttpException;
32  import org.apache.commons.httpclient.HttpMethod;
33  import org.apache.commons.httpclient.methods.GetMethod;
34  
35  import com.sun.syndication.feed.synd.SyndFeed;
36  import com.sun.syndication.fetcher.FetcherEvent;
37  import com.sun.syndication.fetcher.FetcherException;
38  import com.sun.syndication.io.FeedException;
39  import com.sun.syndication.io.SyndFeedInput;
40  
41  /***
42   * @author Nick Lothian
43   */
44  public class HttpClientFeedFetcher extends AbstractFeedFetcher {
45  
46  	private FeedFetcherCache feedInfoCache;
47      private CredentialSupplier credentialSupplier;
48  		
49  	public HttpClientFeedFetcher() {
50  		super();
51  	}
52  	
53  	/***
54  	 * @param cache
55  	 */
56  	public HttpClientFeedFetcher(FeedFetcherCache cache) {
57  		this();
58  		this.feedInfoCache = cache;
59  	}
60  
61  	
62  	public HttpClientFeedFetcher(FeedFetcherCache cache, CredentialSupplier credentialSupplier) {
63  	    this(cache);
64  	    this.credentialSupplier = credentialSupplier;
65  	}
66  	
67      /***
68       * @return Returns the credentialSupplier.
69       */
70      public CredentialSupplier getCredentialSupplier() {
71          return credentialSupplier;
72      }
73      /***
74       * @param credentialSupplier The credentialSupplier to set.
75       */
76      public void setCredentialSupplier(CredentialSupplier credentialSupplier) {
77          this.credentialSupplier = credentialSupplier;
78      }	
79  	
80  	/***
81  	 * @see com.sun.syndication.fetcher.FeedFetcher#retrieveFeed(java.net.URL)
82  	 */
83  	public SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
84  		if (feedUrl == null) {
85  			throw new IllegalArgumentException("null is not a valid URL");
86  		}
87  		// TODO Fix this
88  		//System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.SimpleLog");
89  		HttpClient client = new HttpClient();
90  		
91  		if (getCredentialSupplier() != null) {
92  			client.getState().setAuthenticationPreemptive(true);
93  			// TODO what should realm be here?
94  			Credentials credentials = getCredentialSupplier().getCredentials(null, feedUrl.getHost()); 
95  			if (credentials != null) {
96  			    client.getState().setCredentials(null, feedUrl.getHost(), credentials);
97  			}			
98  		}
99  		
100 		
101 		System.setProperty("httpclient.useragent", getUserAgent());
102 		String urlStr = feedUrl.toString();
103 		if (feedInfoCache != null) {
104 			// get the feed info from the cache
105 		    // Note that syndFeedInfo will be null if it is not in the cache
106 			SyndFeedInfo syndFeedInfo = feedInfoCache.getFeedInfo(feedUrl);			
107 							
108 			// retrieve feed
109 			HttpMethod method = new GetMethod(urlStr);
110 			try {
111 				if (isUsingDeltaEncoding()) {
112 				    method.setRequestHeader("A-IM", "feed");
113 				}	    
114 			    
115 			    if (syndFeedInfo != null) {
116 				    method.setRequestHeader("If-None-Match", syndFeedInfo.getETag());
117 				    
118 				    if (syndFeedInfo.getLastModified() instanceof String) {
119 				        method.setRequestHeader("If-Modified-Since", (String)syndFeedInfo.getLastModified());
120 				    }
121 			    }
122 			    
123 			    method.setFollowRedirects(true);			
124 				
125 				int statusCode = client.executeMethod(method);
126 				fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);				
127 				handleErrorCodes(statusCode);			    
128 			    			    
129 			    SyndFeed feed = retrieveFeed(syndFeedInfo, client, urlStr, method, statusCode);
130 			    				    
131 				syndFeedInfo = buildSyndFeedInfo(feedUrl, urlStr, method, feed, statusCode);
132 				
133 				feedInfoCache.setFeedInfo(new URL(urlStr), syndFeedInfo);	
134 				
135 				// the feed may have been modified to pick up cached values
136 				// (eg - for delta encoding)
137 				feed = syndFeedInfo.getSyndFeed();
138 	
139 				return feed;
140 			} finally {
141 				method.releaseConnection();
142 			}
143 				
144 		} else {
145 		    // cache is not in use		    
146 			HttpMethod method = new GetMethod(urlStr);
147 			try {
148 			    method.setFollowRedirects(true);			
149 				
150 				int statusCode = client.executeMethod(method);
151 				fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);
152 				handleErrorCodes(statusCode);		
153 			    
154 				return retrieveFeed(null, client, urlStr, method, statusCode);
155 			} finally {
156 				method.releaseConnection();
157 			}
158 		}
159 	}
160 
161 
162 	/***
163      * @param feedUrl
164      * @param urlStr
165      * @param method
166      * @param feed
167      * @return
168      * @throws MalformedURLException
169      */
170     private SyndFeedInfo buildSyndFeedInfo(URL feedUrl, String urlStr, HttpMethod method, SyndFeed feed, int statusCode) throws MalformedURLException {
171         SyndFeedInfo syndFeedInfo;
172         syndFeedInfo = new SyndFeedInfo();
173         
174         // this may be different to feedURL because of 3XX redirects
175         syndFeedInfo.setUrl(new URL(urlStr));
176         syndFeedInfo.setId(feedUrl.toString());                					
177                 
178         Header imHeader = method.getResponseHeader("IM");
179         if (imHeader != null && imHeader.getValue().indexOf("feed") >= 0 && isUsingDeltaEncoding() && feedInfoCache != null && statusCode == 226) {
180             // client is setup to use http delta encoding and the server supports it and has returned a delta encoded response
181             // This response only includes new items
182             SyndFeedInfo cachedInfo = feedInfoCache.getFeedInfo(feedUrl);
183 		    if (cachedInfo != null) {
184 			    SyndFeed cachedFeed = cachedInfo.getSyndFeed();
185 			    
186 			    // set the new feed to be the orginal feed plus the new items
187 			    feed = combineFeeds(cachedFeed, feed);			        
188 		    }            
189         }
190         
191         Header lastModifiedHeader = method.getResponseHeader("Last-Modified");
192         if (lastModifiedHeader != null) {
193             syndFeedInfo.setLastModified(lastModifiedHeader.getValue());
194         }
195         
196         Header eTagHeader = method.getResponseHeader("ETag");
197         if (eTagHeader != null) {
198             syndFeedInfo.setETag(eTagHeader.getValue());
199         }
200         
201         syndFeedInfo.setSyndFeed(feed);
202         
203         return syndFeedInfo;
204     }
205 
206     /***
207 	 * @param client
208 	 * @param urlStr
209 	 * @param method
210 	 * @return
211 	 * @throws IOException
212 	 * @throws HttpException
213 	 * @throws FetcherException
214 	 * @throws FeedException
215 	 */
216 	private SyndFeed retrieveFeed(SyndFeedInfo syndFeedInfo, HttpClient client, String urlStr, HttpMethod method, int statusCode) throws IOException, HttpException, FetcherException, FeedException {
217 
218 		if (statusCode == HttpURLConnection.HTTP_NOT_MODIFIED && syndFeedInfo != null) {
219 		    fireEvent(FetcherEvent.EVENT_TYPE_FEED_UNCHANGED, urlStr);
220 		    return syndFeedInfo.getSyndFeed();
221 		}
222 		
223 		fireEvent(FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, urlStr);			
224 		
225 		InputStream stream = null;
226 		if ((method.getResponseHeader("Content-Encoding") != null) && ("gzip".equalsIgnoreCase(method.getResponseHeader("Content-Encoding").getValue()))) {		
227 		    stream = new GZIPInputStream(method.getResponseBodyAsStream());
228 		} else {
229 		    stream = method.getResponseBodyAsStream();
230 		}		
231 		try {				
232 			InputStreamReader reader = new InputStreamReader(stream);
233 			SyndFeedInput input = new SyndFeedInput();
234 			SyndFeed feed = input.build(reader);
235 							
236 			return feed;
237 			
238 		} finally {
239 		    if (stream != null) {
240 		        stream.close();
241 		    }
242 			
243 		}
244 	}
245 	
246     public interface CredentialSupplier {
247         public Credentials getCredentials(String realm, String host);
248     }
249 	
250 
251 }