1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package com.sun.syndication.fetcher.impl;
19
20 import java.io.IOException;
21 import java.io.InputStream;
22 import java.io.InputStreamReader;
23 import java.net.HttpURLConnection;
24 import java.net.MalformedURLException;
25 import java.net.URL;
26 import java.util.zip.GZIPInputStream;
27
28 import org.apache.commons.httpclient.Credentials;
29 import org.apache.commons.httpclient.Header;
30 import org.apache.commons.httpclient.HttpClient;
31 import org.apache.commons.httpclient.HttpException;
32 import org.apache.commons.httpclient.HttpMethod;
33 import org.apache.commons.httpclient.methods.GetMethod;
34
35 import com.sun.syndication.feed.synd.SyndFeed;
36 import com.sun.syndication.fetcher.FetcherEvent;
37 import com.sun.syndication.fetcher.FetcherException;
38 import com.sun.syndication.io.FeedException;
39 import com.sun.syndication.io.SyndFeedInput;
40
41 /***
42 * @author Nick Lothian
43 */
44 public class HttpClientFeedFetcher extends AbstractFeedFetcher {
45
46 private FeedFetcherCache feedInfoCache;
47 private CredentialSupplier credentialSupplier;
48
49 public HttpClientFeedFetcher() {
50 super();
51 }
52
53 /***
54 * @param cache
55 */
56 public HttpClientFeedFetcher(FeedFetcherCache cache) {
57 this();
58 this.feedInfoCache = cache;
59 }
60
61
62 public HttpClientFeedFetcher(FeedFetcherCache cache, CredentialSupplier credentialSupplier) {
63 this(cache);
64 this.credentialSupplier = credentialSupplier;
65 }
66
67 /***
68 * @return Returns the credentialSupplier.
69 */
70 public CredentialSupplier getCredentialSupplier() {
71 return credentialSupplier;
72 }
73 /***
74 * @param credentialSupplier The credentialSupplier to set.
75 */
76 public void setCredentialSupplier(CredentialSupplier credentialSupplier) {
77 this.credentialSupplier = credentialSupplier;
78 }
79
80 /***
81 * @see com.sun.syndication.fetcher.FeedFetcher#retrieveFeed(java.net.URL)
82 */
83 public SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
84 if (feedUrl == null) {
85 throw new IllegalArgumentException("null is not a valid URL");
86 }
87
88
89 HttpClient client = new HttpClient();
90
91 if (getCredentialSupplier() != null) {
92 client.getState().setAuthenticationPreemptive(true);
93
94 Credentials credentials = getCredentialSupplier().getCredentials(null, feedUrl.getHost());
95 if (credentials != null) {
96 client.getState().setCredentials(null, feedUrl.getHost(), credentials);
97 }
98 }
99
100
101 System.setProperty("httpclient.useragent", getUserAgent());
102 String urlStr = feedUrl.toString();
103 if (feedInfoCache != null) {
104
105
106 SyndFeedInfo syndFeedInfo = feedInfoCache.getFeedInfo(feedUrl);
107
108
109 HttpMethod method = new GetMethod(urlStr);
110 try {
111 if (isUsingDeltaEncoding()) {
112 method.setRequestHeader("A-IM", "feed");
113 }
114
115 if (syndFeedInfo != null) {
116 method.setRequestHeader("If-None-Match", syndFeedInfo.getETag());
117
118 if (syndFeedInfo.getLastModified() instanceof String) {
119 method.setRequestHeader("If-Modified-Since", (String)syndFeedInfo.getLastModified());
120 }
121 }
122
123 method.setFollowRedirects(true);
124
125 int statusCode = client.executeMethod(method);
126 fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);
127 handleErrorCodes(statusCode);
128
129 SyndFeed feed = retrieveFeed(syndFeedInfo, client, urlStr, method, statusCode);
130
131 syndFeedInfo = buildSyndFeedInfo(feedUrl, urlStr, method, feed, statusCode);
132
133 feedInfoCache.setFeedInfo(new URL(urlStr), syndFeedInfo);
134
135
136
137 feed = syndFeedInfo.getSyndFeed();
138
139 return feed;
140 } finally {
141 method.releaseConnection();
142 }
143
144 } else {
145
146 HttpMethod method = new GetMethod(urlStr);
147 try {
148 method.setFollowRedirects(true);
149
150 int statusCode = client.executeMethod(method);
151 fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);
152 handleErrorCodes(statusCode);
153
154 return retrieveFeed(null, client, urlStr, method, statusCode);
155 } finally {
156 method.releaseConnection();
157 }
158 }
159 }
160
161
162 /***
163 * @param feedUrl
164 * @param urlStr
165 * @param method
166 * @param feed
167 * @return
168 * @throws MalformedURLException
169 */
170 private SyndFeedInfo buildSyndFeedInfo(URL feedUrl, String urlStr, HttpMethod method, SyndFeed feed, int statusCode) throws MalformedURLException {
171 SyndFeedInfo syndFeedInfo;
172 syndFeedInfo = new SyndFeedInfo();
173
174
175 syndFeedInfo.setUrl(new URL(urlStr));
176 syndFeedInfo.setId(feedUrl.toString());
177
178 Header imHeader = method.getResponseHeader("IM");
179 if (imHeader != null && imHeader.getValue().indexOf("feed") >= 0 && isUsingDeltaEncoding() && feedInfoCache != null && statusCode == 226) {
180
181
182 SyndFeedInfo cachedInfo = feedInfoCache.getFeedInfo(feedUrl);
183 if (cachedInfo != null) {
184 SyndFeed cachedFeed = cachedInfo.getSyndFeed();
185
186
187 feed = combineFeeds(cachedFeed, feed);
188 }
189 }
190
191 Header lastModifiedHeader = method.getResponseHeader("Last-Modified");
192 if (lastModifiedHeader != null) {
193 syndFeedInfo.setLastModified(lastModifiedHeader.getValue());
194 }
195
196 Header eTagHeader = method.getResponseHeader("ETag");
197 if (eTagHeader != null) {
198 syndFeedInfo.setETag(eTagHeader.getValue());
199 }
200
201 syndFeedInfo.setSyndFeed(feed);
202
203 return syndFeedInfo;
204 }
205
206 /***
207 * @param client
208 * @param urlStr
209 * @param method
210 * @return
211 * @throws IOException
212 * @throws HttpException
213 * @throws FetcherException
214 * @throws FeedException
215 */
216 private SyndFeed retrieveFeed(SyndFeedInfo syndFeedInfo, HttpClient client, String urlStr, HttpMethod method, int statusCode) throws IOException, HttpException, FetcherException, FeedException {
217
218 if (statusCode == HttpURLConnection.HTTP_NOT_MODIFIED && syndFeedInfo != null) {
219 fireEvent(FetcherEvent.EVENT_TYPE_FEED_UNCHANGED, urlStr);
220 return syndFeedInfo.getSyndFeed();
221 }
222
223 fireEvent(FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, urlStr);
224
225 InputStream stream = null;
226 if ((method.getResponseHeader("Content-Encoding") != null) && ("gzip".equalsIgnoreCase(method.getResponseHeader("Content-Encoding").getValue()))) {
227 stream = new GZIPInputStream(method.getResponseBodyAsStream());
228 } else {
229 stream = method.getResponseBodyAsStream();
230 }
231 try {
232 InputStreamReader reader = new InputStreamReader(stream);
233 SyndFeedInput input = new SyndFeedInput();
234 SyndFeed feed = input.build(reader);
235
236 return feed;
237
238 } finally {
239 if (stream != null) {
240 stream.close();
241 }
242
243 }
244 }
245
246 public interface CredentialSupplier {
247 public Credentials getCredentials(String realm, String host);
248 }
249
250
251 }