1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package com.sun.syndication.fetcher.impl;
19
20 import java.io.IOException;
21 import java.io.InputStream;
22 import java.net.HttpURLConnection;
23 import java.net.MalformedURLException;
24 import java.net.URL;
25 import java.util.zip.GZIPInputStream;
26
27 import org.apache.commons.httpclient.Credentials;
28 import org.apache.commons.httpclient.Header;
29 import org.apache.commons.httpclient.HttpClient;
30 import org.apache.commons.httpclient.HttpException;
31 import org.apache.commons.httpclient.HttpMethod;
32 import org.apache.commons.httpclient.methods.GetMethod;
33
34 import com.sun.syndication.feed.synd.SyndFeed;
35 import com.sun.syndication.fetcher.FetcherEvent;
36 import com.sun.syndication.fetcher.FetcherException;
37 import com.sun.syndication.io.FeedException;
38 import com.sun.syndication.io.SyndFeedInput;
39 import com.sun.syndication.io.XmlReader;
40
41 /***
42 * @author Nick Lothian
43 */
44 public class HttpClientFeedFetcher extends AbstractFeedFetcher {
45
46 private FeedFetcherCache feedInfoCache;
47 private CredentialSupplier credentialSupplier;
48
49 public HttpClientFeedFetcher() {
50 super();
51 }
52
53 /***
54 * @param cache
55 */
56 public HttpClientFeedFetcher(FeedFetcherCache cache) {
57 this();
58 this.feedInfoCache = cache;
59 }
60
61
62 public HttpClientFeedFetcher(FeedFetcherCache cache, CredentialSupplier credentialSupplier) {
63 this(cache);
64 this.credentialSupplier = credentialSupplier;
65 }
66
67 /***
68 * @return Returns the credentialSupplier.
69 */
70 public CredentialSupplier getCredentialSupplier() {
71 return credentialSupplier;
72 }
73 /***
74 * @param credentialSupplier The credentialSupplier to set.
75 */
76 public void setCredentialSupplier(CredentialSupplier credentialSupplier) {
77 this.credentialSupplier = credentialSupplier;
78 }
79
80 /***
81 * @see com.sun.syndication.fetcher.FeedFetcher#retrieveFeed(java.net.URL)
82 */
83 public SyndFeed retrieveFeed(URL feedUrl) throws IllegalArgumentException, IOException, FeedException, FetcherException {
84 if (feedUrl == null) {
85 throw new IllegalArgumentException("null is not a valid URL");
86 }
87
88
89 HttpClient client = new HttpClient();
90
91 if (getCredentialSupplier() != null) {
92 client.getState().setAuthenticationPreemptive(true);
93
94 Credentials credentials = getCredentialSupplier().getCredentials(null, feedUrl.getHost());
95 if (credentials != null) {
96 client.getState().setCredentials(null, feedUrl.getHost(), credentials);
97 }
98 }
99
100
101 System.setProperty("httpclient.useragent", getUserAgent());
102 String urlStr = feedUrl.toString();
103 if (feedInfoCache != null) {
104
105
106 SyndFeedInfo syndFeedInfo = feedInfoCache.getFeedInfo(feedUrl);
107
108
109 HttpMethod method = new GetMethod(urlStr);
110 method.addRequestHeader("Accept-Encoding", "gzip");
111 try {
112 if (isUsingDeltaEncoding()) {
113 method.setRequestHeader("A-IM", "feed");
114 }
115
116 if (syndFeedInfo != null) {
117 method.setRequestHeader("If-None-Match", syndFeedInfo.getETag());
118
119 if (syndFeedInfo.getLastModified() instanceof String) {
120 method.setRequestHeader("If-Modified-Since", (String)syndFeedInfo.getLastModified());
121 }
122 }
123
124 method.setFollowRedirects(true);
125
126 int statusCode = client.executeMethod(method);
127 fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);
128 handleErrorCodes(statusCode);
129
130 SyndFeed feed = retrieveFeed(syndFeedInfo, urlStr, method, statusCode);
131
132 syndFeedInfo = buildSyndFeedInfo(feedUrl, urlStr, method, feed, statusCode);
133
134 feedInfoCache.setFeedInfo(new URL(urlStr), syndFeedInfo);
135
136
137
138 feed = syndFeedInfo.getSyndFeed();
139
140 return feed;
141 } finally {
142 method.releaseConnection();
143 }
144
145 } else {
146
147 HttpMethod method = new GetMethod(urlStr);
148 try {
149 method.setFollowRedirects(true);
150
151 int statusCode = client.executeMethod(method);
152 fireEvent(FetcherEvent.EVENT_TYPE_FEED_POLLED, urlStr);
153 handleErrorCodes(statusCode);
154
155 return retrieveFeed(null, urlStr, method, statusCode);
156 } finally {
157 method.releaseConnection();
158 }
159 }
160 }
161
162
163 /***
164 * @param feedUrl
165 * @param urlStr
166 * @param method
167 * @param feed
168 * @return
169 * @throws MalformedURLException
170 */
171 private SyndFeedInfo buildSyndFeedInfo(URL feedUrl, String urlStr, HttpMethod method, SyndFeed feed, int statusCode) throws MalformedURLException {
172 SyndFeedInfo syndFeedInfo;
173 syndFeedInfo = new SyndFeedInfo();
174
175
176 syndFeedInfo.setUrl(new URL(urlStr));
177 syndFeedInfo.setId(feedUrl.toString());
178
179 Header imHeader = method.getResponseHeader("IM");
180 if (imHeader != null && imHeader.getValue().indexOf("feed") >= 0 && isUsingDeltaEncoding() && feedInfoCache != null && statusCode == 226) {
181
182
183 SyndFeedInfo cachedInfo = feedInfoCache.getFeedInfo(feedUrl);
184 if (cachedInfo != null) {
185 SyndFeed cachedFeed = cachedInfo.getSyndFeed();
186
187
188 feed = combineFeeds(cachedFeed, feed);
189 }
190 }
191
192 Header lastModifiedHeader = method.getResponseHeader("Last-Modified");
193 if (lastModifiedHeader != null) {
194 syndFeedInfo.setLastModified(lastModifiedHeader.getValue());
195 }
196
197 Header eTagHeader = method.getResponseHeader("ETag");
198 if (eTagHeader != null) {
199 syndFeedInfo.setETag(eTagHeader.getValue());
200 }
201
202 syndFeedInfo.setSyndFeed(feed);
203
204 return syndFeedInfo;
205 }
206
207 /***
208 * @param client
209 * @param urlStr
210 * @param method
211 * @return
212 * @throws IOException
213 * @throws HttpException
214 * @throws FetcherException
215 * @throws FeedException
216 */
217 private SyndFeed retrieveFeed(SyndFeedInfo syndFeedInfo, String urlStr, HttpMethod method, int statusCode) throws IOException, HttpException, FetcherException, FeedException {
218
219 if (statusCode == HttpURLConnection.HTTP_NOT_MODIFIED && syndFeedInfo != null) {
220 fireEvent(FetcherEvent.EVENT_TYPE_FEED_UNCHANGED, urlStr);
221 return syndFeedInfo.getSyndFeed();
222 }
223
224
225 InputStream stream = null;
226 if ((method.getResponseHeader("Content-Encoding") != null) && ("gzip".equalsIgnoreCase(method.getResponseHeader("Content-Encoding").getValue()))) {
227 stream = new GZIPInputStream(method.getResponseBodyAsStream());
228 } else {
229 stream = method.getResponseBodyAsStream();
230 }
231 try {
232 XmlReader reader = null;
233 if (method.getResponseHeader("Content-Type") != null) {
234 reader = new XmlReader(stream, method.getResponseHeader("Content-Type").getValue(), true);
235 } else {
236 reader = new XmlReader(stream, true);
237 }
238 SyndFeed feed = new SyndFeedInput().build(reader);
239
240 fireEvent(FetcherEvent.EVENT_TYPE_FEED_RETRIEVED, urlStr, feed);
241
242 return feed;
243
244 } finally {
245 if (stream != null) {
246 stream.close();
247 }
248
249 }
250 }
251
252 public interface CredentialSupplier {
253 public Credentials getCredentials(String realm, String host);
254 }
255
256
257 }