minreq/response.rs
1use crate::{connection::HttpStream, Error};
2use std::collections::HashMap;
3use std::io::{self, BufReader, Bytes, Read};
4use std::str;
5
6const BACKING_READ_BUFFER_LENGTH: usize = 16 * 1024;
7const MAX_CONTENT_LENGTH: usize = 16 * 1024;
8
9/// An HTTP response.
10///
11/// Returned by [`Request::send`](struct.Request.html#method.send).
12///
13/// # Example
14///
15/// ```no_run
16/// # fn main() -> Result<(), minreq::Error> {
17/// let response = minreq::get("http://example.com").send()?;
18/// println!("{}", response.as_str()?);
19/// # Ok(()) }
20/// ```
21#[derive(Clone, PartialEq, Eq, Debug)]
22pub struct Response {
23 /// The status code of the response, eg. 404.
24 pub status_code: i32,
25 /// The reason phrase of the response, eg. "Not Found".
26 pub reason_phrase: String,
27 /// The headers of the response. The header field names (the
28 /// keys) are all lowercase.
29 pub headers: HashMap<String, String>,
30 /// The URL of the resource returned in this response. May differ from the
31 /// request URL if it was redirected or typo corrections were applied (e.g.
32 /// <http://example.com?foo=bar> would be corrected to
33 /// <http://example.com/?foo=bar>).
34 pub url: String,
35
36 body: Vec<u8>,
37}
38
39impl Response {
40 pub(crate) fn create(mut parent: ResponseLazy, is_head: bool) -> Result<Response, Error> {
41 let mut body = Vec::new();
42 if !is_head && parent.status_code != 204 && parent.status_code != 304 {
43 for byte in &mut parent {
44 let (byte, length) = byte?;
45 body.reserve(length);
46 body.push(byte);
47 }
48 }
49
50 let ResponseLazy {
51 status_code,
52 reason_phrase,
53 headers,
54 url,
55 ..
56 } = parent;
57
58 Ok(Response {
59 status_code,
60 reason_phrase,
61 headers,
62 url,
63 body,
64 })
65 }
66
67 /// Returns the body as an `&str`.
68 ///
69 /// # Errors
70 ///
71 /// Returns
72 /// [`InvalidUtf8InBody`](enum.Error.html#variant.InvalidUtf8InBody)
73 /// if the body is not UTF-8, with a description as to why the
74 /// provided slice is not UTF-8.
75 ///
76 /// # Example
77 ///
78 /// ```no_run
79 /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
80 /// # let url = "http://example.org/";
81 /// let response = minreq::get(url).send()?;
82 /// println!("{}", response.as_str()?);
83 /// # Ok(())
84 /// # }
85 /// ```
86 pub fn as_str(&self) -> Result<&str, Error> {
87 match str::from_utf8(&self.body) {
88 Ok(s) => Ok(s),
89 Err(err) => Err(Error::InvalidUtf8InBody(err)),
90 }
91 }
92
93 /// Returns a reference to the contained bytes of the body. If you
94 /// want the `Vec<u8>` itself, use
95 /// [`into_bytes()`](#method.into_bytes) instead.
96 ///
97 /// # Example
98 ///
99 /// ```no_run
100 /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
101 /// # let url = "http://example.org/";
102 /// let response = minreq::get(url).send()?;
103 /// println!("{:?}", response.as_bytes());
104 /// # Ok(())
105 /// # }
106 /// ```
107 pub fn as_bytes(&self) -> &[u8] {
108 &self.body
109 }
110
111 /// Turns the `Response` into the inner `Vec<u8>`, the bytes that
112 /// make up the response's body. If you just need a `&[u8]`, use
113 /// [`as_bytes()`](#method.as_bytes) instead.
114 ///
115 /// # Example
116 ///
117 /// ```no_run
118 /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
119 /// # let url = "http://example.org/";
120 /// let response = minreq::get(url).send()?;
121 /// println!("{:?}", response.into_bytes());
122 /// // This would error, as into_bytes consumes the Response:
123 /// // let x = response.status_code;
124 /// # Ok(())
125 /// # }
126 /// ```
127 pub fn into_bytes(self) -> Vec<u8> {
128 self.body
129 }
130
131 /// Converts JSON body to a `struct` using Serde.
132 ///
133 /// # Errors
134 ///
135 /// Returns
136 /// [`SerdeJsonError`](enum.Error.html#variant.SerdeJsonError) if
137 /// Serde runs into a problem, or
138 /// [`InvalidUtf8InBody`](enum.Error.html#variant.InvalidUtf8InBody)
139 /// if the body is not UTF-8.
140 ///
141 /// # Example
142 /// In case compiler cannot figure out return type you might need to declare it explicitly:
143 ///
144 /// ```no_run
145 /// use serde_json::Value;
146 ///
147 /// # fn main() -> Result<(), minreq::Error> {
148 /// # let url_to_json_resource = "http://example.org/resource.json";
149 /// // Value could be any type that implements Deserialize!
150 /// let user = minreq::get(url_to_json_resource).send()?.json::<Value>()?;
151 /// println!("User name is '{}'", user["name"]);
152 /// # Ok(())
153 /// # }
154 /// ```
155 #[cfg(feature = "json-using-serde")]
156 pub fn json<'a, T>(&'a self) -> Result<T, Error>
157 where
158 T: serde::de::Deserialize<'a>,
159 {
160 let str = match self.as_str() {
161 Ok(str) => str,
162 Err(_) => return Err(Error::InvalidUtf8InResponse),
163 };
164 match serde_json::from_str(str) {
165 Ok(json) => Ok(json),
166 Err(err) => Err(Error::SerdeJsonError(err)),
167 }
168 }
169}
170
171/// An HTTP response, which is loaded lazily.
172///
173/// In comparison to [`Response`](struct.Response.html), this is
174/// returned from
175/// [`send_lazy()`](struct.Request.html#method.send_lazy), where as
176/// [`Response`](struct.Response.html) is returned from
177/// [`send()`](struct.Request.html#method.send).
178///
179/// In practice, "lazy loading" means that the bytes are only loaded
180/// as you iterate through them. The bytes are provided in the form of
181/// a `Result<(u8, usize), minreq::Error>`, as the reading operation
182/// can fail in various ways. The `u8` is the actual byte that was
183/// read, and `usize` is how many bytes we are expecting to read in
184/// the future (including this byte). Note, however, that the `usize`
185/// can change, particularly when the `Transfer-Encoding` is
186/// `chunked`: then it will reflect how many bytes are left of the
187/// current chunk. The expected size is capped at 16 KiB to avoid
188/// server-side DoS attacks targeted at clients accidentally reserving
189/// too much memory.
190///
191/// # Example
192/// ```no_run
193/// // This is how the normal Response works behind the scenes, and
194/// // how you might use ResponseLazy.
195/// # fn main() -> Result<(), minreq::Error> {
196/// let response = minreq::get("http://example.com").send_lazy()?;
197/// let mut vec = Vec::new();
198/// for result in response {
199/// let (byte, length) = result?;
200/// vec.reserve(length);
201/// vec.push(byte);
202/// }
203/// # Ok(())
204/// # }
205///
206/// ```
207pub struct ResponseLazy {
208 /// The status code of the response, eg. 404.
209 pub status_code: i32,
210 /// The reason phrase of the response, eg. "Not Found".
211 pub reason_phrase: String,
212 /// The headers of the response. The header field names (the
213 /// keys) are all lowercase.
214 pub headers: HashMap<String, String>,
215 /// The URL of the resource returned in this response. May differ from the
216 /// request URL if it was redirected or typo corrections were applied (e.g.
217 /// <http://example.com?foo=bar> would be corrected to
218 /// <http://example.com/?foo=bar>).
219 pub url: String,
220
221 stream: HttpStreamBytes,
222 state: HttpStreamState,
223 max_trailing_headers_size: Option<usize>,
224}
225
226type HttpStreamBytes = Bytes<BufReader<HttpStream>>;
227
228impl ResponseLazy {
229 pub(crate) fn from_stream(
230 stream: HttpStream,
231 max_headers_size: Option<usize>,
232 max_status_line_len: Option<usize>,
233 ) -> Result<ResponseLazy, Error> {
234 let mut stream = BufReader::with_capacity(BACKING_READ_BUFFER_LENGTH, stream).bytes();
235 let ResponseMetadata {
236 status_code,
237 reason_phrase,
238 headers,
239 state,
240 max_trailing_headers_size,
241 } = read_metadata(&mut stream, max_headers_size, max_status_line_len)?;
242
243 Ok(ResponseLazy {
244 status_code,
245 reason_phrase,
246 headers,
247 url: String::new(),
248 stream,
249 state,
250 max_trailing_headers_size,
251 })
252 }
253}
254
255impl Iterator for ResponseLazy {
256 type Item = Result<(u8, usize), Error>;
257
258 fn next(&mut self) -> Option<Self::Item> {
259 use HttpStreamState::*;
260 match self.state {
261 EndOnClose => read_until_closed(&mut self.stream),
262 ContentLength(ref mut length) => read_with_content_length(&mut self.stream, length),
263 Chunked(ref mut expecting_chunks, ref mut length, ref mut content_length) => {
264 read_chunked(
265 &mut self.stream,
266 &mut self.headers,
267 expecting_chunks,
268 length,
269 content_length,
270 self.max_trailing_headers_size,
271 )
272 }
273 }
274 }
275}
276
277impl Read for ResponseLazy {
278 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
279 let mut index = 0;
280 for res in self {
281 // there is no use for the estimated length in the read implementation
282 // so it is ignored.
283 let (byte, _) = res.map_err(|e| match e {
284 Error::IoError(e) => e,
285 _ => io::Error::new(io::ErrorKind::Other, e),
286 })?;
287
288 buf[index] = byte;
289 index += 1;
290
291 // if the buffer is full, it should stop reading
292 if index >= buf.len() {
293 break;
294 }
295 }
296
297 // index of the next byte is the number of bytes thats have been read
298 Ok(index)
299 }
300}
301
302fn read_until_closed(bytes: &mut HttpStreamBytes) -> Option<<ResponseLazy as Iterator>::Item> {
303 if let Some(byte) = bytes.next() {
304 match byte {
305 Ok(byte) => Some(Ok((byte, 1))),
306 Err(err) => Some(Err(Error::IoError(err))),
307 }
308 } else {
309 None
310 }
311}
312
313fn read_with_content_length(
314 bytes: &mut HttpStreamBytes,
315 content_length: &mut usize,
316) -> Option<<ResponseLazy as Iterator>::Item> {
317 if *content_length > 0 {
318 *content_length -= 1;
319
320 if let Some(byte) = bytes.next() {
321 match byte {
322 // Cap Content-Length to 16KiB, to avoid out-of-memory issues.
323 Ok(byte) => return Some(Ok((byte, (*content_length).min(MAX_CONTENT_LENGTH) + 1))),
324 Err(err) => return Some(Err(Error::IoError(err))),
325 }
326 }
327 }
328 None
329}
330
331fn read_trailers(
332 bytes: &mut HttpStreamBytes,
333 headers: &mut HashMap<String, String>,
334 mut max_headers_size: Option<usize>,
335) -> Result<(), Error> {
336 loop {
337 let trailer_line = read_line(bytes, max_headers_size, Error::HeadersOverflow)?;
338 if let Some(ref mut max_headers_size) = max_headers_size {
339 *max_headers_size -= trailer_line.len() + 2;
340 }
341 if let Some((header, value)) = parse_header(trailer_line) {
342 headers.insert(header, value);
343 } else {
344 break;
345 }
346 }
347 Ok(())
348}
349
350fn read_chunked(
351 bytes: &mut HttpStreamBytes,
352 headers: &mut HashMap<String, String>,
353 expecting_more_chunks: &mut bool,
354 chunk_length: &mut usize,
355 content_length: &mut usize,
356 max_trailing_headers_size: Option<usize>,
357) -> Option<<ResponseLazy as Iterator>::Item> {
358 if !*expecting_more_chunks && *chunk_length == 0 {
359 return None;
360 }
361
362 if *chunk_length == 0 {
363 // Max length of the chunk length line is 1KB: not too long to
364 // take up much memory, long enough to tolerate some chunk
365 // extensions (which are ignored).
366
367 // Get the size of the next chunk
368 let length_line = match read_line(bytes, Some(1024), Error::MalformedChunkLength) {
369 Ok(line) => line,
370 Err(err) => return Some(Err(err)),
371 };
372
373 // Note: the trim() and check for empty lines shouldn't be
374 // needed according to the RFC, but we might as well, it's a
375 // small change and it fixes a few servers.
376 let incoming_length = if length_line.is_empty() {
377 0
378 } else {
379 let length = if let Some(i) = length_line.find(';') {
380 length_line[..i].trim()
381 } else {
382 length_line.trim()
383 };
384 match usize::from_str_radix(length, 16) {
385 Ok(length) => length,
386 Err(_) => return Some(Err(Error::MalformedChunkLength)),
387 }
388 };
389
390 if incoming_length == 0 {
391 if let Err(err) = read_trailers(bytes, headers, max_trailing_headers_size) {
392 return Some(Err(err));
393 }
394
395 *expecting_more_chunks = false;
396 headers.insert("content-length".to_string(), (*content_length).to_string());
397 headers.remove("transfer-encoding");
398 return None;
399 }
400 *chunk_length = incoming_length;
401 *content_length += incoming_length;
402 }
403
404 if *chunk_length > 0 {
405 *chunk_length -= 1;
406 if let Some(byte) = bytes.next() {
407 match byte {
408 Ok(byte) => {
409 // If we're at the end of the chunk...
410 if *chunk_length == 0 {
411 //...read the trailing \r\n of the chunk, and
412 // possibly return an error instead.
413
414 // TODO: Maybe this could be written in a way
415 // that doesn't discard the last ok byte if
416 // the \r\n reading fails?
417 if let Err(err) = read_line(bytes, Some(2), Error::MalformedChunkEnd) {
418 return Some(Err(err));
419 }
420 }
421
422 return Some(Ok((byte, (*chunk_length).min(MAX_CONTENT_LENGTH) + 1)));
423 }
424 Err(err) => return Some(Err(Error::IoError(err))),
425 }
426 }
427 }
428
429 None
430}
431
432enum HttpStreamState {
433 // No Content-Length, and Transfer-Encoding != chunked, so we just
434 // read unti lthe server closes the connection (this should be the
435 // fallback, if I read the rfc right).
436 EndOnClose,
437 // Content-Length was specified, read that amount of bytes
438 ContentLength(usize),
439 // Transfer-Encoding == chunked, so we need to save two pieces of
440 // information: are we expecting more chunks, how much is there
441 // left of the current chunk, and how much have we read? The last
442 // number is needed in order to provide an accurate Content-Length
443 // header after loading all the bytes.
444 Chunked(bool, usize, usize),
445}
446
447// This struct is just used in the Response and ResponseLazy
448// constructors, but not in their structs, for api-cleanliness
449// reasons. (Eg. response.status_code is much cleaner than
450// response.meta.status_code or similar.)
451struct ResponseMetadata {
452 status_code: i32,
453 reason_phrase: String,
454 headers: HashMap<String, String>,
455 state: HttpStreamState,
456 max_trailing_headers_size: Option<usize>,
457}
458
459fn read_metadata(
460 stream: &mut HttpStreamBytes,
461 mut max_headers_size: Option<usize>,
462 max_status_line_len: Option<usize>,
463) -> Result<ResponseMetadata, Error> {
464 let line = read_line(stream, max_status_line_len, Error::StatusLineOverflow)?;
465 let (status_code, reason_phrase) = parse_status_line(&line);
466
467 let mut headers = HashMap::new();
468 loop {
469 let line = read_line(stream, max_headers_size, Error::HeadersOverflow)?;
470 if line.is_empty() {
471 // Body starts here
472 break;
473 }
474 if let Some(ref mut max_headers_size) = max_headers_size {
475 *max_headers_size -= line.len() + 2;
476 }
477 if let Some(header) = parse_header(line) {
478 headers.insert(header.0, header.1);
479 }
480 }
481
482 let mut chunked = false;
483 let mut content_length = None;
484 for (header, value) in &headers {
485 // Handle the Transfer-Encoding header
486 if header.to_lowercase().trim() == "transfer-encoding"
487 && value.to_lowercase().trim() == "chunked"
488 {
489 chunked = true;
490 }
491
492 // Handle the Content-Length header
493 if header.to_lowercase().trim() == "content-length" {
494 match str::parse::<usize>(value.trim()) {
495 Ok(length) => content_length = Some(length),
496 Err(_) => return Err(Error::MalformedContentLength),
497 }
498 }
499 }
500
501 let state = if chunked {
502 HttpStreamState::Chunked(true, 0, 0)
503 } else if let Some(length) = content_length {
504 HttpStreamState::ContentLength(length)
505 } else {
506 HttpStreamState::EndOnClose
507 };
508
509 Ok(ResponseMetadata {
510 status_code,
511 reason_phrase,
512 headers,
513 state,
514 max_trailing_headers_size: max_headers_size,
515 })
516}
517
518fn read_line(
519 stream: &mut HttpStreamBytes,
520 max_len: Option<usize>,
521 overflow_error: Error,
522) -> Result<String, Error> {
523 let mut bytes = Vec::with_capacity(32);
524 for byte in stream {
525 match byte {
526 Ok(byte) => {
527 if let Some(max_len) = max_len {
528 if bytes.len() >= max_len {
529 return Err(overflow_error);
530 }
531 }
532 if byte == b'\n' {
533 if let Some(b'\r') = bytes.last() {
534 bytes.pop();
535 }
536 break;
537 } else {
538 bytes.push(byte);
539 }
540 }
541 Err(err) => return Err(Error::IoError(err)),
542 }
543 }
544 String::from_utf8(bytes).map_err(|_error| Error::InvalidUtf8InResponse)
545}
546
547fn parse_status_line(line: &str) -> (i32, String) {
548 // sample status line format
549 // HTTP/1.1 200 OK
550 let mut status_code = String::with_capacity(3);
551 let mut reason_phrase = String::with_capacity(2);
552
553 let mut spaces = 0;
554
555 for c in line.chars() {
556 if spaces >= 2 {
557 reason_phrase.push(c);
558 }
559
560 if c == ' ' {
561 spaces += 1;
562 } else if spaces == 1 {
563 status_code.push(c);
564 }
565 }
566
567 if let Ok(status_code) = status_code.parse::<i32>() {
568 return (status_code, reason_phrase);
569 }
570
571 (503, "Server did not provide a status line".to_string())
572}
573
574fn parse_header(mut line: String) -> Option<(String, String)> {
575 if let Some(location) = line.find(':') {
576 // Trim the first character of the header if it is a space,
577 // otherwise return everything after the ':'. This should
578 // preserve the behavior in versions <=2.0.1 in most cases
579 // (namely, ones where it was valid), where the first
580 // character after ':' was always cut off.
581 let value = if let Some(sp) = line.get(location + 1..location + 2) {
582 if sp == " " {
583 line[location + 2..].to_string()
584 } else {
585 line[location + 1..].to_string()
586 }
587 } else {
588 line[location + 1..].to_string()
589 };
590
591 line.truncate(location);
592 // Headers should be ascii, I'm pretty sure. If not, please open an issue.
593 line.make_ascii_lowercase();
594 return Some((line, value));
595 }
596 None
597}