1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
use std::io::{self, BufRead};
use std::error::Error;
use std::fmt;
use std::str;
use super::*;
/// Wraps a `std::io::BufRead` buffered byte stream and decode it as UTF-8.
pub struct BufReadDecoder<B: BufRead> {
buf_read: B,
bytes_consumed: usize,
incomplete: Incomplete,
}
#[derive(Debug)]
pub enum BufReadDecoderError<'a> {
/// Represents one UTF-8 error in the byte stream.
///
/// In lossy decoding, each such error should be replaced with U+FFFD.
/// (See `BufReadDecoder::next_lossy` and `BufReadDecoderError::lossy`.)
InvalidByteSequence(&'a [u8]),
/// An I/O error from the underlying byte stream
Io(io::Error),
}
impl<'a> BufReadDecoderError<'a> {
/// Replace UTF-8 errors with U+FFFD
pub fn lossy(self) -> Result<&'static str, io::Error> {
match self {
BufReadDecoderError::Io(error) => Err(error),
BufReadDecoderError::InvalidByteSequence(_) => Ok(REPLACEMENT_CHARACTER),
}
}
}
impl<'a> fmt::Display for BufReadDecoderError<'a> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
BufReadDecoderError::InvalidByteSequence(bytes) => {
write!(f, "invalid byte sequence: {:02x?}", bytes)
}
BufReadDecoderError::Io(ref err) => write!(f, "underlying bytestream error: {}", err),
}
}
}
impl<'a> Error for BufReadDecoderError<'a> {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match *self {
BufReadDecoderError::InvalidByteSequence(_) => None,
BufReadDecoderError::Io(ref err) => Some(err),
}
}
}
impl<B: BufRead> BufReadDecoder<B> {
/// This is to `Read::read_to_string` what `String::from_utf8_lossy` is to `String::from_utf8`.
pub fn read_to_string_lossy(buf_read: B) -> io::Result<String> {
let mut decoder = Self::new(buf_read);
let mut string = String::new();
while let Some(result) = decoder.next_lossy() {
string.push_str(result?)
}
Ok(string)
}
pub fn new(buf_read: B) -> Self {
Self {
buf_read,
bytes_consumed: 0,
incomplete: Incomplete::empty(),
}
}
/// Same as `BufReadDecoder::next_strict`, but replace UTF-8 errors with U+FFFD.
pub fn next_lossy(&mut self) -> Option<io::Result<&str>> {
self.next_strict().map(|result| result.or_else(|e| e.lossy()))
}
/// Decode and consume the next chunk of UTF-8 input.
///
/// This method is intended to be called repeatedly until it returns `None`,
/// which represents EOF from the underlying byte stream.
/// This is similar to `Iterator::next`,
/// except that decoded chunks borrow the decoder (~iterator)
/// so they need to be handled or copied before the next chunk can start decoding.
pub fn next_strict(&mut self) -> Option<Result<&str, BufReadDecoderError>> {
enum BytesSource {
BufRead(usize),
Incomplete,
}
macro_rules! try_io {
($io_result: expr) => {
match $io_result {
Ok(value) => value,
Err(error) => return Some(Err(BufReadDecoderError::Io(error)))
}
}
}
let (source, result) = loop {
if self.bytes_consumed > 0 {
self.buf_read.consume(self.bytes_consumed);
self.bytes_consumed = 0;
}
let buf = try_io!(self.buf_read.fill_buf());
// Force loop iteration to go through an explicit `continue`
enum Unreachable {}
let _: Unreachable = if self.incomplete.is_empty() {
if buf.is_empty() {
return None // EOF
}
match str::from_utf8(buf) {
Ok(_) => {
break (BytesSource::BufRead(buf.len()), Ok(()))
}
Err(error) => {
let valid_up_to = error.valid_up_to();
if valid_up_to > 0 {
break (BytesSource::BufRead(valid_up_to), Ok(()))
}
match error.error_len() {
Some(invalid_sequence_length) => {
break (BytesSource::BufRead(invalid_sequence_length), Err(()))
}
None => {
self.bytes_consumed = buf.len();
self.incomplete = Incomplete::new(buf);
// need more input bytes
continue
}
}
}
}
} else {
if buf.is_empty() {
break (BytesSource::Incomplete, Err(())) // EOF with incomplete code point
}
let (consumed, opt_result) = self.incomplete.try_complete_offsets(buf);
self.bytes_consumed = consumed;
match opt_result {
None => {
// need more input bytes
continue
}
Some(result) => {
break (BytesSource::Incomplete, result)
}
}
};
};
let bytes = match source {
BytesSource::BufRead(byte_count) => {
self.bytes_consumed = byte_count;
let buf = try_io!(self.buf_read.fill_buf());
&buf[..byte_count]
}
BytesSource::Incomplete => {
self.incomplete.take_buffer()
}
};
match result {
Ok(()) => Some(Ok(unsafe { str::from_utf8_unchecked(bytes) })),
Err(()) => Some(Err(BufReadDecoderError::InvalidByteSequence(bytes))),
}
}
}