Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions Lib/test/test_csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -857,7 +857,6 @@ class TestEscapedExcel(TestCsvBase):
def test_escape_fieldsep(self):
self.writerAssertEqual([['abc,def']], 'abc\\,def\r\n')

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_read_escape_fieldsep(self):
self.readerAssertEqual('abc\\,def\r\n', [['abc,def']])

Expand All @@ -881,7 +880,6 @@ class TestQuotedEscapedExcel(TestCsvBase):
def test_write_escape_fieldsep(self):
self.writerAssertEqual([['abc,def']], '"abc,def"\r\n')

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_read_escape_fieldsep(self):
self.readerAssertEqual('"abc\\,def"\r\n', [['abc,def']])

Expand Down
121 changes: 85 additions & 36 deletions crates/stdlib/src/csv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -797,45 +797,27 @@ mod _csv {
}

fn to_reader(&self) -> csv_core::Reader {
let mut builder = csv_core::ReaderBuilder::new();
let mut reader = match &self.dialect {
DialectItem::Str(name) => {
let dialect = match &self.dialect {
DialectItem::Str(name) => GLOBAL_HASHMAP.lock().get(name).copied(),
DialectItem::Obj(obj) => Some(*obj),
DialectItem::None => {
let g = GLOBAL_HASHMAP.lock();
if let Some(dialect) = g.get(name) {
let mut builder = builder
.delimiter(dialect.delimiter)
.double_quote(dialect.doublequote);
if let Some(t) = dialect.quotechar {
builder = builder.quote(t);
}
builder
// RustPython todo
// todo! Perfecting the remaining attributes.
} else {
&mut builder
}
}
DialectItem::Obj(obj) => {
let mut builder = builder
.delimiter(obj.delimiter)
.double_quote(obj.doublequote);
if let Some(t) = obj.quotechar {
builder = builder.quote(t);
}
builder
Some(*g.get("excel").unwrap())
}
_ => {
let name = "excel";
let g = GLOBAL_HASHMAP.lock();
let dialect = g.get(name).unwrap();
let mut builder = builder
.delimiter(dialect.delimiter)
.double_quote(dialect.doublequote);
if let Some(quotechar) = dialect.quotechar {
builder = builder.quote(quotechar);
}
builder
};

let mut builder = csv_core::ReaderBuilder::new();
let mut reader = if let Some(dialect) = dialect {
let mut builder = builder
.delimiter(dialect.delimiter)
.double_quote(dialect.doublequote)
.escape(dialect.escapechar);
if let Some(quotechar) = dialect.quotechar {
builder = builder.quote(quotechar);
}
builder
} else {
&mut builder
};

if let Some(t) = self.delimiter {
Expand Down Expand Up @@ -972,6 +954,67 @@ mod _csv {

impl SelfIter for Reader {}

fn read_quote_none_record(
input: &[u8],
dialect: PyDialect,
field_limit: isize,
vm: &VirtualMachine,
) -> PyResult<Vec<PyObjectRef>> {
let mut fields = vec![Vec::new()];
let mut escaped = false;
let mut after_delimiter = false;

for (index, &byte) in input.iter().enumerate() {
if escaped {
fields.last_mut().unwrap().push(byte);
escaped = false;
after_delimiter = false;
} else if dialect.skipinitialspace && after_delimiter && byte == b' ' {
continue;
} else if dialect.escapechar == Some(byte) {
escaped = true;
} else if byte == dialect.delimiter {
fields.push(Vec::new());
after_delimiter = true;
} else if matches!(byte, b'\r' | b'\n') {
if !input[index..]
.iter()
.all(|&byte| matches!(byte, b'\r' | b'\n'))
{
return Err(new_csv_error(
vm,
concat!(
"new-line character seen in unquoted field",
" - do you need to open the file in universal-newline mode?"
),
));
}
break;
} else {
fields.last_mut().unwrap().push(byte);
after_delimiter = false;
}
}

// CPython treats an escape character at the end of an iterator item
// as escaping the implicit newline at the end of that item.
if escaped {
fields.last_mut().unwrap().push(b'\n');
}

fields
.into_iter()
.map(|field| {
if field.len() > field_limit as usize {
return Err(new_csv_error(vm, "filed too long to read"));
}
let field = core::str::from_utf8(&field)
.map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
Ok(vm.ctx.new_str(field).into())
})
.collect()
}

Comment thread
coderabbitai[bot] marked this conversation as resolved.
impl IterNext for Reader {
fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> {
let string = raise_if_stop!(zelf.iter.next(vm)?);
Expand Down Expand Up @@ -1003,6 +1046,12 @@ mod _csv {
let mut output_ends_offset = 0;
let field_limit = GLOBAL_FIELD_LIMIT.lock().to_owned();

if zelf.dialect.quoting == QuoteStyle::None && zelf.dialect.escapechar.is_some() {
let out = read_quote_none_record(input, zelf.dialect, field_limit, vm)?;
*line_num += 1;
return Ok(PyIterReturn::Return(vm.ctx.new_list(out).into()));
}

#[inline]
fn trim_spaces(input: &[u8]) -> &[u8] {
let trimmed_start = input.iter().position(|&x| x != b' ').unwrap_or(input.len());
Expand Down
13 changes: 13 additions & 0 deletions extra_tests/snippets/stdlib_csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,3 +134,16 @@ def test_quote_none_writer_without_quotechar():


test_quote_none_writer_without_quotechar()


def test_quote_none_reader_skipinitialspace_escapechar():
reader = csv.reader(
["a, b,\\ c,d"],
quoting=csv.QUOTE_NONE,
escapechar="\\",
skipinitialspace=True,
)
assert list(reader) == [["a", "b", " c", "d"]]


test_quote_none_reader_skipinitialspace_escapechar()
Loading