From 9421f9ef62f5569e069f8b651eaeb6e5fa2eb89c Mon Sep 17 00:00:00 2001 From: chuanghiduoc Date: Fri, 23 Oct 2026 16:55:51 +0700 Subject: [PATCH] fix(csv-parse): wait for full CRLF before auto-detecting record delimiter --- packages/csv-parse/lib/api/index.js | 18 +++++++------- packages/csv-parse/test/option.encoding.ts | 28 ++++++++++++++++++++++ 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/packages/csv-parse/lib/api/index.js b/packages/csv-parse/lib/api/index.js index ca97838cd..17d59ffe8 100644 --- a/packages/csv-parse/lib/api/index.js +++ b/packages/csv-parse/lib/api/index.js @@ -48,21 +48,21 @@ const transform = function (original_options = {}) { const { quoting, needMoreDataSize, recordDelimiterMaxLength } = this.state; const numOfCharLeft = bufLen - i - 1; + // When the record delimiter has not been discovered yet, auto-detection + // needs enough bytes to distinguish "\r\n" from "\r" in the current + // encoding. This also applies to the delimiter following a closing quote. + const recordDelimiterLookahead = + recordDelimiterMaxLength === 0 + ? Buffer.from("\r\n", encoding).length + : recordDelimiterMaxLength; const requiredLength = Math.max( needMoreDataSize, // Skip if the remaining buffer smaller than record delimiter - // If "record_delimiter" is yet to be discovered: - // 1. It is equals to `[]` and "recordDelimiterMaxLength" equals `0` - // 2. We set the length to windows line ending in the current encoding - // Note, that encoding is known from user or bom discovery at that point - // recordDelimiterMaxLength, - recordDelimiterMaxLength === 0 - ? Buffer.from("\r\n", encoding).length - : recordDelimiterMaxLength, + recordDelimiterLookahead, // Skip if remaining buffer can be an escaped quote quoting ? (escape === null ? 0 : escape.length) + quote.length : 0, // Skip if remaining buffer can be record delimiter following the closing quote - quoting ? quote.length + recordDelimiterMaxLength : 0, + quoting ? quote.length + recordDelimiterLookahead : 0, ); return numOfCharLeft < requiredLength; }, diff --git a/packages/csv-parse/test/option.encoding.ts b/packages/csv-parse/test/option.encoding.ts index 1d3c53491..55099c39b 100644 --- a/packages/csv-parse/test/option.encoding.ts +++ b/packages/csv-parse/test/option.encoding.ts @@ -63,6 +63,34 @@ describe("Option `encoding`", function () { }); }); + describe("auto record delimiter", function () { + it("utf16le with a quoted record split across writes", function (next) { + const input = Buffer.from('"b"\r\n', "utf16le"); + const parser = parse({ encoding: "utf16le" }, (err, records) => { + if (err) return next(err); + records.should.eql([["b"]]); + next(); + }); + for (let i = 0; i < input.length; i++) { + parser.write(input.subarray(i, i + 1)); + } + parser.end(); + }); + + it("utf16le with multiple quoted records split across writes", function (next) { + const input = Buffer.from('"b"\r\n"c"\r\n', "utf16le"); + const parser = parse({ encoding: "utf16le" }, (err, records) => { + if (err) return next(err); + records.should.eql([["b"], ["c"]]); + next(); + }); + for (let i = 0; i < input.length; i++) { + parser.write(input.subarray(i, i + 1)); + } + parser.end(); + }); + }); + describe("with BOM", function () { it("utf16le auto detected", function (next) { const parser = parse(