-
-
Notifications
You must be signed in to change notification settings - Fork 33
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Fix maxBuffer
bug with TextDecoder()
#105
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,34 +1,45 @@ | ||
export const getStreamContents = async (stream, {init, convertChunk, getSize, addChunk, finalize}, {maxBuffer = Number.POSITIVE_INFINITY} = {}) => { | ||
export const getStreamContents = async (stream, {init, convertChunk, getSize, addChunk, getFinalChunk, finalize}, {maxBuffer = Number.POSITIVE_INFINITY} = {}) => { | ||
if (!isAsyncIterable(stream)) { | ||
throw new Error('The first argument must be a Readable, a ReadableStream, or an async iterable.'); | ||
} | ||
|
||
let length = 0; | ||
let contents = init(); | ||
const textDecoder = new TextDecoder(); | ||
const state = init(); | ||
state.length = 0; | ||
|
||
try { | ||
for await (const chunk of stream) { | ||
const chunkType = getChunkType(chunk); | ||
const convertedChunk = convertChunk[chunkType](chunk, textDecoder); | ||
const chunkSize = getSize(convertedChunk); | ||
|
||
if (length + chunkSize > maxBuffer) { | ||
throw new MaxBufferError(); | ||
} | ||
|
||
const newLength = length + chunkSize; | ||
contents = addChunk(convertedChunk, contents, newLength, length); | ||
length = newLength; | ||
const convertedChunk = convertChunk[chunkType](chunk, state); | ||
appendChunk({convertedChunk, state, getSize, addChunk, maxBuffer}); | ||
} | ||
|
||
return finalize(contents, length, textDecoder); | ||
appendFinalChunk({state, convertChunk, getSize, addChunk, getFinalChunk, maxBuffer}); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The final chunk (
|
||
return finalize(state); | ||
} catch (error) { | ||
error.bufferedData = finalize(contents, length, textDecoder); | ||
error.bufferedData = finalize(state); | ||
throw error; | ||
} | ||
}; | ||
|
||
const appendFinalChunk = ({state, getSize, addChunk, getFinalChunk, maxBuffer}) => { | ||
const convertedChunk = getFinalChunk(state); | ||
if (convertedChunk !== undefined) { | ||
appendChunk({convertedChunk, state, getSize, addChunk, maxBuffer}); | ||
} | ||
}; | ||
|
||
const appendChunk = ({convertedChunk, state, getSize, addChunk, maxBuffer}) => { | ||
const chunkSize = getSize(convertedChunk); | ||
const newLength = state.length + chunkSize; | ||
|
||
if (newLength > maxBuffer) { | ||
throw new MaxBufferError(); | ||
} | ||
|
||
state.contents = addChunk(convertedChunk, state, newLength); | ||
state.length = newLength; | ||
}; | ||
|
||
const isAsyncIterable = stream => typeof stream === 'object' && stream !== null && typeof stream[Symbol.asyncIterator] === 'function'; | ||
|
||
const getChunkType = chunk => { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -112,6 +112,11 @@ test('get stream with truncated UTF-8 sequences', async t => { | |
t.is(result, `${multiByteString.slice(0, -1)}${INVALID_UTF8_MARKER}`); | ||
}); | ||
|
||
test('handles truncated UTF-8 sequences over maxBuffer', async t => { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This test succeeds with the current PR, and fails without it. |
||
const maxBuffer = multiByteString.length - 1; | ||
await t.throwsAsync(setupString(multiByteBuffer.slice(0, -1), {maxBuffer}), {instanceOf: MaxBufferError}); | ||
}); | ||
|
||
test('get stream with invalid UTF-8 sequences', async t => { | ||
const result = await setupString(multiByteBuffer.slice(1, 2)); | ||
t.is(result, INVALID_UTF8_MARKER); | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
getFinalChunk()
is only used bygetString()
. It is anoop
otherwise.