Skip to content

Commit bccf566

Browse files
danmardmcppcheckclaude
authored
Fix #739 (Add a lookahead buffer to TokenList::Stream) (#740)
Co-authored-by: Daniel Marjamäki <daniel.marjamaki@cppcheck.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 178e71d commit bccf566

2 files changed

Lines changed: 185 additions & 58 deletions

File tree

‎simplecpp.cpp‎

Lines changed: 84 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -295,68 +295,61 @@ class simplecpp::TokenList::Stream {
295295
public:
296296
virtual ~Stream() = default;
297297

298-
virtual int get() = 0;
299-
virtual int peek() = 0;
300-
virtual void unget() = 0;
301-
virtual bool good() = 0;
302-
298+
/**
299+
* Read the next character. CR and CR LF are returned as LF.
300+
* At the end of the input, EOF converted to unsigned char is returned
301+
* and good() becomes false.
302+
*/
303303
unsigned char readChar() {
304-
auto ch = static_cast<unsigned char>(get());
305-
306-
// For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the
307-
// character is non-ASCII character then replace it with 0xff
308-
if (isUtf16) {
309-
const auto ch2 = static_cast<unsigned char>(get());
310-
const int ch16 = makeUtf16Char(ch, ch2);
311-
ch = static_cast<unsigned char>(((ch16 >= 0x80) ? 0xff : ch16));
304+
int ch = getUnit();
305+
if (ch == EOF) {
306+
atEnd = true;
307+
lastChar = EOF;
308+
return static_cast<unsigned char>(EOF);
312309
}
313310

314311
// Handling of newlines..
315312
if (ch == '\r') {
316313
ch = '\n';
317-
318-
int ch2 = get();
319-
if (isUtf16) {
320-
const int c2 = get();
321-
ch2 = makeUtf16Char(ch2, c2);
322-
}
323-
324-
if (ch2 != '\n')
325-
ungetChar();
314+
const int ch2 = getUnit();
315+
if (ch2 != '\n' && ch2 != EOF)
316+
buffer.push_back(static_cast<char>(ch2));
326317
}
327318

328-
return ch;
319+
lastChar = ch;
320+
return static_cast<unsigned char>(ch);
329321
}
330322

323+
/** The next character, without reading it. EOF at the end of the input. */
331324
int peekChar() {
332-
int ch = peek();
325+
const int ch = getUnit();
333326
if (ch == EOF)
334-
return ch;
335-
336-
// For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the
337-
// character is non-ASCII character then replace it with 0xff
338-
if (isUtf16) {
339-
(void)get();
340-
const auto ch2 = static_cast<unsigned char>(peek());
341-
unget();
342-
const int ch16 = makeUtf16Char(ch, ch2);
343-
ch = (ch16 >= 0x80) ? 0xff : ch16;
344-
}
345-
346-
// Handling of newlines..
347-
if (ch == '\r')
348-
ch = '\n';
349-
350-
return ch;
327+
return EOF;
328+
buffer.push_back(static_cast<char>(ch));
329+
return (ch == '\r') ? '\n' : ch;
351330
}
352331

332+
/** Put back the character that was last read with readChar() */
353333
void ungetChar() {
354-
unget();
355-
if (isUtf16)
356-
unget();
334+
if (lastChar != EOF)
335+
buffer.push_back(static_cast<char>(lastChar));
336+
lastChar = EOF;
337+
}
338+
339+
/**
340+
* false when readChar() has reached the end of the input.
341+
* Unlike std::istream::good(), peekChar() at the end of the input does not make it false.
342+
*/
343+
bool good() const {
344+
return !atEnd;
357345
}
358346

359347
protected:
348+
/** Raw input, used only by the Stream class. Call readChar(), peekChar() and ungetChar() instead. */
349+
virtual int get() = 0;
350+
virtual int peek() = 0;
351+
virtual void unget() = 0;
352+
360353
void init() {
361354
// initialize since we use peek() in getAndSkipBOM()
362355
isUtf16 = false;
@@ -365,6 +358,38 @@ class simplecpp::TokenList::Stream {
365358
}
366359

367360
private:
361+
/**
362+
* Read the next character from the input, or take it from the buffer.
363+
* Newlines are not converted.
364+
*/
365+
int getUnit() {
366+
if (!buffer.empty()) {
367+
const auto ch = static_cast<unsigned char>(buffer.back());
368+
buffer.pop_back();
369+
return ch;
370+
}
371+
if (inputEnd)
372+
return EOF;
373+
const int ch = get();
374+
if (ch == EOF) {
375+
inputEnd = true;
376+
return EOF;
377+
}
378+
if (!isUtf16)
379+
return ch;
380+
381+
// For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the
382+
// character is non-ASCII character then replace it with 0xff
383+
const int ch2 = get();
384+
if (ch2 == EOF) {
385+
// a dangling last byte is dropped
386+
inputEnd = true;
387+
return EOF;
388+
}
389+
const int ch16 = makeUtf16Char(static_cast<unsigned char>(ch), static_cast<unsigned char>(ch2));
390+
return (ch16 >= 0x80) ? 0xff : ch16;
391+
}
392+
368393
inline int makeUtf16Char(const unsigned char ch, const unsigned char ch2) const {
369394
return (bom == 0xfeff) ? (ch<<8 | ch2) : (ch2<<8 | ch);
370395
}
@@ -400,6 +425,15 @@ class simplecpp::TokenList::Stream {
400425
}
401426

402427
unsigned short bom;
428+
429+
/** characters that are read but not used yet. The next character is at the back. */
430+
std::string buffer;
431+
/** the last character returned by readChar(), EOF if ungetChar() has nothing to put back */
432+
int lastChar{EOF};
433+
/** readChar() has returned EOF */
434+
bool atEnd{false};
435+
/** get() has returned EOF */
436+
bool inputEnd{false};
403437
protected:
404438
bool isUtf16;
405439
};
@@ -414,6 +448,7 @@ namespace {
414448
init();
415449
}
416450

451+
protected:
417452
int get() override {
418453
return istr.get();
419454
}
@@ -423,9 +458,6 @@ namespace {
423458
void unget() override {
424459
istr.unget();
425460
}
426-
bool good() override {
427-
return istr.good();
428-
}
429461

430462
private:
431463
std::istream &istr;
@@ -441,28 +473,25 @@ namespace {
441473
init();
442474
}
443475

476+
protected:
444477
int get() override {
445478
if (pos >= size)
446-
return lastStatus = EOF;
479+
return EOF;
447480
return str[pos++];
448481
}
449482
int peek() override {
450483
if (pos >= size)
451-
return lastStatus = EOF;
484+
return EOF;
452485
return str[pos];
453486
}
454487
void unget() override {
455488
--pos;
456489
}
457-
bool good() override {
458-
return lastStatus != EOF;
459-
}
460490

461491
private:
462492
const unsigned char *str;
463493
const std::size_t size;
464494
std::size_t pos{};
465-
int lastStatus{};
466495
};
467496

468497
class FileStream : public simplecpp::TokenList::Stream {
@@ -489,8 +518,9 @@ namespace {
489518
file = nullptr;
490519
}
491520

521+
protected:
492522
int get() override {
493-
lastStatus = lastCh = fgetc(file);
523+
lastCh = fgetc(file);
494524
return lastCh;
495525
}
496526
int peek() override {
@@ -502,9 +532,6 @@ namespace {
502532
void unget() override {
503533
unget_internal(lastCh);
504534
}
505-
bool good() override {
506-
return lastStatus != EOF;
507-
}
508535

509536
private:
510537
void unget_internal(int ch) {
@@ -519,7 +546,6 @@ namespace {
519546

520547
FILE *file;
521548
int lastCh{};
522-
int lastStatus{};
523549
};
524550
}
525551

‎test.cpp‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99
#include <cstdint>
1010
#include <cstdlib>
1111
#include <cstring>
12+
#include <cstdio>
1213
#include <exception>
14+
#include <fstream>
1315
#include <iostream>
1416
#include <list>
1517
#include <sstream>
@@ -142,6 +144,20 @@ static std::string readfile(const char code[], std::size_t size, const simplecpp
142144
return makeTokenList(code,size,files,std::string(),dui,outputList).stringify();
143145
}
144146

147+
/** Write the code to a temporary file and read it with TokenList(filename), so FileStream is used */
148+
static std::string readfileFromFile(const char code[], std::size_t size, simplecpp::OutputList *outputList = nullptr)
149+
{
150+
const std::string filename = "testrunner_filestream.tmp";
151+
{
152+
std::ofstream f(filename, std::ios::binary);
153+
f.write(code, static_cast<std::streamsize>(size));
154+
}
155+
std::vector<std::string> files;
156+
std::string ret = simplecpp::TokenList(filename, files, {}, outputList).stringify();
157+
std::remove(filename.c_str());
158+
return ret;
159+
}
160+
145161
static std::string preprocess(const char code[], std::size_t size, const simplecpp::DUI &dui, simplecpp::OutputList *outputList, std::list<simplecpp::MacroUsage> *macroUsage = nullptr, std::list<simplecpp::IfCond> *ifCond = nullptr, const std::string &file = std::string())
146162
{
147163
std::vector<std::string> files;
@@ -3981,6 +3997,11 @@ static void readfile_char_error()
39813997

39823998
readfile("A = 's\n'", {}, &outputList);
39833999
ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList));
4000+
outputList.clear();
4001+
4002+
// the ' at the end of the file is not dropped after the peek for a digit separator
4003+
readfile("A = 1'", {}, &outputList);
4004+
ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList));
39844005
}
39854006

39864007
static void readfile_string()
@@ -4168,6 +4189,72 @@ static void readfile_no_eof_newline()
41684189
readfile(code, sizeof(code)-1, {}, &outputList);
41694190
ASSERT_EQUALS("", toString(outputList));
41704191
}
4192+
{
4193+
// CR is a newline
4194+
const char code[] = "x\r";
4195+
simplecpp::OutputList outputList;
4196+
readfile(code, sizeof(code)-1, {}, &outputList);
4197+
ASSERT_EQUALS("", toString(outputList));
4198+
}
4199+
{
4200+
const char code[] = "x\r\n";
4201+
simplecpp::OutputList outputList;
4202+
readfile(code, sizeof(code)-1, {}, &outputList);
4203+
ASSERT_EQUALS("", toString(outputList));
4204+
}
4205+
}
4206+
4207+
static void readfile_cr_newlines()
4208+
{
4209+
// CR, CR LF and LF are all newlines
4210+
ASSERT_EQUALS("a\nb\nc", readfile("a\rb\rc\r"));
4211+
ASSERT_EQUALS("a\nb\nc", readfile("a\r\nb\r\nc\r\n"));
4212+
ASSERT_EQUALS("a\nb\nc\n\n\nd", readfile("a\rb\r\nc\n\r\rd\n"));
4213+
ASSERT_EQUALS("# define A 1\nA", readfile("#define A 1\rA\r"));
4214+
}
4215+
4216+
static void readfile_filestream()
4217+
{
4218+
// the edge cases from the other readfile tests, read from a file
4219+
{
4220+
const char code[] = "a\rb\rc\r";
4221+
simplecpp::OutputList outputList;
4222+
ASSERT_EQUALS("a\nb\nc", readfileFromFile(code, sizeof(code)-1, &outputList));
4223+
ASSERT_EQUALS("", toString(outputList));
4224+
}
4225+
{
4226+
const char code[] = "a\r\nb\r\nc\r\n";
4227+
ASSERT_EQUALS("a\nb\nc", readfileFromFile(code, sizeof(code)-1));
4228+
}
4229+
{
4230+
const char code[] = "a\rb\r\nc\n\r\rd\n";
4231+
ASSERT_EQUALS("a\nb\nc\n\n\nd", readfileFromFile(code, sizeof(code)-1));
4232+
}
4233+
{
4234+
const char code[] = "1'";
4235+
simplecpp::OutputList outputList;
4236+
(void)readfileFromFile(code, sizeof(code)-1, &outputList);
4237+
ASSERT_EQUALS("file0,1,syntax_error,No pair for character (\'). Can't process file. File is either invalid or unicode, which is currently not supported.\n", toString(outputList));
4238+
}
4239+
{
4240+
// UTF-16LE with CR newlines
4241+
const char code[] = "\xff\xfe\x78\x00\x0d\x00\x79\x00\x0d\x00";
4242+
simplecpp::OutputList outputList;
4243+
ASSERT_EQUALS("x\ny", readfileFromFile(code, sizeof(code)-1, &outputList));
4244+
ASSERT_EQUALS("", toString(outputList));
4245+
}
4246+
{
4247+
// UTF-16BE with CR newlines
4248+
const char code[] = "\xfe\xff\x00\x78\x00\x0d\x00\x79\x00\x0d";
4249+
ASSERT_EQUALS("x\ny", readfileFromFile(code, sizeof(code)-1));
4250+
}
4251+
{
4252+
// UTF-16 with an odd number of bytes, the dangling last byte is dropped
4253+
const char code[] = "\xff\xfe\x31\x00\x0a";
4254+
simplecpp::OutputList outputList;
4255+
ASSERT_EQUALS("1", readfileFromFile(code, sizeof(code)-1, &outputList));
4256+
ASSERT_EQUALS("file0,1,portability_no_eof_newline,No newline at end of file is undefined behavior in C.\n", toString(outputList));
4257+
}
41714258
}
41724259

41734260
static void stringify1()
@@ -4364,6 +4451,18 @@ static void unicode()
43644451
const char code[] = "\xff\xfe\x0d\x00\x0a\x00\x2f\x00\x2f\x00\x31\x00\x0d\x00\x0a\x00";
43654452
ASSERT_EQUALS("\n//1", readfile(code, sizeof(code)-1));
43664453
}
4454+
{
4455+
// CR newlines
4456+
const char code[] = "\xff\xfe\x78\x00\x0d\x00\x79\x00\x0d\x00";
4457+
ASSERT_EQUALS("x\ny", readfile(code, sizeof(code)-1));
4458+
}
4459+
{
4460+
// odd number of bytes, the dangling last byte is dropped
4461+
const char code[] = "\xff\xfe\x31\x00\x0a";
4462+
simplecpp::OutputList outputList;
4463+
ASSERT_EQUALS("1", readfile(code, sizeof(code)-1, {}, &outputList));
4464+
ASSERT_EQUALS("file0,1,portability_no_eof_newline,No newline at end of file is undefined behavior in C.\n", toString(outputList));
4465+
}
43674466
}
43684467

43694468
static void unicode_invalid()
@@ -5335,6 +5434,8 @@ static void runTests(int argc, char **argv, Input input)
53355434
TEST_CASE(readfile_error);
53365435
TEST_CASE(readfile_file_not_found);
53375436
TEST_CASE(readfile_no_eof_newline);
5437+
TEST_CASE(readfile_cr_newlines);
5438+
TEST_CASE(readfile_filestream);
53385439

53395440
TEST_CASE(stringify1);
53405441

0 commit comments

Comments
 (0)