diff --git a/IMFBoundary.cpp b/IMFBoundary.cpp index 4c42f1f..776bb77 100644 --- a/IMFBoundary.cpp +++ b/IMFBoundary.cpp @@ -1,30 +1,49 @@ #include "IMFBoundary.h" #include "Exception.h" -#include namespace coreutils { - IMFBoundary(StreamReader &reader, ZString boundary) : boundary(boundary) { - - } - - int IMFBoundary::parse() { - switch(state) { - case PREFIX: - if(buffer.remaining() > 3) - if(reader.nextIf("\r\n--")) - state = BOUNDARY; - case BOUNDARY: - while(buffer.getTokenInclude("")); - if(buffer.eod()) - return 2; - else if(buffer.remaining() > 1) - if(buffer.ifNext("--")) - state = LAST_BOUNDARY; - case LAST_BOUNDARY: - return 0; + IMFBoundary::IMFBoundary(StreamReader &reader, ZString boundary) : Parser(reader), boundary(boundary) {} + + Parser::Status IMFBoundary::parse() { + MString &buffer = reader.getBuffer(); + switch (state) { + + case PREFIX: { + MString marker; + marker << "\r\n--"; + marker << boundary; + int pos = buffer.find(marker); + if (pos < 0) + return Status::INCOMPLETE; + if (!buffer.ifNext(marker)) + throw coreutils::Exception("IMFBoundary: boundary marker vanished between find and ifNext."); + state = TAIL; + return Status::DATA_AVAIL; + } + + case TAIL: + // ZString::getLength() is already cursor-relative (bytes remaining + // from the cursor to the end) - do not subtract the cursor offset + // again here. + if (buffer.getLength() < 2) + return Status::INCOMPLETE; + if (buffer.ifNext(ZString("--"))) { + last = true; + return Status::COMPLETE; + } + if (buffer.ifNext(ZString("\r\n"))) { + last = false; + return Status::COMPLETE; + } + throw coreutils::Exception("IMFBoundary: malformed boundary line - expected CRLF or trailing --."); + } - + throw coreutils::Exception("IMFBoundary: unreachable state."); } - -} \ No newline at end of file + + bool IMFBoundary::isLast() { + return last; + } + +} diff --git a/IMFBoundary.h b/IMFBoundary.h index 477bf0e..42d1a3d 100644 --- a/IMFBoundary.h +++ b/IMFBoundary.h @@ -1,21 +1,44 @@ #ifndef __IMFBoundary_h__ # define __IMFBoundary_h__ -#include "StreamReader.h" #include "Parser.h" namespace coreutils { - + + /// + /// IMFBoundary detects a MIME multipart boundary line in the shared + /// StreamReader - either an ordinary boundary ("\r\n--" + boundary + + /// "\r\n", meaning another part follows) or the closing boundary + /// ("\r\n--" + boundary + "--", meaning this was the last part). + /// Resumable: if the boundary text has been seen but there isn't yet + /// enough buffered data to tell an ordinary boundary from the closing + /// one, parse() reports INCOMPLETE without committing the cursor past + /// the ambiguous point. + /// + class IMFBoundary : public Parser { - + public: + IMFBoundary(StreamReader &reader, ZString boundary); - virtual int parse(); - + /// + /// Valid once doParse() has returned Status::COMPLETE - true if this + /// was the closing boundary (no further parts follow). + /// + + bool isLast(); + + protected: + + Status parse() override; + private: + + enum State { PREFIX, TAIL } state = PREFIX; ZString boundary; - + bool last = false; + }; } diff --git a/IMFFormData.cpp b/IMFFormData.cpp index 6115b3d..00ea205 100644 --- a/IMFFormData.cpp +++ b/IMFFormData.cpp @@ -3,9 +3,15 @@ #include "IMFText.h" namespace coreutils { - + IMFFormData::IMFFormData(int fd, ZString boundary) { - IMFHeader header(fd); + // TODO: IMFHeader now reads through a StreamReader (see the Parser/ + // StreamReader redesign) instead of a raw fd. This constructor - and + // IMFMultipart, which still constructs IMFFormData from a bare fd - + // needs migrating to StreamReader together before header parsing can + // be wired back in here. Out of scope for that redesign pass; this + // class was already an unfinished stub (constructed objects were + // discarded, nothing was stored) before this change. IMFText text(fd, boundary); } diff --git a/IMFFormData.h b/IMFFormData.h index 6d93e67..5453a42 100644 --- a/IMFFormData.h +++ b/IMFFormData.h @@ -15,5 +15,3 @@ namespace coreutils { } #endif - - diff --git a/IMFHeader.cpp b/IMFHeader.cpp index df639b0..1635b65 100644 --- a/IMFHeader.cpp +++ b/IMFHeader.cpp @@ -1,32 +1,77 @@ #include "IMFHeader.h" #include "Exception.h" -#include "Log.h" -#include namespace coreutils { - IMFHeader::IMFHeader(int fd) {} + IMFHeader::IMFHeader(StreamReader &reader) : Parser(reader) {} + + Parser::Status IMFHeader::parse() { + MString &buffer = reader.getBuffer(); + + switch (state) { + + case NAME: { + if (buffer.eod()) + return Status::INCOMPLETE; + if (buffer.getChar() == '\r') { + // ZString::getLength() is already cursor-relative. + if (buffer.getLength() < 2) + return Status::INCOMPLETE; + if (!buffer.ifNext(ZString("\r\n"))) + throw coreutils::Exception("IMFHeader: expected CRLF at end of header block."); + return Status::COMPLETE; + } + name << buffer.getTokenInclude("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-0123456789"); + if (buffer.eod()) + return Status::INCOMPLETE; + if (!buffer.ifNext(ZString(":"))) + throw coreutils::Exception("IMFHeader: expected ':' in header name."); + state = SKIP_SPACE; + return Status::DATA_AVAIL; + } + + case SKIP_SPACE: + while (!buffer.eod() && buffer.getChar() == ' ') + buffer.nextChar(); + if (buffer.eod()) + return Status::INCOMPLETE; + state = FIELD; + return Status::DATA_AVAIL; - virtual int parse() { - switch(state) { - case NAME: - name << buffer.getTokenInclude("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-0123456789"); - if(buffer.ifNext(":")) - state = FIELD; - else - return 2; case FIELD: - field << buffer.getTokenInclude("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-;'\"0123456789"); - if(!buffer.ifNext("\r\n")) - if(buffer.eod()) - return 2; - // Right here is where we should check if next line is an extension to the previous line. - - - + field << buffer.getTokenExclude("\r\n"); + if (buffer.eod()) + return Status::INCOMPLETE; + if (!buffer.ifNext(ZString("\r\n"))) + throw coreutils::Exception("IMFHeader: expected CRLF at end of header field."); + state = FOLD_CHECK; + return Status::DATA_AVAIL; + + case FOLD_CHECK: + if (buffer.eod()) + return Status::INCOMPLETE; + if (buffer.getChar() == ' ' || buffer.getChar() == '\t') { + buffer.nextChar(); + field << " "; + state = SKIP_SPACE; + return Status::DATA_AVAIL; + } + commitField(); + state = NAME; + return Status::DATA_AVAIL; + } - - + throw coreutils::Exception("IMFHeader: unreachable state."); } -} \ No newline at end of file + void IMFHeader::commitField() { + headers[name] = field; + name = MString(); + field = MString(); + } + + std::map &IMFHeader::getHeaders() { + return headers; + } + +} diff --git a/IMFHeader.h b/IMFHeader.h index 5c4f567..4bcf59c 100644 --- a/IMFHeader.h +++ b/IMFHeader.h @@ -1,26 +1,48 @@ #ifndef __IMFHeader_h__ # define __IMFHeader_h__ -# include "StreamReader.h" # include "Parser.h" # include "MString.h" -# include "IMFHeaderField.h" -# include +# include namespace coreutils { - + + /// + /// IMFHeader parses a full block of IMF/MIME-style headers (RFC 5322 + /// "name: value" pairs, one per line, with folded continuation lines - + /// any line starting with a space or tab is appended to the previous + /// field's value) from the shared StreamReader, stopping at the blank + /// line that terminates the header block. Resumable across any number of + /// StreamReader::feed() calls - partially-read names/values accumulate + /// in the name/field members rather than being discarded between calls. + /// + class IMFHeader : public Parser { - + public: + IMFHeader(StreamReader &reader); - virtual int parse(); - + /// + /// Valid once doParse() has returned Status::COMPLETE - every header + /// field parsed, keyed by field name. + /// + + std::map &getHeaders(); + + protected: + + Status parse() override; + private: + + enum State { NAME, SKIP_SPACE, FIELD, FOLD_CHECK } state = NAME; MString name; MString field; std::map headers; - + + void commitField(); + }; } diff --git a/LineParser.cpp b/LineParser.cpp index 57a841d..415a624 100644 --- a/LineParser.cpp +++ b/LineParser.cpp @@ -1,18 +1,26 @@ #include "LineParser.h" -#include "Exception.h" -#include namespace coreutils { - LineParser::LineParser(StreamReader &reader) { - - } - - int LineParser::parse() { - + LineParser::LineParser(StreamReader &reader) : Parser(reader) {} - - + Parser::Status LineParser::parse() { + MString &buffer = reader.getBuffer(); + char *lineStart = buffer.getCursor(); + ZString term("\n"); + int pos = buffer.find(term); + if (pos < 0) + return Status::INCOMPLETE; + char *lineEnd = buffer.getCursor(); + if (lineEnd > lineStart && *(lineEnd - 1) == '\r') + --lineEnd; + line = ZString(lineStart, lineEnd - lineStart); + buffer.setCursor(buffer.getCursor() + 1); + return Status::COMPLETE; } - -} \ No newline at end of file + + ZString LineParser::getLine() { + return line; + } + +} diff --git a/LineParser.h b/LineParser.h index d9be254..2aa6ead 100644 --- a/LineParser.h +++ b/LineParser.h @@ -1,18 +1,40 @@ #ifndef __LineParser_h__ # define __LineParser_h__ -#include "StreamReader.h" #include "Parser.h" namespace coreutils { - + + /// + /// LineParser builds one line of text at a time from the shared + /// StreamReader - terminated by LF or CRLF, either accepted, the + /// terminator itself is not included in getLine(). Resumable: if no + /// terminator has arrived yet, parse() reports INCOMPLETE and leaves the + /// cursor exactly where it started, so the same unconsumed bytes are + /// re-scanned (along with whatever new data arrived) on the next call. + /// + class LineParser : public Parser { - + public: + LineParser(StreamReader &reader); - int parse() override; - + /// + /// Valid once doParse() has returned Status::COMPLETE - the line that + /// was parsed, with its terminator stripped. + /// + + ZString getLine(); + + protected: + + Status parse() override; + + private: + + ZString line; + }; } diff --git a/Parser.cpp b/Parser.cpp index 9b2686f..18457a1 100644 --- a/Parser.cpp +++ b/Parser.cpp @@ -1,19 +1,21 @@ #include "Parser.h" -#include "StreamReader.h" namespace coreutils { - Parser::Parser(int socket) : socket(socket) {} + Parser::Parser(StreamReader &reader) : reader(reader) {} Parser::~Parser() {} - Status Parser::doParse() { + Parser::Status Parser::doParse() { + Status status; + do { + status = parse(); + } while (status == Status::DATA_AVAIL); + return status; } - bool Parser::checkParse(Parser &parser); - - Status Parser::parse() { - return Status::COMPLETE; + bool Parser::checkParse(Parser &parser) { + return parser.doParse() == Status::COMPLETE; } - + } diff --git a/Parser.h b/Parser.h index 845637c..b13c11d 100644 --- a/Parser.h +++ b/Parser.h @@ -1,70 +1,80 @@ #ifndef __Parser_h__ #define __Parser_h__ -#include -#include +#include "StreamReader.h" namespace coreutils { /// - /// Use the StreamReader to read data from a socket and buffer it for use by - /// various stream parsers such as IMFMessage. The reader maintains data - /// consistency regardless of the packet delivery for the stream. + /// Parser is the base class for building a protocol/object parser that reads + /// from a shared StreamReader's buffer. A subclass implements parse() as a + /// resumable state machine (typically a switch on an internal state member) + /// that consumes as much of the buffer as it currently can and reports how + /// the parse should proceed via Status: + /// + /// - COMPLETE: the object this Parser builds is fully parsed and ready. + /// - INCOMPLETE: not enough data is currently buffered to make more + /// progress; the caller should wait for more data + /// (StreamReader::feed()) before calling doParse() again. + /// The subclass's state member must reflect exactly where + /// parsing should resume - and must NOT have consumed any + /// cursor position it can't yet commit to (see individual + /// Parser subclasses for how they guarantee this). + /// - DATA_AVAIL: this call made progress and there is more buffered data + /// this Parser can still work through right now, without + /// waiting for new data to arrive. + /// + /// Call doParse() (not parse() directly) to drive a Parser - it loops on + /// DATA_AVAIL automatically and returns once the result is COMPLETE or + /// INCOMPLETE. /// - class Parser : public MString { + class Parser { public: - /// - /// - /// - - Parser(int socket); - - /// - /// - /// + Parser(StreamReader &reader); enum class Status { - COMPLETE, - INCOMPLETE, - DATA_AVAIL + COMPLETE, + INCOMPLETE, + DATA_AVAIL }; - - /// - /// - /// virtual ~Parser(); /// - /// doParse is the entry and re-entry point method to continue the parsing process. - /// - - Status doParse(); - - /// - /// Inside of parse() you can call another parser object with the checkParse method to - /// determine if the parsing should branch to the new passed parsing object. + /// doParse is the entry and re-entry point method to continue the + /// parsing process. Safe to call repeatedly (e.g. once per + /// StreamReader::feed()) - each call resumes exactly where the + /// previous one left off. /// - bool checkParse(Parser &parser); <--- can be called in parse to see if the parsing should branch + Status doParse(); /// + /// Inside of parse() you can call another parser object with the + /// checkParse method to attempt parsing with a candidate sub-parser + /// sharing the same StreamReader. Returns true only once that + /// sub-parser fully completes - retain the same sub-parser instance + /// across re-entries into your own parse() (e.g. as a member) so it + /// can resume, rather than constructing a new one on every call. /// - /// - - virtual Status parse(); - + + bool checkParse(Parser &parser); + protected: - MString buffer; - - private: - int socket; - Parser *parser; - vector chain; - + + /// + /// parse() performs one resumable step of parsing against reader's + /// buffer. Must be implemented by every concrete Parser to build its + /// specific object. + /// + + virtual Status parse() = 0; + + StreamReader &reader; + }; } diff --git a/StreamReader.cpp b/StreamReader.cpp new file mode 100644 index 0000000..fb8f842 --- /dev/null +++ b/StreamReader.cpp @@ -0,0 +1,27 @@ +#include "StreamReader.h" + +namespace coreutils { + + StreamReader::StreamReader() {} + + void StreamReader::feed(ZString &data) { + buffer.write(data); + } + + MString &StreamReader::getBuffer() { + return buffer; + } + + void StreamReader::compact() { + int parsedLength = buffer.getCursor() - buffer.getData(); + if (parsedLength > 0) { + buffer.remove(0, parsedLength); + buffer.setCursor(buffer.getData()); + } + } + + bool StreamReader::hasUnparsedData() { + return !buffer.eod(); + } + +} diff --git a/StreamReader.h b/StreamReader.h new file mode 100644 index 0000000..fe18b81 --- /dev/null +++ b/StreamReader.h @@ -0,0 +1,66 @@ +#ifndef __StreamReader_h__ +#define __StreamReader_h__ + +#include "MString.h" + +namespace coreutils { + + /// + /// StreamReader accumulates data delivered in arbitrary-sized chunks (e.g. from + /// core::Socket::onDataReceived) into one continuous, growable buffer, and exposes + /// it to a chain of Parser objects so they can build higher-level protocol objects + /// (a header, a line, a MIME part, etc.) without needing to worry about where one + /// network packet ended and the next began - a "complete" unit may span any number + /// of feed() calls. + /// + /// Parsers consume data through the ZString-style cursor on getBuffer(). Once a + /// Parser has fully consumed a unit (advanced the cursor past it), call compact() + /// to drop the already-parsed prefix so the buffer doesn't grow unboundedly for the + /// life of a long-running connection. + /// + + class StreamReader { + + public: + + StreamReader(); + + /// + /// Append newly-arrived data to the end of the buffer. Does not affect the + /// cursor - existing parse position is preserved across a feed(). + /// + + void feed(ZString &data); + + /// + /// The accumulated buffer Parsers read from and advance the cursor over. + /// + + MString &getBuffer(); + + /// + /// Drop the already-parsed prefix (everything before the cursor) and reset + /// the cursor to the new start of the buffer. Call this once a Parser has + /// completed a unit and before waiting for more data, so the buffer only + /// ever grows by the amount of not-yet-parsed data. + /// + + void compact(); + + /// + /// True if there is unparsed data currently sitting in the buffer that a + /// Parser hasn't consumed yet (i.e. more progress may be possible without + /// waiting for the next feed()). + /// + + bool hasUnparsedData(); + + private: + + MString buffer; + + }; + +} + +#endif diff --git a/testing/parser_test.cpp b/testing/parser_test.cpp index c37465c..300e457 100644 --- a/testing/parser_test.cpp +++ b/testing/parser_test.cpp @@ -1,14 +1,73 @@ #include -#include "../MString.h" +#include +#include +#include "../StreamReader.h" #include "../LineParser.h" +#include "../IMFHeader.h" +#include "../IMFBoundary.h" -int main(int argc, char **argv) { +using namespace coreutils; - coreutils::LineParser line(0); - Parse.Status status = line.doParse(); - +static int checks = 0; +#define CHECK(cond) do { checks++; if(!(cond)) { std::cerr << "FAIL line " << __LINE__ << ": " #cond << std::endl; return 1; } } while(0) - +void feedStr(StreamReader &r, const char *s) { + ZString z((char *)s, strlen(s)); + r.feed(z); +} + +// The whole point of StreamReader/Parser: a "line" (or header, or boundary) +// can arrive split across any number of socket packets, and parsing must +// resume cleanly from wherever it left off once more data shows up. + +int test_lineparser_split_across_feeds() { + StreamReader reader; + feedStr(reader, "partial li"); + LineParser lp(reader); + CHECK(lp.doParse() == Parser::Status::INCOMPLETE); + feedStr(reader, "ne\r\n"); + CHECK(lp.doParse() == Parser::Status::COMPLETE); + ZString line = lp.getLine(); + CHECK(line.getLength() == 12); + CHECK(strncmp(line.getData(), "partial line", 12) == 0); return 0; } +int test_imfheader_folding_and_split_feeds() { + StreamReader reader; + IMFHeader header(reader); + feedStr(reader, "Subject: line one\r\n "); + CHECK(header.doParse() == Parser::Status::INCOMPLETE); + feedStr(reader, "line two\r\n\r\n"); + CHECK(header.doParse() == Parser::Status::COMPLETE); + auto &fields = header.getHeaders(); + CHECK(fields.size() == 1); + MString subj = fields[MString("Subject")]; + CHECK(subj.getLength() == 17); + CHECK(strncmp(subj.getData(), "line one line two", subj.getLength()) == 0); + return 0; +} + +int test_imfboundary_split_across_feeds() { + ZString b((char *)"XYZ", 3); + StreamReader reader; + IMFBoundary bound(reader, b); + feedStr(reader, "\r\n--X"); + CHECK(bound.doParse() == Parser::Status::INCOMPLETE); + feedStr(reader, "YZ--"); + CHECK(bound.doParse() == Parser::Status::COMPLETE); + CHECK(bound.isLast() == true); + return 0; +} + +int main() { + int rc = 0; + rc |= test_lineparser_split_across_feeds(); + rc |= test_imfheader_folding_and_split_feeds(); + rc |= test_imfboundary_split_across_feeds(); + if (rc == 0) + std::cout << "ALL " << checks << " CHECKS PASSED" << std::endl; + else + std::cout << "SOME CHECKS FAILED (see above)" << std::endl; + return rc; +}