Add HTTPRequest/HTTPResponse: real HTTP parsing on the Parser/StreamReader stack

Stage 2 of the APIServer plan. This codebase has only ever sat behind a
CGI gateway - IMFRequest/IMFResponse/IMFMessage predate the
StreamReader/Parser rebuild and are stubs (IMFResponse's header-emission
calls addHeader()/output(), neither of which exist on the class;
IMFRequest is a fragile non-resumable 3-token line split). Nothing in
this repo has ever actually parsed a raw HTTP request off a socket.

HTTPRequest : public Parser fills that gap the same way every other
Parser here is built: REQUEST_LINE -> HEADERS -> BODY -> DONE, chaining
LineParser (request line) and IMFHeader (headers, reused as-is - HTTP
header syntax is IMF-derived, nothing email-specific in IMFHeader
itself) via checkParse() over one shared StreamReader, exactly the
pattern CGIFormData already established for chaining IMFHeader/
IMFBoundary. The BODY state dispatches on Content-Type as CGIFormData
itself does for CGI bodies - application/x-www-form-urlencoded and
multipart/form-data delegate directly to CGIFormData (reused, not
reimplemented), anything else is captured as a raw body (e.g. JSON).
Query string is split off the path before anything else sees it.
getHeader() does a case-insensitive lookup (HTTP header names are
case-insensitive per RFC 7230, unlike IMFHeader's own std::map, which
compares byte-for-byte) since a route handler constantly needs this and
getting it wrong would be a silent framing bug, not a cosmetic one.

The boundary-extraction code mirrors __jet.cpp's own fix exactly (same
cursor-mutation trap: ZString::find() leaves the searched string's
cursor at the match, and CGIFormData's constructor copy-constructs its
own contentType member from whatever's passed in, inheriting that same
moved cursor unless reset() first - would otherwise silently miss its
own startsWith() dispatch checks).

HTTPResponse is a small, new, output-only builder (status line + a real
headers container + body, Content-Length auto-computed unless set
explicitly) - deliberately not patched from IMFResponse, which has
nothing safe to extend.

v1 limitations, flagged not fixed (matching the standard already set by
other Parser subclasses here for known-but-out-of-scope gaps): chunked
Transfer-Encoding isn't supported, and repeated header names last-wins
(inherited from IMFHeader's own std::map).

Verified live via testing/http_test.cpp (34 assertions, new - now wired
into testing/compile): byte-at-a-time GET with a query string and
case-insensitive header lookup; a GET with no query string; a POST with
an urlencoded body deliberately split mid-body (not just mid-header)
across two feed() calls, confirming the CGIFormData delegation resumes
correctly; a raw JSON POST body; testing/http_request.data (a real
fixture, previously unused - normalized from its stored LF line endings
to real CRLF, since HTTP requires CRLF and IMFHeader enforces that
strictly) fed byte-at-a-time; and HTTPResponse::serialize() both with an
auto-computed and an explicitly-set Content-Length (no duplicate
header). Also reran the full existing testing/compile suite
(zstring/mstring/jstring/mfile/jfile) against the rebuilt libCoreUtils.a
to confirm no regression from the two new files.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

.gitignore

@@ -9,3 +9,4 @@ build/
 .history/
 testing/mstring_test
 testing/zstring_test
+testing/http_test

HTTPRequest.cpp

@@ -0,0 +1,127 @@
+#include "HTTPRequest.h"
+#include "Exception.h"
+#include <strings.h>
+
+namespace coreutils {
+
+   HTTPRequest::HTTPRequest(StreamReader &reader) : Parser(reader) {}
+
+   HTTPRequest::~HTTPRequest() {
+      delete lineParser;
+      delete headerParser;
+      delete formParser;
+   }
+
+   Parser::Status HTTPRequest::parse() {
+      MString &buffer = reader.getBuffer();
+
+      switch (state) {
+
+       case REQUEST_LINE: {
+          if (!lineParser)
+             lineParser = new LineParser(reader);
+          if (!checkParse(*lineParser))
+             return Status::INCOMPLETE;
+          {
+             ZString line = lineParser->getLine();
+             ZString sp(" ");
+             std::vector<ZString> &parts = line.split(sp);
+             if (parts.size() != 3)
+                throw coreutils::Exception("HTTPRequest: malformed request line "
+                                            "(expected \"METHOD /path HTTP/1.1\").");
+             method = MString(parts[0]);
+             path = MString(parts[1]);
+             version = MString(parts[2]);
+          }
+          delete lineParser;
+          lineParser = NULL;
+          {
+             // Split the query string off before anything else sees `path` -
+             // a route table (see the APIServer plan) should never have to
+             // deal with a trailing "?a=1&b=2" itself.
+             int qpos = path.find(ZString("?"));
+             if (qpos != -1) {
+                queryString = MString(path.substring(qpos + 1));
+                path = MString(path.substring(0, qpos));
+             }
+          }
+          state = HEADERS;
+          return Status::DATA_AVAIL;
+       }
+
+       case HEADERS:
+          if (!headerParser)
+             headerParser = new IMFHeader(reader);
+          if (!checkParse(*headerParser))
+             return Status::INCOMPLETE;
+          headers = headerParser->getHeaders();
+          delete headerParser;
+          headerParser = NULL;
+          state = BODY;
+          return Status::DATA_AVAIL;
+
+       case BODY: {
+          MString contentLengthHeader = getHeader("Content-Length");
+          if (contentLengthHeader.getLength() == 0) {
+             if (getHeader("Transfer-Encoding").getLength() > 0)
+                throw coreutils::Exception("HTTPRequest: chunked Transfer-Encoding "
+                                            "is not supported (Content-Length only).");
+             state = DONE;
+             return Status::COMPLETE;
+          }
+          int contentLength = contentLengthHeader.asInteger();
+          MString contentType = getHeader("Content-Type");
+          if (contentType.startsWith("application/x-www-form-urlencoded") ||
+              contentType.startsWith("multipart/form-data")) {
+             if (!formParser) {
+                // Mirrors __jet.cpp's identical CGI-path fix: find() leaves
+                // contentType's own cursor at the match, and CGIFormData's
+                // constructor copy-constructs its own contentType member
+                // from whatever we pass, inheriting that same moved cursor
+                // - its startsWith() dispatch checks are cursor-relative
+                // and would silently miss (falling through to UNKNOWN)
+                // without the reset().
+                int boundaryPos = contentType.find(ZString("boundary="));
+                if (boundaryPos != -1)
+                   boundary = MString(contentType.substring(boundaryPos + 9));
+                contentType.reset();
+                formParser = new CGIFormData(reader, contentType, contentLength, boundary);
+             }
+             if (!checkParse(*formParser))
+                return Status::INCOMPLETE;
+             state = DONE;
+             return Status::COMPLETE;
+          }
+          if ((int)buffer.getLength() < contentLength)
+             return Status::INCOMPLETE;
+          body = MString(ZString(buffer.getCursor(), contentLength));
+          buffer.setCursor(buffer.getCursor() + contentLength);
+          state = DONE;
+          return Status::COMPLETE;
+       }
+
+       case DONE:
+          return Status::COMPLETE;
+
+      }
+      throw coreutils::Exception("HTTPRequest: unreachable state.");
+   }
+
+   MString HTTPRequest::getMethod() { return method; }
+   MString HTTPRequest::getPath() { return path; }
+   MString HTTPRequest::getQueryString() { return queryString; }
+   MString HTTPRequest::getVersion() { return version; }
+   MString HTTPRequest::getBody() { return body; }
+   std::map<MString, MString> &HTTPRequest::getHeaders() { return headers; }
+   CGIFormData *HTTPRequest::getFormData() { return formParser; }
+
+   MString HTTPRequest::getHeader(const char *name) {
+      for (auto &pair : headers) {
+         MString key = pair.first;
+         if (strcasecmp(key.c_str(), name) == 0)
+            return pair.second;
+      }
+      return MString();
+   }
+
+}

HTTPRequest.h

@@ -0,0 +1,109 @@
+#ifndef __HTTPRequest_h__
+#define __HTTPRequest_h__
+
+#include "Parser.h"
+#include "LineParser.h"
+#include "IMFHeader.h"
+#include "CGIFormData.h"
+#include "MString.h"
+#include <map>
+
+namespace coreutils {
+
+   ///
+   /// HTTPRequest parses a full HTTP/1.1 request - request line, headers,
+   /// and (for a Content-Length body) either a raw body or, for
+   /// application/x-www-form-urlencoded / multipart/form-data, a parsed
+   /// CGIFormData - off a shared StreamReader. Resumable across any number
+   /// of StreamReader::feed() calls, exactly like every other Parser in
+   /// this subsystem: built by chaining LineParser (request line) and
+   /// IMFHeader (headers) via checkParse(), the same way CGIFormData itself
+   /// chains IMFHeader/IMFBoundary over one shared StreamReader.
+   ///
+   /// This codebase predates ever parsing raw HTTP off a socket (it has
+   /// only ever sat behind a CGI gateway) - IMFRequest/IMFResponse/
+   /// IMFMessage are stubs from before this Parser/StreamReader subsystem
+   /// existed and are not built on here.
+   ///
+   /// v1 limitations, deliberately not addressed (flagged, not fixed -
+   /// matching the standard already set by other Parser subclasses here for
+   /// known-but-out-of-scope gaps):
+   ///  - chunked Transfer-Encoding is not supported, only Content-Length.
+   ///  - repeated header names (e.g. multiple Cookie: lines) last-wins -
+   ///    the same limitation IMFHeader's std::map already has, inherited
+   ///    here since HEADERS delegates to IMFHeader directly.
+   ///
+
+   class HTTPRequest : public Parser {
+
+    public:
+
+      HTTPRequest(StreamReader &reader);
+      virtual ~HTTPRequest();
+
+      ///
+      /// Valid once doParse() has returned Status::COMPLETE.
+      ///
+
+      MString getMethod();
+      MString getPath();
+      MString getQueryString();
+      MString getVersion();
+
+      ///
+      /// Case-insensitive header lookup (HTTP header names are
+      /// case-insensitive per RFC 7230, unlike IMFHeader's underlying
+      /// std::map, which compares byte-for-byte) - returns an empty
+      /// MString if the header wasn't present. Valid once the HEADERS
+      /// state has completed (i.e. from BODY parsing onward).
+      ///
+
+      MString getHeader(const char *name);
+
+      ///
+      /// Every header as parsed, keyed exactly as received (not
+      /// case-normalized) - prefer getHeader() for lookups.
+      ///
+
+      std::map<MString, MString> &getHeaders();
+
+      ///
+      /// Valid once doParse() reports COMPLETE. Populated only when the
+      /// request had a body that was neither application/
+      /// x-www-form-urlencoded nor multipart/form-data (e.g. a raw JSON
+      /// body) - empty otherwise, including when there was no body at all.
+      ///
+
+      MString getBody();
+
+      ///
+      /// Valid once doParse() reports COMPLETE, when the body's
+      /// Content-Type was application/x-www-form-urlencoded or
+      /// multipart/form-data - NULL otherwise (including no body at all).
+      /// Owned by this HTTPRequest - do not delete.
+      ///
+
+      CGIFormData *getFormData();
+
+    protected:
+
+      Status parse() override;
+
+    private:
+
+      enum State { REQUEST_LINE, HEADERS, BODY, DONE } state = REQUEST_LINE;
+
+      MString method, path, queryString, version;
+      std::map<MString, MString> headers;
+      MString body;
+      MString boundary;
+
+      LineParser *lineParser = NULL;
+      IMFHeader *headerParser = NULL;
+      CGIFormData *formParser = NULL;
+
+   };
+
+}
+
+#endif

HTTPResponse.cpp

@@ -0,0 +1,53 @@
+#include "HTTPResponse.h"
+#include <strings.h>
+
+namespace coreutils {
+
+   HTTPResponse::HTTPResponse() {
+      statusText = "OK";
+   }
+
+   void HTTPResponse::setStatus(int code, ZString text) {
+      statusCode = code;
+      statusText = text;
+   }
+
+   void HTTPResponse::setHeader(ZString name, ZString value) {
+      headers[MString(name)] = MString(value);
+   }
+
+   void HTTPResponse::setBody(ZString bodyIn) {
+      body = bodyIn;
+   }
+
+   MString HTTPResponse::serialize() {
+      bool hasContentLength = false;
+      for (auto &pair : headers) {
+         MString key = pair.first;
+         if (strcasecmp(key.c_str(), "Content-Length") == 0)
+            hasContentLength = true;
+      }
+
+      MString out;
+      out << "HTTP/1.1 ";
+      out << MString(std::to_string(statusCode));
+      out << " ";
+      out << statusText;
+      out << "\r\n";
+      for (auto &pair : headers) {
+         out << pair.first;
+         out << ": ";
+         out << pair.second;
+         out << "\r\n";
+      }
+      if (!hasContentLength) {
+         out << "Content-Length: ";
+         out << MString(std::to_string(body.getLength()));
+         out << "\r\n";
+      }
+      out << "\r\n";
+      out << body;
+      return out;
+   }
+
+}

HTTPResponse.h

@@ -0,0 +1,51 @@
+#ifndef __HTTPResponse_h__
+#define __HTTPResponse_h__
+
+#include "MString.h"
+#include <map>
+
+namespace coreutils {
+
+   ///
+   /// HTTPResponse builds a raw HTTP/1.1 response (status line + headers +
+   /// body) ready to write to a socket. Output-only, not a Parser - there's
+   /// nothing to resume here, the whole thing is built and serialized in
+   /// one pass once a handler has finished producing its body.
+   ///
+   /// Deliberately new rather than built on/patched from IMFResponse -
+   /// IMFResponse's own header-emission code is dead (calls addHeader()/
+   /// output(), neither of which exist on that class) and it never grew a
+   /// real headers container, so there's nothing there safe to extend.
+   ///
+
+   class HTTPResponse {
+
+    public:
+
+      HTTPResponse();
+
+      void setStatus(int code, ZString text);
+      void setHeader(ZString name, ZString value);
+      void setBody(ZString body);
+
+      ///
+      /// Serializes the full response - status line, headers (an explicit
+      /// Content-Length header is respected as-is; otherwise one is
+      /// computed from the body automatically), the blank line, and the
+      /// body - ready to write to a socket.
+      ///
+
+      MString serialize();
+
+    private:
+
+      int statusCode = 200;
+      MString statusText;
+      std::map<MString, MString> headers;
+      MString body;
+
+   };
+
+}
+
+#endif

testing/compile

@@ -4,3 +4,4 @@ g++ -g -std=c++20 -o mstring_test mstring_test.cpp -I.. -L.. -lCoreUtils -lb64
 g++ -o jstring_test jstring_test.cpp -I.. -L.. -lCoreUtils
 g++ -g -std=c++20 -o mfile_test mfile_test.cpp -I.. -L.. -lCoreUtils
 g++ -g -std=c++20 -o jfile_test jfile_test.cpp -I.. -L.. -lCoreUtils
+g++ -g -std=c++23 -o http_test http_test.cpp -I.. -L.. -lCoreUtils

testing/http_test.cpp

@@ -0,0 +1,179 @@
+#include <iostream>
+#include <cassert>
+#include <cstring>
+#include <fstream>
+#include <sstream>
+#include "../StreamReader.h"
+#include "../HTTPRequest.h"
+#include "../HTTPResponse.h"
+
+using namespace coreutils;
+
+static int checks = 0;
+#define CHECK(cond) do { checks++; if(!(cond)) { std::cerr << "FAIL line " << __LINE__ << ": " #cond << std::endl; return 1; } } while(0)
+
+// Feeds a StreamReader one byte at a time and drives a Parser's doParse()
+// after every single byte - the harshest fragmentation test used elsewhere
+// in this subsystem (see CGIFormData's byte-at-a-time stress test) - to
+// prove HTTPRequest is genuinely resumable across arbitrary split points,
+// not just "works when handed the whole request at once".
+static Parser::Status feedByteAtATime(StreamReader &reader, Parser &parser, const std::string &data) {
+   Parser::Status status = Parser::Status::INCOMPLETE;
+   for (size_t i = 0; i < data.size() && status != Parser::Status::COMPLETE; ++i) {
+      ZString chunk((char *)data.data() + i, 1);
+      reader.feed(chunk);
+      status = parser.doParse();
+   }
+   return status;
+}
+
+int test_get_request_byte_at_a_time() {
+   StreamReader reader;
+   HTTPRequest req(reader);
+   std::string raw =
+      "GET /hello?x=1&y=two HTTP/1.1\r\n"
+      "Host: example.com\r\n"
+      "cookie: session=abc123\r\n"
+      "\r\n";
+   CHECK(feedByteAtATime(reader, req, raw) == Parser::Status::COMPLETE);
+   CHECK(req.getMethod() == "GET");
+   CHECK(req.getPath() == "/hello");
+   CHECK(req.getQueryString() == "x=1&y=two");
+   CHECK(req.getVersion() == "HTTP/1.1");
+   CHECK(req.getHeader("Host") == "example.com");
+   // Case-insensitive lookup: header arrived lowercase ("cookie:"), looked
+   // up with the canonical HTTP capitalization ("Cookie") - real clients
+   // and servers disagree on casing constantly, this must not care.
+   CHECK(req.getHeader("Cookie") == "session=abc123");
+   CHECK(req.getBody().getLength() == 0);
+   CHECK(req.getFormData() == NULL);
+   return 0;
+}
+
+int test_get_request_no_query_string() {
+   StreamReader reader;
+   HTTPRequest req(reader);
+   std::string raw = "GET /plain HTTP/1.1\r\nHost: h\r\n\r\n";
+   CHECK(feedByteAtATime(reader, req, raw) == Parser::Status::COMPLETE);
+   CHECK(req.getPath() == "/plain");
+   CHECK(req.getQueryString().getLength() == 0);
+   return 0;
+}
+
+int test_post_urlencoded_body_split_across_feeds() {
+   StreamReader reader;
+   HTTPRequest req(reader);
+   std::string raw =
+      "POST /submit HTTP/1.1\r\n"
+      "Host: example.com\r\n"
+      "Content-Type: application/x-www-form-urlencoded\r\n"
+      "Content-Length: 14\r\n"
+      "\r\n"
+      "name=A%20B&x=1";
+   // Split mid-body, not just mid-header - the whole point of building this
+   // on the same StreamReader/checkParse() chain CGIFormData already uses.
+   size_t splitAt = raw.size() - 5;
+   ZString part1((char *)raw.data(), splitAt);
+   reader.feed(part1);
+   CHECK(req.doParse() == Parser::Status::INCOMPLETE);
+   ZString part2((char *)raw.data() + splitAt, raw.size() - splitAt);
+   reader.feed(part2);
+   CHECK(req.doParse() == Parser::Status::COMPLETE);
+   CHECK(req.getMethod() == "POST");
+   CHECK(req.getFormData() != NULL);
+   MString name = (*req.getFormData())["name"];
+   CHECK(name == "A B");
+   MString x = (*req.getFormData())["x"];
+   CHECK(x == "1");
+   CHECK(req.getBody().getLength() == 0);   // consumed by the form parser, not the raw-body path
+   return 0;
+}
+
+int test_post_raw_json_body() {
+   StreamReader reader;
+   HTTPRequest req(reader);
+   std::string body = "{\"a\":1}";
+   std::string raw =
+      "POST /api HTTP/1.1\r\n"
+      "Host: example.com\r\n"
+      "Content-Type: application/json\r\n"
+      "Content-Length: " + std::to_string(body.size()) + "\r\n"
+      "\r\n" + body;
+   CHECK(feedByteAtATime(reader, req, raw) == Parser::Status::COMPLETE);
+   CHECK(req.getFormData() == NULL);
+   CHECK(req.getBody() == body.c_str());
+   return 0;
+}
+
+int test_real_fixture_file() {
+   // testing/http_request.data is a real captured request, but stored with
+   // plain LF line endings (a text-editing convenience, confirmed via
+   // `file`/`xxd` - not a literal wire capture); real HTTP requires CRLF
+   // per RFC 7230, and IMFHeader (which HTTPRequest's HEADERS state reuses
+   // as-is) enforces that strictly. Normalize LF -> CRLF before feeding,
+   // same as any real client would actually send it.
+   std::ifstream in("http_request.data");
+   CHECK((bool)in);
+   std::stringstream ss;
+   ss << in.rdbuf();
+   std::string raw = ss.str();
+   std::string crlf;
+   crlf.reserve(raw.size() + 16);
+   for (char c : raw) {
+      if (c == '\n') crlf += '\r';
+      crlf += c;
+   }
+
+   StreamReader reader;
+   HTTPRequest req(reader);
+   CHECK(feedByteAtATime(reader, req, crlf) == Parser::Status::COMPLETE);
+   CHECK(req.getMethod() == "GET");
+   CHECK(req.getPath() == "/");
+   CHECK(req.getHeader("Host") == "localhost:8080");
+   CHECK(req.getHeader("Cookie") == "_ga=GA1.1.1017526412.1628195401");
+   return 0;
+}
+
+int test_http_response_serialize() {
+   HTTPResponse resp;
+   resp.setStatus(200, "OK");
+   resp.setHeader("Content-Type", "text/plain");
+   resp.setBody("hello");
+   MString out = resp.serialize();
+   std::string s(out.getData(), out.getLength());
+   CHECK(s.find("HTTP/1.1 200 OK\r\n") == 0);
+   CHECK(s.find("Content-Type: text/plain\r\n") != std::string::npos);
+   CHECK(s.find("Content-Length: 5\r\n") != std::string::npos);
+   CHECK(s.find("\r\n\r\nhello") != std::string::npos);
+   return 0;
+}
+
+int test_http_response_explicit_content_length_respected() {
+   HTTPResponse resp;
+   resp.setStatus(404, "Not Found");
+   resp.setHeader("Content-Length", "0");
+   resp.setBody("");
+   MString out = resp.serialize();
+   std::string s(out.getData(), out.getLength());
+   CHECK(s.find("HTTP/1.1 404 Not Found\r\n") == 0);
+   int firstCL = s.find("Content-Length:");
+   int lastCL = s.rfind("Content-Length:");
+   CHECK(firstCL == lastCL);   // no duplicate Content-Length header
+   return 0;
+}
+
+int main() {
+   int rc = 0;
+   rc |= test_get_request_byte_at_a_time();
+   rc |= test_get_request_no_query_string();
+   rc |= test_post_urlencoded_body_split_across_feeds();
+   rc |= test_post_raw_json_body();
+   rc |= test_real_fixture_file();
+   rc |= test_http_response_serialize();
+   rc |= test_http_response_explicit_content_length_respected();
+   if (rc == 0)
+      std::cout << "ALL " << checks << " CHECKS PASSED" << std::endl;
+   else
+      std::cout << "SOME CHECKS FAILED (see above)" << std::endl;
+   return rc;
+}