JetCore/jetc/JetParser.cpp

#include "JetParser.h"
#include <cctype>
#include <stdexcept>
#include <set>
#include <cstdio>
#include <fstream>
#include <sstream>
#include <cstdio>

namespace jetc {

   bool Parser::starts(const std::string &lit) const {
      return s.compare(pos, lit.size(), lit) == 0;
   }

   bool Parser::isNameChar(char c) {
      return std::isalnum((unsigned char)c) || c == '-' || c == '_';
   }

   void Parser::skipWs() {
      while (!eof() && std::isspace((unsigned char)s[pos])) pos++;
   }

   std::string Parser::readName() {
      size_t start = pos;
      while (!eof() && isNameChar(s[pos])) pos++;
      return s.substr(start, pos - start);
   }

   std::string Parser::peekTagName(size_t at) const {
      size_t p = at + 1;
      size_t start = p;
      while (p < s.size() && isNameChar(s[p])) p++;
      return s.substr(start, p - start);
   }

   namespace {
      // jet-1.4 (the language this parser also has to accept, alongside
      // real 2.0 source) was fully case-insensitive on tag names - the
      // production source mixes case freely, even within one tag's own
      // body (e.g. <SET ...> and <set ...> side by side). Tag names are
      // matched case-insensitively throughout this parser for that
      // reason; nothing else (attribute values, text content) is
      // affected by this helper.
      bool ciEquals(const char *a, size_t alen, const std::string &b) {
         if (alen != b.size()) return false;
         for (size_t i = 0; i < alen; i++)
            if (std::tolower((unsigned char)a[i]) != std::tolower((unsigned char)b[i]))
               return false;
         return true;
      }
   }

   bool Parser::matchTagOpen(const std::string &name, size_t at) const {
      if (s.compare(at, 1, "<") != 0) return false;
      size_t p = at + 1;
      size_t nstart = p;
      while (p < s.size() && isNameChar(s[p])) p++;
      if (!ciEquals(s.data() + nstart, p - nstart, name)) return false;
      if (p >= s.size()) return false;
      char c = s[p];
      return c == ' ' || c == '>' || c == '/';
   }

   bool Parser::matchTagClose(const std::string &name, size_t at) const {
      if (s.compare(at, 2, "</") != 0) return false;
      size_t p = at + 2;
      size_t nstart = p;
      while (p < s.size() && isNameChar(s[p])) p++;
      if (!ciEquals(s.data() + nstart, p - nstart, name)) return false;
      return p < s.size() && s[p] == '>';
   }

   namespace {
      // jet-1.4's real parser knew which tags never take a body (its own
      // "void element" list) - kept here, ahead of skipOpenTag/
      // skipEmbeddedTag/parseFallback, since all of them need it.
      bool isVoidTag(const std::string &name) {
         static const std::set<std::string> voidTags = {
            // jet's own built-ins that never take a body. exclude is
            // NOT here - __exclude.cpp's own constructor requires a
            // real body (throws "exclude must have a container"
            // otherwise), it was previously miscategorized as void.
            "set", "include", "cookie", "header", "read", "write",
            "call", "system", "dump",
            // <container> (see parseContainerSlot/Tag::renderContainer) -
            // real content always writes it as a bare marker with no
            // separate closing tag, even when it carries its own
            // attributes (e.g. SESSIONCTL.tag's own
            // <CONTAINER EVAL="NO">) - without this it was previously only
            // reachable via the generic FALLBACK path (scanBody would
            // incorrectly scan forward for a "</container>" that never
            // exists, consuming unrelated following content).
            "container",
            // standard HTML5 void elements - real 1.4-era content (e.g.
            // SESSIONCTL.tag) writes these the classic HTML way too,
            // bare '>' and never a closing tag.
            "br", "img", "input", "hr", "meta", "link", "area", "base",
            "col", "embed", "source", "track", "wbr",
            // <else> is a marker inside an <if>/<while>/<until> body
            // (scanBody's own lookForElse handles it at the top level) -
            // it never has its own close tag, so when it shows up
            // embedded (inside a quoted attribute or another tag's
            // attribute region, same as <if> does in <linkbutton>) it
            // needs the same void treatment.
            "else"
         };
         std::string lower = name;
         for (char &c : lower) c = (char)std::tolower((unsigned char)c);
         return voidTags.count(lower) != 0;
      }
   }

   // Real 1.4 content nests real jet tags in two places a plain
   // quote/angle-bracket scanner can't handle on its own:
   //  1. Inside a quoted attribute value - SESSIONCTL.tag's <linkbutton>
   //     has HREF="javascript: <IF VALUE1="..." VALUE2="" TYPE=NE> ...
   //     </IF> ... " - the outer string and the embedded tag's own
   //     attributes reuse the same quote character with no escaping.
   //  2. Inside another tag's OWN opening-tag attribute region, before
   //     its closing '>' - linkbutton's <A ...> has a conditional
   //     <if value1="#[CLASS]" value2="" type="ne">CLASS="#[CLASS]"</if>
   //     sitting between its other attributes and its own '>' - the
   //     literal '>' that ends the embedded <if ...>'s OWN opening tag
   //     would otherwise be mistaken for <A>'s closing '>'.
   // skipEmbeddedTag() skips one such nested tag as a whole unit -
   // its own attributes (recursing through skipOpenTag, so an embedded
   // tag's embedded tag works too) and, if it isn't a void tag and wasn't
   // self-closed, its real body up through its own matching close tag
   // (via the same matchTagOpen/matchTagClose depth-tracking scanBody
   // uses at the top level). skipOpenTag's own attribute scan and
   // skipQuotedValue both call this for anything that looks like a
   // nested tag, instead of treating it as ordinary content.
   void Parser::skipEmbeddedTag(size_t &p) {
      size_t nameStart = p + 1;
      size_t nameEnd = nameStart;
      while (nameEnd < s.size() && isNameChar(s[nameEnd])) nameEnd++;
      std::string tagName = s.substr(nameStart, nameEnd - nameStart);
      bool selfClosing = skipOpenTag(p);
      if (!selfClosing && !isVoidTag(tagName)) {
         auto span = scanBody(p, tagName, false);
         p = span.first + 3 + tagName.size();
      }
   }

   void Parser::skipQuotedValue(size_t &p) {
      p++; // opening '"'
      while (p < s.size()) {
         if (s[p] == '"') { p++; return; }
         if (s[p] == '\\' && p + 1 < s.size()) { p += 2; continue; }
         if (s[p] == '<' && p + 1 < s.size() && isNameChar(s[p + 1])) {
            skipEmbeddedTag(p);
            continue;
         }
         if (s[p] == '<' && s.compare(p, 2, "</") == 0) {
            p += 2;
            while (p < s.size() && isNameChar(s[p])) p++;
            if (p < s.size() && s[p] == '>') p++;
            continue;
         }
         p++;
      }
   }

   // Advances p from '<' of "<comment" (any case - matchTagOpen already
   // confirmed the match before this is called) past the whole
   // <comment>...</comment> construct, including any nested <comment>
   // tags, counted the same case-insensitive way everything else in this
   // file matches tag names. Deliberately does NOT call skipOpenTag/
   // skipQuotedValue/skipEmbeddedTag or any other quote- or tag-aware
   // scanning on the body text - a comment containing an explanatory '"'
   // or '\"' (about the code itself, say) previously fell through to
   // parseFallback()'s generic path, which DOES apply those rules while
   // hunting for this same closing tag, and silently mislocated it
   // exactly like the already-documented JS-comment-corrupts-jetc bug
   // class, just triggered from inside a <comment> instead of a //
   // comment. This only ever advances past '<'/'>' boundaries it
   // recognizes as comment open/close - nothing inside can affect it.
   void Parser::skipCommentTag(size_t &p) {
      p += 8; // "<comment" (8 chars regardless of case)
      bool selfClosing = false;
      while (p < s.size() && s[p] != '>') {
         if (s[p] == '/' && p + 1 < s.size() && s[p + 1] == '>') selfClosing = true;
         p++;
      }
      if (p < s.size()) p++; // past '>'
      if (selfClosing) return;
      int depth = 1;
      while (p < s.size()) {
         if (matchTagOpen("comment", p)) {
            depth++;
            p++;
            continue;
         }
         if (matchTagClose("comment", p)) {
            p += 2; // "</"
            while (p < s.size() && isNameChar(s[p])) p++;
            if (p < s.size() && s[p] == '>') p++;
            depth--;
            if (depth == 0) return;
            continue;
         }
         p++;
      }
   }

   bool Parser::skipOpenTag(size_t &p) {
      p++; // '<'
      while (p < s.size() && isNameChar(s[p])) p++;
      while (p < s.size()) {
         if (s[p] == '"') {
            skipQuotedValue(p);
            continue;
         }
         if (s.compare(p, 2, "/>") == 0) { p += 2; return true; }
         if (s[p] == '>') { p += 1; return false; }
         if (s[p] == '<' && p + 1 < s.size() && isNameChar(s[p + 1])) {
            skipEmbeddedTag(p);
            continue;
         }
         p++;
      }
      throw std::runtime_error("jetc: unterminated tag near end of input");
   }

   std::map<std::string, std::string> Parser::parseAttrs(bool &hasContainer) {
      std::map<std::string, std::string> attrs;
      while (true) {
         skipWs();
         if (eof()) throw std::runtime_error("jetc: unterminated tag near end of input");
         if (starts("/>")) { pos += 2; hasContainer = false; break; }
         if (starts(">"))  { pos += 1; hasContainer = true;  break; }
         std::string aname = readName();
         if (aname.empty()) throw std::runtime_error("jetc: malformed attribute near offset " + std::to_string(pos));
         if (starts("=")) {
            pos += 1;
            std::string val;
            if (!eof() && s[pos] == '"') {
               pos++;
               while (!eof() && s[pos] != '"') {
                  if (s[pos] == '\\' && pos + 1 < s.size()) { val += s[pos]; val += s[pos + 1]; pos += 2; continue; }
                  val += s[pos++];
               }
               if (!eof()) pos++; // closing quote
            } else if (!eof() && s[pos] == '\'') {
               // Single-quoted value - real 1.4 content writes both
               // (SESSIONCTL.tag's own name='tmp_#[name]' alongside
               // ordinary name="..." elsewhere in the same file).
               pos++;
               while (!eof() && s[pos] != '\'') {
                  if (s[pos] == '\\' && pos + 1 < s.size()) { val += s[pos]; val += s[pos + 1]; pos += 2; continue; }
                  val += s[pos++];
               }
               if (!eof()) pos++; // closing quote
            } else {
               // Unquoted bareword value - real 1.4 content writes this
               // too (type=eq, format=tobrowser, TYPE=NE throughout
               // SESSIONCTL.tag) - the runtime interpreter's own
               // attribute scanning already tolerates it; jetc's own
               // parser previously didn't, since nothing had ever
               // recursively parsed a tag definition's own body deeply
               // enough to reach one before (see parseTagDef). Runs to
               // the next whitespace or the tag's own closing '>'/'/>' .
               while (!eof() && !std::isspace((unsigned char)s[pos]) && s[pos] != '>' &&
                      !(s[pos] == '/' && pos + 1 < s.size() && s[pos + 1] == '>')) {
                  val += s[pos++];
               }
            }
            attrs[aname] = val;
         }
      }
      return attrs;
   }

   std::pair<size_t, size_t> Parser::scanBody(size_t bodyStart, const std::string &tagName, bool lookForElse) {
      size_t p = bodyStart;
      int depth = 1;
      size_t elsePos = std::string::npos;
      while (p < s.size()) {
         if (matchTagClose(tagName, p)) {
            depth--;
            if (depth == 0) return {p, elsePos};
            p += 3 + tagName.size();
            continue;
         }
         if (lookForElse && depth == 1 && elsePos == std::string::npos && s.compare(p, 6, "<else>") == 0) {
            elsePos = p;
            p += 6;
            continue;
         }
         if (matchTagOpen(tagName, p)) {
            bool selfClosing = skipOpenTag(p);
            if (!selfClosing) depth++;
            continue;
         }
         if (s[p] == '<') {
            if (s.compare(p, 2, "</") == 0) {
               size_t q = p + 2;
               while (q < s.size() && isNameChar(s[q])) q++;
               if (q < s.size() && s[q] == '>') q++;
               p = q;
            } else {
               skipOpenTag(p);
            }
            continue;
         }
         p++;
      }
      throw std::runtime_error("jetc: unterminated <" + tagName + "> - no matching close tag found");
   }

   std::vector<NodePtr> Parser::parseSub(size_t start, size_t end) {
      // Always topLevel=false - an <if>/<for>/<while> body is never the
      // file's true top level, regardless of what topLevel this Parser
      // itself was constructed with (see parseTagDef/isVoidTag's doc
      // comments - a <tag name="X"> found here stays FALLBACK, unchanged).
      Parser sub(s.substr(start, end - start), baseDir, includeDepth, false);
      return sub.parseNodes();
   }

   std::vector<NodePtr> Parser::parseInclude() {
      pos += 8; // "<include"
      bool hasContainer;
      // <include> is always void, the same way <set> is (see parseSet) -
      // real content writes it bare (<include file="X">, no explicit
      // '/>'), which parseAttrs reports as hasContainer=true since it
      // can't tell that apart from a real container tag on syntax alone.
      auto attrs = parseAttrs(hasContainer);
      (void)hasContainer;
      auto it = attrs.find("file");
      if (it == attrs.end())
         throw std::runtime_error("jetc: <include> must have a file attribute.");
      if (includeDepth > 32)
         throw std::runtime_error("jetc: <include> depth exceeded (possible include cycle) resolving " + it->second);
      std::string fullPath = baseDir.empty() ? it->second : baseDir + "/" + it->second;
      std::ifstream in(fullPath);
      if (!in)
         throw std::runtime_error("jetc: <include> cannot open " + fullPath);
      std::ostringstream ss;
      ss << in.rdbuf();
      // Inherits the includer's own topLevel - an <include> spliced in at
      // the file's true top level reads as if its content were written
      // inline there (see this method's own doc comment), so a <tag
      // name="X"> defined in the included file is just as eligible for
      // compilation as one written directly in the includer.
      Parser sub(ss.str(), baseDir, includeDepth + 1, topLevel);
      return sub.parseNodes();
   }

   NodePtr Parser::parseVar() {
      size_t start = pos;
      pos += 2; // "$[" or "#["
      int depth = 1;
      while (!eof() && depth > 0) {
         if (starts("$[") || starts("#[")) { depth++; pos += 2; continue; }
         if (s[pos] == ']') { depth--; pos++; continue; }
         pos++;
      }
      auto n = std::make_shared<Node>();
      n->kind = NodeKind::VAR;
      n->varText = s.substr(start, pos - start);
      return n;
   }

   NodePtr Parser::parseSet() {
      pos += 4; // "<set"
      bool hasContainer;
      auto attrs = parseAttrs(hasContainer);
      // jet-1.4's <set name=... value=... expr=...> is always a plain
      // attribute-based assignment - it never had a container/body form.
      // Real 1.4 content (config.cfg, SESSIONCTL.tag, throughout) writes
      // it with a bare closing '>' and no explicit </set>, which is
      // syntactically indistinguishable from a (speculative, and per the
      // comment this replaced, never actually implemented) 2.0
      // container-form <set>...</set>. Since the container form was never
      // real to begin with, <set> is always treated as self-closing here,
      // matching 1.4's actual, exclusive behavior rather than guessing.
      (void)hasContainer;
      auto n = std::make_shared<Node>();
      n->kind = NodeKind::SET;
      n->setName = attrs.count("name") ? attrs["name"] : "";
      if (attrs.count("value")) { n->setHasValue = true; n->setValue = attrs["value"]; }
      if (attrs.count("expr"))  { n->setHasExpr = true;  n->setExpr = attrs["expr"]; }
      n->setScope = attrs.count("scope") ? attrs["scope"] : "";
      return n;
   }

   NodePtr Parser::parseIf() {
      pos += 3; // "<if"
      bool hasContainer;
      auto attrs = parseAttrs(hasContainer);
      auto n = std::make_shared<Node>();
      n->kind = NodeKind::IF;
      if (attrs.count("expr")) {
         n->cond.mode = CondMode::EXPR;
         n->cond.expr = attrs["expr"];
      } else {
         n->cond.mode = CondMode::VALUES;
         n->cond.value1 = attrs.count("value1") ? attrs["value1"] : "";
         n->cond.value2 = attrs.count("value2") ? attrs["value2"] : "";
         n->cond.type   = attrs.count("type")   ? attrs["type"]   : "";
      }
      if (hasContainer) {
         auto span = scanBody(pos, "if", true);
         size_t closePos = span.first, elsePos = span.second;
         size_t thenEnd = (elsePos != std::string::npos) ? elsePos : closePos;
         n->thenBody = parseSub(pos, thenEnd);
         if (elsePos != std::string::npos) {
            n->hasElse = true;
            n->elseBody = parseSub(elsePos + 6, closePos);
         }
         pos = closePos + 3 + 2; // "</if>"
      }
      return n;
   }

   NodePtr Parser::parseFor() {
      pos += 4; // "<for"
      bool hasContainer;
      auto attrs = parseAttrs(hasContainer);
      auto n = std::make_shared<Node>();
      n->kind = NodeKind::FOR;
      if (attrs.count("start")) { n->forHasStart = true; n->forStart = attrs["start"]; }
      n->forEnd  = attrs.count("end")  ? attrs["end"]  : "0";
      n->forStep = attrs.count("step") ? attrs["step"] : "1";
      if (attrs.count("name")) { n->forHasName = true; n->forName = attrs["name"]; }
      n->forScope = attrs.count("scope") ? attrs["scope"] : "";
      if (hasContainer) {
         auto span = scanBody(pos, "for", false);
         n->body = parseSub(pos, span.first);
         pos = span.first + 3 + 3; // "</for>"
      }
      return n;
   }

   NodePtr Parser::parseWhile() {
      pos += 6; // "<while"
      bool hasContainer;
      auto attrs = parseAttrs(hasContainer);
      auto n = std::make_shared<Node>();
      n->kind = NodeKind::WHILE;
      if (attrs.count("expr")) {
         n->cond.mode = CondMode::EXPR;
         n->cond.expr = attrs["expr"];
      } else {
         n->cond.mode = CondMode::VALUES;
         n->cond.value1 = attrs.count("value1") ? attrs["value1"] : "";
         n->cond.value2 = attrs.count("value2") ? attrs["value2"] : "";
         n->cond.type   = attrs.count("type")   ? attrs["type"]   : "";
      }
      if (hasContainer) {
         auto span = scanBody(pos, "while", false);
         n->body = parseSub(pos, span.first);
         pos = span.first + 3 + 5; // "</while>"
      }
      return n;
   }

   NodePtr Parser::parseFallback() {
      size_t origStart = pos;
      std::string tagName = peekTagName(pos);
      size_t p = pos;
      bool selfClosing = skipOpenTag(p);
      if (!selfClosing && !isVoidTag(tagName)) {
         auto span = scanBody(p, tagName, false);
         p = span.first + 3 + tagName.size();
      }
      pos = p;
      auto n = std::make_shared<Node>();
      n->kind = NodeKind::FALLBACK;
      n->rawSpan = s.substr(origStart, pos - origStart);
      return n;
   }

   NodePtr Parser::parseTagDef() {
      size_t origStart = pos;
      pos += 4; // "<tag"
      bool hasContainer;
      auto attrs = parseAttrs(hasContainer);
      std::string name = attrs.count("name") ? attrs["name"] : "";
      // A dynamic name ($[...]/#[...]) or a body-less definition can't be
      // (and, per __tag.cpp, isn't even valid as) a compiled definition -
      // fall back to the exact same FALLBACK behavior as before rather
      // than failing to compile. Real content always uses a literal name.
      bool dynamicName = name.find("$[") != std::string::npos || name.find("#[") != std::string::npos;
      if (!hasContainer || name.empty() || dynamicName) {
         if (hasContainer) {
            auto span = scanBody(pos, "tag", false);
            pos = span.first + 3 + 3; // "</tag>"
         }
         auto n = std::make_shared<Node>();
         n->kind = NodeKind::FALLBACK;
         n->rawSpan = s.substr(origStart, pos - origStart);
         return n;
      }
      std::string lowerName = name;
      for (char &c : lowerName) c = (char)std::tolower((unsigned char)c);

      auto span = scanBody(pos, "tag", false);
      size_t closePos = span.first;
      size_t afterClose = closePos + 3 + 3; // "</tag>"

      // Best-effort: this compiler's own modeled grammar is a real subset
      // of what the runtime interpreter tolerates (jet-1.4 content is
      // lenient in ways this parser doesn't fully cover yet - e.g. a
      // completely unrecognized bare tag like SESSIONCTL.tag's own
      // <setcookie ...>, which the interpreter just echoes as literal
      // text rather than treating as a real tag needing a close - see
      // Tag.cpp's own final dispatch fallback). Rather than teach every
      // such case to this parser up front, fall all the way back to the
      // exact same FALLBACK behavior a pre-TAGDEF compiler would have
      // produced for this whole span whenever the body can't be parsed -
      // this tag definition simply isn't compiled (stays fully
      // interpreted, unchanged), with zero risk to anything else in the
      // file. pos is never mutated by the failed parseSub call itself
      // (it operates on its own copied substring), so no explicit
      // rewind is needed before falling through.
      std::vector<NodePtr> body;
      bool bodyOk = true;
      try {
         body = parseSub(pos, closePos);
      } catch (const std::exception &) {
         bodyOk = false;
      }
      if (!bodyOk) {
         pos = afterClose;
         auto n = std::make_shared<Node>();
         n->kind = NodeKind::FALLBACK;
         n->rawSpan = s.substr(origStart, pos - origStart);
         return n;
      }
      pos = afterClose;

      auto n = std::make_shared<Node>();
      n->kind = NodeKind::TAGDEF;
      n->tagDefName = lowerName;
      n->tagDefBody = body;
      // Exact original source text - codegen still emits the same
      // interpretSpan-based raw registration into global.tags alongside
      // the new compiled fast path (see JetCodegen's TAGDEF case), so
      // anything that reaches this tag some other way (a nested/locally-
      // scoped definition shadowing it, or interpreted content elsewhere)
      // keeps working exactly as it does today.
      n->rawSpan = s.substr(origStart, pos - origStart);
      return n;
   }

   NodePtr Parser::parseContainerSlot() {
      pos += 10; // "<container"
      bool hasContainer;
      auto attrs = parseAttrs(hasContainer);
      // Attributes on <container> (eval=/name=/scope=, as real content
      // sometimes writes) are not consulted by the interpreter's own
      // <container> dispatch branch either (see Tag::renderContainer) -
      // matching that exactly rather than inventing new behavior here.
      (void)attrs;
      (void)hasContainer;
      auto n = std::make_shared<Node>();
      n->kind = NodeKind::CONTAINER_SLOT;
      return n;
   }

   std::vector<NodePtr> Parser::parseNodes() {
      std::vector<NodePtr> nodes;
      std::string textBuf;
      auto flush = [&]() {
         if (!textBuf.empty()) {
            auto n = std::make_shared<Node>();
            n->kind = NodeKind::TEXT;
            n->text = textBuf;
            nodes.push_back(n);
            textBuf.clear();
         }
      };
      while (!eof()) {
         if (starts("|>")) {
            flush();
            pos += 2;
            std::string buf;
            while (!eof() && !starts("<|")) buf += s[pos++];
            if (!eof()) pos += 2;
            auto n = std::make_shared<Node>();
            n->kind = NodeKind::TEXT;
            n->text = buf;
            nodes.push_back(n);
            continue;
         }
         if (s[pos] == '<') {
            if (matchTagOpen("set", pos))   { flush(); nodes.push_back(parseSet());   continue; }
            if (matchTagOpen("if", pos))    { flush(); nodes.push_back(parseIf());    continue; }
            if (matchTagOpen("for", pos))   { flush(); nodes.push_back(parseFor());   continue; }
            if (matchTagOpen("while", pos)) { flush(); nodes.push_back(parseWhile()); continue; }
            if (matchTagOpen("include", pos)) {
               flush();
               auto included = parseInclude();
               nodes.insert(nodes.end(), included.begin(), included.end());
               continue;
            }
            if (matchTagOpen("container", pos)) { flush(); nodes.push_back(parseContainerSlot()); continue; }
            if (matchTagOpen("comment", pos)) { flush(); skipCommentTag(pos); continue; }
            if (topLevel && matchTagOpen("tag", pos)) { flush(); nodes.push_back(parseTagDef()); continue; }
            flush();
            nodes.push_back(parseFallback());
            continue;
         }
         if (starts("$[") || starts("#[")) {
            flush();
            nodes.push_back(parseVar());
            continue;
         }
         textBuf += s[pos++];
      }
      flush();
      return nodes;
   }

}