token-parsers.h

//===-- lib/Parser/token-parsers.h ------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#ifndef FORTRAN_PARSER_TOKEN_PARSERS_H_
#define FORTRAN_PARSER_TOKEN_PARSERS_H_
 
// These parsers are driven by the parsers of the Fortran grammar to consume
// the prescanned character stream and recognize context-sensitive tokens.
 
#include "basic-parsers.h"
#include "type-parsers.h"
#include "flang/Parser/char-set.h"
#include "flang/Parser/characters.h"
#include "flang/Parser/instrumented-parser.h"
#include <cstddef>
#include <cstring>
#include <limits>
#include <list>
#include <optional>
#include <string>
 
namespace Fortran::parser {
 
// "xyz"_ch matches one instance of the characters x, y, or z without skipping
// any spaces before or after.  The parser returns the location of the character
// on success.
class AnyOfChars {
public:
  using resultType = const char *;
  constexpr AnyOfChars(const AnyOfChars &) = default;
  constexpr AnyOfChars(SetOfChars set) : set_{set} {}
  std::optional<const char *> Parse(ParseState &state) const {
    if (std::optional<const char *> at{state.PeekAtNextChar()}) {
      if (set_.Has(**at)) {
        state.UncheckedAdvance();
        state.set_anyTokenMatched();
        return at;
      }
    }
    state.Say(MessageExpectedText{set_});
    return std::nullopt;
  }
 
private:
  const SetOfChars set_;
};
 
constexpr AnyOfChars operator""_ch(const char str[], std::size_t n) {
  return AnyOfChars{SetOfChars(str, n)};
}
 
constexpr auto letter{"abcdefghijklmnopqrstuvwxyz"_ch};
constexpr auto digit{"0123456789"_ch};
 
// Skips over optional spaces.  Always succeeds.
struct Space {
  using resultType = Success;
  constexpr Space() {}
  static std::optional<Success> Parse(ParseState &state) {
    while (std::optional<const char *> p{state.PeekAtNextChar()}) {
      if (**p != ' ') {
        break;
      }
      state.UncheckedAdvance();
    }
    return {Success{}};
  }
};
constexpr Space space;
 
// Skips a space that in free form requires a warning if it precedes a
// character that could begin an identifier or keyword.  Always succeeds.
inline void MissingSpace(ParseState &state) {
  if (!state.inFixedForm()) {
    state.Nonstandard(
        LanguageFeature::OptionalFreeFormSpace, "missing space"_port_en_US);
  }
}
 
struct SpaceCheck {
  using resultType = Success;
  constexpr SpaceCheck() {}
  static std::optional<Success> Parse(ParseState &state) {
    if (std::optional<const char *> p{state.PeekAtNextChar()}) {
      char ch{**p};
      if (ch == ' ') {
        state.UncheckedAdvance();
        return space.Parse(state);
      }
      if (IsLegalInIdentifier(ch)) {
        MissingSpace(state);
      }
    }
    return {Success{}};
  }
};
constexpr SpaceCheck spaceCheck;
 
// Matches a token string.  Spaces in the token string denote where
// spaces may appear in the source; they can be made mandatory for
// some free form keyword sequences.  Missing mandatory spaces in free
// form elicit a warning; they are not necessary for recognition.
// Spaces before and after the token are also skipped.
//
// Token strings appear in the grammar as C++ user-defined literals
// like "BIND ( C )"_tok and "SYNC ALL"_sptok.  The _tok suffix is implied
// when a string literal appears before the sequencing operator >> or
// after the sequencing operator /.  The literal "..."_id parses a
// token that cannot be a prefix of a longer identifier.
template <bool MandatoryFreeFormSpace = false, bool MustBeComplete = false>
class TokenStringMatch {
public:
  using resultType = Success;
  constexpr TokenStringMatch(const TokenStringMatch &) = default;
  constexpr TokenStringMatch(const char *str, std::size_t n)
      : str_{str}, bytes_{n} {}
  explicit constexpr TokenStringMatch(const char *str) : str_{str} {}
  std::optional<Success> Parse(ParseState &state) const {
    space.Parse(state);
    const char *start{state.GetLocation()};
    const char *p{str_};
    std::optional<const char *> at; // initially empty
    for (std::size_t j{0}; j < bytes_ && *p != '\0'; ++j, ++p) {
      bool spaceSkipping{*p == ' '};
      if (spaceSkipping) {
        if (j + 1 == bytes_ || p[1] == ' ' || p[1] == '\0') {
          continue; // redundant; ignore
        }
      }
      if (!at) {
        at = nextCh.Parse(state);
        if (!at) {
          return std::nullopt;
        }
      }
      if (spaceSkipping) {
        if (**at == ' ') {
          at = nextCh.Parse(state);
          if (!at) {
            return std::nullopt;
          }
        } else if constexpr (MandatoryFreeFormSpace) {
          MissingSpace(state);
        }
        // 'at' remains full for next iteration
      } else if (**at == ToLowerCaseLetter(*p)) {
        at.reset();
      } else {
        state.Say(start, MessageExpectedText{str_, bytes_});
        return std::nullopt;
      }
    }
    if constexpr (MustBeComplete) {
      if (auto after{state.PeekAtNextChar()}) {
        if (IsLegalInIdentifier(**after)) {
          state.Say(start, MessageExpectedText{str_, bytes_});
          return std::nullopt;
        }
      }
    }
    state.set_anyTokenMatched();
    if (IsLegalInIdentifier(p[-1])) {
      return spaceCheck.Parse(state);
    } else {
      return space.Parse(state);
    }
  }
 
private:
  const char *const str_;
  const std::size_t bytes_{std::string::npos};
};
 
constexpr TokenStringMatch<> operator""_tok(const char str[], std::size_t n) {
  return {str, n};
}
 
constexpr TokenStringMatch<true> operator""_sptok(
    const char str[], std::size_t n) {
  return {str, n};
}
 
constexpr TokenStringMatch<false, true> operator""_id(
    const char str[], std::size_t n) {
  return {str, n};
}
 
template <class PA>
inline constexpr std::enable_if_t<std::is_class_v<PA>,
    SequenceParser<TokenStringMatch<>, PA>>
operator>>(const char *str, const PA &p) {
  return SequenceParser<TokenStringMatch<>, PA>{TokenStringMatch<>{str}, p};
}
 
template <class PA>
inline constexpr std::enable_if_t<std::is_class_v<PA>,
    FollowParser<PA, TokenStringMatch<>>>
operator/(const PA &p, const char *str) {
  return FollowParser<PA, TokenStringMatch<>>{p, TokenStringMatch<>{str}};
}
 
template <class PA> inline constexpr auto parenthesized(const PA &p) {
  return "(" >> p / ")";
}
 
template <class PA> inline constexpr auto bracketed(const PA &p) {
  return "[" >> p / "]";
}
 
template <class PA> inline constexpr auto braced(const PA &p) {
  return "{" >> p / "}";
}
 
// Quoted character literal constants.
struct CharLiteralChar {
  using resultType = std::pair<char, bool /* was escaped */>;
  static std::optional<resultType> Parse(ParseState &state) {
    auto at{state.GetLocation()};
    if (std::optional<const char *> cp{nextCh.Parse(state)}) {
      char ch{**cp};
      if (ch == '\n') {
        state.Say(CharBlock{at, state.GetLocation()},
            "Unclosed character constant"_err_en_US);
        return std::nullopt;
      }
      if (ch == '\\') {
        // Most escape sequences in character literals are processed later,
        // but we have to look for quotes here so that doubled quotes work.
        if (std::optional<const char *> next{state.PeekAtNextChar()}) {
          char escaped{**next};
          if (escaped == '\'' || escaped == '"' || escaped == '\\') {
            state.UncheckedAdvance();
            return std::make_pair(escaped, true);
          }
        }
      }
      return std::make_pair(ch, false);
    }
    return std::nullopt;
  }
};
 
template <char quote> struct CharLiteral {
  using resultType = std::string;
  static std::optional<std::string> Parse(ParseState &state) {
    std::string str;
    static constexpr auto nextch{attempt(CharLiteralChar{})};
    while (auto ch{nextch.Parse(state)}) {
      if (ch->second) {
        str += '\\';
      } else if (ch->first == quote) {
        static constexpr auto doubled{attempt(AnyOfChars{SetOfChars{quote}})};
        if (!doubled.Parse(state)) {
          return str;
        }
      }
      str += ch->first;
    }
    return std::nullopt;
  }
};
 
// Parse "BOZ" binary literal quoted constants.
// As extensions, support X as an alternate hexadecimal marker, and allow
// BOZX markers to appear as suffixes.
struct BOZLiteral {
  using resultType = std::string;
  static std::optional<resultType> Parse(ParseState &state) {
    char base{'\0'};
    auto baseChar{[&base](char ch) -> bool {
      switch (ch) {
      case 'b':
      case 'o':
      case 'z':
        base = ch;
        return true;
      case 'x':
        base = 'z';
        return true;
      default:
        return false;
      }
    }};
 
    space.Parse(state);
    const char *start{state.GetLocation()};
    std::optional<const char *> at{nextCh.Parse(state)};
    if (!at) {
      return std::nullopt;
    }
    if (**at == 'x' &&
        !state.IsNonstandardOk(LanguageFeature::BOZExtensions,
            "nonstandard BOZ literal"_port_en_US)) {
      return std::nullopt;
    }
    if (baseChar(**at)) {
      at = nextCh.Parse(state);
      if (!at) {
        return std::nullopt;
      }
    }
 
    char quote = **at;
    if (quote != '\'' && quote != '"') {
      return std::nullopt;
    }
 
    std::string content;
    while (true) {
      at = nextCh.Parse(state);
      if (!at) {
        return std::nullopt;
      }
      if (**at == quote) {
        break;
      }
      if (**at == ' ') {
        continue;
      }
      if (!IsHexadecimalDigit(**at)) {
        return std::nullopt;
      }
      content += ToLowerCaseLetter(**at);
    }
 
    if (!base) {
      // extension: base allowed to appear as suffix, too
      if (!(at = nextCh.Parse(state)) || !baseChar(**at) ||
          !state.IsNonstandardOk(LanguageFeature::BOZExtensions,
              "nonstandard BOZ literal"_port_en_US)) {
        return std::nullopt;
      }
      spaceCheck.Parse(state);
    }
 
    if (content.empty()) {
      state.Say(start, "no digit in BOZ literal"_err_en_US);
      return std::nullopt;
    }
    return {std::string{base} + '"' + content + '"'};
  }
};
 
// R711 digit-string -> digit [digit]...
// N.B. not a token -- no space is skipped
struct DigitString {
  using resultType = CharBlock;
  static std::optional<resultType> Parse(ParseState &state) {
    if (std::optional<const char *> ch1{state.PeekAtNextChar()}) {
      if (IsDecimalDigit(**ch1)) {
        state.UncheckedAdvance();
        while (std::optional<const char *> p{state.PeekAtNextChar()}) {
          if (!IsDecimalDigit(**p)) {
            break;
          }
          state.UncheckedAdvance();
        }
        return CharBlock{*ch1, state.GetLocation()};
      }
    }
    return std::nullopt;
  }
};
constexpr DigitString digitString;
 
struct SignedIntLiteralConstantWithoutKind {
  using resultType = CharBlock;
  static std::optional<resultType> Parse(ParseState &state) {
    resultType result{state.GetLocation()};
    static constexpr auto sign{maybe("+-"_ch / space)};
    if (sign.Parse(state)) {
      if (auto digits{digitString.Parse(state)}) {
        result.ExtendToCover(*digits);
        return result;
      }
    }
    return std::nullopt;
  }
};
 
struct DigitString64 {
  using resultType = std::uint64_t;
  static std::optional<std::uint64_t> Parse(ParseState &state) {
    std::optional<const char *> firstDigit{digit.Parse(state)};
    if (!firstDigit) {
      return std::nullopt;
    }
    std::uint64_t value = **firstDigit - '0';
    bool overflow{false};
    static constexpr auto getDigit{attempt(digit)};
    while (auto nextDigit{getDigit.Parse(state)}) {
      if (value > std::numeric_limits<std::uint64_t>::max() / 10) {
        overflow = true;
      }
      value *= 10;
      int digitValue = **nextDigit - '0';
      if (value > std::numeric_limits<std::uint64_t>::max() - digitValue) {
        overflow = true;
      }
      value += digitValue;
    }
    if (overflow) {
      state.Say(*firstDigit, "overflow in decimal literal"_err_en_US);
    }
    return {value};
  }
};
constexpr DigitString64 digitString64;
 
// R707 signed-int-literal-constant -> [sign] int-literal-constant
// N.B. Spaces are consumed before and after the sign, since the sign
// and the int-literal-constant are distinct tokens.  Does not
// handle a trailing kind parameter.
static std::optional<std::int64_t> SignedInteger(
    const std::optional<std::uint64_t> &x, Location at, bool negate,
    ParseState &state) {
  if (!x) {
    return std::nullopt;
  }
  std::uint64_t limit{std::numeric_limits<std::int64_t>::max()};
  if (negate) {
    limit = -(limit + 1);
  }
  if (*x > limit) {
    state.Say(at, "overflow in signed decimal literal"_err_en_US);
  }
  std::int64_t value = *x;
  return std::make_optional<std::int64_t>(negate ? -value : value);
}
 
// R710 signed-digit-string -> [sign] digit-string
// N.B. Not a complete token -- no space is skipped.
// Used only in the exponent parts of real literal constants.
struct SignedDigitString {
  using resultType = std::int64_t;
  static std::optional<std::int64_t> Parse(ParseState &state) {
    std::optional<const char *> sign{state.PeekAtNextChar()};
    if (!sign) {
      return std::nullopt;
    }
    bool negate{**sign == '-'};
    if (negate || **sign == '+') {
      state.UncheckedAdvance();
    }
    return SignedInteger(digitString64.Parse(state), *sign, negate, state);
  }
};
 
// Variants of the above for use in FORMAT specifications, where spaces
// must be ignored.
struct DigitStringIgnoreSpaces {
  using resultType = std::uint64_t;
  static std::optional<std::uint64_t> Parse(ParseState &state) {
    static constexpr auto getFirstDigit{space >> digit};
    std::optional<const char *> firstDigit{getFirstDigit.Parse(state)};
    if (!firstDigit) {
      return std::nullopt;
    }
    std::uint64_t value = **firstDigit - '0';
    bool overflow{false};
    static constexpr auto getDigit{space >> attempt(digit)};
    while (auto nextDigit{getDigit.Parse(state)}) {
      if (value > std::numeric_limits<std::uint64_t>::max() / 10) {
        overflow = true;
      }
      value *= 10;
      int digitValue = **nextDigit - '0';
      if (value > std::numeric_limits<std::uint64_t>::max() - digitValue) {
        overflow = true;
      }
      value += digitValue;
    }
    if (overflow) {
      state.Say(*firstDigit, "overflow in decimal literal"_err_en_US);
    }
    return value;
  }
};
 
struct PositiveDigitStringIgnoreSpaces {
  using resultType = std::int64_t;
  static std::optional<std::int64_t> Parse(ParseState &state) {
    Location at{state.GetLocation()};
    return SignedInteger(
        DigitStringIgnoreSpaces{}.Parse(state), at, false /*positive*/, state);
  }
};
 
struct SignedDigitStringIgnoreSpaces {
  using resultType = std::int64_t;
  static std::optional<std::int64_t> Parse(ParseState &state) {
    static constexpr auto getSign{space >> attempt("+-"_ch)};
    bool negate{false};
    if (std::optional<const char *> sign{getSign.Parse(state)}) {
      negate = **sign == '-';
    }
    Location at{state.GetLocation()};
    return SignedInteger(
        DigitStringIgnoreSpaces{}.Parse(state), at, negate, state);
  }
};
 
// Legacy feature: Hollerith literal constants
struct HollerithLiteral {
  using resultType = std::string;
  static std::optional<std::string> Parse(ParseState &state) {
    space.Parse(state);
    const char *start{state.GetLocation()};
    std::optional<std::uint64_t> charCount{
        DigitStringIgnoreSpaces{}.Parse(state)};
    if (!charCount || *charCount < 1) {
      return std::nullopt;
    }
    static constexpr auto letterH{"h"_ch};
    std::optional<const char *> h{letterH.Parse(state)};
    if (!h) {
      return std::nullopt;
    }
    std::string content;
    for (auto j{*charCount}; j-- > 0;) {
      int chBytes{UTF_8CharacterBytes(state.GetLocation())};
      for (int bytes{chBytes}; bytes > 0; --bytes) {
        if (std::optional<const char *> at{nextCh.Parse(state)}) {
          if (chBytes == 1 && !IsPrintable(**at)) {
            state.Say(start, "Bad character in Hollerith"_err_en_US);
            return std::nullopt;
          }
          content += **at;
        } else {
          state.Say(start, "Insufficient characters in Hollerith"_err_en_US);
          return std::nullopt;
        }
      }
    }
    return content;
  }
};
 
struct ConsumedAllInputParser {
  using resultType = Success;
  constexpr ConsumedAllInputParser() {}
  static inline std::optional<Success> Parse(ParseState &state) {
    if (state.IsAtEnd()) {
      return {Success{}};
    }
    return std::nullopt;
  }
};
constexpr ConsumedAllInputParser consumedAllInput;
 
template <char goal> struct SkipPast {
  using resultType = Success;
  constexpr SkipPast() {}
  constexpr SkipPast(const SkipPast &) {}
  static std::optional<Success> Parse(ParseState &state) {
    while (std::optional<const char *> p{state.GetNextChar()}) {
      if (**p == goal) {
        return {Success{}};
      } else if (**p == '\n') {
        break;
      }
    }
    return std::nullopt;
  }
};
 
template <char goal> struct SkipTo {
  using resultType = Success;
  constexpr SkipTo() {}
  constexpr SkipTo(const SkipTo &) {}
  static std::optional<Success> Parse(ParseState &state) {
    while (std::optional<const char *> p{state.PeekAtNextChar()}) {
      if (**p == goal) {
        return {Success{}};
      } else if (**p == '\n') {
        break;
      } else {
        state.UncheckedAdvance();
      }
    }
    return std::nullopt;
  }
};
 
template <char left, char right> struct SkipPastNested {
  using resultType = Success;
  constexpr SkipPastNested() {}
  constexpr SkipPastNested(const SkipPastNested &) {}
  static std::optional<Success> Parse(ParseState &state) {
    int nesting{1};
    while (std::optional<const char *> p{state.GetNextChar()}) {
      if (**p == right) {
        if (!--nesting) {
          return {Success{}};
        }
      } else if (**p == left) {
        ++nesting;
      } else if (**p == '\n') {
        break;
      }
    }
    return std::nullopt;
  }
};
 
// A common idiom in the Fortran grammar is an optional item (usually
// a nonempty comma-separated list) that, if present, must follow a comma
// and precede a doubled colon.  When the item is absent, the comma must
// not appear, and the doubled colons are optional.
//   [[, xyz] ::]     is  optionalBeforeColons(xyz)
//   [[, xyz]... ::]  is  optionalBeforeColons(nonemptyList(xyz))
template <typename PA> inline constexpr auto optionalBeforeColons(const PA &p) {
  using resultType = std::optional<typename PA::resultType>;
  return "," >> construct<resultType>(p) / "::" ||
      ("::"_tok || !","_tok) >> pure<resultType>();
}
template <typename PA>
inline constexpr auto optionalListBeforeColons(const PA &p) {
  using resultType = std::list<typename PA::resultType>;
  return "," >> nonemptyList(p) / "::" ||
      ("::"_tok || !","_tok) >> pure<resultType>();
}
 
// Skip over empty lines, leading spaces, and some compiler directives (viz.,
// the ones that specify the source form) that might appear before the
// next statement.  Skip over empty statements (bare semicolons) when
// not in strict standard conformance mode.  Always succeeds.
struct SkipStuffBeforeStatement {
  using resultType = Success;
  static std::optional<Success> Parse(ParseState &state) {
    if (UserState * ustate{state.userState()}) {
      if (ParsingLog * log{ustate->log()}) {
        // Save memory: vacate the parsing log before each statement unless
        // we're logging the whole parse for debugging.
        if (!ustate->instrumentedParse()) {
          log->clear();
        }
      }
    }
    while (std::optional<const char *> at{state.PeekAtNextChar()}) {
      if (**at == '\n' || **at == ' ') {
        state.UncheckedAdvance();
      } else if (**at == '!') {
        static const char fixed[] = "!dir$ fixed\n", free[] = "!dir$ free\n";
        static constexpr std::size_t fixedBytes{sizeof fixed - 1};
        static constexpr std::size_t freeBytes{sizeof free - 1};
        std::size_t remain{state.BytesRemaining()};
        if (remain >= fixedBytes && std::memcmp(*at, fixed, fixedBytes) == 0) {
          state.set_inFixedForm(true).UncheckedAdvance(fixedBytes);
        } else if (remain >= freeBytes &&
            std::memcmp(*at, free, freeBytes) == 0) {
          state.set_inFixedForm(false).UncheckedAdvance(freeBytes);
        } else {
          break;
        }
      } else if (**at == ';' &&
          state.IsNonstandardOk(
              LanguageFeature::EmptyStatement, "empty statement"_port_en_US)) {
        state.UncheckedAdvance();
      } else {
        break;
      }
    }
    return {Success{}};
  }
};
constexpr SkipStuffBeforeStatement skipStuffBeforeStatement;
 
// R602 underscore -> _
constexpr auto underscore{"_"_ch};
 
// Characters besides letters and digits that may appear in names.
// N.B. Don't accept an underscore if it is immediately followed by a
// quotation mark, so that kindParam_"character literal" is parsed properly.
// PGI and ifort accept '$' in identifiers, even as the initial character.
// Cray and gfortran accept '$', but not as the first character.
// Cray accepts '@' as well.
constexpr auto otherIdChar{underscore / !"'\""_ch ||
    extension<LanguageFeature::PunctuationInNames>(
        "nonstandard usage: punctuation in name"_port_en_US, "$@"_ch)};
 
constexpr auto logicalTRUE{
    (".TRUE."_tok ||
        extension<LanguageFeature::LogicalAbbreviations>(
            "nonstandard usage: .T. spelling of .TRUE."_port_en_US,
            ".T."_tok)) >>
    pure(true)};
constexpr auto logicalFALSE{
    (".FALSE."_tok ||
        extension<LanguageFeature::LogicalAbbreviations>(
            "nonstandard usage: .F. spelling of .FALSE."_port_en_US,
            ".F."_tok)) >>
    pure(false)};
 
// deprecated: Hollerith literals
constexpr auto rawHollerithLiteral{
    deprecated<LanguageFeature::Hollerith>(HollerithLiteral{})};
 
template <typename A> constexpr decltype(auto) verbatim(A x) {
  return sourced(construct<Verbatim>(x));
}
 
} // namespace Fortran::parser
#endif // FORTRAN_PARSER_TOKEN_PARSERS_H_