stedolan-jq/lexer.l

%{
#include <assert.h>
#include "jv_alloc.h"
#include "compile.h"

struct lexer_param;

#include "parser.h"  /* Generated by bison. */

#define YY_USER_ACTION                           \
  do {                                           \
    yylloc->start = yyget_extra(yyscanner);      \
    yylloc->end = yylloc->start + yyleng;        \
    yyset_extra(yylloc->end, yyscanner);         \
  } while (0);

%}

%s IN_PAREN
%s IN_BRACKET
%s IN_BRACE
%s IN_QQINTERP
%x IN_QQSTRING
%{
  static int enter(int opening, int state, yyscan_t yyscanner);
  static int try_exit(int closing, int state, yyscan_t yyscanner);
%}

%option noyywrap nounput noinput nodefault
%option noyyalloc noyyrealloc noyyfree
%option reentrant
%option extra-type="int"
%option bison-bridge bison-locations
%option prefix="jq_yy"
%option stack


%%

"#"[^\r\n]* { /* comments */ }

"!=" { return NEQ; }
"==" { return EQ; }
"as" { return AS; }
"import" { return IMPORT; }
"module" { return MODULE; }
"def" { return DEF; }
"if" { return IF; }
"then" { return THEN; }
"else" { return ELSE; }
"elif" { return ELSE_IF; }
"and" { return AND; }
"or" { return OR; }
"end" { return END; }
"reduce" { return REDUCE; }
"foreach" { return FOREACH; }
"//" { return DEFINEDOR; }
"try" { return TRY; }
"catch" { return CATCH; }
"label" { return LABEL; }
"break" { return BREAK; }
"__loc__" { return LOC; }
"|=" { return SETPIPE; }
"+=" { return SETPLUS; }
"-=" { return SETMINUS; }
"*=" { return SETMULT; }
"/=" { return SETDIV; }
"%=" { return SETMOD; }
"//=" { return SETDEFINEDOR; }
"<=" { return LESSEQ; }
">=" { return GREATEREQ; }
".." { return REC; }
"."|"?"|"="|";"|","|":"|"|"|"+"|"-"|"*"|"/"|"%"|"\$"|"<"|">" { return yytext[0];}

"["|"{"|"(" {
  return enter(yytext[0], YY_START, yyscanner);
}

"]"|"}"|")" {
  return try_exit(yytext[0], YY_START, yyscanner);
}

"@"[a-zA-Z0-9_]+ {
  yylval->literal = jv_string_sized(yytext + 1, yyleng - 1); return FORMAT;
}

[0-9.]+([eE][+-]?[0-9]+)? { 
   yylval->literal = jv_parse_sized(yytext, yyleng); return LITERAL; 
}

"\"" {
  yy_push_state(IN_QQSTRING, yyscanner);
  return QQSTRING_START;
}

<IN_QQSTRING>{
  "\\(" {
    return enter(QQSTRING_INTERP_START, YY_START, yyscanner);
  }
  "\"" {
    yy_pop_state(yyscanner);
    return QQSTRING_END;
  }
  (\\[^u(]|\\u[a-zA-Z0-9]{0,4})+ {
    /* pass escapes to the json parser */
    jv escapes = jv_string_fmt("\"%.*s\"", (int)yyleng, yytext);
    yylval->literal = jv_parse_sized(jv_string_value(escapes), jv_string_length_bytes(jv_copy(escapes)));
    jv_free(escapes);
    return QQSTRING_TEXT;
  }
  [^\\\"]+ {
    yylval->literal = jv_string_sized(yytext, yyleng);
    return QQSTRING_TEXT;
  }
  . {
    return INVALID_CHARACTER;
  }
}
  

([a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]*  { yylval->literal = jv_string(yytext); return IDENT;}
\.[a-zA-Z_][a-zA-Z_0-9]*  { yylval->literal = jv_string(yytext+1); return FIELD;}

[ \n\t]+  {}

. { return INVALID_CHARACTER; }

%%
/* perhaps these should be calls... */
/*
"true" { return TRUE; }
"false" { return FALSE; }
"null" { return NULL; }
*/
static int try_exit(int c, int state, yyscan_t yyscanner) {
  char match = 0;
  int ret;
  switch (state) {
  case IN_PAREN: match = ret = ')'; break;
  case IN_BRACKET: match = ret = ']'; break;
  case IN_BRACE: match = ret = '}'; break;

  case IN_QQINTERP:
    match = ')'; 
    ret = QQSTRING_INTERP_END;
    break;

  default:
    // may not be the best error to give
    return INVALID_CHARACTER;
  }
  assert(match);
  if (match == c) {
    yy_pop_state(yyscanner);
    return ret;
  } else {
    // FIXME: should we pop? Give a better error at least
    return INVALID_CHARACTER;
  }
}

static int enter(int c, int currstate, yyscan_t yyscanner) {
  int state = 0;
  switch (c) {
  case '(': state = IN_PAREN; break;
  case '[': state = IN_BRACKET; break;
  case '{': state = IN_BRACE; break;
  case QQSTRING_INTERP_START: state = IN_QQINTERP; break;
  }
  assert(state);
  yy_push_state(state, yyscanner);
  return c;
}

void* yyalloc(size_t sz, void* extra) {
  return jv_mem_alloc(sz);
}
void* yyrealloc(void* p, size_t sz, void* extra) {
  return jv_mem_realloc(p, sz);
}
void yyfree(void* p, void* extra) {
  jv_mem_free(p);
}