%{
// Copyright (c) 2016 University of Helsinki
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
// See the file COPYING included with this distribution for more
// information.

#define YYDEBUG 1

#include <stdio.h>
#include <assert.h>
#include <iostream>

#include "HfstTransducer.h"
#include "HfstInputStream.h"
#include "HfstXeroxRules.h"

using namespace hfst;
using hfst::HfstTransducer;
using namespace hfst::xeroxRules;
using namespace hfst::implementations;

#include "xre_utils.h"

namespace hfst
{
  namespace xre
  {
    // number of characters read, used for scanning function
    // definition xre for argument symbols
    extern unsigned int cr;
    extern bool harmonize_;
    extern bool harmonize_flags_;
    extern bool allow_extra_text_at_end;

    // to control how many times a warning is given
    bool has_weight_been_zeroed = false;
    float zero_weights(float f)
    {
        if ((! has_weight_been_zeroed) && (f != 0))
        {
            hfst::xre::warn("warning: ignoring weights in rule context\n");
            has_weight_been_zeroed = true;
        }
        return 0;
    }

    bool is_weighted()
    {
        return (hfst::xre::format == hfst::TROPICAL_OPENFST_TYPE ||
                hfst::xre::format == hfst::LOG_OPENFST_TYPE);
    }
  }
}

using hfst::xre::harmonize_;
using hfst::xre::harmonize_flags_;

union YYSTYPE;
struct yy_buffer_state;
typedef yy_buffer_state * YY_BUFFER_STATE;
typedef void * yyscan_t;

extern int xreparse(yyscan_t);
extern int xrelex_init (yyscan_t*);
extern YY_BUFFER_STATE xre_scan_string (const char *, yyscan_t);
extern void xre_delete_buffer (YY_BUFFER_STATE, yyscan_t);
extern int xrelex_destroy (yyscan_t);

extern int xreerror(yyscan_t, const char*);
extern int xreerror(const char*);
int xrelex ( YYSTYPE * , yyscan_t );

%}

%name-prefix="xre"
%define api.pure
%lex-param {void * scanner}
%parse-param {void * scanner}
%error-verbose
%debug

%union {

    int value;
    int* values;
    double weight;
    char* label;

    char *subval1, *subval2;

    hfst::HfstTransducer* transducer;
    hfst::HfstTransducerPair* transducerPair;
    hfst::HfstTransducerPairVector* transducerPairVector;
    hfst::HfstTransducerVector* transducerVector;

   std::pair<hfst::xeroxRules::ReplaceArrow,
             std::vector<hfst::xeroxRules::Rule> >* replaceRuleVectorWithArrow;
   std::pair< hfst::xeroxRules::ReplaceArrow,
              hfst::xeroxRules::Rule>* replaceRuleWithArrow;
   std::pair< hfst::xeroxRules::ReplaceArrow,
              hfst::HfstTransducerPairVector>* mappingVectorWithArrow;
   std::pair< hfst::xeroxRules::ReplaceArrow,
              hfst::HfstTransducerPair>* mappingWithArrow;

   std::pair<hfst::xeroxRules::ReplaceType,
             hfst::HfstTransducerPairVector>* contextWithMark;

   hfst::xeroxRules::ReplaceType replType;
   hfst::xeroxRules::ReplaceArrow replaceArrow;

}

/* some parts have been ripped from foma’s parser now */

%type <replaceArrow> REPLACE_ARROW

%type <transducer> XRE REGEXP1 REGEXP2  REGEXP4 REGEXP5 REGEXP6 REGEXP7
                    REGEXP8 REGEXP9 REGEXP10 REGEXP11 REGEXP12 LABEL
                   REPLACE REGEXP3 SUB1  SUB3 SYMBOL_LIST
%type <replaceRuleVectorWithArrow> PARALLEL_RULES
%type <replaceRuleWithArrow>  RULE
%type <mappingVectorWithArrow> MAPPINGPAIR_VECTOR
%type <mappingWithArrow> MAPPINGPAIR

%type <contextWithMark> CONTEXTS_WITH_MARK
%type <transducerPairVector> CONTEXTS_VECTOR RESTR_CONTEXTS_VECTOR
%type <transducerPair> CONTEXT RESTR_CONTEXT
%type <replType>  CONTEXT_MARK
%type <label>     HALFARC SUB2
%type <label>     SYMBOL_OR_QUOTED

%type <transducerVector> REGEXP_LIST   // function call
%type <label> FUNCTION                 // function call

%nonassoc <weight> WEIGHT
%nonassoc <label> SYMBOL MULTICHAR_SYMBOL CURLY_BRACKETS

%left  CROSS_PRODUCT COMPOSITION LENIENT_COMPOSITION INTERSECTION
       MERGE_RIGHT_ARROW MERGE_LEFT_ARROW
%left  CENTER_MARKER MARKUP_MARKER
%left  SHUFFLE
%right LEFT_RESTRICTION LEFT_ARROW RIGHT_ARROW LEFT_RIGHT_ARROW
%left  REPLACE_RIGHT REPLACE_LEFT
       OPTIONAL_REPLACE_RIGHT OPTIONAL_REPLACE_LEFT
       REPLACE_LEFT_RIGHT OPTIONAL_REPLACE_LEFT_RIGHT
       RTL_LONGEST_MATCH RTL_SHORTEST_MATCH
       LTR_LONGEST_MATCH LTR_SHORTEST_MATCH

%right REPLACE_CONTEXT_UU REPLACE_CONTEXT_LU
       REPLACE_CONTEXT_UL REPLACE_CONTEXT_LL

%left  UNION MINUS UPPER_MINUS LOWER_MINUS UPPER_PRIORITY_UNION
       LOWER_PRIORITY_UNION

%left  IGNORING IGNORE_INTERNALLY LEFT_QUOTIENT

%left  COMMACOMMA

%left  COMMA

%left  BEFORE AFTER

%nonassoc SUBSTITUTE_LEFT TERM_COMPLEMENT
%nonassoc COMPLEMENT CONTAINMENT CONTAINMENT_ONCE CONTAINMENT_OPT
%nonassoc REVERSE INVERT XRE_UPPER XRE_LOWER STAR PLUS
%nonassoc <values> CATENATE_N_TO_K
%nonassoc <value> CATENATE_N CATENATE_N_PLUS CATENATE_N_MINUS

%nonassoc <label> READ_BIN READ_TEXT READ_SPACED READ_PROLOG READ_RE
%nonassoc <label> FUNCTION_NAME   // function call
%token LEFT_BRACKET RIGHT_BRACKET LEFT_PARENTHESIS RIGHT_PARENTHESIS
       LEFT_BRACKET_DOTTED RIGHT_BRACKET_DOTTED SUBVAL
%token EPSILON_TOKEN ANY_TOKEN BOUNDARY_MARKER
%token LEXER_ERROR
%token END_OF_EXPRESSION
%token PAIR_SEPARATOR
%nonassoc <label> QUOTED_LITERAL QUOTED_MULTICHAR_LITERAL
%%

XRE: REGEXP1 { $$ = $1; }
     |
     {
       // only comments
       hfst::xre::contains_only_comments = true;
       return 0;
     }
     ;
REGEXP1: REGEXP2 END_OF_EXPRESSION {
       hfst::xre::last_compiled = $1;
       $$ = hfst::xre::last_compiled;
       if (hfst::xre::allow_extra_text_at_end)
       {
         return 0;
       }
   }
   | REGEXP2 {
        hfst::xre::last_compiled = $1;
        $$ = hfst::xre::last_compiled;
   }
;


REGEXP2: REPLACE {
            $$ = & $1->optimize();
         }
       | REGEXP2 COMPOSITION REPLACE
       {
        if ($1->has_flag_diacritics() && $3->has_flag_diacritics())
          {
            if (! harmonize_flags_)
            {
                 hfst::xre::warn("both composition arguments contain flag "
                                 "diacritics that are not harmonized\n");
            }
            else
            {
                $1->harmonize_flag_diacritics(*$3);
            }
          }

         try
         {
            $$ = & $1->compose(*$3, harmonize_).optimize();
         }
         catch (const FlagDiacriticsAreNotIdentitiesException & )
         {
            xreerror("flag diacritics must be identities in composition "
                     "if flag-is-epsilon is ON.\n"
                     "I.e. only FLAG:FLAG is allowed, not FLAG1:FLAG2, "
                     "FLAG:bar or foo:FLAG\n"
                     "Apply twosided flag-diacritics (tfd) "
                     "before composition.\n");
            YYABORT;
          }
          delete $3;
        }
       | REGEXP2 CROSS_PRODUCT REPLACE {
            $$ = & $1->cross_product(*$3).optimize();
            delete $3;
        }
       | REGEXP2 LENIENT_COMPOSITION REPLACE {
            $$ = & $1->lenient_composition(*$3).optimize();
            delete $3;
        }
       | REGEXP2 MERGE_RIGHT_ARROW REPLACE {
          try
          {
            $$ = & hfst::xre::merge_first_to_second($1, $3)->optimize();
          }
          catch (const TransducersAreNotAutomataException & )
          {
            xreerror("transducers must be automata in merge operation.");
            delete $1;
            delete $3;
            YYABORT;
          }
          delete $1;
       }
       | REGEXP2 MERGE_LEFT_ARROW REPLACE {
            $$ = & hfst::xre::merge_first_to_second($3, $1)->optimize();
            delete $3;
       }
        // substitute
       | SUB1 HALFARC PAIR_SEPARATOR HALFARC COMMA HALFARC PAIR_SEPARATOR HALFARC RIGHT_BRACKET {
            $1->substitute(StringPair($2,$4), StringPair($6,$8));
            $$ = & $1->optimize();
            free($2);
            free($4);
            free($6);
            free($8);
       }
       | SUB1 SUB2 SUB3 {

            StringSet alpha = $1->get_alphabet();
            if (hfst::xre::is_definition($2))
            {
                hfst::xre::warn("using definition as an ordinary label, "
                                "cannot substitute\n");
                $$ = & $1->optimize();
            }
            else if (alpha.find($2) == alpha.end())
            {
                $$ = & $1->optimize();
            }
            else
            {
                alpha = $3->get_alphabet();
                StringPair tmp($2, $2);
                HfstTransducer * tmpTr = new HfstTransducer(*$1);
                bool empty_replace_transducer=false;
                HfstTransducer empty(hfst::xre::format);
                if (empty.compare(*$3))
                {
                    empty_replace_transducer=true;
                }
                if (empty_replace_transducer)
                {
                    // substitute all transitions {b:a, a:b, b:b} with b:b
                    // as they will be removed anyway
                    hfst::xre::set_substitution_function_symbol($2);
                    tmpTr->substitute(&hfst::xre::substitution_function);
                }
                // `[ a:b, b, x y ]
                // substitute b with x | y
                tmpTr->substitute(tmp, *$3, false); // no harmonization
                if (!empty_replace_transducer)
                {
                    // a:b .o. b -> x | y
                    // [[a:b].i .o. b -> x | y].i -
                    // this is for cases when b is on left side
                    // build Replace transducer
                    HfstTransducerPair mappingPair(HfstTransducer($2, $2,
                                                   hfst::xre::format), *$3);
                    HfstTransducerPairVector mappingPairVector;
                    mappingPairVector.push_back(mappingPair);
                    Rule rule(mappingPairVector);
                    HfstTransducer replaceTr(hfst::xre::format);
                    replaceTr = replace(rule, false);
                    // if we are replacing with flag diacritics,
                    // the rule must allow flags to be replaced with themselves
                    StringSet alpha = $3->get_alphabet();
                    for (StringSet::const_iterator it = alpha.begin();
                         it != alpha.end(); it++)
                    {
                        if (FdOperation::is_diacritic(*it))
                        {
                            replaceTr.insert_freely(StringPair(*it, *it), false);
                        }
                    }
                    replaceTr.optimize();
                    tmpTr->compose(replaceTr).optimize();
                    tmpTr->invert().compose(replaceTr).invert();
                }
                if (alpha.find($2) == alpha.end())
                {
                  tmpTr->remove_from_alphabet($2);
                }
                tmpTr->optimize();
                $$ = tmpTr;
                delete $1;
                free($2);
                delete $3;
            }
         }
        ;

SUB1: SUBSTITUTE_LEFT LEFT_BRACKET REPLACE COMMA { $$ = $3; } ; // first argument
SUB2: HALFARC COMMA { $$ = $1; } ;  // symbol that needs to be replaced
SUB3: SYMBOL_LIST RIGHT_BRACKET {  $$ = $1;  }  // symbol list
      |
      RIGHT_BRACKET {
        $$ = new HfstTransducer(hfst::xre::format); // an empty symbol list
        }
      ;

////////////////////////////
// Replace operators
///////////////////////////

REPLACE : REGEXP3 { $$ = $1; }
       |  PARALLEL_RULES
         {
            switch ( $1->first )
            {
            case E_REPLACE_RIGHT:
                $$ = new HfstTransducer( replace( $1->second, false ) );
                break;
            case E_OPTIONAL_REPLACE_RIGHT:
                $$ = new HfstTransducer( replace( $1->second, true ) );
                break;
            case E_REPLACE_LEFT:
                $$ = new HfstTransducer( replace_left( $1->second, false ) );
                break;
            case E_OPTIONAL_REPLACE_LEFT:
                $$ = new HfstTransducer( replace_left( $1->second, true ) );
                break;
            case E_RTL_LONGEST_MATCH:
                $$ = new HfstTransducer(
                    replace_rightmost_longest_match( $1->second ) );
                break;
            case E_RTL_SHORTEST_MATCH:
                $$ = new HfstTransducer(
                    replace_rightmost_shortest_match($1->second) );
                break;
            case E_LTR_LONGEST_MATCH:
                $$ = new HfstTransducer(
                    replace_leftmost_longest_match( $1->second ) );
                break;
            case E_LTR_SHORTEST_MATCH:
                $$ = new HfstTransducer(
                    replace_leftmost_shortest_match( $1->second ) );
                break;
            case E_REPLACE_RIGHT_MARKUP:
                /* fall-through? */
            default:
                xreerror("Unhandled arrow stuff I suppose");
                YYABORT;
                break;
            }
            delete $1;
         }
      ;

PARALLEL_RULES: PARALLEL_RULES COMMACOMMA RULE
         {
            //std::cerr << "parallel_rules: parallel_rules ,, rule"<< std::endl;
            if ($3->first != $1->first)
            {
                delete $3;
                delete $1;
                xreerror("Replace type mismatch in parallel rules");
                YYABORT;
            }
            Rule tmpRule($3->second);
            $1->second.push_back(tmpRule);
            $$ =  new std::pair< ReplaceArrow, std::vector<Rule> > ($3->first,
                                                                    $1->second);
            delete $1; delete $3;
         }
         | RULE
         {
            //std::cerr << "parallel_rules:rule"<< std::endl;
            std::vector<Rule> * ruleVector = new std::vector<Rule>();
            ruleVector->push_back($1->second);
            $$ =  new std::pair< ReplaceArrow, std::vector<Rule> > ($1->first,
                                                                   *ruleVector);
            delete ruleVector;
            delete $1;
         }
         ;

RULE: MAPPINGPAIR_VECTOR
      {
        // std::cerr << "rule: mapping_vector"<< std::endl;
        // HfstTransducer allMappingsDisjuncted
        // = disjunctVectorMembers($1->second);
        Rule rule( $1->second );;
        $$ =  new std::pair< ReplaceArrow, Rule> ($1->first, rule);
        delete $1;
      }
      | MAPPINGPAIR_VECTOR CONTEXTS_WITH_MARK
      {
        //  std::cerr << "rule: mapping_vector contextsWM"<< std::endl;
        //   HfstTransducer allMappingsDisjuncted
        // = disjunctVectorMembers($1->second);
        Rule rule( $1->second, $2->second, $2->first );
        $$ =  new std::pair< ReplaceArrow, Rule> ($1->first, rule);
        delete $1;
        delete $2;
      }
      ;

// Mappings: ( ie. a -> b , c -> d , ... , g -> d)
MAPPINGPAIR_VECTOR: MAPPINGPAIR_VECTOR COMMA MAPPINGPAIR
      {
        // std::cerr << "mapping_vector : mapping_vector comma mapping"
        //           << std::endl;
        // check if new Arrow is the same as the first one
        if ($1->first != $3->first)
        {
            hfst::xre::warn("Replace arrows should be the same. "
                            "Calculated as if all replacements "
                            "had the first arrow.");
        }
        $1->second.push_back($3->second);
        $$ =  new std::pair< ReplaceArrow, HfstTransducerPairVector>($1->first,
                                                                    $1->second);
        delete $1;
        delete $3;
      }
      | MAPPINGPAIR
      {
        // std::cerr << "mapping_vector : mapping"<< std::endl;
        HfstTransducerPairVector * mappingPairVector =
            new HfstTransducerPairVector();
        mappingPairVector->push_back( $1->second );
        $$ =  new std::pair< ReplaceArrow, HfstTransducerPairVector> ($1->first,
                                                        * mappingPairVector);
        delete mappingPairVector;
        delete $1;
      }
      ;


MAPPINGPAIR: REPLACE REPLACE_ARROW REPLACE
      {
        hfst::xre::warn_about_special_symbols_in_replace($1);
        hfst::xre::warn_about_special_symbols_in_replace($3);
        HfstTransducerPair mappingPair(*$1, *$3);
        $$ =  new std::pair< ReplaceArrow, HfstTransducerPair> ($2,
                                                                mappingPair);
        delete $1;
        delete $3;
      }
      | REPLACE REPLACE_ARROW REPLACE MARKUP_MARKER REPLACE
      {
        HfstTransducerPair marks(*$3, *$5);
        HfstTransducerPair tmpMappingPair(*$1,
                                            HfstTransducer(hfst::xre::format));
        HfstTransducerPair mappingPair =
            create_mapping_for_mark_up_replace( tmpMappingPair, marks );
        $$ =  new std::pair< ReplaceArrow, HfstTransducerPair> ($2,
                                                                mappingPair);
        delete $1;
        delete $3;
        delete $5;
      }
      | REPLACE REPLACE_ARROW REPLACE MARKUP_MARKER
      {
        HfstTransducer epsilon(hfst::internal_epsilon, hfst::xre::format);
        HfstTransducerPair marks(*$3, epsilon);
        HfstTransducerPair tmpMappingPair(*$1,
                                            HfstTransducer(hfst::xre::format));
        HfstTransducerPair mappingPair =
                create_mapping_for_mark_up_replace( tmpMappingPair, marks );
        $$ =  new std::pair< ReplaceArrow, HfstTransducerPair> ($2,
                                                                mappingPair);
        delete $1;
        delete $3;
      }
      | REPLACE REPLACE_ARROW MARKUP_MARKER REPLACE
      {
        HfstTransducer epsilon(hfst::internal_epsilon, hfst::xre::format);
        HfstTransducerPair marks(epsilon, *$4);
        HfstTransducerPair tmpMappingPair(*$1,
                                          HfstTransducer(hfst::xre::format));
        HfstTransducerPair mappingPair =
                create_mapping_for_mark_up_replace( tmpMappingPair, marks );
        $$ =  new std::pair< ReplaceArrow, HfstTransducerPair> ($2,
                                                                mappingPair);
        delete $1;
        delete $4;
      }
       | LEFT_BRACKET_DOTTED RIGHT_BRACKET_DOTTED REPLACE_ARROW REPLACE
      {
        HfstTransducer epsilon(hfst::internal_epsilon, hfst::xre::format);
        //HfstTransducer mappingTr(epsilon);
        //mappingTr.cross_product(*$4);
        HfstTransducerPair mappingPair(epsilon, *$4);
        $$ =  new std::pair< ReplaceArrow, HfstTransducerPair> ($3,
                                                                mappingPair);
        delete $4;
      }
      | LEFT_BRACKET_DOTTED REPLACE RIGHT_BRACKET_DOTTED REPLACE_ARROW REPLACE
      {
        HfstTransducerPair mappingPair(*$2, *$5);
        $$ =  new std::pair< ReplaceArrow, HfstTransducerPair> ($4,
                                                                mappingPair);
        delete $2;
        delete $5;
      }
      | REPLACE REPLACE_ARROW LEFT_BRACKET_DOTTED RIGHT_BRACKET_DOTTED
      {
        HfstTransducer epsilon(hfst::internal_epsilon, hfst::xre::format);
        HfstTransducerPair mappingPair(*$1, epsilon);
        $$ =  new std::pair< ReplaceArrow, HfstTransducerPair> ($2,
                                                                mappingPair);
        delete $1;
      }
      | REPLACE REPLACE_ARROW LEFT_BRACKET_DOTTED REPLACE RIGHT_BRACKET_DOTTED
      {
        HfstTransducerPair mappingPair(*$1, *$4);
        $$ =  new std::pair< ReplaceArrow, HfstTransducerPair> ($2, mappingPair);
        delete $1;
        delete $4;
      }
      ;

// Contexts: ( ie. || k _ f , ... , f _ s )
CONTEXTS_WITH_MARK:  CONTEXT_MARK CONTEXTS_VECTOR
      {
        $$ =  new std::pair< ReplaceType, HfstTransducerPairVector> ($1, *$2);
        delete $2;
      }
      ;

CONTEXTS_VECTOR: CONTEXT
      {
        HfstTransducerPairVector * ContextVector = new
        HfstTransducerPairVector();
        ContextVector->push_back(*$1);
        $$ = ContextVector;
        delete $1;
      }

      | CONTEXTS_VECTOR COMMA CONTEXT
      {
        $1->push_back(*$3);
        $$ = $1;
        delete $3;
      }
      ;

CONTEXT: REPLACE CENTER_MARKER REPLACE
      {
        if (hfst::xre::has_non_identity_pairs($1))
        {
            xreerror("Contexts need to be automata");
            YYABORT;
        }
        if (hfst::xre::has_non_identity_pairs($3))
        {
            xreerror("Contexts need to be automata");
            YYABORT;
        }
        HfstTransducer t1(*$1);
        HfstTransducer t2(*$3);
        if (hfst::xre::is_weighted())
        {
            hfst::xre::has_weight_been_zeroed=false;
            t1.transform_weights(&hfst::xre::zero_weights);
        }
        t1.optimize().prune_alphabet(false);
        if (hfst::xre::is_weighted())
        {
            t2.transform_weights(&hfst::xre::zero_weights);
            hfst::xre::has_weight_been_zeroed=false;
        }
        t2.optimize().prune_alphabet(false);
        $$ = new HfstTransducerPair(t1, t2);
        delete $1;
        delete $3;
      }
      | REPLACE CENTER_MARKER
      {
        if (hfst::xre::has_non_identity_pairs($1))
        {
            xreerror("Contexts need to be automata");
            YYABORT;
        }
        HfstTransducer t1(*$1);
        if (hfst::xre::is_weighted())
        {
            hfst::xre::has_weight_been_zeroed=false;
            t1.transform_weights(&hfst::xre::zero_weights);
            hfst::xre::has_weight_been_zeroed=false;
        }
        t1.optimize().prune_alphabet(false);
        HfstTransducer epsilon(hfst::internal_epsilon, hfst::xre::format);
        $$ = new HfstTransducerPair(t1, epsilon);
        delete $1;
      }
      | CENTER_MARKER REPLACE
      {
        if (hfst::xre::has_non_identity_pairs($2)) // if non-identity symbols present..
        {
            xreerror("Contexts need to be automata");
            YYABORT;
        }
        HfstTransducer t1(*$2);
        if (hfst::xre::is_weighted())
        {
            hfst::xre::has_weight_been_zeroed=false;
            t1.transform_weights(&hfst::xre::zero_weights);
            hfst::xre::has_weight_been_zeroed=false;
        }
        t1.optimize().prune_alphabet(false);
        HfstTransducer epsilon(hfst::internal_epsilon, hfst::xre::format);
        $$ = new HfstTransducerPair(epsilon, t1);
        delete $2;
      }
      | CENTER_MARKER
      {
        HfstTransducer epsilon(hfst::internal_epsilon, hfst::xre::format);
        $$ = new HfstTransducerPair(epsilon, epsilon);
      }
      ;

CONTEXT_MARK: REPLACE_CONTEXT_UU
      {
        $$ = REPL_UP;
      }
      | REPLACE_CONTEXT_LU
      {
        $$ = REPL_RIGHT;
      }
      | REPLACE_CONTEXT_UL
      {
        $$ = REPL_LEFT;
      }
      | REPLACE_CONTEXT_LL
      {
        $$ = REPL_DOWN;
      }
      ;

REPLACE_ARROW: REPLACE_RIGHT
      {
        $$ = E_REPLACE_RIGHT;
      }
      | OPTIONAL_REPLACE_RIGHT
      {
        $$ = E_OPTIONAL_REPLACE_RIGHT;
      }
      | RTL_LONGEST_MATCH
      {
        $$ = E_RTL_LONGEST_MATCH;
      }
      | RTL_SHORTEST_MATCH
      {
        $$ = E_RTL_SHORTEST_MATCH;
      }
      | LTR_LONGEST_MATCH
      {
         $$ = E_LTR_LONGEST_MATCH;
      }
      | LTR_SHORTEST_MATCH
      {
        $$ = E_LTR_SHORTEST_MATCH;
      }
      |  REPLACE_LEFT
      {
        $$ =  E_REPLACE_LEFT;
      }
      |   OPTIONAL_REPLACE_LEFT
      {
        $$ = E_OPTIONAL_REPLACE_LEFT;
      }
      ;

////////////////
REGEXP3: REGEXP4 { $$ = $1; }
       | REGEXP3 SHUFFLE REGEXP4 {
            xreerror("No shuffle");
            //$$ = $1;
            delete $3;
            YYABORT;
       }
       | REGEXP3 BEFORE REGEXP4 {
            $$ = new HfstTransducer( before (*$1, *$3) );
            delete $1; delete $3;
       }
       | REGEXP3 AFTER REGEXP4 {
            $$ = new HfstTransducer( after (*$1, *$3) );
            delete $1; delete $3;
       }
       ;

REGEXP4: REGEXP5 { $$ = $1; }
       // restriction rule
       | REGEXP4 RIGHT_ARROW RESTR_CONTEXTS_VECTOR {
            $$ = new HfstTransducer( restriction(*$1, *$3) ) ;
            delete $1;
            delete $3;
       }
       // doesn't exist in xfst
       | REGEXP4 LEFT_ARROW REGEXP5 CENTER_MARKER REGEXP5 {
            xreerror("No Arrows");
            //$$ = $1;
            delete $3;
            delete $5;
            YYABORT;
       }
       // doesn't exist in xfst
       | REGEXP4 LEFT_RIGHT_ARROW REGEXP5 CENTER_MARKER REGEXP5 {
            xreerror("No Arrows");
            //$$ = $1;
            delete $3;
            delete $5;
            YYABORT;
       }
       ;

RESTR_CONTEXTS_VECTOR: RESTR_CONTEXT {
            HfstTransducerPairVector * ContextVector = new HfstTransducerPairVector();
            ContextVector->push_back(*$1);
            $$ = ContextVector;
            delete $1;
      }

      | RESTR_CONTEXTS_VECTOR COMMA RESTR_CONTEXT
      {
            $1->push_back(*$3);
            $$ = $1;
            delete $3;
      }
      ;

RESTR_CONTEXT: REGEXP4 CENTER_MARKER REGEXP4
      {
            $$ = new HfstTransducerPair(*$1, *$3);
            delete $1; delete $3;
      }
      | REGEXP4 CENTER_MARKER
      {
            // std::cerr << "Mapping: \n" << *$1  << std::endl;
            HfstTransducer epsilon(hfst::internal_epsilon, hfst::xre::format);
            // std::cerr << "Epsilon: \n" << epsilon  << std::endl;
            $$ = new HfstTransducerPair(*$1, epsilon);
            delete $1;
      }
      | CENTER_MARKER REGEXP4
      {
            HfstTransducer epsilon(hfst::internal_epsilon, hfst::xre::format);
            $$ = new HfstTransducerPair(epsilon, *$2);
            delete $2;
      }
      | CENTER_MARKER
      {
            HfstTransducer empty(hfst::xre::format);
            $$ = new HfstTransducerPair(empty, empty);
      }
      ;


REGEXP5: REGEXP6 { $$ = $1; }
       | REGEXP5 UNION REGEXP6 {
            $$ = & $1->disjunct(*$3, harmonize_);
            delete $3;
       }
       | REGEXP5 INTERSECTION REGEXP6 {
            // std::cerr << "Intersection: \n"  << std::endl;
            $$ = & $1->intersect(*$3, harmonize_).optimize()
                                .prune_alphabet(false);
            delete $3;
       }
       | REGEXP5 MINUS REGEXP6 {
            $$ = & $1->subtract(*$3, harmonize_).prune_alphabet(false);
            delete $3;
       }
       | REGEXP5 UPPER_MINUS REGEXP6 {
            xreerror("No upper minus");
            //$$ = $1;
            delete $3;
            YYABORT;
       }
       | REGEXP5 LOWER_MINUS REGEXP6 {
            xreerror("No lower minus");
            //$$ = $1;
            delete $3;
            YYABORT;
       }
       | REGEXP5 UPPER_PRIORITY_UNION REGEXP6 {
            $$ = & $1->priority_union(*$3);
            delete $3;
       }
       | REGEXP5 LOWER_PRIORITY_UNION REGEXP6 {
            HfstTransducer* left = new HfstTransducer(*$1);
            HfstTransducer* right =  new HfstTransducer(*$3);
            right->invert();
            left->invert();
            $$ = & (left->priority_union(*right).invert());
            delete $1; delete $3;
       }
       ;

REGEXP6: REGEXP7 { $$ = $1; }
       | REGEXP6 REGEXP7 {
        $$ = & $1->concatenate(*$2, harmonize_);
        delete $2;
       }
       ;

REGEXP7: REGEXP8 { $$ = $1; }
       | REGEXP7 IGNORING REGEXP8 {
            // this is how ignoring is done in foma and xfst
            $1->harmonize(*$3, true /*force harmonization also for foma type*/);
            $$ = & $1->insert_freely(*$3, false); // no harmonization
            delete $3;
       }
       | REGEXP7 IGNORE_INTERNALLY REGEXP8 {
            xreerror("No ignoring internally");
            //$$ = $1;
            delete $3;
            YYABORT;
       }
       | REGEXP7 LEFT_QUOTIENT REGEXP8 {
            xreerror("No left quotient");
            //$$ = $1;
            delete $3;
            YYABORT;
       }
       ;

REGEXP8: REGEXP9 { $$ = $1; }
       | COMPLEMENT REGEXP8 {
        // forbid pair complement (ie ~a:b)
        if (! $2->is_automaton())
        {
            xreerror("Complement operator ~ is defined only for automata\n"
                     "Use expression [[?:?] - A]] instead "
                     "where A is the relation to be complemented.");
            YYABORT;
        }
            HfstTransducer complement =
                HfstTransducer::identity_pair( hfst::xre::format );
            complement.repeat_star().optimize();
            complement.subtract(*$2).prune_alphabet(false);
            $$ = new HfstTransducer(complement);
            delete $2;
       }
       | CONTAINMENT REGEXP8 {
            // std::cerr << "Containment: \n" << std::endl;
            if (hfst::xre::has_non_identity_pairs($2))
            {
                hfst::xre::warn("using transducer that is not an automaton"
                              "in containment\n");
                $$ = hfst::xre::contains($2); // ..resort to simple containment
            }
            else
            {
                $$ = hfst::xre::contains_with_weight($2, 0);
            }
            delete $2;
       }
       | CONTAINMENT WEIGHT REGEXP8 {
            // std::cerr << "Containment: \n" << std::endl;
            if (hfst::xre::has_non_identity_pairs($3))
            {
              xreerror("Containment with weight only works with automata");
              YYABORT;
            }
            $$ = hfst::xre::contains_with_weight($3, hfst::double_to_float($2));
            delete $3;
       }
       | CONTAINMENT_ONCE REGEXP8 {
            //std::cerr << "Contain 1 \n"<< std::endl;

            $$ = hfst::xre::contains_once($2);
            delete $2;
       }
       | CONTAINMENT_OPT REGEXP8 {
            $$ = hfst::xre::contains_once_optional($2);
            delete $2;
       }
       ;

REGEXP9: REGEXP10 { $$ = $1; }
       | REGEXP9 STAR {
            $$ = & $1->repeat_star();
       }
       | REGEXP9 PLUS {
            $$ = & $1->repeat_plus();
       }
       | REGEXP9 REVERSE {
            $$ = & $1->reverse();
       }
       | REGEXP9 INVERT {
            $$ = & $1->invert();
       }
       | REGEXP9 XRE_UPPER {
            $$ = & $1->input_project();
       }
       | REGEXP9 XRE_LOWER {
            $$ = & $1->output_project();
       }
       | REGEXP9 CATENATE_N {
            $$ = & $1->repeat_n($2);
       }
       | REGEXP9 CATENATE_N_PLUS {
            //std::cerr << "value is ::::: \n"<< $2 << std::endl;
            $$ = & $1->repeat_n_plus($2+1);
       }
       | REGEXP9 CATENATE_N_MINUS {
            $$ = & $1->repeat_n_minus($2-1);
       }
       | REGEXP9 CATENATE_N_TO_K {
            $$ = & $1->repeat_n_to_k($2[0], $2[1]);
            free($2);
       }
       ;

REGEXP10: REGEXP11 { $$ = $1; }
        | TERM_COMPLEMENT REGEXP10 {
            HfstTransducer* any = new HfstTransducer(hfst::internal_identity,
                                        hfst::xre::format);
            $$ = & ( any->subtract(*$2));
            delete $2;
        }
        /*
       | SUBSTITUTE_LEFT REGEXP10 COMMA REGEXP10 COMMA REGEXP10 RIGHT_BRACKET {
            xreerror("no substitute");
            //$$ = $2;
            YYABORT;
        }
        */
        ;

REGEXP11: REGEXP12 { $$ = $1; }
        | LEFT_BRACKET REGEXP2 RIGHT_BRACKET {
            $$ = & $2->optimize();
        }
        // [foo]:[bar]
        | LEFT_BRACKET REGEXP2 RIGHT_BRACKET PAIR_SEPARATOR LEFT_BRACKET REGEXP2 RIGHT_BRACKET {
            $$ = & $2->cross_product(*$6);
            delete $6;
        }
        // [foo]:{bar}
        | LEFT_BRACKET REGEXP2 RIGHT_BRACKET PAIR_SEPARATOR CURLY_BRACKETS {
            HfstTransducer * tmp =
                hfst::xre::xfst_curly_label_to_transducer($5,$5);
            free($5);
            $$ = & $2->cross_product(*tmp);
            delete tmp;
        }
        // {foo}:[bar]
        | CURLY_BRACKETS PAIR_SEPARATOR LEFT_BRACKET REGEXP2 RIGHT_BRACKET {
            HfstTransducer * tmp =
                hfst::xre::xfst_curly_label_to_transducer($1,$1);
            free($1);
            $$ = & $4->cross_product(*tmp);
            delete tmp;
        }
        // [foo]:bar
        | LEFT_BRACKET REGEXP2 RIGHT_BRACKET PAIR_SEPARATOR HALFARC {
            HfstTransducer * tmp = hfst::xre::expand_definition($5);
            free($5);
            $$ = & $2->cross_product(*tmp);
            delete tmp;
        }
        // foo:[bar]
        | HALFARC PAIR_SEPARATOR LEFT_BRACKET REGEXP2 RIGHT_BRACKET {
            $$ = hfst::xre::expand_definition($1);
            free($1);
            $$ = & $$->cross_product(*$4);
            delete $4;
        }
        | LEFT_BRACKET REGEXP2 RIGHT_BRACKET WEIGHT {
            $$ = & $2->set_final_weights(hfst::double_to_float($4),
                                         true).optimize();
        }
        | LEFT_PARENTHESIS REGEXP2 RIGHT_PARENTHESIS {
            $$ = & $2->optionalize();
        }
        ;

// building 3rd argument in the substitute list
SYMBOL_LIST: HALFARC {
            if (strcmp($1, hfst::internal_unknown.c_str()) == 0)
              {
                $$ = new HfstTransducer(hfst::internal_identity, hfst::xre::format);
              }
            else
              {
                $$ = new HfstTransducer($1, $1, hfst::xre::format);
              }
            free($1);
        }
        | SYMBOL_LIST HALFARC {
            HfstTransducer * tmp ;
            if (strcmp($2, hfst::internal_unknown.c_str()) == 0)
              {
                 tmp = new HfstTransducer(hfst::internal_identity, hfst::xre::format);
              }
            else
              {
                 tmp = new HfstTransducer($2, $2, hfst::xre::format);
              }
            $1->disjunct(*tmp, false); // do not harmonize
            $$ = & $1->optimize();
            free($2);
            delete tmp;
            }
        ;

REGEXP12: LABEL { $$ = $1; }
        | LABEL WEIGHT {
            $$ = & $1->set_final_weights(hfst::double_to_float($2), true);
        }
        | READ_BIN {
            try {
                hfst::HfstInputStream instream($1);
                $$ = new HfstTransducer(instream);
                instream.close();
                free($1);
            }
            catch (const HfstException & )
            {
                char msg [256];
                sprintf(msg, "Error reading transducer file '%s'.", $1);
                xreerror(msg);
                free($1);
                YYABORT;
            }
        }
        | READ_TEXT {
            FILE * f = NULL;
            f = hfst::hfst_fopen($1, "r");
            free($1);
            if (f == NULL)
            {
                xreerror("File cannot be opened.\n");
                YYABORT;
            }
            else
            {
                HfstBasicTransducer tmp;
                HfstTokenizer tok;
                char line [1000];
                while( fgets(line, 1000, f) != NULL )
                {
                    hfst::xre::strip_newline(line);
                    StringPairVector spv = tok.tokenize(line, false); // XXX
                    tmp.disjunct(spv, 0);
                }
                fclose(f);
                HfstTransducer * retval =
                    new HfstTransducer(tmp, hfst::xre::format);
                retval->optimize();
                $$ = retval;
            }
        }
        | READ_SPACED {
            FILE * f = NULL;
            f = hfst::hfst_fopen($1, "r");
            free($1);
            if (f == NULL)
            {
                xreerror("File cannot be opened.\n");
                YYABORT;
            }
            else
            {
                HfstTokenizer tok;
                HfstBasicTransducer tmp;
                char line [1000];
                while( fgets(line, 1000, f) != NULL )
                {
                    hfst::xre::strip_newline(line);
                    StringPairVector spv =
                        HfstTokenizer::tokenize_space_separated(line);
                    tmp.disjunct(spv, 0);
                }
                fclose(f);
                HfstTransducer * retval =
                    new HfstTransducer(tmp, hfst::xre::format);
                retval->optimize();
                $$ = retval;
            }
        }
        | READ_PROLOG {
            FILE * f = NULL;
            f = hfst::hfst_fopen($1, "r");
            free($1);
            if (f == NULL)
            {
              xreerror("File cannot be opened.\n");
              YYABORT;
            }
            else
            {
                try
                {
                    unsigned int linecount = 0;
                    HfstBasicTransducer tmp =
                        HfstBasicTransducer::read_in_prolog_format(f,
                            linecount);
                    fclose(f);
                    HfstTransducer * retval = new HfstTransducer(tmp, hfst::xre::format);
                    retval->optimize();
                    $$ = retval;
                }
                catch (const HfstException & )
                {
                    fclose(f);
                    xreerror("Error reading prolog file.\n");
                    YYABORT;
                }
            }
        }
        | READ_RE {
            FILE * f = NULL;
            f = hfst::hfst_fopen($1, "r");
            if (f == NULL)
            {
                xreerror("File cannot be opened.\n");
                fclose(f);
                free($1);
                YYABORT;
            }
            else
            {
                fclose(f);
                // read the regex in a string
                std::ifstream ifs($1);
                free($1);
                std::stringstream buffer;
                buffer << ifs.rdbuf();
                char * regex_string = strdup(buffer.str().c_str());
                // create a new scanner for evaluating the regex
                yyscan_t scanner;
                xrelex_init(&scanner);
                YY_BUFFER_STATE bs = xre_scan_string(regex_string, scanner);
                unsigned int chars_read = hfst::xre::cr;
                hfst::xre::cr = 0;
                int parse_retval = xreparse(scanner);
                xre_delete_buffer(bs,scanner);
                xrelex_destroy(scanner);
                free(regex_string);
                hfst::xre::cr = chars_read;
                $$ = hfst::xre::last_compiled;
                if (parse_retval != 0)
                {
                    xreerror("Error parsing regex.\n");
                    YYABORT;
                }
            }
        }
        ;

LABEL: HALFARC {
        if (strcmp($1, hfst::internal_unknown.c_str()) == 0)
          {
            $$ = new HfstTransducer(hfst::internal_identity, hfst::xre::format);
          }
        else
          {
            // HfstTransducer * tmp = new HfstTransducer($1, hfst::xre::format);
            // $$ = hfst::xre::expand_definition(tmp, $1);
            $$ = hfst::xre::expand_definition($1);
          }
        free($1);
     }
     |
     HALFARC PAIR_SEPARATOR HALFARC {
        $$ = hfst::xre::xfst_label_to_transducer($1,$3);
        free($1);
        free($3);
     }
     | HALFARC PAIR_SEPARATOR CURLY_BRACKETS {
        $$ = hfst::xre::xfst_label_to_transducer($1,$1);
        free($1);
        HfstTransducer * tmp = hfst::xre::xfst_curly_label_to_transducer($3,$3);
        free($3);
        $$ = & $$->cross_product(*tmp);
        delete tmp;
     }
     | CURLY_BRACKETS PAIR_SEPARATOR HALFARC {
        HfstTransducer * tmp = hfst::xre::xfst_label_to_transducer($3,$3);
        free($3);
        $$ = hfst::xre::xfst_curly_label_to_transducer($1,$1);
        free($1);
        $$ = & $$->cross_product(*tmp);
        delete tmp;
     }
     | CURLY_BRACKETS {
        $$ = hfst::xre::xfst_curly_label_to_transducer($1,$1);
        free($1);
     }
     | CURLY_BRACKETS PAIR_SEPARATOR CURLY_BRACKETS {
        $$ = hfst::xre::xfst_curly_label_to_transducer($1,$3);
        free($1);
        free($3);
     }
     // function call
     | FUNCTION REGEXP_LIST RIGHT_PARENTHESIS {
        if (! hfst::xre::is_valid_function_call($1, $2))
        {
            delete $1;
            delete $2;
            return EXIT_FAILURE;
        }
        else
        {
            // create a new scanner for evaluating the function
            yyscan_t scanner;
            xrelex_init(&scanner);
            YY_BUFFER_STATE bs =
                xre_scan_string(hfst::xre::get_function_xre($1),
                                scanner);
            // define special variables so that function arguments get
            // the values given in regexp list
            if (! hfst::xre::define_function_args($1, $2))
            {
                xreerror("Could not define function args.\n");
                free($1);
                delete $2;
                YYABORT;
            }
            delete $2;
            // if we are scanning a function definition for argument symbols,
            // do not include the characters read
            // when evaluating functions inside it
            unsigned int chars_read = hfst::xre::cr;
            int parse_retval = xreparse(scanner);
            hfst::xre::cr = chars_read;
            hfst::xre::undefine_function_args($1);
            free($1);
            xre_delete_buffer(bs,scanner);
            xrelex_destroy(scanner);
            $$ = hfst::xre::last_compiled;
            if (parse_retval != 0)
            {
                YYABORT;
            }
        }
      }
      /*
      | QUOTE {
        HfstTokenizer TOK;
        $$ = new HfstTransducer($1, TOK, hfst::xre::format);
        free($1);
      }
      */
      ;

SYMBOL_OR_QUOTED: SYMBOL
     | MULTICHAR_SYMBOL {
        hfst::xre::check_multichar_symbol($1);
        $$ = $1;
     }
     | QUOTED_MULTICHAR_LITERAL {
        hfst::xre::check_multichar_symbol($1);
        $$ = $1;
     }
     | QUOTED_LITERAL
     ;

HALFARC: SYMBOL_OR_QUOTED
     {
        // Symbols of form <foo> are not harmonized in xfst, that is why
        // they need to be escaped as @_<foo>_@.
        // $$ = hfst::xre::escape_enclosing_angle_brackets($1);
        hfst::xre::warn_about_hfst_special_symbol($1);
        hfst::xre::warn_about_xfst_special_symbol($1);
        $$ = $1;
     }
     | EPSILON_TOKEN {
        $$ = strdup(hfst::internal_epsilon.c_str());
     }
     | ANY_TOKEN {
        $$ = strdup(hfst::internal_unknown.c_str());
     }
     | BOUNDARY_MARKER {
        $$ = strdup("@#@");
     }
     ;

REGEXP_LIST: REGEXP_LIST COMMA REGEXP2 {
        $$->push_back(*($3));
        delete $3;
     }
     | REGEXP2 {
        $$ = new hfst::HfstTransducerVector();
        $$->push_back(*($1));
        delete $1;
     }
     ;

FUNCTION: FUNCTION_NAME {
        $$ = strdup($1);
    };

%%

