// Copyright (c) 2016 University of Helsinki
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
// See the file COPYING included with this distribution for more
// information.

/**
 * @file xre.cc
 *
 * @brief implements xre routines.
 */

#include <cassert>
#include <cstdlib>
#include <cstring>
#include <unistd.h>

#include "HfstTransducer.h"
#include "HfstXeroxRules.h"
#include "XreCompiler.h"
#include "xre_utils.h"

using std::map;
using std::ostringstream;
using std::string;

struct yy_buffer_state;
typedef yy_buffer_state *YY_BUFFER_STATE;
typedef void *yyscan_t;
extern int xreparse(yyscan_t);
extern int xrelex_init(yyscan_t *);
extern YY_BUFFER_STATE xre_scan_string(const char *, yyscan_t);
extern void xre_delete_buffer(YY_BUFFER_STATE, yyscan_t);
extern int xrelex_destroy(yyscan_t);
extern char *xreget_text(yyscan_t);

namespace hfst
{
namespace xre
{
extern unsigned int cr; // number of characters read, defined in XreCompiler.cc
extern unsigned int lr; // number of lines read, defined in XreCompiler.cc
bool allow_extra_text_at_end = false;
extern std::ostream *error_;
extern bool verbose_;
extern std::set<std::string> *defined_multichar_symbols_;
}
}

std::ostream *
xreerrstr()
{
    return hfst::xre::XreCompiler::get_stream(hfst::xre::error_);
}

void
xreflush(std::ostream *os)
{
    hfst::xre::XreCompiler::flush(os);
}

#define COLOUR_BOLD "\033[01m"
#define COLOUR_RED "\033[31m"
#define COLOUR_GREEN "\033[32m"
#define COLOUR_YELLOW "\033[33m"
#define COLOUR_BLUE "\033[34m"
#define COLOUR_MAGENTA "\033[35m"
#define COLOUR_CYAN "\033[36m"
#define COLOUR_RESET "\033[0m"

static bool
should_colourise()
{
    if (isatty(1))
    {
        return true;
    }
    else
    {
        return false;
    }
    return false;
}

int
xreerror(yyscan_t scanner, const char *msg)
{
    if (hfst::xre::verbose_)
    {
        const char *scanner_msg = xreget_text(scanner);
        char *buffer = (char *)malloc(strlen(msg) + strlen(hfst::xre::data)
                                      + strlen(scanner_msg) + 100);
        char *p = buffer;
        if (should_colourise())
        {
            int n = sprintf(p, COLOUR_RED);
            assert(n >= 0);
            p += n;
        }
        int n = sprintf(p, "*** xre parsing failed: %s\n", msg);
        assert(n >= 0);
        p += n;
        if (should_colourise())
        {
            n = sprintf(p, COLOUR_RED);
            assert(n >= 0);
            p += n;
        }
        if (strlen(hfst::xre::data) < 60)
        {
            n = sprintf(p, "***    parsing %s [near %s] on line %u\n%c",
                        hfst::xre::data, xreget_text(scanner), hfst::xre::lr,
                        '\0');
        }
        else
        {
            n = sprintf(p, "***    parsing %60s [near %s] on line %u...\n%c",
                        hfst::xre::data, xreget_text(scanner), hfst::xre::lr,
                        '\0');
        }
        if (should_colourise())
        {
            n = sprintf(p, COLOUR_RESET);
            assert(n >= 0);
            p += n;
        }
        std::ostream *err = xreerrstr();
        *(err) << std::string(buffer);
        free(buffer);
        xreflush(err);
    }
    return 0;
}

int
xreerror(const char *msg)
{
    char *buffer = static_cast<char *>(malloc(strlen(msg) + 128));
    char *p = buffer;
    if (should_colourise())
    {
        int n = sprintf(p, COLOUR_RED);
    }
    int n = sprintf(p, "*** xre parsing failed: %s\n", msg);
    if (should_colourise())
    {
        n = sprintf(p, COLOUR_RESET);
    }
    std::ostream *err = xreerrstr();
    *err << std::string(buffer);
    xreflush(err);
    return 0;
}

namespace hfst
{
namespace xre
{

char *data;
std::map<std::string, hfst::HfstTransducer *> definitions;
std::map<std::string, std::string> function_definitions;
std::map<std::string, unsigned int> function_arguments;
std::map<std::string, std::set<string> > symbol_lists;
char *startptr; // changed this to an internal variable in compile functions
hfst::HfstTransducer *last_compiled;
bool contains_only_comments = false;
hfst::ImplementationType format;
size_t len;

bool expand_definitions = false;
bool harmonize_ = true;
bool harmonize_flags_ = false;
// bool verbose_=false;

std::string substitution_function_symbol;

void
set_substitution_function_symbol(const std::string &symbol)
{
    substitution_function_symbol = symbol;
}

bool
substitution_function(const hfst::StringPair &p, hfst::StringPairSet &sps)
{
    if (p.first == substitution_function_symbol
        || p.second == substitution_function_symbol)
    {
        sps.insert(hfst::StringPair(substitution_function_symbol,
                                    substitution_function_symbol));
        return true;
    }
    return false;
}

int *
get_n_to_k(const char *s)
{
    int *rv = static_cast<int *>(malloc(sizeof(int) * 2));
    char *endptr;
    char *finalptr;
    if (*(s + 1) == '{')
    {
        rv[0] = strtol(s + 2, &endptr, 10);
        rv[1] = strtol(endptr + 1, &finalptr, 10);
        assert(*finalptr == '}');
    }
    else
    {
        rv[0] = strtol(s + 1, &endptr, 10);
        rv[1] = strtol(endptr + 1, &finalptr, 10);
        assert(*finalptr == '\0');
    }
    return rv;
}

int
getinput(char *buf, int maxlen)
{
    int retval = 0;
    if (maxlen > (int)len)
    {
        maxlen = hfst::size_t_to_int(len);
    }
    memcpy(buf, data, maxlen);
    data += maxlen;
    len -= maxlen;
    retval = maxlen;
    return retval;
}

char *
strip_newline(char *s)
{
    for (unsigned int pos = 0; s[pos] != '\0'; pos++)
    {
        if (s[pos] == '\n' || s[pos] == '\r')
            s[pos] = '\0';
    }
    return s;
}

void
count_lines(const char *s)
{
    const char *c = s;
    while (*c != '\0')
    {
        if (*c == '\n')
        {
            hfst::xre::lr += 1;
        }
        else if (*c == '\r')
        {
            c++;
            if (*c == '\n')
            {
                hfst::xre::cr += 1;
            }
            else
            {
                c--;
            }
            hfst::xre::lr += 1;
        }
        hfst::xre::cr += 1;
        c++;
    }
}

char *
strip_curly(const char *s)
{
    const char *c = s;
    char *stripped = (char *)calloc(sizeof(char), strlen(s) + 1);
    int i = 0;
    while (*c != '\0')
    {
        // If first char is { or last one }, then skip it
        if ((*c == '{' && i == 0) || (*c == '}' && *(c + 1) == '\0'))
        {
            if (*(c + 1) == '\0')
            {
                break;
            }
            else
            {
                stripped[i] = *(c + 1);
                i++;
                c += 2;
            }
        }
        else
        {
            stripped[i] = *c;
            i++;
            c++;
        }
    }
    stripped[i] = '\0';
    return stripped;
}

char *
strip_percents(const char *s)
{
    const char *c = s;
    char *stripped = (char *)calloc(sizeof(char), strlen(s) + 1);
    size_t i = 0;
    while (*c != '\0')
    {
        if (*c == '%')
        {
            if (*(c + 1) == '\0')
            {
                break;
            }
            else
            {
                stripped[i] = *(c + 1);
                i++;
                c += 2;
            }
        }
        else
        {
            stripped[i] = *c;
            i++;
            c++;
        }
    }
    stripped[i] = '\0';
    return stripped;
}

char *
add_percents(const char *s)
{
    char *ns = static_cast<char *>(malloc(sizeof(char) * strlen(s) * 2 + 1));
    char *p = ns;
    while (*s != '\0')
    {
        if ((*s == '@') || (*s == '-') || (*s == ' ') || (*s == '|')
            || (*s == '!') || (*s == ':') || (*s == ';') || (*s == '0')
            || (*s == '\\') || (*s == '&') || (*s == '?') || (*s == '$')
            || (*s == '+') || (*s == '*') || (*s == '/') || (*s == '/')
            || (*s == '_') || (*s == '(') || (*s == ')') || (*s == '{')
            || (*s == '}') || (*s == '[') || (*s == ']'))
        {
            *p = '%';
            p++;
        }
        *p = *s;
        p++;
        s++;
    }
    *p = '\0';
    return ns;
}

char *
get_quoted(const char *s)
{
    const char *qstart = strchr((char *)s, '"') + 1;
    const char *qend = strrchr((char *)s, '"');
    char *qpart = strdup(qstart);
    *(qpart + (size_t)(qend - qstart)) = '\0';
    return qpart;
}

char *
parse_quoted(const char *s, unsigned int &length)
{
    std::ostream *err = xreerrstr();

    char *quoted = get_quoted(s);

    char *rv = static_cast<char *>(malloc(sizeof(char) * strlen(quoted) + 1));
    char *p = quoted;
    char *r = rv;
    while (*p != '\0')
    {
        if (*p == '\n' || *p == '\r')
        {
            throw "Unescaped newline characters found inside quoted string.";
        }
        else if (*p != '\\')
        {
            *r = *p;
            r++;
            p++;
        }
        else if (*p == '\\')
        {
            switch (*(p + 1))
            {
            case '0':
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
                *err << "*** XRE unimplemented: parse octal escape in "
                     << std::string(p);
                xreflush(err);
                *r = '\0';
                p = p + 5;
                break;
            case 'a':
                *r = '\a';
                r++;
                p = p + 2;
                break;
            case 'b':
                *r = '\b';
                r++;
                p = p + 2;
                break;
            case 'f':
                *r = '\f';
                r++;
                p = p + 2;
                break;
            case 'n':
                *r = '\n';
                r++;
                p = p + 2;
                break;
            case 'r':
                *r = '\r';
                r++;
                p = p + 2;
                break;
            case 't':
                *r = '\t';
                r++;
                p = p + 2;
                break;
            case 'u':
                *err << "Unimplemented: parse unicode escapes in "
                     << std::string(p);
                xreflush(err);
                *r = '\0';
                r++;
                p = p + 6;
                break;
            case 'v':
                *r = '\v';
                r++;
                p = p + 2;
                break;
            case 'x':
            {
                char *endp;
                int i = strtol(p + 2, &endp, 10);
                if (0 < i && i <= 127)
                {
                    *r = static_cast<char>(i);
                }
                else
                {
                    *err << "*** XRE unimplemented: parse \\x" << i
                         << std::endl;
                    xreflush(err);
                    // fprintf(stderr, "*** XRE unimplemented: "
                    //         "parse \\x%d\n", i);
                    *r = '\0';
                }
                r++;
                assert(endp != p);
                p = endp;
                break;
            }
            case '\0':
                *err << "End of line after \\ escape" << std::endl;
                xreflush(err);
                // fprintf(stderr, "End of line after \\ escape\n");
                *r = '\0';
                r++;
                p++;
                break;
            default:
                *r = *(p + 1);
                r++;
                p += 2;
                break;
            }
        }
    }
    *r = '\0';
    free(quoted);

    length = hfst::HfstTokenizer::check_utf8_correctness_and_calculate_length(
        std::string(rv));

    return rv;
}

// If \a str is of form "@_<foo>_@", insert pair ("@_<foo>_@", "<foo>") into \a
// substitutions.
static void
insert_angle_bracket_substitutions(
    const std::string &str, hfst::HfstSymbolSubstitutions &substitutions)
{
    if (str.length() < 6)
        return;
    if (str.substr(0, 3) == "@_<" && str.substr(str.length() - 3, 3) == ">_@")
    {
        std::string substituting_str = str.substr(2, str.length() - 4);
        substitutions.insert(StringPair(str, substituting_str));
    }
}

char *
escape_enclosing_angle_brackets(char *s)
{
    if (s[0] != '<')
        return s;
    unsigned int i = 0;
    while (s[i] != '\0')
    {
        i++;
    }
    i--;
    if (s[i] != '>')
        return s;

    std::string retval
        = std::string("@_") + std::string(s) + std::string("_@");
    free(s);
    return strdup(retval.c_str());
}

HfstTransducer *
unescape_enclosing_angle_brackets(HfstTransducer *t)
{
    hfst::HfstSymbolSubstitutions substitutions;
    StringSet alpha = t->get_alphabet();
    for (StringSet::const_iterator it = alpha.begin(); it != alpha.end(); it++)
    {
        insert_angle_bracket_substitutions(*it, substitutions);
    }
    if (substitutions.size() == 0)
        return t;

    t->substitute(substitutions);
    t->optimize();
    return t;
}

double
get_weight(const char *s)
{
    double rv = -3.1415;
    const char *weightstart = s;
    while ((*weightstart != '\0')
           && ((*weightstart == ' ') || (*weightstart == '\t')
               || (*weightstart == ';')))
    {
        weightstart++;
    }
    char *endp;
    rv = strtod(weightstart, &endp);
    assert(endp != weightstart);
    return rv;
}

HfstTransducer *
compile(const string &xre, map<string, HfstTransducer *> &defs,
        map<string, string> &func_defs, map<string, unsigned int> func_args,
        map<string, std::set<string> > &lists, ImplementationType impl)
{
    // lock here?
    data = strdup(xre.c_str());
    // use an internal variable startptr_ instead of global startptr
    char *startptr_ = data;
    len = strlen(data);
    definitions = defs;
    function_definitions = func_defs;
    function_arguments = func_args;
    symbol_lists = lists;
    format = impl;

    contains_only_comments = false;

    yyscan_t scanner;
    xrelex_init(&scanner);
    YY_BUFFER_STATE bs = xre_scan_string(startptr_, scanner);

    int parse_retval = xreparse(scanner);

    xre_delete_buffer(bs, scanner);
    xrelex_destroy(scanner);

    free(startptr_);
    data = 0;
    len = 0;
    if (parse_retval == 0 && !contains_only_comments) // if (yynerrs == 0)
    {
        HfstTransducer *rv = new HfstTransducer(*last_compiled);
        delete last_compiled;
        return rv;
    }
    else
    {
        return NULL;
    }
}

// todo: Contains lots of code copied directly from compile(...), should be
// rewritten..
HfstTransducer *
compile_first(const string &xre, map<string, HfstTransducer *> &defs,
              map<string, string> &func_defs,
              map<string, unsigned int> func_args,
              map<string, std::set<string> > &lists, ImplementationType impl,
              unsigned int &chars_read)
{
    // lock here?
    data = strdup(xre.c_str());
    // use an internal variable startptr_ instead of global startptr
    char *startptr_ = data;
    len = strlen(data);
    definitions = defs;
    function_definitions = func_defs;
    function_arguments = func_args;
    symbol_lists = lists;
    format = impl;

    contains_only_comments = false;

    yyscan_t scanner;
    xrelex_init(&scanner);
    YY_BUFFER_STATE bs = xre_scan_string(startptr_, scanner);

    bool tmp = hfst::xre::allow_extra_text_at_end;
    hfst::xre::allow_extra_text_at_end = true;
    hfst::xre::cr = 0;
    hfst::xre::lr = 1;

    int parse_retval = xreparse(scanner);
    chars_read = hfst::xre::cr;
    hfst::xre::allow_extra_text_at_end = tmp;

    xre_delete_buffer(bs, scanner);
    xrelex_destroy(scanner);

    free(startptr_);

    data = 0;
    len = 0;
    if (parse_retval == 0 && !contains_only_comments) // if (yynerrs == 0)
    {
        HfstTransducer *rv = new HfstTransducer(*last_compiled);
        delete last_compiled;
        return rv;
    }
    else
    {
        return NULL;
    }
}

bool
is_valid_function_call(const char *name,
                       const std::vector<HfstTransducer> *args)
{
    std::map<std::string, std::string>::const_iterator name2xre
        = function_definitions.find(name);
    std::map<std::string, unsigned int>::const_iterator name2args
        = function_arguments.find(name);

    if (name2xre == function_definitions.end()
        || name2args == function_arguments.end())
    {
        std::ostream *err = xreerrstr();
        *err << "No such function defined: '" << name << "'" << std::endl;
        xreflush(err);
        // fprintf(stderr, "No such function defined: '%s'\n", name);
        return false;
    }

    unsigned int number_of_args = name2args->second;

    if (number_of_args != args->size())
    {
        std::ostream *err = xreerrstr();
        *err << "Wrong number of arguments: function '" << name << "' expects "
             << (int)number_of_args << ", " << (int)args->size() << " given"
             << std::endl;
        xreflush(err);
        // fprintf(stderr, "Wrong number of arguments: function '%s' expects
        // %i, %i given\n",
        //        name, (int)number_of_args, (int)args->size());
        return false;
    }

    return true;
}

const char *
get_function_xre(const char *name)
{
    std::map<std::string, std::string>::const_iterator it
        = function_definitions.find(name);
    if (it == function_definitions.end())
    {
        return NULL;
    }
    return it->second.c_str();
}

bool
define_function_args(const char *name, const std::vector<HfstTransducer> *args)
{
    if (!is_valid_function_call(name, args))
    {
        return false;
    }
    unsigned int arg_number = 1;
    for (std::vector<HfstTransducer>::const_iterator it = args->begin();
         it != args->end(); it++)
    {
        ostringstream os;
        os << arg_number;
        std::string function_arg = "@" + std::string(name) + os.str() + "@";
        definitions[function_arg] = new HfstTransducer(*it);
        // fprintf(stderr, "defined function arg: '%s', %i:\n", name,
        // arg_number); // DEBUG std::cerr << *it << std::endl;
        arg_number++;
    }
    return true;
}

void
undefine_function_args(const char *name)
{
    std::map<std::string, unsigned int>::const_iterator it
        = function_arguments.find(name);
    if (it == function_arguments.end())
    {
        return;
    }
    for (unsigned int arg_number = 1; arg_number <= it->second; arg_number++)
    {
        ostringstream os;
        os << arg_number;
        std::string function_arg = "@" + std::string(name) + os.str() + "@";
        delete definitions[function_arg];
        definitions.erase(function_arg);
        // fprintf(stderr, "undefined function arg: '%s', %i:\n", name,
        // arg_number); // DEBUG
    }
}

bool
is_definition(const char *symbol)
{
    std::string symbol_(symbol);
    if (definitions.find(symbol_) == definitions.end())
        return false;
    return true;
}

HfstTransducer *
expand_definition(const char *symbol)
{
    if (expand_definitions)
    {
        for (std::map<std::string, hfst::HfstTransducer *>::const_iterator it
             = definitions.begin();
             it != definitions.end(); it++)
        {
            if (strcmp(it->first.c_str(), symbol) == 0)
            {
                return new HfstTransducer(*(it->second));
            }
        }
    }
    return new HfstTransducer(symbol, symbol, hfst::xre::format);
}

HfstTransducer *
expand_definition(HfstTransducer *tr, const char *symbol)
{
    if (expand_definitions)
    {
        for (std::map<std::string, hfst::HfstTransducer *>::const_iterator it
             = definitions.begin();
             it != definitions.end(); it++)
        {
            if (strcmp(it->first.c_str(), symbol) == 0)
            {
                StringSet alpha = it->second->get_alphabet();
                tr->substitute(hfst::StringPair(symbol, symbol), *(it->second),
                               false); // do not harmonize
                if (alpha.find(symbol) == alpha.end())
                    tr->remove_from_alphabet(symbol);
                // if (it->second->get_alphabet().find(symbol) !=
                // it->second->get_alphabet().end())
                //   std::cerr << "WARN: symbol '" << std::string(symbol) << "'
                //   was removed" << std::endl;
                break;
            }
        }
    }
    return tr;
}

/*static const char * get_print_format(const char * symbol)
{
if (strcmp(hfst::internal_identity.c_str(), symbol) == 0)
  return "?";
if (strcmp(hfst::internal_unknown.c_str(), symbol) == 0)
  return "?";
if (strcmp(hfst::internal_epsilon.c_str(), symbol) == 0)
  return "0";
return symbol;
}*/

HfstTransducer *
xfst_curly_label_to_transducer(const char *input, const char *output)
{
    HfstTransducer *retval = NULL;

    if (strcmp(input, hfst::internal_unknown.c_str()) == 0)
    {
        HfstTokenizer tok;
        StringVector sv = tok.tokenize_one_level(output, false);
        std::string first_token(sv.at(0));
        retval = new HfstTransducer(hfst::internal_unknown, first_token,
                                    hfst::xre::format);

        for (StringVector::const_iterator it = sv.begin(); it != sv.end();
             it++)
        {
            HfstTransducer tmp(*it, first_token, hfst::xre::format);
            retval->disjunct(tmp, false);
        }
        for (StringVector::const_iterator it = ++(sv.begin()); it != sv.end();
             it++)
        {
            HfstTransducer tmp(hfst::internal_epsilon, *it, hfst::xre::format);
            retval->concatenate(tmp, false);
        }
    }
    else if (strcmp(output, hfst::internal_unknown.c_str()) == 0)
    {
        HfstTokenizer tok;
        StringVector sv = tok.tokenize_one_level(input, false);
        std::string first_token(sv.at(0));
        retval = new HfstTransducer(first_token, hfst::internal_unknown,
                                    hfst::xre::format);

        for (StringVector::const_iterator it = sv.begin(); it != sv.end();
             it++)
        {
            HfstTransducer tmp(first_token, *it, hfst::xre::format);
            retval->disjunct(tmp, false);
        }
        for (StringVector::const_iterator it = ++(sv.begin()); it != sv.end();
             it++)
        {
            HfstTransducer tmp(*it, hfst::internal_epsilon, hfst::xre::format);
            retval->concatenate(tmp, false);
        }
    }
    else
    {
        std::string istr(input);
        std::string ostr(output);
        HfstTokenizer tok;
        tok.add_multichar_symbol(hfst::internal_epsilon);
        retval = new HfstTransducer(istr, ostr, tok, hfst::xre::format);
    }

    retval->minimize(); // it should be safe to minimize
    return retval;
}

HfstTransducer *
xfst_label_to_transducer(const char *input, const char *output)
{
    HfstTransducer *retval = NULL;

    bool input_is_definition = is_definition(input);
    bool output_is_definition = is_definition(output);
    bool input_is_unknown
        = (strcmp(input, hfst::internal_unknown.c_str()) == 0);
    bool output_is_unknown
        = (strcmp(output, hfst::internal_unknown.c_str()) == 0);

    // definitions -> use cross-product
    if (input_is_definition || output_is_definition)
    {
        HfstTransducer *tmp
            = NULL; // temporary transducer for cross-product calculation
        if (input_is_unknown)
        {
            retval = new HfstTransducer(hfst::internal_identity,
                                        hfst::xre::format);
            tmp = expand_definition(output);
        }
        else if (output_is_unknown)
        {
            tmp = new HfstTransducer(hfst::internal_identity,
                                     hfst::xre::format);
            retval = expand_definition(input);
        }
        else // neither is unknown
        {
            retval = expand_definition(input);
            tmp = expand_definition(output);
        }
        retval->cross_product(*tmp);
        delete tmp;
        return retval;
    }

    // no definitions
    if (input_is_unknown && output_is_unknown)
    {
        retval = new HfstTransducer(hfst::internal_unknown,
                                    hfst::internal_unknown, hfst::xre::format);
        HfstTransducer id(hfst::internal_identity, hfst::internal_identity,
                          hfst::xre::format);
        retval->disjunct(id).minimize(); // it should be safe to minimize
    }
    else if (input_is_unknown)
    {
        retval = new HfstTransducer(hfst::internal_unknown, output,
                                    hfst::xre::format);
        HfstTransducer output_tr(output, output, hfst::xre::format);
        retval->disjunct(output_tr)
            .minimize(); // it should be safe to minimize
    }
    else if (output_is_unknown)
    {
        retval = new HfstTransducer(input, hfst::internal_unknown,
                                    hfst::xre::format);
        HfstTransducer input_tr(input, input, hfst::xre::format);
        retval->disjunct(input_tr).minimize(); // it should be safe to minimize
    }
    else
    {
        retval = new HfstTransducer(input, output, hfst::xre::format);
    }
    return retval;
}

HfstTransducer *
contains_twolc(const HfstTransducer *t)
{
    // marker = [0:M ?]*
    HfstTransducer marker("@_EPSILON_SYMBOL_@", "M", t->get_type());
    HfstTransducer id("@_IDENTITY_SYMBOL_@", t->get_type());
    marker.concatenate(id)
        .repeat_star()
        .minimize(); // it should be safe to minimize

    // the rule
    HfstTransducer right_context(*t);
    right_context.insert_freely(StringPair("M", "@_EPSILON_SYMBOL_@"))
        .optimize();
    right_context.insert_freely(StringPair("M", "M")).optimize();
    HfstTransducer left_context("@_EPSILON_SYMBOL_@", t->get_type());
    HfstTransducerPair context(left_context, right_context);

    StringPairSet mappings;
    mappings.insert(StringPair("M", "M"));

    // alphabet list possible mappings: M:M, M:0,
    StringPairSet alphabet;
    alphabet.insert(StringPair("M", "M"));
    alphabet.insert(StringPair("M", "@_EPSILON_SYMBOL_@"));
    alphabet.insert(StringPair("@_IDENTITY_SYMBOL_@", "@_IDENTITY_SYMBOL_@"));

    // and identity pairs of symbols that occur in t
    HfstBasicTransducer basic(*t);
    StringPairSet symbol_pairs = basic.get_transition_pairs();
    for (StringPairSet::const_iterator it = symbol_pairs.begin();
         it != symbol_pairs.end(); it++)
    {
        alphabet.insert(*it);
        alphabet.insert(StringPair(it->first, it->first));
        alphabet.insert(StringPair(it->second, it->second));
    }

    HfstTransducer rule
        = hfst::rules::two_level_if_and_only_if(context, mappings, alphabet);

    return new HfstTransducer(rule);

    marker.compose(rule);
    marker.optimize();

    return new HfstTransducer(marker);
}

HfstTransducer *
contains(const HfstTransducer *t)
{
    HfstTransducer any(hfst::internal_identity, hfst::xre::format);
    any.repeat_star().minimize(); // it should be safe to minimize
    HfstTransducer *retval = new HfstTransducer(any);
    retval->concatenate(*t).concatenate(any);
    retval->optimize();
    return retval;
}

// [ 0::weight -> 0 || _ [t] ] - [?* - $[t]]
HfstTransducer *
contains_with_weight(const HfstTransducer *t, float weight)
{
    HfstTransducer weighted_epsilon(hfst::internal_epsilon, hfst::xre::format);
    weighted_epsilon.set_final_weights(weight);
    HfstTransducer epsilon(hfst::internal_epsilon, hfst::xre::format);

    // mapping: 0::weight -> 0
    HfstTransducerPair mappingPair(weighted_epsilon, epsilon);
    HfstTransducerPairVector mappingPairVector;
    mappingPairVector.push_back(mappingPair);

    // context: 0 _ [t]
    HfstTransducerPair contextPair(epsilon, *t);
    HfstTransducerPairVector contextPairVector;
    contextPairVector.push_back(contextPair);

    // weighted_rule = [ 0::weight @-> 0 || _ [t] ]
    // (add weight for each occurrence of t)
    hfst::xeroxRules::Rule rule(mappingPairVector, contextPairVector,
                                hfst::xeroxRules::REPL_UP);
    HfstTransducer weighted_rule(hfst::xre::format);
    weighted_rule = replace(rule, false);

    // note that replace_leftmost_longest_match does not work correctly in the
    // case of epsilons

    // noT = ?* - $[t]
    // (strings that do not contain t)
    HfstTransducer noT(hfst::internal_identity, t->get_type());
    noT.repeat_star().minimize(); // it should be safe to minimize
    HfstTransducer *oneOrMoreT = contains(t);
    noT.subtract(*oneOrMoreT);
    noT.optimize();
    delete oneOrMoreT;

    // return [weighted_rule - noT]
    // (subtract strings that do not contain t from weighted rule)
    weighted_rule.subtract(noT);
    weighted_rule.optimize();
    return new HfstTransducer(weighted_rule);
}

// [$c - $[ [[?+ c ?*] & [c ?*]] | [[c ?+] & [c]]]]
HfstTransducer *
contains_once(const HfstTransducer *c)
{
    // any_star = [?*]
    HfstTransducer any_star(hfst::internal_identity, hfst::xre::format);
    any_star.repeat_star().minimize(); // it should be safe to minimize

    // any_plus = [?+]
    HfstTransducer any_plus(hfst::internal_identity, hfst::xre::format);
    any_plus.repeat_plus().minimize(); // it should be safe to minimize

    // t1 = [?+ c ?*]
    HfstTransducer *t1 = new HfstTransducer(any_plus);
    t1->concatenate(*c);
    t1->optimize();
    t1->concatenate(any_star);
    t1->optimize();

    // t2 = [c ?*]
    HfstTransducer t2(*c);
    t2.concatenate(any_star);
    t2.optimize();

    // t1 = [[?+ c ?*] & [c ?*]]
    t1->intersect(t2);

    // t3 = [[c ?+] & c]
    HfstTransducer t3(*c);
    t3.concatenate(any_plus);
    t3.optimize();
    t3.intersect(*c);
    t3.optimize();

    // t1 = [t1 | t3]
    t1->disjunct(t3);
    t1->optimize();

    // cont_t1 = $[t1]
    HfstTransducer *cont_t1 = contains(t1);
    delete t1;
    // cont_c = $[c]
    HfstTransducer *cont_c = contains(c);

    // $[c] - $[t1]
    cont_c->subtract(*cont_t1);
    cont_c->optimize();
    delete cont_t1;
    return cont_c;
}

HfstTransducer *
contains_once_optional(const HfstTransducer *t)
{
    // neg_t = ~$[t]
    HfstTransducer *cont_t = contains(t);
    HfstTransducer neg_t(hfst::internal_identity, hfst::xre::format);
    neg_t.repeat_star();
    neg_t.optimize();
    neg_t.subtract(*cont_t);
    neg_t.optimize();
    delete cont_t;

    HfstTransducer *retval = contains_once(t);
    retval->disjunct(neg_t);
    retval->optimize();
    return retval;
}

HfstTransducer *
merge_first_to_second(HfstTransducer *tr1, HfstTransducer *tr2)
{
    // Merge operation creates an XreCompiler that needs this information
    // below. Otherwise, it will overwrite all this.
    struct XreConstructorArguments args(
        hfst::xre::definitions, hfst::xre::function_definitions,
        hfst::xre::function_arguments, hfst::xre::symbol_lists,
        hfst::xre::format);

    tr1->optimize();
    tr2->merge(*tr1, args);
    return tr2;
}

void
warn(const char *msg)
{
    if (!verbose_)
        return;

    std::ostream *err = xreerrstr();
    *err << msg;
    xreflush(err);
}

void
warn_about_xfst_special_symbol(const char *symbol)
{
    if (strcmp("all", symbol) == 0)
    {
        if (verbose_)
        {
            warn("warning: symbol 'all' has no special meaning in hfst\n");
        }
        return;
    }

    if (symbol[0] != '<')
        return;
    unsigned int max_index = 1;
    while (symbol[max_index] != '\0')
    {
        max_index++;
    }
    max_index--;
    if (max_index < 1)
        return;

    if (symbol[max_index] != '>')
        return;
    if (!verbose_)
        return;
    std::ostream *err = xreerrstr();
    *err << "warning: '" << symbol << " ' is an ordinary symbol in hfst"
         << std::endl;
    xreflush(err);
}

void
warn_about_hfst_special_symbol(const char *symbol)
{
    if (symbol[0] != '@')
        return;
    if (symbol[1] != '_')
        return;
    unsigned int max_index = 2;
    while (symbol[max_index] != '\0')
    {
        max_index++;
    }
    max_index--;
    if (max_index < 3)
        return;
    if (symbol[max_index] != '@')
        return;
    if (symbol[max_index - 1] != '_')
        return;
    if (!verbose_)
        return;
    if (hfst::xre::verbose_)
    {
        std::ostream *err = xreerrstr();
        *err << "warning: '" << symbol << "' is not an ordinary symbol in hfst"
             << std::endl;
        xreflush(err);
    }
}

void
warn_about_special_symbols_in_replace(HfstTransducer *t)
{
    if (!verbose_)
        return;

    std::ostream *err = xreerrstr();

    StringSet alphabet = t->get_alphabet();
    for (StringSet::const_iterator it = alphabet.begin(); it != alphabet.end();
         it++)
    {
        if (HfstTransducer::is_special_symbol(*it)
            && *it != hfst::internal_epsilon && *it != hfst::internal_unknown
            && *it != hfst::internal_identity)
        {
            *err << "warning: using special symbol '" << *it
                 << "' in replace rule, use substitute instead" << std::endl;
        }
    }
    xreflush(err);
}

void
check_multichar_symbol(const char *symbol)
{
    if (defined_multichar_symbols_ == NULL)
        return;

    if (defined_multichar_symbols_->find(std::string(symbol))
        == defined_multichar_symbols_->end())
    {
        std::ostream *err = xreerrstr();
        *err << "warning: multichar symbol '" << symbol
             << "' used but not defined" << std::endl;
        xreflush(err);
    }
}

bool
has_non_identity_pairs(const HfstTransducer *t)
{
    hfst::implementations::HfstBasicTransducer basic(*t);
    StringPairSet sps = basic.get_transition_pairs();
    for (StringPairSet::const_iterator it = sps.begin(); it != sps.end(); it++)
    {
        if (it->first != it->second)
        {
            return true;
        }
    }
    return false;
}

}
}
