// Copyright (c) 2016 University of Helsinki
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
// See the file COPYING included with this distribution for more
// information.

#include "HfstTokenizer.h"
#include "HfstFlagDiacritics.h"
#include <cassert>
#include <iostream>
#include <string.h>
#include <string>

#include <unicode/ubrk.h>
#include <unicode/ustring.h>

#ifndef MAIN_TEST

using std::string;
namespace hfst
{
bool
MultiCharSymbolTrie::is_end_of_string(const char *p) const
{
    return *(p + 1) == 0;
}

void
MultiCharSymbolTrie::set_symbol_end(const char *p)
{
    is_leaf[(unsigned char)(*p)] = true;
}

bool
MultiCharSymbolTrie::is_symbol_end(const char *p) const
{
    return is_leaf[(unsigned char)(*p)];
}

void
MultiCharSymbolTrie::init_symbol_rests(const char *p)
{
    if (symbol_rests[(unsigned char)(*p)] == NULL)
    {
        symbol_rests[(unsigned char)(*p)] = new MultiCharSymbolTrie();
    }
}

void
MultiCharSymbolTrie::add_symbol_rest(const char *p)
{
    symbol_rests[(unsigned char)(*p)]->add(p + 1);
}

MultiCharSymbolTrie *
MultiCharSymbolTrie::get_symbol_rest_trie(const char *p) const
{
    return symbol_rests[(unsigned char)(*p)];
}

MultiCharSymbolTrie::MultiCharSymbolTrie(void)
    : symbol_rests(UCHAR_MAX, (MultiCharSymbolTrie *)NULL),
      is_leaf(UCHAR_MAX, false)
{
}

MultiCharSymbolTrie::~MultiCharSymbolTrie(void)
{
    for (MultiCharSymbolTrieVector::iterator it = symbol_rests.begin();
         it != symbol_rests.end(); ++it)
    {
        delete *it;
    }
}

void
MultiCharSymbolTrie::add(const char *p)
{
    if (is_end_of_string(p))
    {
        set_symbol_end(p);
    }
    else
    {
        init_symbol_rests(p);
        add_symbol_rest(p);
    }
}

const char *
MultiCharSymbolTrie::find(const char *p) const
{
    MultiCharSymbolTrie *symbol_rest_trie = get_symbol_rest_trie(p);
    if (symbol_rest_trie == NULL)
    {
        if (is_symbol_end(p))
        {
            return p + 1;
        }
        return NULL;
    }
    const char *symbol_end = symbol_rest_trie->find(p + 1);
    if (symbol_end == NULL)
    {
        if (is_symbol_end(p))
        {
            return p + 1;
        }
    }
    return symbol_end;
}

HfstTokenizer::HfstTokenizer() {}

int
HfstTokenizer::get_next_symbol_size(const char *symbol,
                                    bool split_characters) const
{
    if (!*symbol)
    {
        return 0;
    }

    const char *multi_char_symbol_end = multi_char_symbols.find(symbol);

    /* The string begins with a multi character symbol */
    if (multi_char_symbol_end != NULL)
    {
        return (int)(multi_char_symbol_end - symbol);
    }
    /* take next combining grapheme cluster */
    if (!split_characters)
    {
        UErrorCode status = U_ZERO_ERROR;
        UChar *ICUdata = (UChar *)malloc(sizeof(UChar) * (strlen(symbol) + 1));
        int32_t length = 0;
        ICUdata = u_strFromUTF8(ICUdata, strlen(symbol) + 1, &length, symbol,
                                -1, &status);
        if (U_FAILURE(status))
        {
            fprintf(stderr, "ICU error converting UTF-8 %s to UChars: %s\n",
                    symbol, u_errorName(status));
        }
        status = U_ZERO_ERROR;
        UBreakIterator *graphemes;
        graphemes = ubrk_open(UBRK_CHARACTER, "C", NULL, -1, &status);
        if (U_FAILURE(status))
        {
            fprintf(stderr,
                    "ICU error trying to open grapheme segmenter: %s\n",
                    u_errorName(status));
        }
        status = U_ZERO_ERROR;
        ubrk_setText(graphemes, ICUdata, length, &status);
        if (U_FAILURE(status))
        {
            fprintf(stderr, "ICU error trying to get graphemes from %s: %s\n",
                    symbol, u_errorName(status));
        }
        status = U_ZERO_ERROR;
        int32_t begin = ubrk_first(graphemes);
        int32_t end = ubrk_next(graphemes);
        if (begin == end)
        {
            return 0;
        }
        if (end == UBRK_DONE)
        {
            return 0;
        }
        char *grapheme = (char *)malloc(sizeof(char) * (end - begin) * 4 + 1);
        grapheme = u_strToUTF8(grapheme, (end - begin) * 4 + 1, &length,
                               &ICUdata[begin], end - begin, &status);
        if (U_FAILURE(status))
        {
            fprintf(stderr, "ICU error getting UTF-8 from grpaheme: %s\n",
                    u_errorName(status));
        }
        return (strlen(grapheme)); // strlen is number of bytes
    }
    /* split_characters => take next UTF-8 only */
    else
    {
        if ((128 & *symbol) == 0)
        {
            return 1;
        }
        else if ((32 & *symbol) == 0)
        {
            return 2;
        }
        else if ((16 & *symbol) == 0)
        {
            return 3;
        }
        else
        {
            return 4;
        }
    }
}

bool
HfstTokenizer::is_skip_symbol(hfst::String &s) const
{
    return (s.empty()) || (skip_symbol_set.find(s) != skip_symbol_set.end());
}

void
HfstTokenizer::add_multichar_symbol(const std::string &symbol)
{
    if (symbol.empty())
    {
        return;
    }
    multi_char_symbols.add(symbol.c_str());
}

void
HfstTokenizer::add_skip_symbol(const std::string &symbol)
{
    if (symbol.empty())
    {
        return;
    }
    multi_char_symbols.add(symbol.c_str());
    skip_symbol_set.insert(symbol.c_str());
}

StringPairVector
HfstTokenizer::tokenize(const std::string &input_string,
                        bool split_characters) const
{
    check_utf8_correctness(input_string);
    StringPairVector spv;
    const char *s = input_string.c_str();
    while (*s)
    {
        int symbol_size = get_next_symbol_size(s, split_characters);
        std::string symbol(s, 0, symbol_size);
        s += symbol_size;
        if (is_skip_symbol(symbol))
        {
            continue;
        }
        spv.push_back(StringPair(symbol, symbol));
    }
    return spv;
}

StringVector
HfstTokenizer::tokenize_one_level(const std::string &input_string,
                                  bool split_characters) const
{
    check_utf8_correctness(input_string);

    StringVector sv;
    const char *s = input_string.c_str();
    while (*s)
    {
        int symbol_size = get_next_symbol_size(s, split_characters);
        std::string symbol(s, 0, symbol_size);
        s += symbol_size;
        if (is_skip_symbol(symbol))
        {
            continue;
        }
        sv.push_back(symbol);
    }
    return sv;
}

StringPairVector
HfstTokenizer::tokenize_space_separated(const std::string &str)
{
    check_utf8_correctness(str);

    StringPairVector retval;
    size_t pos = 0;
    // position where a symbol begins, not yet defined
    size_t symbol_pos = std::string::npos;

    while (pos < str.size())
    {
        // end of symbol reached
        if (str[pos] == ' ' && symbol_pos != std::string::npos)
        {
            std::string symbol(str, symbol_pos, pos - symbol_pos);
            retval.push_back(StringPair(symbol, symbol));
            symbol_pos = std::string::npos; // next symbol not yet found
        }
        // next symbol found
        else if (str[pos] != ' ' && symbol_pos == std::string::npos)
        {
            symbol_pos = pos;
        }
        else
        {
        }
        ++pos;
    }

    // last symbol
    if (symbol_pos != std::string::npos)
    {
        std::string symbol(str, symbol_pos, std::string::npos);
        retval.push_back(StringPair(symbol, symbol));
    }

    return retval;
}

StringPairVector
HfstTokenizer::tokenize(const std::string &input_string,
                        const std::string &output_string,
                        bool split_characters) const
{
    check_utf8_correctness(input_string);
    check_utf8_correctness(output_string);

    StringPairVector spv;

    StringPairVector input_spv
        = tokenize(input_string.c_str(), split_characters);
    StringPairVector output_spv
        = tokenize(output_string.c_str(), split_characters);

    if (input_spv.size() < output_spv.size())
    {
        StringPairVector::iterator jt = output_spv.begin();
        for (StringPairVector::iterator it = input_spv.begin();
             it != input_spv.end(); ++it)
        {
            spv.push_back(StringPair(it->first, jt->first));
            ++jt;
        }
        for (; jt != output_spv.end(); ++jt)
        {
            spv.push_back(StringPair(internal_epsilon, jt->first));
        }
    }
    else
    {
        StringPairVector::iterator it = input_spv.begin();
        for (StringPairVector::iterator jt = output_spv.begin();
             jt != output_spv.end(); ++jt)
        {
            spv.push_back(StringPair(it->first, jt->first));
            ++it;
        }
        for (; it != input_spv.end(); ++it)
        {
            spv.push_back(StringPair(it->first, internal_epsilon));
        }
    }
    return spv;
}

StringPairVector
HfstTokenizer::tokenize(
    const std::string &input_string, const std::string &output_string,
    bool split_characters,
    void (*warn_about_pair)(
        const std::pair<std::string, std::string> &symbol_pair)) const
{
    check_utf8_correctness(input_string);
    check_utf8_correctness(output_string);

    StringPairVector spv;

    StringPairVector input_spv
        = tokenize(input_string.c_str(), split_characters);
    StringPairVector output_spv
        = tokenize(output_string.c_str(), split_characters);

    if (input_spv.size() < output_spv.size())
    {
        StringPairVector::iterator jt = output_spv.begin();
        for (StringPairVector::iterator it = input_spv.begin();
             it != input_spv.end(); ++it)
        {
            StringPair sp(it->first, jt->first);
            warn_about_pair(sp);
            spv.push_back(sp);
            ++jt;
        }
        for (; jt != output_spv.end(); ++jt)
        {
            StringPair sp(internal_epsilon, jt->first);
            warn_about_pair(sp);
            spv.push_back(sp);
        }
    }
    else
    {
        StringPairVector::iterator it = input_spv.begin();
        for (StringPairVector::iterator jt = output_spv.begin();
             jt != output_spv.end(); ++jt)
        {
            StringPair sp(it->first, jt->first);
            warn_about_pair(sp);
            spv.push_back(sp);
            ++it;
        }
        for (; it != input_spv.end(); ++it)
        {
            StringPair sp(it->first, internal_epsilon);
            warn_about_pair(sp);
            spv.push_back(sp);
        }
    }
    return spv;
}

StringPairVector
HfstTokenizer::tokenize_and_align_flag_diacritics(
    const std::string &input_string, const std::string &output_string,
    bool split_characters,
    void (*warn_about_pair)(
        const std::pair<std::string, std::string> &symbol_pair)) const
{
    check_utf8_correctness(input_string);
    check_utf8_correctness(output_string);

    StringPairVector spv;

    StringPairVector input_spv
        = tokenize(input_string.c_str(), split_characters);
    StringPairVector output_spv
        = tokenize(output_string.c_str(), split_characters);

    assert(input_spv.size() > 0 && output_spv.size() > 0);
    StringPairVector::const_iterator it = input_spv.begin();
    StringPairVector::const_iterator jt = output_spv.begin();

    // proceed until both token vectors are exhausted
    while (it != input_spv.end() || jt != output_spv.end())
    {
        StringPair sp("", ""); // string pair to push back to the result
        StringPair sp_cont(
            "", ""); // possible continuation in case of missaligned flags

        if (it == input_spv.end())
        {
            if (FdOperation::is_diacritic(
                    jt->first)) // copy diacritic to other side
            {
                sp = StringPair(jt->first, jt->first);
            }
            else // pad input with epsilons
            {
                sp = StringPair(internal_epsilon, jt->first);
            }
            jt++;
        }
        else if (jt == output_spv.end())
        {
            if (FdOperation::is_diacritic(
                    it->first)) // copy diacritic to other side
            {
                sp = StringPair(it->first, it->first);
            }
            else // pad output with epsilons
            {
                sp = StringPair(it->first, internal_epsilon);
            }
            it++;
        }
        else
        {
            // take from both vectors (cases foo:bar, foo:foo, flag1:flag1)
            if ((!FdOperation::is_diacritic(it->first)
                 && !FdOperation::is_diacritic(jt->first))
                || *it == *jt)
            {
                sp = StringPair(it->first, jt->first);
            }
            // take first from first vector and then from second
            // (cases flag1:flag2, flag1::bar, foo:flag2)
            else
            {
                StringPair wrong_pair(it->first, jt->first);
                warn_about_pair(wrong_pair);
                sp = StringPair(it->first, it->first);
                sp_cont = StringPair(jt->first, jt->first);
            }
            it++;
            jt++;
        }

        spv.push_back(sp);
        if (sp_cont.first.size() != 0 && sp_cont.second.size() != 0)
        {
            spv.push_back(sp_cont);
        }
    }

    return spv;
}

void
HfstTokenizer::check_utf8_correctness(const std::string &input_string)
{
    (void)check_utf8_correctness_and_calculate_length(input_string);
}

unsigned int
HfstTokenizer::check_utf8_correctness_and_calculate_length(
    const std::string &input_string)
{
    UErrorCode status = U_ZERO_ERROR;
    int32_t length = 0;
    u_strFromUTF8(NULL, 0, &length, input_string.c_str(), -1, &status);
    if (status == U_BUFFER_OVERFLOW_ERROR)
    {
        // this is dumb
        status = U_ZERO_ERROR;
    }
    if (U_FAILURE(status))
    {
        std::cerr << "DEBUG: " << status << ": " << u_errorName(status)
                  << std::endl;
        HFST_THROW_MESSAGE(IncorrectUtf8CodingException, u_errorName(status));
    }
    else
    {
        return length;
    }
}

}

#else // MAIN_TEST was defined

#include <iostream>

int
main(int argc, char *argv[])
{
    std::cout << "Unit tests for " __FILE__ ":" << std::endl;

    std::cout << "ok" << std::endl;
    return 0;
}

#endif // MAIN_TEST
