#!/usr/bin/python3
# -*- coding: UTF-8 -*-

import re
from urllib.parse import urlparse

from .getter import Getter
from .logger import Logger
from .tools import Tools
from ..config import Configuration

class Robots(object):
    base_path = ''
    robots_txt = ''
    word_list = []
    uri_list = []

    def __init__(self, base_path):
        self.base_path = base_path

        if Configuration.norobots:
            return

        if Configuration.case_insensitive:
            self.base_path = base_path.lower()

        rUri = urlparse(self.base_path)

        try:
            Logger.pl('{+} {W}Getting informations from /robots.txt at {C}%s{W} ' % rUri.netloc)

            proxy1=Tools.get_proxy(Configuration.proxy)

            url1 = "%s://%s/robots.txt" % (rUri.scheme, rUri.netloc)
            r1 = Getter.general_request(url1, proxy1, "GET")
            if r1 is not None and r1.status_code == 200 and len(r1.text) > 0:
                self.robots_txt += str(r1.text)

                if Configuration.proxy_report_to != '':
                    try:
                        proxy=Tools.get_proxy(Configuration.proxy_report_to)
                        
                        Getter.general_request(url1, proxy, "GET")

                    except Exception as e:
                        print(e)
                        pass

                try:
                    if Configuration.db is not None:
                        Configuration.db.insertUri(url1, r1.status_code, len(r1.text))
                except Exception:
                    pass

            url2 = "%s://%s%s/robots.txt" % (rUri.scheme, rUri.netloc, "/" + rUri.path.strip("/"))
            url2 = url2.replace("//robots.txt", "/robots.txt")
            if url1 != url2:
                r1 = Getter.general_request(url2, proxy1, "GET")
                if r1 is not None and r1.status_code == 200 and len(r1.text) > 0:
                    self.robots_txt += str(r1.text)
                    
                    if Configuration.proxy_report_to != '':
                        try:
                            proxy=Tools.get_proxy(Configuration.proxy_report_to)
                            
                            Getter.general_request(url2, proxy, "GET")

                        except Exception:
                            pass

                    try:
                        if Configuration.db is not None:
                            Configuration.db.insertUri(url2, r1.status_code, len(r1.text))
                    except Exception:
                        pass

            self.parse()
        except Exception as e:
            print(e)
            pass

    def parse(self):
        paths = []
        lines = self.robots_txt.replace("\r", "\n").split("\n")
        rUri = urlparse(self.base_path)
        for line in lines:
            if line.endswith('\n'):
                line = line[:-1]

            if Configuration.case_insensitive:
                line = line.lower()

            if 'disallow:' in line.lower() or 'allow:' in line.lower():
                m = re.search("(?P<url>/[^\s]+)", line)
                if m is not None:
                    path = m.group("url")
                    path = ''.join(filter(Tools.permited_char, path)).strip()
                    if path not in paths:
                        paths.append(path.strip("/"))

        for path in paths:
            vp = "%s://%s/%s" % (rUri.scheme, rUri.netloc, path) 
            if self.base_path in vp:
                if path not in self.uri_list:
                    self.uri_list.append(path)
            
            parts = path.split("/")
            for p in parts:
                if p not in self.word_list and p not in self.uri_list:
                    self.word_list.append(p)

        if len(self.word_list) > 0 or len(self.uri_list) > 0:
            Logger.pl('{+} {W}Loaded {O}%s{W} path(es) and {O}%s{W} unique word(s) from /robots.txt' % (len(self.uri_list), len(self.word_list)))
            for p in self.uri_list:
                Logger.pl('==> ENTRY: /%s' % (p))
        
        Logger.pl(" ")

    def getWordList(self):
        return self.word_list

    def getUriList(self):
        return self.uri_list
