# AUTOGENERATED! DO NOT EDIT! File to edit: 02_wikiparser.ipynb (unless otherwise specified).

__all__ = ['WikiParser']

# Cell
from fastcore.test import *
import requests
import time
import chardet
import re
import pyprind
import pickle
from unicodedata import normalize
from collections import OrderedDict
from .word import *
from .wordtype import *
from .config import *

# Internal Cell
def remove_duplicates_keep_order(seq):
    seen = set()
    seen_add = seen.add
    return [x for x in seq if not (x in seen or seen_add(x))]

# Internal Cell
def load_words_from_file(path):
    text = read_textfile(path)
    text = normalize('NFC', text)
    text = text.split('\n')
    mylist = []
    for line in text:
        f_line = __filter_line(line)
        for word in f_line:
            word = word.strip()
            if (word=="\n" or len(word) < 2 or word[0]=="#"):
                pass
            elif word[-1]=="\n":
                mylist.append(word[:-1])
            else:
                mylist.append(word)
    return mylist

# Internal Cell
def __filter_line(line):
    mylist = []
    matches = re.findall("[^\d\W]+", line, re.UNICODE)
    if matches:
        for match in matches:
            if len(match) > 1:
                mylist.append(match)
    return mylist

# Internal Cell
def read_textfile(url):
    with open(url, "rb") as file:
        rawdata = file.read()
        guess = chardet.detect(rawdata)
        encoding = guess['encoding']
        confidence = guess['confidence']
        if (float(confidence) > 0.7):
            text = rawdata.decode(encoding)
            return text
        return None

# Internal Cell
def is_special(word, key, message, print=print):
    check = re.findall(key, word.wikitext, re.UNICODE)
    if check:
        s = f"'{word.title}'"
        print(s+message)
        return True
    return False

# Internal Cell
def is_different_spelling(word, print=print):
    check = re.findall(r'Wortart', word.wikitext, re.UNICODE)
    if not check:
        print("No 'Wortart' attribute, maybe uncommon spelling")
        return True
    return False

# Internal Cell
def is_word_derivate(word_obj):
    if len(word_obj.word_types)==1:
        wtype = word_obj.word_types[0]
        matches = re.findall(regex_basic_word, word_obj.wikitext, re.UNICODE)
        if (matches):
            return (True, matches[0])
    return (False, None)

# Cell
class WikiParser:
    def __init__(self, verbose=False):
        self.verbose = verbose
        self.words = []
        self.not_found = []
        self.sleep_len = 0.3

    def print_verbose(self, text, end='\n'):
        if self.verbose:
            print (text, end=end)

    def create_parsing_url(self, word):
        word_uri = requests.utils.quote(word)
        url = wiki_site+wiki_options+word_uri
        return url

    def create_normal_url(self, word):
        url = wiki_site+wiki_normal+word
        return url

    def fetch_wiki_data(self, word):
        url_wiki = self.create_parsing_url(word)
        url = self.create_normal_url(word)

        self.print_verbose ("URL: {0}".format(url))
        self.print_verbose ("URL: {0}".format(url_wiki), end='\n\n')

        response = requests.get(url_wiki)
        json_obj = response.json()

        if 'error' in json_obj:
            self.print_verbose(f"Word '{word}' doesn't exist in the wiktionary")
            return None
        elif 'parse' in json_obj:
            title = json_obj['parse']['title']
            pageid = json_obj['parse']['pageid']
            wikitext = json_obj['parse']['wikitext']['*']
            word = Word(title, pageid, wikitext, url)
            return word

    def clear(self):
        self.words.clear()

    def __fetch_words(self, wordlist):
        valid_words = []
        not_found = []
        wordlist = remove_duplicates_keep_order(wordlist) # remove duplicates
        bar = pyprind.ProgBar(len(wordlist), bar_char='█', width=50, title="Downloading words")
        for word in wordlist:
            self.print_verbose (f'fetching the word: {word}\n')
            word_obj = self.fetch_wiki_data(word)
            if word_obj:
                word_obj = self.filter_language(word_obj, "Deutsch")
                if word_obj:
                    valid_words.append(word_obj)
                else:
                    self.print_verbose (f'There is not a german word named {word} on the wiktionary')
                    not_found.append(word) # add to non-existant words
            else:
                not_found.append(word) # add to non-existant words

            bar.update(item_id=word+"            ")
            time.sleep(self.sleep_len)
        return (valid_words, not_found)

    def filter_language(self, word, lang):
        languages = self.extract_languages(word)
        for key in languages:
            if key==lang:
                word.wikitext=languages[key]
                return word
        return None

    def extract_languages(self, word):
        text = word.wikitext
        matches = re.findall(r'(==[\w.\[\]®\<\>\/\(\)! ]*\(\{\{\w*\|(\w*)\}\}\)[ ]*==)', text, re.UNICODE)
        languages = {}
        if matches:
            for i, match in enumerate(matches):
                self.print_verbose ("extract_languages | language found: %s " %match[1])
                if (len(matches)-i > 1):
                    lan = match[1]
                    next_match = matches[i+1]
                    index = text.index(match[0])
                    next_index = text.index(next_match[0])
                    languages[lan] = text[index:next_index]
                else:
                    index = text.index(match[0])
                    languages[match[1]] = text[index:len(text)]
            return languages
        raise WikiParsingError("parsing error in 'extract languages' function")

    def parse(self, wordlist, sleep=0.3):
        assert isinstance(wordlist, list),"Please pass a list of words"
        self.sleep_len = sleep
        self.words, self.not_found = self.__fetch_words(wordlist)
        self.cut_into_word_types(self.words)
        self.fix_spelling()
        self.fix_derivates_single()
        self.fix_derivatives_multi()
        self.remove_remaining_derivatives()
        self.remove_words_without_wordtypes()
        self.remove_main_wikitext()
        self.parse_substantives()
        self.parse_verbs()
        self.parse_adjektives()
        self.parse_IPAS()
        self.parse_pronomens()
        self.parse_examples()

        if self.not_found:
            not_found_words = ", ".join(self.not_found)
            print (f"Didn't find words: {not_found_words}")

    def get_wortart(self, cut):
        matches = re.findall(regex_word_type, cut, re.UNICODE)
        if matches:
            name = '/'.join(matches)
            return name
        raise WikiParsingError("Uncommon error, no 'Wortart' pattern")
        return None

    def cut_into_word_types(self, wordlist):
        for word_obj in wordlist:
            indexes = []
            matches = re.finditer(regex_wortart, word_obj.wikitext, re.MULTILINE)
            matches_exist = sum(1 for _ in re.finditer(regex_wortart, word_obj.wikitext, re.MULTILINE))
            if matches_exist:
                for match in matches:
                    indexes.append(match.start())
                indexes.append(len(word_obj.wikitext))
                cuts = [word_obj.wikitext[a[0]:a[1]] for a in zip(indexes[:-1],indexes[1:])]
                for cut in cuts:
                    excess_found = re.findall(regex_discard, cut, re.MULTILINE)
                    if excess_found:
                        cut_index = cut.find(excess_found[0])
                        cut = cut[0:cut_index]
                    w_type = self.get_wortart(cut)
                    word_type = WordType(word_obj.title, w_type, cut)
                    word_obj.word_types.append(word_type)
            else:
                self.print_verbose(f"{word_obj} doesn't mach wortart pattern")
                w = WordType(word_obj.title, "Wrong Spelling" , word_obj.wikitext)
                word_obj.word_types.append(w)

    def fix_spelling(self):
        to_remove = []
        to_parse =  []
        for word_obj in self.words:
            if 'Wrong Spelling' in word_obj.word_types[0].name:
                to_remove.append(word_obj)
                wikitext = word_obj.word_types[0].wikitext
                matches = re.findall(regex_proper_spelling, wikitext, re.UNICODE)
                to_parse.append(matches[0])
        if to_parse:
            self.words = [w for w in self.words if w not in to_remove]
            words, not_found = self.__fetch_words(to_parse)
            self.not_found.extend(not_found)
            self.words.extend(words)
            self.cut_into_word_types(words)

    def remove_remaining_derivatives(self):
        for word_obj in self.words:
            to_remove = []
            w_types = word_obj.word_types
            for w_type in w_types:
                w_type_split = w_type.name.split('/')
                for w in w_type_split:
                    if w in complex_kinds:
                        to_remove.append(w_type)
            w_types = [w for w in w_types if w not in to_remove]
            word_obj.word_types = w_types

    def fix_derivatives_multi(self):
        to_parse = []

        for word_obj in self.words:
            to_remove = []
            w_types = word_obj.word_types
            if len(w_types) > 1:
                for w_type in w_types:
                    w_type_split = w_type.name.split('/')
                    for w in w_type_split:
                        if w in complex_kinds:
                            matches = re.findall(regex_basic_word, w_type.wikitext, re.UNICODE)
                            to_remove.append(w_type)
                            if (matches):
                                to_parse.append(matches[0])
            w_types = [w for w in w_types if w not in to_remove]
            word_obj.word_types = w_types

        to_parse = remove_duplicates_keep_order(to_parse)
        if to_parse:
            words, not_found = self.__fetch_words(to_parse)
            self.not_found.extend(not_found)
            self.words.extend(words)
            self.cut_into_word_types(words)

    def remove_words_without_wordtypes(self):
        remove = []
        for w in self.words:
            if not w.word_types:
                remove.append(w)
        self.words = [w for w in self.words if w not in remove]

    def fix_derivates_single(self):
        to_parse, to_remove = [], []

        for word_obj in self.words:
            is_derivate, right_word = is_word_derivate(word_obj)
            if is_derivate:
                to_remove.append(word_obj)
                to_parse.append(right_word)

        to_parse = remove_duplicates_keep_order(to_parse)

        if to_parse:
            self.words = [w for w in self.words if w not in to_remove]
            words, not_found = self.__fetch_words(to_parse)
            self.not_found.extend(not_found)
            self.words.extend(words)
            self.cut_into_word_types(words)

    def remove_main_wikitext(self):
        for word_obj in self.words:
            word_obj.wikitext = []

    def parse_substantive(self, wikitext):

        def get_determiner(string):
            if (string=='m'):
                return 'der'
            elif (string=='f'):
                return 'die'
            elif (string=='n'):
                return 'das'
            else:
                print ("Huge error in get_determiner")
                return None

        substantive_singular = []
        substantive_plural = []
        has_genus = True
        # find if there is genus at all:
        find_wordgenus = u'\|Genus'
        matches = re.findall(find_wordgenus, wikitext, re.UNICODE)
        if matches:
            # find words with no singular:
            find_genuszero = u'\|Genus\s*=0'
            matches = re.findall(find_genuszero, wikitext, re.UNICODE)
            if matches:
                self.print_verbose ("there is a 0 genus")
                find_genus0singular = u'Nominativ Singular\s*=\s*(\w+)'
                matches = re.findall(find_genus0singular, wikitext, re.UNICODE)
                if matches:
                    substantive_singular.append(matches)
                    has_genus=False
                else:
                    has_genus=False
                    substantive_singular.append('—')
        else:
            has_genus=False
            self.print_verbose ("No word genus at all!")

        if has_genus:
            self.print_verbose ('there is genus')
            find_genus = u'\|Genus\s*\d*=(\w)'
            matches = re.findall(find_genus, wikitext, re.UNICODE)
            if matches: ## there is genus
                num_matches = len(matches)
                self.print_verbose (f"there are {num_matches} matches (genus):")
                for i, genus in enumerate(matches):
                    self.print_verbose (f'index: {i}, genus: {genus}')
                    if (num_matches==1):
                        singular = u'Nominativ Singular\s*=\s*(\w+)'
                    else: # multiple matches
                        singular = u'Nominativ Singular\s*'+str(i+1)+'=\s*(\w+)'
                    matches = re.search(singular, wikitext, re.UNICODE)
                    if matches:
                        word = (matches.group(1))
                        word = get_determiner(genus)+" "+word
                        substantive_singular.append(word)
                    else:
                        singular = u'Nominativ Singular\s*=\s*(\w+)'
                        matches = re.search(singular, wikitext, re.UNICODE)
                        if matches:
                            word = (matches.group(1))
                            word = get_determiner(genus)+" "+word
                            substantive_singular.append(word)

        # finding all plurals
        find_num_plurals = u'Nominativ Plural\s*\d*=\s*([\w]+)'
        matches = re.findall(find_num_plurals, wikitext, re.UNICODE)
        if matches: ## there are plurals
            num_matches = len(matches)
            self.print_verbose (f"there are {num_matches} matches (plural):")
            for i, plural in enumerate(matches):
                self.print_verbose  (f'index: {i}, plural: {plural}')
                if (num_matches==1):
                    plural_search = u'Nominativ Plural\s*=\s*([\w]+)'
                else:
                    plural_search = u'Nominativ Plural\s*'+str(i+1)+'\s*=\s*([\w]+)'
                matches = re.search(plural_search, wikitext, re.UNICODE)
                if matches:
                    word = (matches.group(1))
                    word = "die "+word
                    substantive_plural.append(word)
                else:
                    plural_search = u'Nominativ Plural\s*=\s*([\w]+)'
                    matches = re.search(plural_search, wikitext, re.UNICODE)
                    if matches:
                        word = (matches.group(1))
                        word = "die "+word
                        substantive_plural.append(word)
        else:
            self.print_verbose ("no plurals I guess")
            if (len(substantive_singular)!=0):
                substantive_plural.append("—")
        if (len(substantive_singular)==0 and len(substantive_plural)==0):
            self.print_verbose ("the word is not a substantive")
            return None
        return (substantive_singular, substantive_plural)

    def parse_substantives(self):
        for word_obj in self.words:
            for w_type in word_obj.word_types:
                for w in w_type.name.split('/'):
                    if w in ['Substantiv', 'Abkürzung']:
                        res = self.parse_substantive(w_type.wikitext)
                        if res:
                            w_type.substantiv = res

    def parse_verb(self, wikitext):
        praesens_ich, praeteritum_ich, partizip_ii, hilfsverb = [],[],[],[]

        find_praesens_ich = u'\|Pr\wsens_ich\**\s*=([\w ]+)'
        matches = re.findall(find_praesens_ich, wikitext, re.UNICODE)
        if matches: #there is at least one praesens ich
            self.print_verbose (f"there are {len(matches)} matches (praesens ich):")
            praesens_ich = matches

        find_praeteritum_ich = u'\|Pr\wteritum_ich\**\s*=\s*([\w ]+)'
        matches = re.findall(find_praeteritum_ich, wikitext, re.UNICODE)
        if matches: #there is at least one praeteritum_ich
            self.print_verbose (f"there are {len(matches)} matches (praeteritum ich):")
            praeteritum_ich = matches


        find_partizip_ii = u'\|Partizip II\**\s*=\s*([\w ]+)'
        matches = re.findall(find_partizip_ii, wikitext, re.UNICODE)
        if matches: #there is at least one partizip_ii
            self.print_verbose (f"there are {len(matches)} matches (partizip ii):")
            partizip_ii = matches

        find_hilfsverb = u'\|Hilfsverb\**\s*=\s*([\w ]+)'
        matches = re.findall(find_hilfsverb, wikitext, re.UNICODE)
        if matches: #there is at least one hilfsverb
            self.print_verbose (f"there are {len(matches)} matches (hilfsverb):")
            hilfsverb = matches

        if (len(praesens_ich)==0 and len(praeteritum_ich)==0 and len(partizip_ii)==0 and len(hilfsverb)==0):
            self.print_verbose ("the word is not a substantive")
            return None
        return (praesens_ich, praeteritum_ich, partizip_ii, hilfsverb)

    def parse_adjektive(self, wikitext):
        positiv, komparativ, superlativ = "—","—","—"

        find_positiv = u'\|Positiv\**=([\w\<\>]+)'
        matches = re.findall(find_positiv, wikitext, re.UNICODE)
        if matches: #there is at least one positiv
            self.print_verbose (f"there are {len(matches)} matches (positiv):")
            positiv = matches

        find_komparativ = u'\|Komparativ\**=([\w\<\>]+)'
        matches = re.findall(find_komparativ, wikitext, re.UNICODE)
        if matches: #there is at least one komparativ
            self.print_verbose (f"there are {len(matches)} matches (komparativ):")
            komparativ = matches

        find_superlativ = u'\|Superlativ\**=([\w\<\>]+)'
        matches = re.findall(find_superlativ, wikitext, re.UNICODE)
        if matches: #there is at least one superlativ
            self.print_verbose (f"there are {len(matches)} matches (superlativ):")
            matches = ["am "+m for m in matches]
            superlativ = matches

        if (positiv=="—" and komparativ=="—" and superlativ=="—"):
            self.print_verbose ("the word is not a adjektive")
            return None
        return (positiv, komparativ, superlativ)

    def parse_adjektives(self):
        for word_obj in self.words:
            for w_type in word_obj.word_types:
                for w in w_type.name.split('/'):
                    if w in ['Adjektiv', 'Adverb']:
                        res = self.parse_adjektive(w_type.wikitext)
                        if res:
                            w_type.adjektiv = res

    def parse_verbs(self):
        for word_obj in self.words:
            for w_type in word_obj.word_types:
                for w in w_type.name.split('/'):
                    if w in ['Verb']:
                        res = self.parse_verb(w_type.wikitext)
                        if res:
                            w_type.verb = res

    def parse_IPA(self, wikitext):
        IPA = []
        find_ipa = u':{{IPA}}[\d\D]*?{{Lautschrift[\d\D]*?}}'
        matches = re.findall(find_ipa, wikitext, re.UNICODE)
        if not matches:
            self.print_verbose ("The word has no IPA!")
            return None
        self.print_verbose ("the word has an IPA!")

        find_ipa_forms = u'{{Lautschrift\|([\d\D]*?)}}'
        matches = re.findall(find_ipa_forms, wikitext, re.UNICODE)
        if matches: #there is at least one IPA
            self.print_verbose (f"there are {len(matches)} matches (IPA):")
            #matches = [simplify_ipa(m) for m in matches] #simplify IPA
            IPA = matches
        if (len(IPA)==0):
            self.print_verbose ("The word doesn't have an IPA")
            return None
        return IPA

    def parse_IPAS(self):
        for word_obj in self.words:
            for w_type in word_obj.word_types:
                res = self.parse_IPA(w_type.wikitext)
                if res:
                    w_type.IPA = res

    def parse_pronomen(self, wikitext):
        dic = dict()
        counter = 0
        l1 = ['Nominativ', 'Genitiv', 'Dativ', 'Akkusativ']
        l2 = ['Singular', 'Plural']
        l3 = ['m','f','n']
        types = []
        for a in l1:
            for b in l2:
                    for c in l3:
                        if (b!="Plural"):
                            types.append(a+" "+b+" "+c)
                        else:
                            types.append(a+" "+b)
                            break

        for typ in types:
            search = u"\|"+typ+"\s*\**=\s*([\w ]+)"
            matches = re.findall(search, wikitext, re.UNICODE)
            if matches:
                dic[typ]=matches
            else:
                dic[typ]="—"
                counter +=1
        if (counter==16):
            return None
        return dic

    def parse_pronomens(self):
        for word_obj in self.words:
            for w_type in word_obj.word_types:
                for w in w_type.name.split('/'):
                    if 'pronomen' in str.lower(w):
                        res = self.parse_pronomen(w_type.wikitext)
                        if res:
                            w_type.pronomen = res

    def cut_examples(self, wikitext):
        find_cut = u'{{Beispiele}}([\s\S]+?)[= ]*[\n]+{{[\w ]+}}'
        matches = re.search(find_cut, wikitext, re.UNICODE)
        if matches:
            return matches.group(1)
        else:
            find_cut = u'{{Beispiele}}([\s\S]+)'
            matches = re.search(find_cut, wikitext, re.UNICODE)
            if matches:
                return matches.group(1)
        raise WikiParsingError("Problem in cut_examples(wikitext)")

    def clean_example(self, wikitext):
        text = wikitext
        search_reference = u'<[Rr]ef[\s\S]*?>[\s\S]*?</[rR]ef>'
        matches = re.findall(search_reference, wikitext, re.UNICODE)
        if matches:
            for m in matches:
                text = text.replace(m,"")

        search_reference_2 = u'\<[Rr]ef[\s\S]*?\/\>'
        matches = re.findall(search_reference_2, wikitext, re.UNICODE)
        if matches:
            for m in matches:
                text = text.replace(m,"")

        text = text.replace("''","")
        return text

    def parse_examples_from_wordtype(self, wikitext):
        ex = []
        search_examples = u'{{Beispiele}}'
        matches = re.findall(search_examples, wikitext, re.UNICODE)
        if matches:
            self.print_verbose ("There are examples")
            cut = self.cut_examples(wikitext)
            search_example = u'\:\[[\d]+\]\s+(.*)'
            matches = re.findall(search_example, cut, re.UNICODE)
            if matches:
                len_examples = len(matches)
                self.print_verbose (f'number of examples: {len_examples}')
                for example in matches:
                    example_clean = self.clean_example(example)
                    ex.append(example_clean)
            return ex
        else:
            self.print_verbose ("No examples")
            return None

    def parse_examples(self):
        for word_obj in self.words:
            for w_type in word_obj.word_types:
                res = self.parse_examples_from_wordtype(w_type.wikitext)
                if res:
                    w_type.examples = res

    def parse3(self, wordlist, sleep=0.3):
        self.parse_substantives()
        self.parse_verbs()
        self.parse_adjektives()
        self.parse_IPAS()
        self.parse_pronomens()
        self.parse_examples()

        if self.not_found:
            not_found_words = ", ".join(self.not_found)
            print (f"Didn't find words: {not_found_words}")