api.py

from flask import Flask, request, jsonify, render_template
from sklearn.externals import joblib
import traceback
import pandas as pd
import numpy as np
import sys
import nltk
from nltk.stem import WordNetLemmatizer
from nltk.corpus import wordnet
import re
import collections
from collections import defaultdict
from sklearn import ensemble
from sklearn.model_selection import GridSearchCV
from sklearn.externals import joblib
import json
from sklearn.linear_model import LinearRegression

app = Flask(__name__)


def sentence_to_wordlist(raw_sentence):

    clean_sentence = re.sub("[^a-zA-Z0-9]", " ", raw_sentence)
    tokens = nltk.word_tokenize(clean_sentence)

    return tokens


def tokenize(essay):
    stripped_essay = essay.strip()

    tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
    raw_sentences = tokenizer.tokenize(stripped_essay)

    tokenized_sentences = []
    for raw_sentence in raw_sentences:
        if len(raw_sentence) > 0:
            tokenized_sentences.append(sentence_to_wordlist(raw_sentence))

    return tokenized_sentences


def avg_word_len(essay):

    clean_essay = re.sub(r'\W', ' ', essay)
    words = nltk.word_tokenize(clean_essay)

    return sum(len(word) for word in words) / len(words)


def word_count(essay):

    clean_essay = re.sub(r'\W', ' ', essay)
    words = nltk.word_tokenize(clean_essay)

    return len(words)


def char_count(essay):

    clean_essay = re.sub(r'\s', '', str(essay).lower())

    return len(clean_essay)


def sent_count(essay):

    sentences = nltk.sent_tokenize(essay)

    return len(sentences)


def punctuation_count(essay):

    clean_essay = re.sub(r'[a-zA-Z0-9]', ' ', essay)
    punctuations = nltk.word_tokenize(clean_essay)

    return len(punctuations)


def count_lemmas(essay):

    tokenized_sentences = tokenize(essay)

    lemmas = []
    wordnet_lemmatizer = WordNetLemmatizer()

    for sentence in tokenized_sentences:
        tagged_tokens = nltk.pos_tag(sentence)

        for token_tuple in tagged_tokens:

            pos_tag = token_tuple[1]

            if pos_tag.startswith('N'):
                pos = wordnet.NOUN
                lemmas.append(wordnet_lemmatizer.lemmatize(
                    token_tuple[0], pos))
            elif pos_tag.startswith('J'):
                pos = wordnet.ADJ
                lemmas.append(wordnet_lemmatizer.lemmatize(
                    token_tuple[0], pos))
            elif pos_tag.startswith('V'):
                pos = wordnet.VERB
                lemmas.append(wordnet_lemmatizer.lemmatize(
                    token_tuple[0], pos))
            elif pos_tag.startswith('R'):
                pos = wordnet.ADV
                lemmas.append(wordnet_lemmatizer.lemmatize(
                    token_tuple[0], pos))
            else:
                pos = wordnet.NOUN
                lemmas.append(wordnet_lemmatizer.lemmatize(
                    token_tuple[0], pos))

    lemma_count = len(set(lemmas))

    return lemma_count


def count_spell_error(essay):

    clean_essay = re.sub(r'\W', ' ', str(essay).lower())
    clean_essay = re.sub(r'[0-9]', '', clean_essay)

    data = open('C:\\Users\\ig\\Documents\\PY\\big.txt').read()

    words_ = re.findall('[a-z]+', data.lower())

    word_dict = collections.defaultdict(lambda: 0)

    for word in words_:
        word_dict[word] += 1

    clean_essay = re.sub(r'\W', ' ', str(essay).lower())
    clean_essay = re.sub(r'[0-9]', '', clean_essay)

    mispell_count = 0

    words = clean_essay.split()

    for word in words:
        if not word in word_dict:
            mispell_count += 1

    return mispell_count


def count_pos(essay):

    tokenized_sentences = tokenize(essay)

    noun_count = 0
    adj_count = 0
    verb_count = 0
    adv_count = 0
    pronoun_count = 0
    preposition_count = 0

    for sentence in tokenized_sentences:
        tagged_tokens = nltk.pos_tag(sentence)

        for token_tuple in tagged_tokens:
            pos_tag = token_tuple[1]

            if pos_tag.startswith('N'):
                noun_count += 1
            elif pos_tag.startswith('J'):
                adj_count += 1
            elif pos_tag.startswith('V'):
                verb_count += 1
            elif pos_tag.startswith('R'):
                adv_count += 1
            elif pos_tag.startswith('P'):
                pronoun_count += 1
            elif pos_tag.startswith('I'):
                preposition_count += 1

    return noun_count, adj_count, verb_count, adv_count, pronoun_count, preposition_count


def extract_features(data):

    features = data.copy()

    features['char_count'] = features['essay'].apply(char_count)

    features['word_count'] = features['essay'].apply(word_count)

    features['sent_count'] = features['essay'].apply(sent_count)

    features['avg_word_len'] = features['essay'].apply(avg_word_len)

    features['lemma_count'] = features['essay'].apply(count_lemmas)

    features['spell_err_count'] = features['essay'].apply(count_spell_error)

    features['punctuation_count'] = features['essay'].apply(punctuation_count)

    features['noun_count'], features['adj_count'], features['verb_count'], features['adv_count'], features[
        'pronoun_count'], features['preposition_count'] = zip(*features['essay'].map(count_pos))

    return features


@app.route('/predict', methods=['POST'])
def predict():

    json_ = request.get_json(force=True)
    print(json_)
    essay = pd.DataFrame(json_)
    f1 = extract_features(essay)
    X = f1.iloc[:, 1:].values

    prediction = list(linear_regressor.predict(X))
    print(prediction)

    return jsonify({'prediction': str(prediction)})


@app.route('/enter', methods=['GET', 'POST'])
def enter():
    return render_template('form.html')


if __name__ == '__main__':
    try:
        port = int(sys.argv[1])
    except:
        port = 12345

    linear_regressor = joblib.load("model.pkl")

    app.run(port=port, debug=True)