%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import tensorflow as tf
import re
import os

file_names = ['Youtube01-Psy.csv','Youtube02-KatyPerry.csv','Youtube03-LMFAO.csv',
              'Youtube04-Eminem.csv','Youtube05-Shakira.csv']
filepath = 'c:/users/michael/documents/github_data/data'
dataframes = [pd.read_csv(os.path.join(filepath,file_name)) for file_name in file_names]
dataframes[0].head(3)

for content in dataframes[0].CONTENT[14:20]:
    print(content)

def format_sentence(s):
    ret_sent = re.sub('http\S+','WEBLINK',s)
    ret_sent = re.sub(r'\\ufeff','',ret_sent)
    return ret_sent

from sklearn.model_selection import train_test_split
data = pd.concat(dataframes).reset_index(drop=True)
X_train,X_test,y_train,y_test = train_test_split(data.CONTENT,data.CLASS)

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
import nltk
from nltk.stem.porter import PorterStemmer
def port_tokenizer(text):
    words = re.split(r'\s+',text.lower())
    ps = PorterStemmer()
    return [ps.stem(word) for word in words if not word in nltk.corpus.stopwords.words('english') and word]
count_vect = CountVectorizer(preprocessor=format_sentence,tokenizer=port_tokenizer,min_df=5)
tfidf_vect = TfidfVectorizer(preprocessor=format_sentence,tokenizer=port_tokenizer,min_df=5)
X_train_bow = count_vect.fit_transform(X_train)
X_train_tfidf = tfidf_vect.fit_transform(X_train)
print(X_train_bow.shape)

from sklearn.metrics import f1_score

from sklearn.linear_model import LogisticRegression
from sklearn.grid_search import GridSearchCV
course_params = {'C':[2**x for x in range(-6,15,2)]}
gs = GridSearchCV(LogisticRegression(class_weight='balanced'),course_params,n_jobs=-1,cv=5,scoring='f1')
gs.fit(X_train_bow,y_train)
fine_params = {'C':[gs.best_params_['C']*(2**x) for x in np.arange(-1.75,2,.25)]}
gs = GridSearchCV(LogisticRegression(class_weight='balanced'),fine_params,n_jobs=-1,cv=5,scoring='f1')
gs.fit(X_train_bow,y_train)
print(gs.best_params_)
print(gs.best_score_)
sample_sizes = range(100,1000,10)
train_bow,test_bow = list(),list()
lr_bow = LogisticRegression(class_weight='balanced',C=gs.best_params_['C'])
for sample_size in sample_sizes:
    xs,ys= X_train[:sample_size],y_train[:sample_size]
    count_vect = CountVectorizer(preprocessor=format_sentence,tokenizer=port_tokenizer)
    xs_bow = count_vect.fit_transform(xs)
    xs_train,xs_test,ys_train,ys_test = train_test_split(xs_bow,ys)
    lr_bow.fit(xs_train,ys_train)
    train_bow.append(f1_score(lr_bow.predict(xs_train),ys_train))
    test_bow.append(f1_score(lr_bow.predict(xs_test),ys_test))
plt.plot(sample_sizes,train_bow,c='r',label='training set')
plt.plot(sample_sizes,test_bow,c='b',label='test set')
plt.xlabel('no of samples')
plt.ylabel('f1 score')
plt.title('Bag of words, logistic regression learning curve')
plt.legend(loc='lower right')
plt.show()
Xs_test = count_vect.transform(X_test)
print("Test Set f1-score: {:.3}".format(f1_score(lr_bow.predict(Xs_test),y_test)))

course_params = {'C':[2**x for x in range(-6,15,2)]}
gs = GridSearchCV(LogisticRegression(class_weight='balanced'),course_params,n_jobs=-1,cv=5,scoring='f1')
gs.fit(X_train_tfidf,y_train)
fine_params = {'C':[gs.best_params_['C']*(2**x) for x in np.arange(-1.75,2,.25)]}
gs = GridSearchCV(LogisticRegression(class_weight='balanced'),fine_params,n_jobs=-1,cv=5,scoring='f1')
gs.fit(X_train_tfidf,y_train)
print(gs.best_params_)
print(gs.best_score_)
sample_sizes = range(100,1000,10)
train_tfidf,test_tfidf = list(),list()
lr_tfidf = LogisticRegression(class_weight='balanced',C=gs.best_params_['C'])
for sample_size in sample_sizes:
    xs,ys= X_train[:sample_size],y_train[:sample_size]
    tfidf_vect = TfidfVectorizer(preprocessor=format_sentence,tokenizer=port_tokenizer)
    xs_tfidf = tfidf_vect.fit_transform(xs)
    xs_train,xs_test,ys_train,ys_test = train_test_split(xs_tfidf,ys)
    lr_tfidf.fit(xs_train,ys_train)
    train_tfidf.append(f1_score(lr_tfidf.predict(xs_train),ys_train))
    test_tfidf.append(f1_score(lr_tfidf.predict(xs_test),ys_test))
plt.plot(sample_sizes,train_bow,c='r',label='training set')
plt.plot(sample_sizes,test_bow,c='b',label='test set')
plt.xlabel('no of samples')
plt.ylabel('f1 score')
plt.legend(loc='lower right')
plt.title('TFIDF vectors, logistic regression learning curve')
plt.show()
Xs_test = tfidf_vect.transform(X_test)
print("Test Set f1-score: {:.3}".format(f1_score(lr_tfidf.predict(Xs_test),y_test)))

from sklearn.base import BaseEstimator
from sklearn.model_selection import cross_val_score
class BayesianSpamClassifier(BaseEstimator):
    """Provides a bayesian estimator of whether or """
    def __init__(self,min_df=5):
        self.min_df = min_df
    def fit(self,X,y):
        spam = X[y.values.astype(bool)]>0
        ham = X[~y.values.astype(bool)]>0
        self.ham= ham
        self.spam=spam
        self.valid_indices = (X.sum(axis=0)>=self.min_df).astype(bool)
        prob_word_given_spam = (spam.sum(axis=0)+1)/((spam.sum(axis=0)+1).sum())
        self.prob_word_given_spam = prob_word_given_spam[self.valid_indices]
        prob_word_given_ham = (ham.sum(axis=0)+1)/((ham.sum(axis=0)+1).sum())
        self.prob_word_given_ham = prob_word_given_ham[self.valid_indices]
        self.prob_spam = y.sum()/y.shape[0]
        self.prob_ham = 1-self.prob_spam
    def predict(self,X):
        prob_ham = self.prob_ham
        prob_spam = self.prob_spam
        prob_word_given_spam = self.prob_word_given_spam
        prob_word_given_ham = self.prob_word_given_ham
        X = X[:,self.valid_indices]>0
        log_prob_ratio_spam_ham = X.dot(np.log((prob_word_given_spam.T)/(prob_word_given_ham.T)))+np.log(prob_spam/prob_ham)
        return log_prob_ratio_spam_ham>0
cv = cross_val_score(BayesianSpamClassifier(2),X_train_bow.toarray(),y_train,
                     cv=10,scoring='f1')
print('the f1 score on the training set is: {:.2f} +/- {:.2f}'.format(cv.mean(),cv.std()))

train_bow=list()
test_bow=list()
for sample_size in sample_sizes:
    xs,ys= X_train[:sample_size],y_train[:sample_size]
    bc_bow =BayesianSpamClassifier(2)
    count_vect = CountVectorizer(preprocessor=format_sentence,tokenizer=port_tokenizer)
    xs_bow = count_vect.fit_transform(xs)
    xs_train,xs_test,ys_train,ys_test = train_test_split(xs_bow,ys)
    bc_bow.fit(xs_train.toarray(),ys_train)
    train_bow.append(f1_score(bc_bow.predict(xs_train.toarray()),ys_train))
    test_bow.append(f1_score(bc_bow.predict(xs_test.toarray()),ys_test))
plt.plot(sample_sizes,train_bow,c='r',label='training set')
plt.plot(sample_sizes,test_bow,c='b',label='test set')
plt.xlabel('no of samples')
plt.ylabel('f1 score')
plt.title('Naive bayes learning curve')
plt.legend(loc='lower right')
plt.show()
xs_test = count_vect.transform(X_test)
print("The f1 score on the test set is {:.2f}".format(f1_score(bc_bow.predict(xs_test),y_test)))

from gensim.models import Word2Vec
format_sentence, port_tokenizer
def process_data(X,y,embedding_dim=10,sentence_length=10):
    """Loads the text data and processes it into a padded array, and returns
    the array along with sentence lengths
    
    Parameters:
    --------------
    X: iterable array, shape = [n_samples,]
        entries consist of unprocessed sentences (i.e. text)
    y: iterable array, shape = [n_samples], 
        entries consist of 0 or 1
    embedding_dim: int, default 10: the size of word vectors in the model
    sentence_length: int, default 10: the maximum sentence length to consider,
                     longer sentences will be shortened
    
    Returns:
    tuple consisting of X,y,length
    X: array, shape = [n_samples,sentence_length,embedding_dim] 
    y: array, shape = [n_samples,2]
    length:, array, shape = [n_samples]
    """
    if(type(X)!=type(pd.Series())):
        X = pd.Series(X)
    if(type(y)!=type(pd.Series())):
        y =pd.Series(y)
    X = X.apply(format_sentence)
    X = X.apply(port_tokenizer)
    wv=Word2Vec(X,size=embedding_dim).wv
    X = X.apply(lambda ls:[list(wv[word]) for word in ls if
                                              word in wv.vocab])
    lengths = X.apply(lambda x:len(x) if len(x)<sentence_length else sentence_length)
    pad = [0. for i in range(embedding_dim)]
    X = X.apply(lambda ls:[ls[i] if i<len(ls) else pad for i in range(sentence_length)])
    X = X[lengths>0]
    y = y[lengths>0]
    lengths = lengths[lengths>0]
    #One hot encode y as [ham, spam]
    y = y.apply(lambda x:[0,1] if x else [1,0])
    return list(X),list(y),list(lengths)

class Config():
    """ A class to hold the settings for neural network.
    """
    learning_rate=.005
    batch_size=100
    max_epochs=1000

    #length of word vectors
    n_input=20
    #max no of words per sentence
    n_steps=10
    n_classes=2
    #size of hidden RNN layer(i.e. length of sentence features)
    n_hidden=20
    #regularization parameter
    l2=.01

class RNN(BaseEstimator):
    """ A class for a simple variable length RNN to classify sentences.  The sentences
    must be preprocessed into arrays of word vectors prior to using the RNN.

    Parameters:
    -------------
    x_train: array, shape = [n_samples,n_steps,n_input]
        training data, n_steps refers to the number of words per sentence,
        n_input refers to the number of features per word.   Sentences smaller
        then n_steps must be padded with vectors to ensure each sample contains
        an array of the same size.

    x_test: array, shape = [n_samples,n_steps,n_input]
        test data, same conditions as apply to x_train

    y_train: array shape = [n_samples,n_classes]
        test data.  The data must be in the form of one hot vectors for each
        predicted class

    y_test: array shape = [n_samples,n_classes]
        test data, same conditions as apply to y_train

    length_train: int array or list, shape = [n_samples]
        The length of each training sentence before padding.

    length_test: int array or list, shape = [n_samples]
        The lengths of each test sentence before padding

    self.config: object which holds configuration parameters
        config.learning rate: float, default: .001
            The learning rate used in minibatch stochastic gradient descent
        config.batch_size: int, default: 100
            The size of minibatches fed into the NN for training.
        config.max_epochs: int, default: 50
            number of times to iterate over the training data for stochastic gradient descent
        config.n_input: int, default: 10
            length of embedding vectors for words
        config.n_steps: int, default: 10
            maximum number of words per sentence
        config.n_classes: int default: 2
            number of classes to predict
        config.n_hidden: int, default=10,
            size of vectors in RNN, represents the number of features per sentence
    """
    def __init__(self,config,cell_type='rnn'):
        self.config=config
        self.cell_type=cell_type

    def add_placeholders(self):
        """Initializes the placeholders to be used in the calculation"""
        n_steps = self.config.n_steps
        n_input = self.config.n_input
        n_classes = self.config.n_classes
        
        self.x_input = tf.placeholder(tf.float32,shape=[None,n_steps,n_input])
        self.y_input  = tf.placeholder(tf.float32,shape=[None,n_classes])
        self.length_input = tf.placeholder(tf.int32,shape=[None])

    def get_batch(self,i,X,y,lengths):
        """ returns the ith mini-batch of the training data
        Parameters:
        -------------
        i: int, which batch of training data to return
        X: arraylike, shape = [n_samples,n_steps,n_input]
            The training data
        y: arraylike, int, shape = [n_samples,n_classes]
            the class labels
        lengths: arraylike, int, shape = [n_samples]
            the lengths of each encoded sentence before 
            padding is added

        Returns:
        --------------
        tuple consisting of batches for X,y, and length
        """
        batch_size = self.config.batch_size
        return X[i*batch_size:(i+1)*batch_size],\
                 y[i*batch_size:(i+1)*batch_size],\
                  lengths[i*batch_size:(i+1)*batch_size]

    def initialize_variables(self):
        """ Initializes the variables to be used in the tensorflow graph.
        """
        n_hidden = self.config.n_hidden
        n_classes = self.config.n_classes
        
        with tf.variable_scope('RNN'):
            weights = tf.get_variable('W',shape=[n_hidden,n_classes])
            biases = tf.get_variable('b',shape=[n_classes])

    def predict(self,X,lengths,reuse_var=False):
        """returns the logits predicted for each sentence

        Parameters:
        ---------------
        X, arraylike, shape=[n_samples,n_steps,n_features]
            preprocessed sentence arrays
        lengths, arraylike, shape=[n_samples] list of words
            per sentence before padding is added

        Returns:
        --------------
        logits, arraylike, shape = [n_samples,n_classes]
            predicted logits for each class
        """
        n_steps = self.config.n_steps
        n_hidden = self.config.n_hidden
        with tf.variable_scope('RNN',reuse=reuse_var):
            if self.cell_type=='rnn':
                rnn_cell = tf.contrib.rnn.BasicRNNCell(n_hidden,reuse=reuse_var)
            elif self.cell_type=='lstm':
                rnn_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden,reuse=reuse_var,forget_bias=.6)
            outputs,states = tf.nn.dynamic_rnn(rnn_cell,
                                sequence_length=lengths,inputs=X,
                                time_major=False,dtype=tf.float32)

        with tf.variable_scope('RNN',reuse=True):
            W = tf.get_variable('W')
            b = tf.get_variable('b')
        output_list = tf.unstack(outputs,n_steps,1)
        batch_size = tf.shape(outputs)[0]
        last_rnn_output = tf.gather_nd(outputs,tf.stack([tf.range(
            batch_size),lengths-1],axis=1))
        return tf.matmul(last_rnn_output,W)+b
    
    def get_cost(self,pred,y):
        cost=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
                logits=pred,labels=y))
        return cost
    
    def optimize(self,pred,y):
        """Adds the optimization layer to the graph.

        Parameters:
        ---------------
        pred: arraylike, shape = [n_samples,n_classes]
            logits for predicted class labels
        y: arraylike, shape = [n_samples,n_classes]
            one hot vector corresponding to actual values.
        returns:
        ------------
        tuple consisting of cost and optimizer,
        cost: float, total cost associated with predictions and actual values,
            uses cross entropy to compute the cost.
        optimizer:
            Adam optimizer to reduce the cost.
        """
        cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
            logits=pred,labels=y))
        with tf.variable_scope('RNN',reuse=True):
            cost += self.config.l2*tf.nn.l2_loss(tf.get_variable('W'))
        optimizer = tf.train.AdamOptimizer(learning_rate=self.config.\
                                           learning_rate).minimize(cost)
        return cost,optimizer


    def train(self,X,y,lengths,verbose=False):
        """Trains the neural network given a batch of training data (X,y,length)
        Parameters:
        --------------
        X: array, shape = [n_samples,n_steps,n_input]
            training data, n_steps refers to the number of words per sentence,
            n_input refers to the number of features per word.   Sentences smaller
            then n_steps must be padded with vectors to ensure each sample contains
            an array of the same size.

        y: array, shape = [n_samples,n_classes]
            test data.  The data must be in the form of one hot vectors for each
            predicted class

        length: array or list, int, shape = [n_samples]
            The length of each training sentence before padding.
        """
        self.add_placeholders()
        self.initialize_variables()
        logits = self.predict(self.x_input,self.length_input)
        cost,optimizer = self.optimize(logits,self.y_input)
        y_pred = tf.argmax(logits,1)
        labels  = tf.argmax(self.y_input,1)
        
        #Create streaming measurements to find accuracy, precision and recall
        with tf.name_scope('measurements'):
            precision,prec_update = tf.contrib.metrics.streaming_precision(y_pred,labels)
            recall,recall_update = tf.contrib.metrics.streaming_recall(y_pred,labels)
            accuracy,accuracy_update = tf.contrib.metrics.streaming_accuracy(y_pred,labels)
        #Store variables related to metrics in a list, which allows all measuremnt variables
        #to be reset
        stream_vars = [i for i in tf.local_variables() if i.name.split('/')[0] == 'measurements']
        reset_measurements = [tf.variables_initializer(stream_vars)]
        
        self.saver = tf.train.Saver()
        
        with tf.Session() as sess:
            sess.run(tf.global_variables_initializer())
            sess.run(tf.local_variables_initializer())

            for epoch in range(config.max_epochs):
                av_loss = []
                for i in range(len(X)//self.config.batch_size):
                    batch_x,batch_y,batch_length = self.get_batch(i,X,y,lengths)
                    fd = {self.x_input:batch_x,self.y_input:batch_y,
                          self.length_input:batch_length}
                    loss,*_ = sess.run((cost,optimizer,
                                        prec_update,recall_update,
                                        accuracy_update)
                                       ,feed_dict=fd)
                    av_loss.append(loss)
                loss = np.array(av_loss).mean()
                acc,p,r = sess.run((accuracy,precision,recall))
                sess.run(reset_measurements)
                f1_score = 2*p*r/(p+r)
                if verbose and epoch%100==0:
                    print("Epoch {}, Minibatch Loss={:.6f}, training accuracy={:.5f}, f1={:.5f}"\
                              .format(epoch,loss,acc,f1_score))
            self.saver.save(sess,'/temp/model.ckpt')
            
    def test(self,X,y,lengths):
        logits = self.predict(self.x_input,self.length_input,reuse_var=True)
        cost = self.get_cost(logits,self.y_input)

        y_pred = tf.argmax(logits,1)
        labels  = tf.argmax(self.y_input,1)
        
        precision,prec_update = tf.contrib.metrics.streaming_precision(y_pred,labels)
        recall,recall_update = tf.contrib.metrics.streaming_recall(y_pred,labels)
        accuracy,accuracy_update = tf.contrib.metrics.streaming_accuracy(y_pred,labels)
        
        with tf.Session() as sess:
            self.saver.restore(sess, "/temp/model.ckpt")
            print("Model restored.")
            sess.run(tf.local_variables_initializer())
            cost = sess.run(cost,feed_dict={self.x_input:X,
                                                    self.y_input:y,
                                                    self.length_input:lengths})
            sess.run((prec_update,recall_update,accuracy_update),feed_dict={self.x_input:X,
                                                    self.y_input:y,
                                                    self.length_input:lengths})
            acc,p,r = sess.run((accuracy,precision,recall))
            f1_score = 2*p*r/(p+r)
            print("Testing Loss={:.6f}, testing accuracy= {:.5f}, f1={:.5f} ".format(cost,acc,f1_score))
tf.reset_default_graph()
config = Config()
config.n_input = 50
config.n_hidden=50
rnn = RNN(config)
X,y,lengths = process_data(data.CONTENT,data.CLASS,embedding_dim =config.n_input)
X_train,X_test,y_train,y_test,lengths_train,lengths_test = \
        train_test_split(X,y,lengths,train_size=.8,random_state=2)
rnn.train(X_train,y_train,lengths_train,verbose=True)
rnn.test(X_test,y_test,lengths_test)

for n_hidden in [5,10,20,50]:
    tf.reset_default_graph()
    config = Config()
    config.n_input = 50
    config.n_hidden=n_hidden
    rnn = RNN(config)
    print('RNN cell, n_hidden = {}'.format(n_hidden))
    rnn.train(X_train,y_train,lengths_train,verbose=True)
    rnn.test(X_test,y_test,lengths_test)

%%html