!git clone https://github.com/shadab4150/tez.git

tez_path = '/content/tez/'
import sys
sys.path.append(tez_path)

from google.colab import files

uploaded = files.upload()

for fn in uploaded.keys():
  print('User uploaded file "{name}" with length {length} bytes'.format(name=fn, length=len(uploaded[fn])))
  
# Then move kaggle.json into the folder where the API expects to find it.
!mkdir -p ~/.kaggle/ && mv kaggle.json ~/.kaggle/ && chmod 600 ~/.kaggle/kaggle.json

!pip uninstall kaggle -y
!pip install -qq kaggle

!kaggle competitions download -c quora-question-pairs

!unzip '/content/quora-question-pairs.zip'
!unzip '/content/train.csv.zip'
!unzip '/content/test.csv.zip'

!pip install -qq transformers

import tez
import transformers
import torch
import sys
import torch.nn as nn
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import sklearn
from pathlib import Path

trn_df = pd.read_csv('train.csv')

path = Path('/content')

trn_df.sample(n=5)

trn_df.info()

from sklearn.model_selection import StratifiedKFold

trn_df['kfold'] = -1
trn_df = trn_df.sample(frac=1.)
y = trn_df['is_duplicate'].values
skf = StratifiedKFold(n_splits=20)
for fld, (trn_,val_) in enumerate(skf.split(X=trn_df,y=y)):
    trn_df.loc[val_,'kfold'] = fld

sns.countplot(x=trn_df[trn_df.kfold==1].is_duplicate);

import re
import nltk
nltk.download('stopwords')
STOPWORDS = nltk.corpus.stopwords.words('english')

def text_to_wordlist(text, remove_stopwords=False, stem_words=False):
    # Clean the text, with the option to remove stopwords and to stem words.
    
    # Convert words to lower case and split them
    text = text.lower().split()

    # Optionally, remove stop words
    if remove_stopwords:
        text = [w for w in text if not w in STOPWORDS]
    
    text = " ".join(text)
    
    # Remove punctuation from text
    # text = "".join([c for c in text if c not in punctuation])

    # Clean the text
    text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text)
    text = re.sub(r"what's", "what is ", text)
    text = re.sub(r"\'s", " ", text)
    text = re.sub(r"\'ve", " have ", text)
    text = re.sub(r"can't", "cannot ", text)
    text = re.sub(r"n't", " not ", text)
    text = re.sub(r"i'm", "i am ", text)
    text = re.sub(r"\'re", " are ", text)
    text = re.sub(r"\'d", " would ", text)
    text = re.sub(r"\'ll", " will ", text)
    text = re.sub(r",", " ", text)
    text = re.sub(r"\.", " ", text)
    text = re.sub(r"!", " ! ", text)
    text = re.sub(r"\/", " ", text)
    text = re.sub(r"\^", " ^ ", text)
    text = re.sub(r"\+", " + ", text)
    text = re.sub(r"\-", " - ", text)
    text = re.sub(r"\=", " = ", text)
    text = re.sub(r"'", " ", text)
    text = re.sub(r":", " : ", text)
    text = re.sub(r"(\d+)(k)", r"\g<1>000", text)
    text = re.sub(r" e g ", " eg ", text)
    text = re.sub(r" b g ", " bg ", text)
    text = re.sub(r" u s ", " american ", text)
    # text = re.sub(r"\0s", "0", text) # It doesn't make sense to me
    text = re.sub(r" 9 11 ", "911", text)
    text = re.sub(r"e - mail", "email", text)
    text = re.sub(r"j k", "jk", text)
    text = re.sub(r"\s{2,}", " ", text)
    
    # Optionally, shorten words to their stems
    if stem_words:
        text = text.split()
        stemmer = SnowballStemmer('english')
        stemmed_words = [stemmer.stem(word) for word in text]
        text = " ".join(stemmed_words)
    
    # Return a list of words
    return(text)

val_df = trn_df[trn_df.kfold==3].copy() # valid
train_df = trn_df[trn_df.kfold<3].copy() # train 
ts_df = trn_df[trn_df.kfold==8].copy() # test

train_df.shape,val_df.shape

train_df['question1'] = train_df['question1'].apply(lambda x: text_to_wordlist(str(x), remove_stopwords=False))
train_df['question2'] = train_df['question2'].apply(lambda x: text_to_wordlist(str(x), remove_stopwords=False))
val_df['question1'] = val_df['question1'].apply(lambda x: text_to_wordlist(str(x), remove_stopwords=False))
val_df['question2'] = val_df['question2'].apply(lambda x: text_to_wordlist(str(x), remove_stopwords=False))

train_df.head()

from transformers import AdamW,get_linear_schedule_with_warmup
import sklearn.metrics as sklm

class BERTDataset:
    def __init__(self, texts1,texts2, targets,max_len=64):
        #Question 1st
        self.texts1 = texts1
        # Question 2nd
        self.texts2 = texts2
        # target is duplicate
        self.targets = targets
        # tokenizer from transformers
        self.tokenizer = transformers.BertTokenizer.from_pretrained(
            "bert-base-cased",
            do_lower_case=False
        )
        self.max_len = max_len
    
    def __len__(self):
        return min(len(self.texts1),len(self.texts2))

    def __getitem__(self,idx):
        
        text1 = str(self.texts1[idx])
        text2 = str(self.texts2[idx])
        # a simple text split and join to clean
        text1 = " ".join(text1.split())
        text2 = " ".join(text2.split())
        # tokenize
        inputs = self.tokenizer.encode_plus(
            text1,
            text2,
            add_special_tokens=True,
            max_length=self.max_len,
            padding="max_length",
            truncation=True
            )
        # return from dataset
        #After tokenizing, bert excepts input_ids for text, there mask as well token type ids.
        resp = {
            "ids" :  torch.tensor(inputs["input_ids"],dtype=torch.long),
            "mask" : torch.tensor(inputs["attention_mask"],dtype=torch.long),
            "token_type_ids" : torch.tensor(inputs["token_type_ids"],dtype=torch.long),
            "targets" : torch.tensor(self.targets[idx],dtype=torch.float)
        }

        return resp

train_dataset = BERTDataset(train_df.question1.values,train_df.question2.values,train_df.is_duplicate.values)
valid_dataset = BERTDataset(val_df.question1.values,val_df.question2.values,val_df.is_duplicate.values)

test_dataset = BERTDataset(ts_df.question1.values,ts_df.question2.values,ts_df.is_duplicate.values)

train_dataset[0]

class QuestionModel(tez.Model):
    
    def __init__(self,num_classes,num_train_steps):
        super().__init__()
        # Load Bert Pretrained Model
        self.bert = transformers.BertModel.from_pretrained(
            "bert-base-cased",return_dict=False)
        # A Dropout Before final Output
        self.bert_drop = nn.Dropout(0.3)
        # Final layer out.
        self.out = nn.Linear(768,num_classes)
        self.num_train_steps = num_train_steps
        self.step_scheduler_after = "batch"

    def fetch_optimizer(self):
        # optimiser
        opt = AdamW(self.parameters(),lr=1e-4)
        return opt

    def fetch_scheduler(self):
        # learning rate scheduler
        sch = get_linear_schedule_with_warmup(
            self.optimizer,num_warmup_steps=0,num_training_steps=self.num_train_steps
            )
        return sch

    def loss(self,outputs,targets):
        # loss function
        if targets is None:
            return None
        
        return nn.BCEWithLogitsLoss()(outputs, targets.view(-1,1))

    def monitor_metrics(self,outputs,targets):
        #metrics to monitor model performance
        if targets is None:
            return {}
        outputs = torch.sigmoid(outputs).cpu().detach().numpy()
        targets = targets.cpu().detach().numpy()
        acc = sklm.accuracy_score(targets,outputs>=0.5)
        try:
            score1 = sklm.roc_auc_score(targets, outputs)
            score2 = sklm.log_loss(targets, outputs)
        except:
            score1 = 0.5
            score2 = 0.5
        return {"roc_auc": score1,"accuracy":acc} 

    def forward(self,ids,mask,token_type_ids,targets=None):
        # model input and output
        _,x1 = self.bert(ids,attention_mask=mask,token_type_ids=token_type_ids)
        x = self.bert_drop(x1)
        x = self.out(x)
        
        if targets is not None:
            loss = self.loss(x,targets)
            met = self.monitor_metrics(x,targets)
            return x, loss, met
        
        return x,-1, {}

TRAIN_BS  = 16 # train batch size
EPOCHS = 10 # number of epochs to train
n_train_steps = int(len(train_df) / TRAIN_BS* EPOCHS);n_train_steps

model = QuestionModel(num_classes=1,num_train_steps=n_train_steps)

es = tez.callbacks.EarlyStopping(monitor="valid_accuracy", model_path="model.bin")

model.fit(train_dataset,
          valid_dataset=valid_dataset,
          device="cuda",
          epochs=2, 
          callbacks=[es],
          train_bs=32,
          fp16=True)

from tqdm.notebook import tqdm

preds = model.predict(test_dataset, batch_size=64, n_jobs=-1, device="cuda")
y_pred = []
for p in tqdm(preds):
    y_pred += list(p.flatten())
y_pred = torch.sigmoid(torch.tensor(y_pred)).numpy()

sklm.accuracy_score(ts_df.is_duplicate.values,y_pred>=0.5)

import gc
model = None
gc.collect()