iac_v1,_iac_v2,_and_tweets_a1,_a2,_a3,_a4.py

# -*- coding: utf-8 -*-
"""IAC-V1, IAC-V2, and Tweets A1, A2, A3, A4

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1-HBqf2EdGxikz0HhJSdN9a3EGY2QoZ7a

## Imports:
"""

from google.colab import drive
drive.mount('/content/drive')

!pip install transformers
!pip install emoji
!pip install optuna
!pip install evaluate
!pip install -q bitsandbytes
!pip install pytorch_metric_learning
!pip install sentencepiece
!pip install rouge_score
!pip install accelerate -U
import pandas as pd
import numpy as np

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.utils.data import DataLoader, TensorDataset

import transformers
from transformers import AutoTokenizer, AutoModelForCausalLM, AutoModel

import bitsandbytes as bnb

from tqdm import tqdm
from operator import itemgetter
from pytorch_metric_learning.losses import NTXentLoss

"""## IAC-V1:

### A1:
"""

import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
import torch
from torch import nn
from torch.utils.data import Dataset
from transformers import Trainer,TrainingArguments
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score
from scipy import stats

class FFNNClassifier(nn.Module):
  def __init__(self, embed_dim, hidden_dim, num_classes):
    super(FFNNClassifier, self).__init__()
    self.fc1 = nn.Linear(embed_dim, hidden_dim)
    self.relu = nn.ReLU()
    self.fc2 = nn.Linear(hidden_dim, num_classes)

  def forward(self, embeddings, labels=None):
    batch_size, seq_len, embed_dim = embeddings.shape
    embeddings = embeddings.view(batch_size * seq_len, embed_dim)
    embeddings = embeddings.float()
    out = self.fc1(embeddings)
    out = self.relu(out)
    logits = self.fc2(out)
    logits = logits.view(batch_size, seq_len, -1)
    return {'logits': logits}

class SarcasmDataset(torch.utils.data.Dataset):
  def __init__(self, embeddings, labels):
    self.embeddings = embeddings
    self.labels = labels

  def __getitem__(self, idx):
    item = {'embeddings': self.embeddings[idx], 'labels': torch.tensor(self.labels[idx])}
    return item

  def __len__(self):
    return len(self.labels)

def compute_metrics(p):
  pred, labels = p
  aggregated_logits = stats.mode(pred, axis=1)[0]
  probabilities = np.exp(aggregated_logits) / np.sum(np.exp(aggregated_logits), axis=-1, keepdims=True)
  pred = np.argmax(probabilities, axis=-1)
  accuracy = accuracy_score(y_true=labels, y_pred=pred)
  f1 = f1_score(y_true=labels, y_pred=pred, average='binary')
  precision = precision_score(y_true=labels, y_pred=pred, average='binary')
  recall = recall_score(y_true=labels, y_pred=pred, average='binary')
  return {"accuracy": accuracy, "f1": f1, "precision": precision, "recall": recall}

class CustomTrainer(Trainer):
  def compute_loss(self, model, inputs, return_outputs=False):
    embeddings = inputs.get("embeddings")
    outputs = model(embeddings)
    logits = outputs['logits']
    labels = inputs.get("labels").to(logits.device)
    probabilities = torch.nn.functional.softmax(logits, dim=-1)
    sentence_probabilities = torch.mean(probabilities, dim=1)
    sentence_logits = torch.log(sentence_probabilities)
    loss_fct = nn.CrossEntropyLoss()
    loss = loss_fct(sentence_logits, labels)
    return (loss, outputs) if return_outputs else loss

import pickle
from transformers import AutoTokenizer, TrainingArguments, Trainer, AdamW
from torch import stack

# Load word-level embeddings
with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/train_word", "rb") as fp:
  train_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/test_word", "rb") as fp:
  test_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/val_word", "rb") as fp:
  val_embeddings = pickle.load(fp)

# Load data
train = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V1/train.csv')
test = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V1/test.csv')
val = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V1/val.csv')

train_embeddings = [embedding.astype(np.float32) for embedding in train_embeddings]
test_embeddings = [embedding.astype(np.float32) for embedding in test_embeddings]
val_embeddings = [embedding.astype(np.float32) for embedding in val_embeddings]

train_labels = train['sarcastic'].values.tolist()
test_labels = test['sarcastic'].values.tolist()
val_labels = val['sarcastic'].values.tolist()

train_dataset = SarcasmDataset(train_embeddings, train_labels)
test_dataset = SarcasmDataset(test_embeddings, test_labels)
val_dataset = SarcasmDataset(val_embeddings, val_labels)

embed_dim = 768
hidden_dim = 128
num_classes = 2

model = FFNNClassifier(embed_dim, hidden_dim, num_classes)

training_args = TrainingArguments(
  output_dir='./res', evaluation_strategy="steps", num_train_epochs=5, per_device_train_batch_size=32,
  per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01,logging_dir='./logs4',
  load_best_model_at_end=True,
)

trainer = CustomTrainer(
  model=model, args=training_args, train_dataset=train_dataset,
  eval_dataset=val_dataset,
  compute_metrics=compute_metrics,
)

trainer.train()

# Evaluate on test dataset after training
trainer.evaluate(test_dataset)

"""### A2 RoBERTa:"""

import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
import torch
from torch import nn
from torch.utils.data import Dataset
from transformers import Trainer,TrainingArguments
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score

class FFNNClassifier(nn.Module):
  def __init__(self, embed_dim, hidden_dim, num_classes):
    super(FFNNClassifier, self).__init__()
    self.fc1 = nn.Linear(embed_dim, hidden_dim)
    self.relu = nn.ReLU()
    self.fc2 = nn.Linear(hidden_dim, num_classes)

  def forward(self, embeddings, labels=None):
    out = self.fc1(embeddings)
    out = self.relu(out)
    logits = self.fc2(out)
    return {'logits': logits}

class SarcasmDataset(torch.utils.data.Dataset):
  def __init__(self, embeddings, labels):
    self.embeddings = embeddings
    self.labels = labels

  def __getitem__(self, idx):
    item = {'embeddings': self.embeddings[idx], 'labels': torch.tensor(self.labels[idx])}
    return item

  def __len__(self):
    return len(self.labels)

def compute_metrics(p):
  pred, labels = p
  pred = np.argmax(pred, axis=1)

  accuracy = accuracy_score(y_true=labels, y_pred=pred)
  f1 = f1_score(labels, pred)
  precision = precision_score(labels, pred)
  recall = recall_score(labels, pred)

  return {"accuracy": accuracy, "f1_score": f1, "precision": precision, "recall": recall}

class CustomTrainer(Trainer):
  def compute_loss(self, model, inputs, return_outputs=False):
    embeddings = inputs.get("embeddings")
    outputs = model(embeddings)
    logits = outputs['logits']
    labels = inputs.get("labels").to(logits.device)
    loss_fct = nn.CrossEntropyLoss()
    loss = loss_fct(logits.view(-1, num_classes), labels.view(-1))
    return (loss, outputs) if return_outputs else loss

import pickle
from transformers import AutoTokenizer, TrainingArguments, Trainer, AdamW
from torch import stack

# Load sentence-level embeddings
with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/train_vanilla", "rb") as fp:
  vanilla_train = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/test_vanilla", "rb") as fp:
  vanilla_test = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/val_vanilla", "rb") as fp:
  vanilla_val = pickle.load(fp)

# Load data
train = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V1/train.csv')
test = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V1/test.csv')
val = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V1/val.csv')

train_labels = train['sarcastic'].values.tolist()
test_labels = test['sarcastic'].values.tolist()
val_labels = val['sarcastic'].values.tolist()

train_dataset = SarcasmDataset(vanilla_train, train_labels)
test_dataset = SarcasmDataset(vanilla_test, test_labels)
val_dataset = SarcasmDataset(vanilla_val, val_labels)

embed_dim = 768
hidden_dim = 128
num_classes = 2

model = FFNNClassifier(embed_dim, hidden_dim, num_classes)

training_args = TrainingArguments(
  output_dir='./res', evaluation_strategy="steps", num_train_epochs=5, per_device_train_batch_size=32,
  per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01,logging_dir='./logs4',
  load_best_model_at_end=True,
)

trainer = CustomTrainer(
  model=model, args=training_args, train_dataset=train_dataset,
  eval_dataset=val_dataset,
  compute_metrics=compute_metrics,
)

trainer.train()

# Evaluate on test dataset after training
trainer.evaluate(test_dataset)

"""### A2 BERTweet:"""

import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
import torch
from torch import nn
from torch.utils.data import Dataset
from transformers import Trainer,TrainingArguments
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score

class FFNNClassifier(nn.Module):
  def __init__(self, embed_dim, hidden_dim, num_classes):
    super(FFNNClassifier, self).__init__()
    self.fc1 = nn.Linear(embed_dim, hidden_dim)
    self.relu = nn.ReLU()
    self.fc2 = nn.Linear(hidden_dim, num_classes)

  def forward(self, embeddings, labels=None):
    out = self.fc1(embeddings)
    out = self.relu(out)
    logits = self.fc2(out)
    return {'logits': logits}

class SarcasmDataset(torch.utils.data.Dataset):
  def __init__(self, embeddings, labels):
    self.embeddings = embeddings
    self.labels = labels

  def __getitem__(self, idx):
    item = {'embeddings': self.embeddings[idx], 'labels': torch.tensor(self.labels[idx])}
    return item

  def __len__(self):
    return len(self.labels)

def compute_metrics(p):
  pred, labels = p
  pred = np.argmax(pred, axis=1)

  accuracy = accuracy_score(y_true=labels, y_pred=pred)
  f1 = f1_score(labels, pred)
  precision = precision_score(labels, pred)
  recall = recall_score(labels, pred)

  return {"accuracy": accuracy, "f1_score": f1, "precision": precision, "recall": recall}

class CustomTrainer(Trainer):
  def compute_loss(self, model, inputs, return_outputs=False):
    embeddings = inputs.get("embeddings")
    outputs = model(embeddings)
    logits = outputs['logits']
    labels = inputs.get("labels").to(logits.device)
    loss_fct = nn.CrossEntropyLoss()
    loss = loss_fct(logits.view(-1, num_classes), labels.view(-1))
    return (loss, outputs) if return_outputs else loss

import pickle
from transformers import AutoTokenizer, TrainingArguments, Trainer, AdamW
from torch import stack

# Load sentence-level embeddings
with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/BERTweet_train", "rb") as fp:
  train_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/BERTweet_test", "rb") as fp:
  test_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/BERTweet_val", "rb") as fp:
  val_embeddings = pickle.load(fp)

# Load data
train = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V1/train.csv')
test = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V1/test.csv')
val = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V1/val.csv')

train_labels = train['sarcastic'].values.tolist()
test_labels = test['sarcastic'].values.tolist()
val_labels = val['sarcastic'].values.tolist()

train_dataset = SarcasmDataset(train_embeddings, train_labels)
test_dataset = SarcasmDataset(test_embeddings, test_labels)
val_dataset = SarcasmDataset(val_embeddings, val_labels)

embed_dim = 768
hidden_dim = 128
num_classes = 2

model = FFNNClassifier(embed_dim, hidden_dim, num_classes)

training_args = TrainingArguments(
  output_dir='./res', evaluation_strategy="steps", num_train_epochs=5, per_device_train_batch_size=32,
  per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01,logging_dir='./logs4',
  load_best_model_at_end=True,
)

trainer = CustomTrainer(
  model=model, args=training_args, train_dataset=train_dataset,
  eval_dataset=val_dataset,
  compute_metrics=compute_metrics,
)

trainer.train()

trainer.evaluate(test_dataset)

"""### A3:"""

import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
import torch
from torch import nn
from torch.utils.data import Dataset
from transformers import Trainer,TrainingArguments
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score

class FFNNClassifier(nn.Module):
  def __init__(self, embed_dim, hidden_dim, num_classes):
    super(FFNNClassifier, self).__init__()
    self.fc1 = nn.Linear(embed_dim, hidden_dim)
    self.relu = nn.ReLU()
    self.fc2 = nn.Linear(hidden_dim, num_classes)

  def forward(self, embeddings, labels=None):
    out = self.fc1(embeddings)
    out = self.relu(out)
    logits = self.fc2(out)
    return {'logits': logits}

class SarcasmDataset(torch.utils.data.Dataset):
  def __init__(self, embeddings, labels):
    self.embeddings = embeddings
    self.labels = labels

  def __getitem__(self, idx):
    item = {'embeddings': self.embeddings[idx], 'labels': torch.tensor(self.labels[idx])}
    return item

  def __len__(self):
    return len(self.labels)

def compute_metrics(p):
  pred, labels = p
  pred = np.argmax(pred, axis=1)

  accuracy = accuracy_score(y_true=labels, y_pred=pred)
  f1 = f1_score(labels, pred)
  precision = precision_score(labels, pred)
  recall = recall_score(labels, pred)

  return {"accuracy": accuracy, "f1_score": f1, "precision": precision, "recall": recall}

class CustomTrainer(Trainer):
  def compute_loss(self, model, inputs, return_outputs=False):
    embeddings = inputs.get("embeddings")
    outputs = model(embeddings)
    logits = outputs['logits']
    labels = inputs.get("labels").to(logits.device)
    loss_fct = nn.CrossEntropyLoss()
    loss = loss_fct(logits.view(-1, num_classes), labels.view(-1))
    return (loss, outputs) if return_outputs else loss

import pickle
from transformers import AutoTokenizer, TrainingArguments, Trainer, AdamW
from torch import stack

# Load sentence-level embeddings
with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/train_BERTweet_sentence", "rb") as fp:
  train_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/test_BERTweet_sentence", "rb") as fp:
  test_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/val_BERTweet_sentence", "rb") as fp:
  val_embeddings = pickle.load(fp)

# Load data
train = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V1/train.csv')
test = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V1/test.csv')
val = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V1/val.csv')

train_labels = train['sarcastic'].values.tolist()
test_labels = test['sarcastic'].values.tolist()
val_labels = val['sarcastic'].values.tolist()

train_dataset = SarcasmDataset(train_embeddings, train_labels)
test_dataset = SarcasmDataset(test_embeddings, test_labels)
val_dataset = SarcasmDataset(val_embeddings, val_labels)

embed_dim = 768
hidden_dim = 128
num_classes = 2

model = FFNNClassifier(embed_dim, hidden_dim, num_classes)

training_args = TrainingArguments(
  output_dir='./res', evaluation_strategy="steps", num_train_epochs=5, per_device_train_batch_size=32,
  per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01,logging_dir='./logs4',
  load_best_model_at_end=True,
)

trainer = CustomTrainer(
  model=model, args=training_args, train_dataset=train_dataset,
  eval_dataset=val_dataset,
  compute_metrics=compute_metrics,
)

trainer.train()

# Evaluate on test dataset after training
trainer.evaluate(test_dataset)

"""### A4:"""

# Import necessary libraries
import pandas as pd
import torch
from torch import nn
from torch.utils.data import Dataset
from transformers import AdamW, AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments
from sklearn.metrics import accuracy_score, f1_score
from sklearn.model_selection import train_test_split
import pickle
from transformers import AutoTokenizer, TrainingArguments, Trainer, AdamW
from torch import stack

# Define the SarcasmDataset class
class SarcasmDataset(Dataset):
  def __init__(self, encodings, labels, word_embeddings, simclr_embeddings, vanilla_embeddings):
    self.encodings = encodings
    self.labels = labels
    self.word_embeddings = word_embeddings
    self.simclr_embeddings = simclr_embeddings
    self.vanilla_embeddings = vanilla_embeddings

  def __getitem__(self, idx):
    item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
    item['labels'] = torch.tensor(self.labels[idx])
    item['embeddings'] = torch.cat([torch.flatten(torch.tensor(self.word_embeddings[idx])), self.simclr_embeddings[idx], self.vanilla_embeddings[idx]], dim=-1)
    return item

  def __len__(self):
    return len(self.labels)

class CustomModel(AutoModelForSequenceClassification):
  def __init__(self, config):
    super(CustomModel, self).__init__(config)
    self.roberta = AutoModelForSequenceClassification.from_pretrained(MODEL, config=config)
    self.embeddings = nn.Linear(50 * config.hidden_size + 2 * config.hidden_size, config.hidden_size)
    self.dropout = nn.Dropout(0.2)
    self.concat = nn.Linear(config.hidden_size * 2, config.hidden_size)
    self.classifier = nn.Linear(config.hidden_size, config.num_labels)

  def forward(self, input_ids, attention_mask=None, embeddings=None, labels=None):
    outputs = self.roberta(input_ids, attention_mask=attention_mask)
    pooled_output = outputs[0]
    embeddings = self.embeddings(embeddings)
    pooled_output = torch.cat([pooled_output, embeddings], dim=-1)
    pooled_output = self.dropout(pooled_output)
    logits = self.classifier(pooled_output)

    if labels is not None:
      loss_fct = FBetaLoss(beta=1)
      loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
      return loss
    else:
      return logits

class FBetaLoss(nn.Module):
  def __init__(self, beta=1):
    super(FBetaLoss, self).__init__()
    self.beta = beta

  def forward(self, input, target):
    precision = (input * target).sum() / input.sum()
    recall = (input * target).sum() / target.sum()
    fbeta = (1 + self.beta**2) * precision * recall / (self.beta**2 * precision + recall + 1e-15)
    return 1 - fbeta

from sklearn.metrics import precision_score, recall_score

def compute_metrics(p):
  pred, labels = p
  pred = np.argmax(pred, axis=1)

  accuracy = accuracy_score(y_true=labels, y_pred=pred)
  f1 = f1_score(y_true=labels, y_pred=pred)
  precision = precision_score(y_true=labels, y_pred=pred)
  recall = recall_score(y_true=labels, y_pred=pred)

  return {"accuracy": accuracy, "f1_score": f1, "precision": precision, "recall": recall}

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/train_BERTweet_sentence", "rb") as fp:
  train_simclr_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/test_BERTweet_sentence", "rb") as fp:
  test_simclr_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/val_BERTweet_sentence", "rb") as fp:
  val_simclr_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/train_vanilla", "rb") as fp:
  vanilla_train = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/test_vanilla", "rb") as fp:
  vanilla_test = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/val_vanilla", "rb") as fp:
  vanilla_val = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/train_word", "rb") as fp:
  train_word_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/test_word", "rb") as fp:
  test_word_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V1/val_word", "rb") as fp:
  val_word_embeddings = pickle.load(fp)

train = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V1/train.csv')
test = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V1/test.csv')
val = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V1/val.csv')

train_tweets = train['tweet'].values.tolist()
train_labels = train['sarcastic'].values.tolist()
test_tweets = test['tweet'].values.tolist()
test_labels = test['sarcastic'].values.tolist()
val_tweets = val['tweet'].values.tolist()
val_labels = val['sarcastic'].values.tolist()

MODEL = "vinai/bertweet-base"
tokenizer = AutoTokenizer.from_pretrained(MODEL)

train_encodings = tokenizer(train_tweets, truncation=True, padding=True,return_tensors = 'pt')
test_encodings = tokenizer(test_tweets, truncation=True, padding=True,return_tensors = 'pt')
val_encodings = tokenizer(val_tweets, truncation=True, padding=True,return_tensors = 'pt')

train_dataset = SarcasmDataset(train_encodings, train_labels, train_word_embeddings, train_simclr_embeddings, vanilla_train)
test_dataset = SarcasmDataset(test_encodings, test_labels, test_word_embeddings, test_simclr_embeddings, vanilla_test)
val_dataset = SarcasmDataset(val_encodings, val_labels, val_word_embeddings, val_simclr_embeddings, vanilla_val)

model = CustomModel.from_pretrained(MODEL, num_labels=2)

training_args = TrainingArguments(
  output_dir='./results',
  num_train_epochs=5,
  evaluation_strategy="steps",
  eval_steps=20,
  learning_rate=1e-5,
  per_device_train_batch_size=16,
  per_device_eval_batch_size=16,
  warmup_steps=500,
  weight_decay=0.01,
  logging_dir='./logs',
)

model = CustomModel.from_pretrained(MODEL)

optimizer = AdamW(model.parameters(), lr=1e-5)

trainer = Trainer(
  model=model,
  args=training_args,
  train_dataset=train_dataset,
  eval_dataset=val_dataset,
  compute_metrics=compute_metrics,
  optimizers=(optimizer, None),
)

trainer.train()
trainer.evaluate(test_dataset)

"""## Twitter:

### A1:
"""

import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
import torch
from torch import nn
from torch.utils.data import Dataset
from transformers import Trainer,TrainingArguments
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score
from scipy import stats

class FFNNClassifier(nn.Module):
  def __init__(self, embed_dim, hidden_dim, num_classes):
    super(FFNNClassifier, self).__init__()
    self.fc1 = nn.Linear(embed_dim, hidden_dim)
    self.relu = nn.ReLU()
    self.fc2 = nn.Linear(hidden_dim, num_classes)

  def forward(self, embeddings, labels=None):
    batch_size, seq_len, embed_dim = embeddings.shape
    embeddings = embeddings.view(batch_size * seq_len, embed_dim)
    embeddings = embeddings.float()
    out = self.fc1(embeddings)
    out = self.relu(out)
    logits = self.fc2(out)
    logits = logits.view(batch_size, seq_len, -1)
    return {'logits': logits}

class SarcasmDataset(torch.utils.data.Dataset):
  def __init__(self, embeddings, labels):
    self.embeddings = embeddings
    self.labels = labels

  def __getitem__(self, idx):
    item = {'embeddings': self.embeddings[idx], 'labels': torch.tensor(self.labels[idx])}
    return item

  def __len__(self):
    return len(self.labels)

def compute_metrics(p):
  pred, labels = p
  aggregated_logits = stats.mode(pred, axis=1)[0]
  probabilities = np.exp(aggregated_logits) / np.sum(np.exp(aggregated_logits), axis=-1, keepdims=True)
  pred = np.argmax(probabilities, axis=-1)
  accuracy = accuracy_score(y_true=labels, y_pred=pred)
  f1 = f1_score(y_true=labels, y_pred=pred, average='binary')
  precision = precision_score(y_true=labels, y_pred=pred, average='binary')
  recall = recall_score(y_true=labels, y_pred=pred, average='binary')
  return {"accuracy": accuracy, "f1": f1, "precision": precision, "recall": recall}

class CustomTrainer(Trainer):
  def compute_loss(self, model, inputs, return_outputs=False):
    embeddings = inputs.get("embeddings")
    outputs = model(embeddings)
    logits = outputs['logits']
    labels = inputs.get("labels").to(logits.device)
    probabilities = torch.nn.functional.softmax(logits, dim=-1)
    sentence_probabilities = torch.mean(probabilities, dim=1)
    sentence_logits = torch.log(sentence_probabilities)
    loss_fct = nn.CrossEntropyLoss()
    loss = loss_fct(sentence_logits, labels)
    return (loss, outputs) if return_outputs else loss

import pickle
from transformers import AutoTokenizer, TrainingArguments, Trainer, AdamW
from torch import stack

# Load sentence-level embeddings
with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/train_word", "rb") as fp:
  train_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/test_word", "rb") as fp:
  test_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/val_word", "rb") as fp:
  val_embeddings = pickle.load(fp)

# Load data
train = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted Twitter/train.csv')
test = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted Twitter/test.csv')
val = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted Twitter/val.csv')

train_labels = train['sarcastic'].values.tolist()
test_labels = test['sarcastic'].values.tolist()
val_labels = val['sarcastic'].values.tolist()

train_dataset = SarcasmDataset(train_embeddings, train_labels)
test_dataset = SarcasmDataset(test_embeddings, test_labels)
val_dataset = SarcasmDataset(val_embeddings, val_labels)

embed_dim = 768
hidden_dim = 128
num_classes = 2

model = FFNNClassifier(embed_dim, hidden_dim, num_classes)

training_args = TrainingArguments(
  output_dir='./res', evaluation_strategy="steps", num_train_epochs=5, per_device_train_batch_size=32,
  per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01,logging_dir='./logs4',
  load_best_model_at_end=True,
)

trainer = CustomTrainer(
  model=model, args=training_args, train_dataset=train_dataset,
  eval_dataset=val_dataset,
  compute_metrics=compute_metrics,
)

trainer.train()

# Evaluate on test dataset after training
trainer.evaluate(test_dataset)

"""### A2 RoBERTa:"""

import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
import torch
from torch import nn
from torch.utils.data import Dataset
from transformers import Trainer,TrainingArguments
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score

class FFNNClassifier(nn.Module):
  def __init__(self, embed_dim, hidden_dim, num_classes):
    super(FFNNClassifier, self).__init__()
    self.fc1 = nn.Linear(embed_dim, hidden_dim)
    self.relu = nn.ReLU()
    self.fc2 = nn.Linear(hidden_dim, num_classes)

  def forward(self, embeddings, labels=None):
    out = self.fc1(embeddings)
    out = self.relu(out)
    logits = self.fc2(out)
    return {'logits': logits}

class SarcasmDataset(torch.utils.data.Dataset):
  def __init__(self, embeddings, labels):
    self.embeddings = embeddings
    self.labels = labels

  def __getitem__(self, idx):
    item = {'embeddings': self.embeddings[idx], 'labels': torch.tensor(self.labels[idx])}
    return item

  def __len__(self):
    return len(self.labels)

def compute_metrics(p):
  pred, labels = p
  pred = np.argmax(pred, axis=1)

  accuracy = accuracy_score(y_true=labels, y_pred=pred)
  f1 = f1_score(labels, pred)
  precision = precision_score(labels, pred)
  recall = recall_score(labels, pred)

  return {"accuracy": accuracy, "f1_score": f1, "precision": precision, "recall": recall}

class CustomTrainer(Trainer):
  def compute_loss(self, model, inputs, return_outputs=False):
    embeddings = inputs.get("embeddings")
    outputs = model(embeddings)
    logits = outputs['logits']
    labels = inputs.get("labels").to(logits.device)
    loss_fct = nn.CrossEntropyLoss()
    loss = loss_fct(logits.view(-1, num_classes), labels.view(-1))
    return (loss, outputs) if return_outputs else loss

import pickle
from transformers import AutoTokenizer, TrainingArguments, Trainer, AdamW
from torch import stack

# Load sentence-level embeddings
with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/train_vanilla", "rb") as fp:
  vanilla_train = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/test_vanilla", "rb") as fp:
  vanilla_test = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/train_vanilla", "rb") as fp:
  vanilla_val = pickle.load(fp)

# Load data
train = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted Twitter/train.csv')
test = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted Twitter/test.csv')
val = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted Twitter/val.csv')

train_labels = train['sarcastic'].values.tolist()
test_labels = test['sarcastic'].values.tolist()
val_labels = val['sarcastic'].values.tolist()

train_dataset = SarcasmDataset(vanilla_train, train_labels)
test_dataset = SarcasmDataset(vanilla_test, test_labels)
val_dataset = SarcasmDataset(vanilla_val, val_labels)

embed_dim = 768
hidden_dim = 128
num_classes = 2

model = FFNNClassifier(embed_dim, hidden_dim, num_classes)

training_args = TrainingArguments(
  output_dir='./res', evaluation_strategy="steps", num_train_epochs=5, per_device_train_batch_size=32,
  per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01,logging_dir='./logs4',
  load_best_model_at_end=True,
)

trainer = CustomTrainer(
  model=model, args=training_args, train_dataset=train_dataset,
  eval_dataset=val_dataset,
  compute_metrics=compute_metrics,
)

trainer.train()

# Evaluate on test dataset after training
trainer.evaluate(test_dataset)

"""### A2 BERTweet:"""

import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
import torch
from torch import nn
from torch.utils.data import Dataset
from transformers import Trainer,TrainingArguments
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score

class FFNNClassifier(nn.Module):
  def __init__(self, embed_dim, hidden_dim, num_classes):
    super(FFNNClassifier, self).__init__()
    self.fc1 = nn.Linear(embed_dim, hidden_dim)
    self.relu = nn.ReLU()
    self.fc2 = nn.Linear(hidden_dim, num_classes)

  def forward(self, embeddings, labels=None):
    out = self.fc1(embeddings)
    out = self.relu(out)
    logits = self.fc2(out)
    return {'logits': logits}

class SarcasmDataset(torch.utils.data.Dataset):
  def __init__(self, embeddings, labels):
    self.embeddings = embeddings
    self.labels = labels

  def __getitem__(self, idx):
    item = {'embeddings': self.embeddings[idx], 'labels': torch.tensor(self.labels[idx])}
    return item

  def __len__(self):
    return len(self.labels)

def compute_metrics(p):
  pred, labels = p
  pred = np.argmax(pred, axis=1)

  accuracy = accuracy_score(y_true=labels, y_pred=pred)
  f1 = f1_score(labels, pred)
  precision = precision_score(labels, pred)
  recall = recall_score(labels, pred)

  return {"accuracy": accuracy, "f1_score": f1, "precision": precision, "recall": recall}

class CustomTrainer(Trainer):
  def compute_loss(self, model, inputs, return_outputs=False):
    embeddings = inputs.get("embeddings")
    outputs = model(embeddings)
    logits = outputs['logits']
    labels = inputs.get("labels").to(logits.device)
    loss_fct = nn.CrossEntropyLoss()
    loss = loss_fct(logits.view(-1, num_classes), labels.view(-1))
    return (loss, outputs) if return_outputs else loss

import pickle
from transformers import AutoTokenizer, TrainingArguments, Trainer, AdamW
from torch import stack

# Load sentence-level embeddings
with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/BERTweet_train", "rb") as fp:
  train_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/BERTweet_test", "rb") as fp:
  test_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/BERTweet_val", "rb") as fp:
  val_embeddings = pickle.load(fp)

# Load data
train = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted Twitter/train.csv')
test = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted Twitter/test.csv')
val = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted Twitter/val.csv')

train_labels = train['sarcastic'].values.tolist()
test_labels = test['sarcastic'].values.tolist()
val_labels = val['sarcastic'].values.tolist()

train_dataset = SarcasmDataset(train_embeddings, train_labels)
test_dataset = SarcasmDataset(test_embeddings, test_labels)
val_dataset = SarcasmDataset(val_embeddings, val_labels)

embed_dim = 768
hidden_dim = 128
num_classes = 2

model = FFNNClassifier(embed_dim, hidden_dim, num_classes)

training_args = TrainingArguments(
  output_dir='./res', evaluation_strategy="steps", num_train_epochs=5, per_device_train_batch_size=32,
  per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01,logging_dir='./logs4',
  load_best_model_at_end=True,
)

trainer = CustomTrainer(
  model=model, args=training_args, train_dataset=train_dataset,
  eval_dataset=val_dataset,
  compute_metrics=compute_metrics,
)

trainer.train()

# Evaluate on test dataset after training
trainer.evaluate(test_dataset)

"""### A3:"""

import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
import torch
from torch import nn
from torch.utils.data import Dataset
from transformers import Trainer,TrainingArguments
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score

class FFNNClassifier(nn.Module):
  def __init__(self, embed_dim, hidden_dim, num_classes):
    super(FFNNClassifier, self).__init__()
    self.fc1 = nn.Linear(embed_dim, hidden_dim)
    self.relu = nn.ReLU()
    self.fc2 = nn.Linear(hidden_dim, num_classes)

  def forward(self, embeddings, labels=None):
    out = self.fc1(embeddings)
    out = self.relu(out)
    logits = self.fc2(out)
    return {'logits': logits}

class SarcasmDataset(torch.utils.data.Dataset):
  def __init__(self, embeddings, labels):
    self.embeddings = embeddings
    self.labels = labels

  def __getitem__(self, idx):
    item = {'embeddings': self.embeddings[idx], 'labels': torch.tensor(self.labels[idx])}
    return item

  def __len__(self):
    return len(self.labels)

def compute_metrics(p):
  pred, labels = p
  pred = np.argmax(pred, axis=1)

  accuracy = accuracy_score(y_true=labels, y_pred=pred)
  f1 = f1_score(labels, pred)
  precision = precision_score(labels, pred)
  recall = recall_score(labels, pred)

  return {"accuracy": accuracy, "f1_score": f1, "precision": precision, "recall": recall}

class CustomTrainer(Trainer):
  def compute_loss(self, model, inputs, return_outputs=False):
    embeddings = inputs.get("embeddings")
    outputs = model(embeddings)
    logits = outputs['logits']
    labels = inputs.get("labels").to(logits.device)
    loss_fct = nn.CrossEntropyLoss()
    loss = loss_fct(logits.view(-1, num_classes), labels.view(-1))
    return (loss, outputs) if return_outputs else loss

import pickle
from transformers import AutoTokenizer, TrainingArguments, Trainer, AdamW
from torch import stack

# Load sentence-level embeddings
with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/train_BERTweet_sentence", "rb") as fp:
  train_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/test_BERTweet_sentence", "rb") as fp:
  test_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/val_BERTweet_sentence", "rb") as fp:
  val_embeddings = pickle.load(fp)

# Load data
train = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted Twitter/train.csv')
test = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted Twitter/test.csv')
val = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted Twitter/val.csv')

train_labels = train['sarcastic'].values.tolist()
test_labels = test['sarcastic'].values.tolist()
val_labels = val['sarcastic'].values.tolist()

train_dataset = SarcasmDataset(train_embeddings, train_labels)
test_dataset = SarcasmDataset(test_embeddings, test_labels)
val_dataset = SarcasmDataset(val_embeddings, val_labels)

embed_dim = 768
hidden_dim = 128
num_classes = 2

model = FFNNClassifier(embed_dim, hidden_dim, num_classes)

training_args = TrainingArguments(
  output_dir='./res', evaluation_strategy="steps", num_train_epochs=5, per_device_train_batch_size=32,
  per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01,logging_dir='./logs4',
  load_best_model_at_end=True,
)

trainer = CustomTrainer(
  model=model, args=training_args, train_dataset=train_dataset,
  eval_dataset=val_dataset,
  compute_metrics=compute_metrics,
)

trainer.train()

# Evaluate on test dataset after training
trainer.evaluate(test_dataset)

"""### A4:"""

# Import necessary libraries
import pandas as pd
import torch
from torch import nn
from torch.utils.data import Dataset
from transformers import AdamW, AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments
from sklearn.metrics import accuracy_score, f1_score
from sklearn.model_selection import train_test_split
import pickle
from transformers import AutoTokenizer, TrainingArguments, Trainer, AdamW
from torch import stack

# Define the SarcasmDataset class
class SarcasmDataset(Dataset):
  def __init__(self, encodings, labels, word_embeddings, simclr_embeddings, vanilla_embeddings):
    self.encodings = encodings
    self.labels = labels
    self.word_embeddings = word_embeddings
    self.simclr_embeddings = simclr_embeddings
    self.vanilla_embeddings = vanilla_embeddings

  def __getitem__(self, idx):
    item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
    item['labels'] = torch.tensor(self.labels[idx])
    item['embeddings'] = torch.cat([torch.flatten(torch.tensor(self.word_embeddings[idx])), self.simclr_embeddings[idx], self.vanilla_embeddings[idx]], dim=-1)
    return item

  def __len__(self):
    return len(self.labels)

# Define the CustomModel class
class CustomModel(AutoModelForSequenceClassification):
  def __init__(self, config):
    super(CustomModel, self).__init__(config)
    self.roberta = AutoModelForSequenceClassification.from_pretrained(MODEL, config=config)
    self.embeddings = nn.Linear(50 * config.hidden_size + 2 * config.hidden_size, config.hidden_size)
    self.dropout = nn.Dropout(0.2)
    self.concat = nn.Linear(config.hidden_size * 2, config.hidden_size)
    self.classifier = nn.Linear(config.hidden_size, config.num_labels)

  def forward(self, input_ids, attention_mask=None, embeddings=None, labels=None):
    outputs = self.roberta(input_ids, attention_mask=attention_mask)
    pooled_output = outputs[0]
    embeddings = self.embeddings(embeddings)
    pooled_output = torch.cat([pooled_output, embeddings], dim=-1)
    pooled_output = self.dropout(pooled_output)
    logits = self.classifier(pooled_output)

    if labels is not None:
      loss_fct = FBetaLoss(beta=1)
      loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
      return loss
    else:
      return logits

class FBetaLoss(nn.Module):
  def __init__(self, beta=1):
    super(FBetaLoss, self).__init__()
    self.beta = beta

  def forward(self, input, target):
    precision = (input * target).sum() / input.sum()
    recall = (input * target).sum() / target.sum()
    fbeta = (1 + self.beta**2) * precision * recall / (self.beta**2 * precision + recall + 1e-15)
    return 1 - fbeta

from sklearn.metrics import precision_score, recall_score

def compute_metrics(p):
  pred, labels = p
  pred = np.argmax(pred, axis=1)

  accuracy = accuracy_score(y_true=labels, y_pred=pred)
  f1 = f1_score(y_true=labels, y_pred=pred)
  precision = precision_score(y_true=labels, y_pred=pred)
  recall = recall_score(y_true=labels, y_pred=pred)

  return {"accuracy": accuracy, "f1_score": f1, "precision": precision, "recall": recall}

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/train_BERTweet_sentence", "rb") as fp:
  train_simclr_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/test_BERTweet_sentence", "rb") as fp:
  test_simclr_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/val_BERTweet_sentence", "rb") as fp:
  val_simclr_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/train_vanilla", "rb") as fp:
  vanilla_train = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/test_vanilla", "rb") as fp:
  vanilla_test = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/train_vanilla", "rb") as fp:
  vanilla_val = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/train_word", "rb") as fp:
  train_word_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/test_word", "rb") as fp:
  test_word_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/Twitter/val_word", "rb") as fp:
  val_word_embeddings = pickle.load(fp)

train = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted Twitter/train.csv')
test = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted Twitter/test.csv')
val = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted Twitter/val.csv')

train_tweets = train['tweet'].values.tolist()
train_labels = train['sarcastic'].values.tolist()
test_tweets = test['tweet'].values.tolist()
test_labels = test['sarcastic'].values.tolist()
val_tweets = val['tweet'].values.tolist()
val_labels = val['sarcastic'].values.tolist()

MODEL = "vinai/bertweet-base"
tokenizer = AutoTokenizer.from_pretrained(MODEL)

train_encodings = tokenizer(train_tweets, truncation=True, padding=True,return_tensors = 'pt')
test_encodings = tokenizer(test_tweets, truncation=True, padding=True,return_tensors = 'pt')
val_encodings = tokenizer(val_tweets, truncation=True, padding=True,return_tensors = 'pt')

train_dataset = SarcasmDataset(train_encodings, train_labels, train_word_embeddings, train_simclr_embeddings, vanilla_train)
test_dataset = SarcasmDataset(test_encodings, test_labels, test_word_embeddings, test_simclr_embeddings, vanilla_test)
val_dataset = SarcasmDataset(val_encodings, val_labels, val_word_embeddings, val_simclr_embeddings, vanilla_val)

model = CustomModel.from_pretrained(MODEL, num_labels=2)

training_args = TrainingArguments(
  output_dir='./results',
  num_train_epochs=5,
  evaluation_strategy="steps",
  eval_steps=20,
  learning_rate=1e-5,
  per_device_train_batch_size=32,
  per_device_eval_batch_size=32,
  warmup_steps=500,
  weight_decay=0.01,
  logging_dir='./logs',
)

model = CustomModel.from_pretrained(MODEL)

optimizer = AdamW(model.parameters(), lr=1e-5)

trainer = Trainer(
  model=model,
  args=training_args,
  train_dataset=train_dataset,
  eval_dataset=val_dataset,
  compute_metrics=compute_metrics,
  optimizers=(optimizer, None),
)

trainer.train()
trainer.evaluate(test_dataset)

"""## IAC-V2:

### A1:
"""

import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
import torch
from torch import nn
from torch.utils.data import Dataset
from transformers import Trainer,TrainingArguments
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score
from scipy import stats

class FFNNClassifier(nn.Module):
  def __init__(self, embed_dim, hidden_dim, num_classes):
    super(FFNNClassifier, self).__init__()
    self.fc1 = nn.Linear(embed_dim, hidden_dim)
    self.relu = nn.ReLU()
    self.fc2 = nn.Linear(hidden_dim, num_classes)

  def forward(self, embeddings, labels=None):
    batch_size, seq_len, embed_dim = embeddings.shape
    embeddings = embeddings.view(batch_size * seq_len, embed_dim)
    embeddings = embeddings.float()
    out = self.fc1(embeddings)
    out = self.relu(out)
    logits = self.fc2(out)
    logits = logits.view(batch_size, seq_len, -1)
    return {'logits': logits}

class SarcasmDataset(torch.utils.data.Dataset):
  def __init__(self, embeddings, labels):
    self.embeddings = embeddings
    self.labels = labels

  def __getitem__(self, idx):
    item = {'embeddings': self.embeddings[idx], 'labels': torch.tensor(self.labels[idx])}
    return item

  def __len__(self):
    return len(self.labels)

def compute_metrics(p):
  pred, labels = p
  aggregated_logits = stats.mode(pred, axis=1)[0]
  probabilities = np.exp(aggregated_logits) / np.sum(np.exp(aggregated_logits), axis=-1, keepdims=True)
  pred = np.argmax(probabilities, axis=-1)
  accuracy = accuracy_score(y_true=labels, y_pred=pred)
  f1 = f1_score(y_true=labels, y_pred=pred, average='binary')
  precision = precision_score(y_true=labels, y_pred=pred, average='binary')
  recall = recall_score(y_true=labels, y_pred=pred, average='binary')
  return {"accuracy": accuracy, "f1": f1, "precision": precision, "recall": recall}

class CustomTrainer(Trainer):
  def compute_loss(self, model, inputs, return_outputs=False):
    embeddings = inputs.get("embeddings")
    outputs = model(embeddings)
    logits = outputs['logits']
    labels = inputs.get("labels").to(logits.device)
    probabilities = torch.nn.functional.softmax(logits, dim=-1)
    sentence_probabilities = torch.mean(probabilities, dim=1)
    sentence_logits = torch.log(sentence_probabilities)
    loss_fct = nn.CrossEntropyLoss()
    loss = loss_fct(sentence_logits, labels)
    return (loss, outputs) if return_outputs else loss

import pickle
from transformers import AutoTokenizer, TrainingArguments, Trainer, AdamW
from torch import stack

# Load word-level embeddings
with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/train_word", "rb") as fp:
  train_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/test_word", "rb") as fp:
  test_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/val_word", "rb") as fp:
  val_embeddings = pickle.load(fp)

# Load data
train = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V2/train.csv')
test = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V2/test.csv')
val = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V2/val.csv')

train_labels = train['sarcastic'].values.tolist()
test_labels = test['sarcastic'].values.tolist()
val_labels = val['sarcastic'].values.tolist()

train_dataset = SarcasmDataset(train_embeddings, train_labels)
test_dataset = SarcasmDataset(test_embeddings, test_labels)
val_dataset = SarcasmDataset(val_embeddings, val_labels)

embed_dim = 768
hidden_dim = 128
num_classes = 2

model = FFNNClassifier(embed_dim, hidden_dim, num_classes)

training_args = TrainingArguments(
  output_dir='./res', evaluation_strategy="steps", num_train_epochs=5, per_device_train_batch_size=32,
  per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01,logging_dir='./logs4',
  load_best_model_at_end=True,
)

trainer = CustomTrainer(
  model=model, args=training_args, train_dataset=train_dataset,
  eval_dataset=val_dataset,
  compute_metrics=compute_metrics,
)

trainer.train()

# Evaluate on test dataset after training
trainer.evaluate(test_dataset)

"""### A2 RoBERTa:"""

import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
import torch
from torch import nn
from torch.utils.data import Dataset
from transformers import Trainer,TrainingArguments
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score

class FFNNClassifier(nn.Module):
  def __init__(self, embed_dim, hidden_dim, num_classes):
    super(FFNNClassifier, self).__init__()
    self.fc1 = nn.Linear(embed_dim, hidden_dim)
    self.relu = nn.ReLU()
    self.fc2 = nn.Linear(hidden_dim, num_classes)

  def forward(self, embeddings, labels=None):
    out = self.fc1(embeddings)
    out = self.relu(out)
    logits = self.fc2(out)
    return {'logits': logits}

class SarcasmDataset(torch.utils.data.Dataset):
  def __init__(self, embeddings, labels):
    self.embeddings = embeddings
    self.labels = labels

  def __getitem__(self, idx):
    item = {'embeddings': self.embeddings[idx], 'labels': torch.tensor(self.labels[idx])}
    return item

  def __len__(self):
    return len(self.labels)

def compute_metrics(p):
  pred, labels = p
  pred = np.argmax(pred, axis=1)

  accuracy = accuracy_score(y_true=labels, y_pred=pred)
  f1 = f1_score(labels, pred)
  precision = precision_score(labels, pred)
  recall = recall_score(labels, pred)

  return {"accuracy": accuracy, "f1_score": f1, "precision": precision, "recall": recall}

class CustomTrainer(Trainer):
  def compute_loss(self, model, inputs, return_outputs=False):
    embeddings = inputs.get("embeddings")
    outputs = model(embeddings)
    logits = outputs['logits']
    labels = inputs.get("labels").to(logits.device)
    loss_fct = nn.CrossEntropyLoss()
    loss = loss_fct(logits.view(-1, num_classes), labels.view(-1))
    return (loss, outputs) if return_outputs else loss

import pickle
from transformers import AutoTokenizer, TrainingArguments, Trainer, AdamW
from torch import stack

# Load sentence-level embeddings
with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/train_vanilla", "rb") as fp:
  vanilla_train = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/test_vanilla", "rb") as fp:
  vanilla_test = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/val_vanilla", "rb") as fp:
  vanilla_val = pickle.load(fp)

# Load data
train = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V2/train.csv')
test = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V2/test.csv')
val = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V2/val.csv')

train_labels = train['sarcastic'].values.tolist()
test_labels = test['sarcastic'].values.tolist()
val_labels = val['sarcastic'].values.tolist()

train_dataset = SarcasmDataset(vanilla_train, train_labels)
test_dataset = SarcasmDataset(vanilla_test, test_labels)
val_dataset = SarcasmDataset(vanilla_val, val_labels)

embed_dim = 768
hidden_dim = 128
num_classes = 2

model = FFNNClassifier(embed_dim, hidden_dim, num_classes)

training_args = TrainingArguments(
  output_dir='./res', evaluation_strategy="steps", num_train_epochs=5, per_device_train_batch_size=32,
  per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01,logging_dir='./logs4',
  load_best_model_at_end=True,
)

trainer = CustomTrainer(
  model=model, args=training_args, train_dataset=train_dataset,
  eval_dataset=val_dataset,
  compute_metrics=compute_metrics,
)

trainer.train()

# Evaluate on test dataset after training
trainer.evaluate(test_dataset)

"""### A2 BERTweet:"""

import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
import torch
from torch import nn
from torch.utils.data import Dataset
from transformers import Trainer,TrainingArguments
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score

class FFNNClassifier(nn.Module):
  def __init__(self, embed_dim, hidden_dim, num_classes):
    super(FFNNClassifier, self).__init__()
    self.fc1 = nn.Linear(embed_dim, hidden_dim)
    self.relu = nn.ReLU()
    self.fc2 = nn.Linear(hidden_dim, num_classes)

  def forward(self, embeddings, labels=None):
    out = self.fc1(embeddings)
    out = self.relu(out)
    logits = self.fc2(out)
    return {'logits': logits}

class SarcasmDataset(torch.utils.data.Dataset):
  def __init__(self, embeddings, labels):
    self.embeddings = embeddings
    self.labels = labels

  def __getitem__(self, idx):
    item = {'embeddings': self.embeddings[idx], 'labels': torch.tensor(self.labels[idx])}
    return item

  def __len__(self):
    return len(self.labels)

def compute_metrics(p):
  pred, labels = p
  pred = np.argmax(pred, axis=1)

  accuracy = accuracy_score(y_true=labels, y_pred=pred)
  f1 = f1_score(labels, pred)
  precision = precision_score(labels, pred)
  recall = recall_score(labels, pred)

  return {"accuracy": accuracy, "f1_score": f1, "precision": precision, "recall": recall}

class CustomTrainer(Trainer):
  def compute_loss(self, model, inputs, return_outputs=False):
    embeddings = inputs.get("embeddings")
    outputs = model(embeddings)
    logits = outputs['logits']
    labels = inputs.get("labels").to(logits.device)
    loss_fct = nn.CrossEntropyLoss()
    loss = loss_fct(logits.view(-1, num_classes), labels.view(-1))
    return (loss, outputs) if return_outputs else loss

import pickle
from transformers import AutoTokenizer, TrainingArguments, Trainer, AdamW
from torch import stack

# Load sentence-level embeddings
with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/BERTweet_train", "rb") as fp:
  train_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/BERTweet_test", "rb") as fp:
  test_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/BERTweet_val", "rb") as fp:
  val_embeddings = pickle.load(fp)

# Load data
train = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V2/train.csv')
test = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V2/test.csv')
val = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V2/val.csv')

train_labels = train['sarcastic'].values.tolist()
test_labels = test['sarcastic'].values.tolist()
val_labels = val['sarcastic'].values.tolist()

train_dataset = SarcasmDataset(train_embeddings, train_labels)
test_dataset = SarcasmDataset(test_embeddings, test_labels)
val_dataset = SarcasmDataset(val_embeddings, val_labels)

embed_dim = 768
hidden_dim = 128
num_classes = 2

model = FFNNClassifier(embed_dim, hidden_dim, num_classes)

training_args = TrainingArguments(
  output_dir='./res', evaluation_strategy="steps", num_train_epochs=5, per_device_train_batch_size=32,
  per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01,logging_dir='./logs4',
  load_best_model_at_end=True,
)

trainer = CustomTrainer(
  model=model, args=training_args, train_dataset=train_dataset,
  eval_dataset=val_dataset,
  compute_metrics=compute_metrics,
)

trainer.train()

# Evaluate on test dataset after training
trainer.evaluate(test_dataset)

"""### A3:"""

import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')
import torch
from torch import nn
from torch.utils.data import Dataset
from transformers import Trainer,TrainingArguments
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score

class FFNNClassifier(nn.Module):
  def __init__(self, embed_dim, hidden_dim, num_classes):
    super(FFNNClassifier, self).__init__()
    self.fc1 = nn.Linear(embed_dim, hidden_dim)
    self.relu = nn.ReLU()
    self.fc2 = nn.Linear(hidden_dim, num_classes)

  def forward(self, embeddings, labels=None):
    out = self.fc1(embeddings)
    out = self.relu(out)
    logits = self.fc2(out)
    return {'logits': logits}

class SarcasmDataset(torch.utils.data.Dataset):
  def __init__(self, embeddings, labels):
    self.embeddings = embeddings
    self.labels = labels

  def __getitem__(self, idx):
    item = {'embeddings': self.embeddings[idx], 'labels': torch.tensor(self.labels[idx])}
    return item

  def __len__(self):
    return len(self.labels)

def compute_metrics(p):
  pred, labels = p
  pred = np.argmax(pred, axis=1)

  accuracy = accuracy_score(y_true=labels, y_pred=pred)
  f1 = f1_score(labels, pred)
  precision = precision_score(labels, pred)
  recall = recall_score(labels, pred)

  return {"accuracy": accuracy, "f1_score": f1, "precision": precision, "recall": recall}

class CustomTrainer(Trainer):
  def compute_loss(self, model, inputs, return_outputs=False):
    embeddings = inputs.get("embeddings")
    outputs = model(embeddings)
    logits = outputs['logits']
    labels = inputs.get("labels").to(logits.device)
    loss_fct = nn.CrossEntropyLoss()
    loss = loss_fct(logits.view(-1, num_classes), labels.view(-1))
    return (loss, outputs) if return_outputs else loss

import pickle
from transformers import AutoTokenizer, TrainingArguments, Trainer, AdamW
from torch import stack

# Load sentence-level embeddings
with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/train_BERTweet_sentence", "rb") as fp:
  train_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/test_BERTweet_sentence", "rb") as fp:
  test_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/val_BERTweet_sentence", "rb") as fp:
  val_embeddings = pickle.load(fp)

# Load data
train = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V2/train.csv')
test = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V2/test.csv')
val = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V2/val.csv')

train_labels = train['sarcastic'].values.tolist()
test_labels = test['sarcastic'].values.tolist()
val_labels = val['sarcastic'].values.tolist()

train_dataset = SarcasmDataset(train_embeddings, train_labels)
test_dataset = SarcasmDataset(test_embeddings, test_labels)
val_dataset = SarcasmDataset(val_embeddings, val_labels)

embed_dim = 768
hidden_dim = 128
num_classes = 2

model = FFNNClassifier(embed_dim, hidden_dim, num_classes)

training_args = TrainingArguments(
  output_dir='./res', evaluation_strategy="steps", num_train_epochs=5, per_device_train_batch_size=32,
  per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01,logging_dir='./logs4',
  load_best_model_at_end=True,
)

trainer = CustomTrainer(
  model=model, args=training_args, train_dataset=train_dataset,
  eval_dataset=val_dataset,
  compute_metrics=compute_metrics,
)

trainer.train()

# Evaluate on test dataset after training
trainer.evaluate(test_dataset)

"""### A4:"""

# Import necessary libraries
import pandas as pd
import torch
from torch import nn
from torch.utils.data import Dataset
from transformers import AdamW, AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments
from sklearn.metrics import accuracy_score, f1_score
from sklearn.model_selection import train_test_split
import pickle
from transformers import AutoTokenizer, TrainingArguments, Trainer, AdamW
from torch import stack

# Define the SarcasmDataset class
class SarcasmDataset(Dataset):
  def __init__(self, encodings, labels, word_embeddings, simclr_embeddings, vanilla_embeddings):
    self.encodings = encodings
    self.labels = labels
    self.word_embeddings = word_embeddings
    self.simclr_embeddings = simclr_embeddings
    self.vanilla_embeddings = vanilla_embeddings

  def __getitem__(self, idx):
    item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
    item['labels'] = torch.tensor(self.labels[idx])
    item['embeddings'] = torch.cat([torch.flatten(torch.tensor(self.word_embeddings[idx])), self.simclr_embeddings[idx], self.vanilla_embeddings[idx]], dim=-1)
    return item

  def __len__(self):
    return len(self.labels)

class CustomModel(AutoModelForSequenceClassification):
  def __init__(self, config):
    super(CustomModel, self).__init__(config)
    self.roberta = AutoModelForSequenceClassification.from_pretrained(MODEL, config=config)
    self.embeddings = nn.Linear(50 * config.hidden_size + 2 * config.hidden_size, config.hidden_size)
    self.dropout = nn.Dropout(0.2)
    self.concat = nn.Linear(config.hidden_size * 2, config.hidden_size)
    self.classifier = nn.Linear(config.hidden_size, config.num_labels)

  def forward(self, input_ids, attention_mask=None, embeddings=None, labels=None):
    outputs = self.roberta(input_ids, attention_mask=attention_mask)
    pooled_output = outputs[0]
    embeddings = self.embeddings(embeddings)
    pooled_output = torch.cat([pooled_output, embeddings], dim=-1)
    pooled_output = self.dropout(pooled_output)
    logits = self.classifier(pooled_output)

    if labels is not None:
      loss_fct = FBetaLoss(beta=1)
      loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
      return loss
    else:
      return logits

class FBetaLoss(nn.Module):
  def __init__(self, beta=1):
    super(FBetaLoss, self).__init__()
    self.beta = beta

  def forward(self, input, target):
    precision = (input * target).sum() / input.sum()
    recall = (input * target).sum() / target.sum()
    fbeta = (1 + self.beta**2) * precision * recall / (self.beta**2 * precision + recall + 1e-15)
    return 1 - fbeta

from sklearn.metrics import precision_score, recall_score

def compute_metrics(p):
  pred, labels = p
  pred = np.argmax(pred, axis=1)

  accuracy = accuracy_score(y_true=labels, y_pred=pred)
  f1 = f1_score(y_true=labels, y_pred=pred)
  precision = precision_score(y_true=labels, y_pred=pred)
  recall = recall_score(y_true=labels, y_pred=pred)

  return {"accuracy": accuracy, "f1_score": f1, "precision": precision, "recall": recall}

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/train_BERTweet_sentence", "rb") as fp:
  train_simclr_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/test_BERTweet_sentence", "rb") as fp:
  test_simclr_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/val_BERTweet_sentence", "rb") as fp:
  val_simclr_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/train_vanilla", "rb") as fp:
  vanilla_train = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/test_vanilla", "rb") as fp:
  vanilla_test = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/val_vanilla", "rb") as fp:
  vanilla_val = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/train_word", "rb") as fp:
  train_word_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/test_word", "rb") as fp:
  test_word_embeddings = pickle.load(fp)

with open("/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/Embeddings/IAC-V2/val_word", "rb") as fp:
  val_word_embeddings = pickle.load(fp)

train = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V2/train.csv')
test = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V2/test.csv')
val = pd.read_csv('/content/drive/MyDrive/Contrastive NLP Stuff/Data/SOTA Data/CSV Data/Formatted IAC-V2/val.csv')

train_tweets = train['tweet'].values.tolist()
train_labels = train['sarcastic'].values.tolist()
test_tweets = test['tweet'].values.tolist()
test_labels = test['sarcastic'].values.tolist()
val_tweets = val['tweet'].values.tolist()
val_labels = val['sarcastic'].values.tolist()

MODEL = "vinai/bertweet-base"
tokenizer = AutoTokenizer.from_pretrained(MODEL)

train_encodings = tokenizer(train_tweets, truncation=True, padding=True,return_tensors = 'pt')
test_encodings = tokenizer(test_tweets, truncation=True, padding=True,return_tensors = 'pt')
val_encodings = tokenizer(val_tweets, truncation=True, padding=True,return_tensors = 'pt')

train_dataset = SarcasmDataset(train_encodings, train_labels, train_word_embeddings, train_simclr_embeddings, vanilla_train)
test_dataset = SarcasmDataset(test_encodings, test_labels, test_word_embeddings, test_simclr_embeddings, vanilla_test)
val_dataset = SarcasmDataset(val_encodings, val_labels, val_word_embeddings, val_simclr_embeddings, vanilla_val)

model = CustomModel.from_pretrained(MODEL, num_labels=2)

training_args = TrainingArguments(
  output_dir='./results',
  num_train_epochs=5,
  evaluation_strategy="steps",
  eval_steps=20,
  learning_rate=1e-5,
  per_device_train_batch_size=32,
  per_device_eval_batch_size=32,
  warmup_steps=500,
  weight_decay=0.01,
  logging_dir='./logs',
)

model = CustomModel.from_pretrained(MODEL)

optimizer = AdamW(model.parameters(), lr=1e-5)

trainer = Trainer(
  model=model,
  args=training_args,
  train_dataset=train_dataset,
  eval_dataset=val_dataset,
  compute_metrics=compute_metrics,
  optimizers=(optimizer, None),
)

trainer.train()
trainer.evaluate(test_dataset)