Spaces:

Ashish-R
/

LoRAFromScratchSentimentAnalysis

Sleeping

LoRAFromScratchSentimentAnalysis / baseline.py

Ashish Reddy

committing

a090db7 over 1 year ago

2.94 kB

import torch
from transformers import (
    AutoTokenizer,
    AutoModelForSequenceClassification
)
from datasets import load_dataset
from torch.utils.data import DataLoader

"""
---- Device ----
"""

if torch.cuda.is_available():
    device = torch.device('cuda')
    print("Using CUDA (GPU)")
elif torch.backends.mps.is_available() and torch.backends.mps.is_built():
    device = torch.device('mps')
    print("Using MPS (Apple Silicon GPU)")
else:
    device = torch.device('cpu')
    print("Using device's CPU")

"""
--- Model ---
"""

model_ckpt = "distilbert-base-uncased"

print(f"--- Loading pre-trained model and tokenizer: {model_ckpt.upper()} ---")

tok = AutoTokenizer.from_pretrained(model_ckpt)
model = AutoModelForSequenceClassification.from_pretrained(model_ckpt)
model.to(device)
print(f"Model moved to {device}")


"""
--- Data Prep ---
"""

print("\n--- Loading and preparing IMDB dataset ---")
imdb_dataset = load_dataset("imdb")
"""
DatasetDict({
    train: Dataset({
        features: ['text', 'label'],
        num_rows: 25000
    })
    test: Dataset({
        features: ['text', 'label'],
        num_rows: 25000
    })
    unsupervised: Dataset({
        features: ['text', 'label'],
        num_rows: 50000
    })
})
"""

def tokenize_fn(examples):
    return tok(examples["text"], padding="max_length", truncation=True)

tokenized_datasets = imdb_dataset.map(tokenize_fn, batched=True)

tokenized_datasets = tokenized_datasets.remove_columns(["text"])
tokenized_datasets = tokenized_datasets.rename_column("label", "labels")
tokenized_datasets.set_format("torch")


if __name__ == '__main__':

    small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000)) # Select random 1000 test datasets
    eval_dataloader = DataLoader(small_eval_dataset, batch_size=8) # Convert them into 8 batches --> 125 ['labels', 'token_ids', 'attention_mask'] examples in each batch

    print("\n--- Evaluating baseline model performance ---")
    model.eval() 
    num_correct = 0
    num_samples = 0

    with torch.no_grad(): # Disable gradient calculation for inference (No backprop)
        for batch in eval_dataloader:
            batch = {k: v.to(device) for k, v in batch.items()}

            outputs = model(**batch) # Forward pass
            logits = outputs.logits  # Logits
            
            predictions = torch.argmax(logits, dim=-1) # Highest logit score
            
            # Compare predictions to true labels
            num_correct += (predictions == batch["labels"]).sum().item()
            num_samples += batch["labels"].size(0)

    accuracy = num_correct / num_samples
    print(f"Baseline Accuracy on 1000 samples: {accuracy:.4f}") # Around 0.4880 --> 48% accurate (For 1000 testing examples) [As it plays the game of guessing, it always is around the 50% mark as the model isn't still trained and you can expect the output to be always positive or always negative]

	import torch
	from transformers import (
	AutoTokenizer,
	AutoModelForSequenceClassification
	)
	from datasets import load_dataset
	from torch.utils.data import DataLoader

	"""
	---- Device ----
	"""

	if torch.cuda.is_available():
	device = torch.device('cuda')
	print("Using CUDA (GPU)")
	elif torch.backends.mps.is_available() and torch.backends.mps.is_built():
	device = torch.device('mps')
	print("Using MPS (Apple Silicon GPU)")
	else:
	device = torch.device('cpu')
	print("Using device's CPU")

	"""
	--- Model ---
	"""

	model_ckpt = "distilbert-base-uncased"

	print(f"--- Loading pre-trained model and tokenizer: {model_ckpt.upper()} ---")

	tok = AutoTokenizer.from_pretrained(model_ckpt)
	model = AutoModelForSequenceClassification.from_pretrained(model_ckpt)
	model.to(device)
	print(f"Model moved to {device}")


	"""
	--- Data Prep ---
	"""

	print("\n--- Loading and preparing IMDB dataset ---")
	imdb_dataset = load_dataset("imdb")
	"""
	DatasetDict({
	train: Dataset({
	features: ['text', 'label'],
	num_rows: 25000
	})
	test: Dataset({
	features: ['text', 'label'],
	num_rows: 25000
	})
	unsupervised: Dataset({
	features: ['text', 'label'],
	num_rows: 50000
	})
	})
	"""

	def tokenize_fn(examples):
	return tok(examples["text"], padding="max_length", truncation=True)

	tokenized_datasets = imdb_dataset.map(tokenize_fn, batched=True)

	tokenized_datasets = tokenized_datasets.remove_columns(["text"])
	tokenized_datasets = tokenized_datasets.rename_column("label", "labels")
	tokenized_datasets.set_format("torch")


	if __name__ == '__main__':

	small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000)) # Select random 1000 test datasets
	eval_dataloader = DataLoader(small_eval_dataset, batch_size=8) # Convert them into 8 batches --> 125 ['labels', 'token_ids', 'attention_mask'] examples in each batch

	print("\n--- Evaluating baseline model performance ---")
	model.eval()
	num_correct = 0
	num_samples = 0

	with torch.no_grad(): # Disable gradient calculation for inference (No backprop)
	for batch in eval_dataloader:
	batch = {k: v.to(device) for k, v in batch.items()}

	outputs = model(**batch) # Forward pass
	logits = outputs.logits # Logits

	predictions = torch.argmax(logits, dim=-1) # Highest logit score

	# Compare predictions to true labels
	num_correct += (predictions == batch["labels"]).sum().item()
	num_samples += batch["labels"].size(0)

	accuracy = num_correct / num_samples
	print(f"Baseline Accuracy on 1000 samples: {accuracy:.4f}") # Around 0.4880 --> 48% accurate (For 1000 testing examples) [As it plays the game of guessing, it always is around the 50% mark as the model isn't still trained and you can expect the output to be always positive or always negative]