Upload 3 files

tiny-gpt2-fine-tuning/chat.py

@@ -0,0 +1,41 @@
+from transformers import GPT2Tokenizer, GPT2LMHeadModel
+import torch
+
+# Load the fine-tuned model and tokenizer
+model = GPT2LMHeadModel.from_pretrained("./fine-tuned-gpt2")
+tokenizer = GPT2Tokenizer.from_pretrained("./fine-tuned-gpt2")
+
+# Function to generate a response
+def generate_response(prompt, max_length=100, temperature=0.7, top_k=50, top_p=0.95):
+    # Encode the input prompt
+    input_ids = tokenizer.encode(prompt, return_tensors='pt')
+    
+    # Create an attention mask
+    attention_mask = (input_ids != tokenizer.pad_token_id).long()
+    
+    with torch.no_grad():
+        output = model.generate(
+            input_ids,
+            attention_mask=attention_mask,  # Add attention mask
+            max_length=max_length,  # Adjust as needed
+            temperature=temperature,  # Sampling temperature
+            top_k=top_k,  # Top-k sampling
+            top_p=top_p,  # Top-p (nucleus) sampling
+            do_sample=True,  # Enable sampling
+            pad_token_id=tokenizer.eos_token_id  # Handle padding token
+        )
+    
+    # Decode the output
+    response = tokenizer.decode(output[0], skip_special_tokens=True)
+    return response
+
+# Chat with the model
+if __name__ == "__main__":
+    print("Chat with the model (type 'exit' to quit)")
+    while True:
+        prompt = input("You: ")
+        if prompt.lower() in ["exit", "quit", "bye"]:
+            print("Goodbye!")
+            break
+        response = generate_response(prompt)
+        print("Bot:", response)

tiny-gpt2-fine-tuning/fine-tuning.py

@@ -0,0 +1,71 @@
+from transformers import GPT2Tokenizer, GPT2LMHeadModel, TrainingArguments, Trainer
+from datasets import Dataset
+import torch
+
+# Load the pre-trained tokenizer and model
+model_name = "sshleifer/tiny-gpt2"
+tokenizer = GPT2Tokenizer.from_pretrained(model_name)
+model = GPT2LMHeadModel.from_pretrained(model_name)
+
+# Set the pad token to be the same as the eos token
+tokenizer.pad_token = tokenizer.eos_token
+
+# Function to read the text file and return a list of texts
+def read_text_file(file_path):
+    with open(file_path, "r") as file:
+        text = file.read()
+    return text.splitlines()
+
+# Load and process your data
+file_path = "data.txt"  # Replace with the path to your data.txt
+texts = read_text_file(file_path)
+
+# Create a Dataset object
+dataset = Dataset.from_dict({"text": texts})
+
+# Split the dataset into training and evaluation sets
+train_testdatasets = dataset.train_test_split(test_size=0.1)
+train_dataset = train_testdatasets["train"]
+eval_dataset = train_testdatasets["test"]
+
+# Tokenize the dataset and prepare labels
+def tokenize_function(examples):
+    encoding = tokenizer(examples["text"], padding="max_length", truncation=True, max_length=128)
+    encoding["labels"] = encoding["input_ids"].copy()  # Labels are the same as input_ids
+    return encoding
+
+tokenized_train_dataset = train_dataset.map(tokenize_function, batched=True, remove_columns=["text"])
+tokenized_eval_dataset = eval_dataset.map(tokenize_function, batched=True, remove_columns=["text"])
+
+# Set up training arguments
+training_args = TrainingArguments(
+    output_dir="./results",            # Output directory for model predictions and checkpoints
+    overwrite_output_dir=True,         # Overwrite the content of the output directory
+    num_train_epochs=5,                # Number of training epochs
+    per_device_train_batch_size=2,     # Batch size per GPU/TPU core/CPU for training
+    save_steps=10_000,                 # Save checkpoint every 10,000 steps
+    save_total_limit=2,                # Limit the total amount of checkpoints, delete the older ones
+    logging_dir='./logs',              # Directory for storing logs
+    logging_steps=500,                 # Log every 500 steps
+    eval_strategy="epoch",       # Evaluate at the end of each epoch
+    learning_rate = 0.01,
+)
+
+# Initialize the Trainer
+trainer = Trainer(
+    model=model,                        # The pre-trained model to be fine-tuned
+    args=training_args,                 # Training arguments
+    train_dataset=tokenized_train_dataset,   # Tokenized training dataset
+    eval_dataset=tokenized_eval_dataset,          
+)
+
+# Train the model
+trainer.train()
+
+# Save the fine-tuned model and tokenizer
+model.save_pretrained("./fine-tuned-gpt2")
+tokenizer.save_pretrained("./fine-tuned-gpt2")
+
+# Evaluate the model
+results = trainer.evaluate()
+print(results)

tiny-gpt2-fine-tuning/fine-tuning2.py

@@ -0,0 +1,111 @@
+from transformers import GPT2Tokenizer, GPT2LMHeadModel, TrainingArguments, Trainer, DataCollatorForLanguageModeling
+from datasets import Dataset
+import torch
+import numpy as np
+from sklearn.model_selection import train_test_split
+from transformers import EarlyStoppingCallback
+import os
+
+# Load the tokenizer and model dynamically
+model_name = "sshleifer/tiny-gpt2"  # Replace with any GPT-2 model (e.g., "gpt2" or "gpt2-medium")
+tokenizer = GPT2Tokenizer.from_pretrained(model_name)
+model = GPT2LMHeadModel.from_pretrained(model_name)
+
+# Ensure the tokenizer has a pad token
+if tokenizer.pad_token is None:
+    tokenizer.pad_token = tokenizer.eos_token
+
+# Function to read and process the text file into a list of texts
+def read_text_file(file_path):
+    with open(file_path, "r") as file:
+        text = file.read()
+    return text.splitlines()
+
+# Load and process your data
+file_path = "data.txt"  # Replace with the path to your data.txt
+texts = read_text_file(file_path)
+
+# Shuffle and split the dataset more robustly
+train_texts, eval_texts = train_test_split(texts, test_size=0.1, random_state=42)
+train_dataset = Dataset.from_dict({"text": train_texts})
+eval_dataset = Dataset.from_dict({"text": eval_texts})
+
+# Tokenize the dataset and prepare labels
+def tokenize_function(examples):
+    encoding = tokenizer(examples["text"], padding="max_length", truncation=True, max_length=332)
+    encoding["labels"] = encoding["input_ids"].copy()  # Labels are the same as input_ids
+    return encoding
+
+# Tokenize datasets
+tokenized_train_dataset = train_dataset.map(tokenize_function, batched=True, remove_columns=["text"])
+tokenized_eval_dataset = eval_dataset.map(tokenize_function, batched=True, remove_columns=["text"])
+
+# Data collator for language modeling to handle dynamic padding
+data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+
+# Set up advanced training arguments
+training_args = TrainingArguments(
+    output_dir="./results",                # Output directory for model predictions and checkpoints
+    overwrite_output_dir=True,             # Overwrite the content of the output directory
+    num_train_epochs=10,                    # Number of training epochs
+    per_device_train_batch_size=2,         # Batch size per GPU/TPU core/CPU for training
+    per_device_eval_batch_size=2,          # Batch size per GPU/TPU core/CPU for evaluation
+    save_steps=2_000,                      # Save checkpoint every 2,000 steps
+    save_total_limit=2,                    # Limit the total amount of checkpoints, delete the older ones
+    logging_dir='./logs',                  # Directory for storing logs
+    logging_steps=10,                     # Log every 100 steps
+    eval_strategy="steps",                 # Evaluate every X steps (set by `evaluation_strategy` argument)          # Perform evaluation every N steps
+    learning_rate=0.01,                    # Lower learning rate for fine-tuning
+    weight_decay=0.01,                     # Weight decay to prevent overfitting
+    warmup_steps=64,                      # Number of steps to perform learning rate warmup
+    fp16=True,                                                          # Use 16-bit precision if supported by GPU
+    load_best_model_at_end=True,           # Load the best model when training is finished
+    metric_for_best_model="loss",          # Metric to use to compare the best model
+    greater_is_better=False,               # Set to False as we're minimizing loss
+    disable_tqdm=False,                    # Show progress bar
+)
+
+# Early stopping to prevent overfitting
+early_stopping = EarlyStoppingCallback(early_stopping_patience=3, early_stopping_threshold=0.01)
+
+# Custom evaluation metrics (e.g., perplexity)
+def compute_metrics(eval_pred):
+    logits, labels = eval_pred
+    # Convert numpy arrays to PyTorch tensors
+    logits = torch.tensor(logits)
+    labels = torch.tensor(labels)
+    
+    # Shift the logits and labels for loss calculation
+    shift_logits = logits[..., :-1, :].contiguous()
+    shift_labels = labels[..., 1:].contiguous()
+    
+    # Calculate perplexity
+    loss_fct = torch.nn.CrossEntropyLoss(ignore_index=-100)
+    loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+    perplexity = torch.exp(loss)
+    
+    return {"perplexity": perplexity.item()}
+
+# Initialize the Trainer with advanced settings
+trainer = Trainer(
+    model=model,                              # The pre-trained model to be fine-tuned
+    args=training_args,                       # Training arguments
+    train_dataset=tokenized_train_dataset,    # Tokenized training dataset
+    eval_dataset=tokenized_eval_dataset,      # Tokenized evaluation dataset
+    data_collator=data_collator,              # Data collator for dynamic padding
+    compute_metrics=compute_metrics,          # Custom evaluation metrics
+    callbacks=[early_stopping],               # Early stopping callback
+)
+
+# Train the model
+trainer.train()
+
+# Save the fine-tuned model and tokenizer
+model_output_dir = "./fine-tuned-gpt2"
+os.makedirs(model_output_dir, exist_ok=True)
+model.save_pretrained(model_output_dir)
+tokenizer.save_pretrained(model_output_dir)
+
+# Evaluate the model
+results = trainer.evaluate()
+print(f"Evaluation Results: {results}")