From cec8bbcf6c6b70092de14ebbbd6f4c0ec2ddb884 Mon Sep 17 00:00:00 2001 From: Gokul Mohanarangan Date: Tue, 25 Jul 2023 10:22:46 +0530 Subject: [PATCH] move all experiments to trials --- trials/bert.py | 43 +++++++++++++++++ trials/pegasus.py | 33 +++++++++++++ trials/t5.py | 27 +++++++++++ trials/vicuna.py | 44 +++++++++++++++++ trials/youtube_scraping.py | 98 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 245 insertions(+) create mode 100644 trials/bert.py create mode 100644 trials/pegasus.py create mode 100644 trials/t5.py create mode 100644 trials/vicuna.py create mode 100644 trials/youtube_scraping.py diff --git a/trials/bert.py b/trials/bert.py new file mode 100644 index 00000000..a79bb76d --- /dev/null +++ b/trials/bert.py @@ -0,0 +1,43 @@ +import torch +from transformers import BertTokenizer, BertModel +from sentence_transformers import SentenceTransformer +from sklearn.metrics.pairwise import cosine_similarity + +# Load the pre-trained BERT model and tokenizer +model_name = "bert-base-uncased" +model = BertModel.from_pretrained(model_name) +tokenizer = BertTokenizer.from_pretrained(model_name) + +# Set the device to use +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model.to(device) + +# Load the SentenceTransformer model +sentence_transformer_model = SentenceTransformer('average_word_embeddings_glove.6B.300d') + +# Define the input text +text = "Your input text to be summarized goes here." + +# Tokenize the text +tokens = tokenizer.tokenize(text) +input_ids = tokenizer.convert_tokens_to_ids(tokens) +input_ids = torch.tensor([input_ids]).to(device) + +# Get the BERT model output +with torch.no_grad(): + outputs = model(input_ids)[0] # Extract the last hidden states + +# Calculate sentence embeddings +sentence_embeddings = outputs.mean(dim=1).squeeze().cpu().numpy() +input_text_embedding = sentence_transformer_model.encode([text])[0] + +# Calculate cosine similarity between sentences and input text +similarity_scores = cosine_similarity([input_text_embedding], sentence_embeddings) + +# Sort the sentences by similarity scores in descending order +sorted_sentences = [sent for _, sent in sorted(zip(similarity_scores[0], sentences), reverse=True)] + +# Choose the top sentences as the summary +num_summary_sentences = 2 # Adjust as needed +summary = ". ".join(sorted_sentences[:num_summary_sentences]) +print("Summary:", summary) diff --git a/trials/pegasus.py b/trials/pegasus.py new file mode 100644 index 00000000..884ed3ee --- /dev/null +++ b/trials/pegasus.py @@ -0,0 +1,33 @@ +from transformers import PegasusForConditionalGeneration, PegasusTokenizer +import torch +# Load the Pegasus model and tokenizer +model_name = "google/pegasus-large" +model = PegasusForConditionalGeneration.from_pretrained(model_name) +tokenizer = PegasusTokenizer.from_pretrained(model_name) + +# Set the device to use +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model.to(device) + +sample_chunks = ["You all just came off of your incredible Google Cloud next conference where you released a wide variety of functionality and features and new products across artisan television and also across the entire sort of cloud ecosystem . You want to just first by walking through , first start by walking through all the innovations that you sort of released and what you 're excited about when you come to Google Cloud ? Now our vision is super simple . If you look at what smartphones did for a consumer , you know they took a computer and internet browser , a communication device , and a camera , and made it so that it 's in everybody 's pocket , so it really brought computation to every person . We feel that , you know , our , what we 're trying to do is take all the technological innovation that Google 's doing , but make it super simple so that everyone can consume it . And so that includes our global data center footprint , all the new types of hardware and large-scale systems we work on , the software that we 're making available for people to do high-scale computation , tools for data processing , tools for cybersecurity , processing , tools for cyber security , tools for machine learning , but make it so simple that everyone can use it . And every step that we do to simplify things for people , we think adoption can grow . And so that 's a lot of what we 've done these last three , four years , and we made a number of announcements that next in machine learning and AI in particular , you know , we look at our work as four elements , how we take our large-scale compute systems that were building for AI and how we make that available to everybody . Second , what we 're doing with the software stacks and top of it , things like jacks and other things and how we 're making those available to everybody . Third is advances because different people have different levels of expertise . Some people say I need the hardware to build my own large language model or algorithm . Other people say , look , I really need to use a building block . You guys give me . So , 30s we 've done a lot with AutoML and we announce new capability for image , video , and translation to make it available to everybody . And then lastly , we 're also building completely packaged solutions for some areas and we announce some new stuff . ", + " We 're joined next by Thomas Curian , CEO of Google Cloud , and Alexander Wang , CEO and founder of Scale AI . Thomas joined Google in November 2018 as the CEO of Google Cloud . Prior to Google , Thomas spent 22 years at Oracle , where most recently he was president of product development . Before that , Thomas worked at McKinsey as a business analyst and engagement manager . His nearly 30 years of experience have given him a deep knowledge of engineering enterprise relationships and leadership of large organizations . Thomas 's degrees include an MBA in administration and management from Stanford University , as an RJ Miller scholar and a BSEE in electrical engineering and computer science from Princeton University , where he graduated suma cum laude . Thomas serves as a member of the Stanford graduate School of Business Advisory Council and Princeton University School of Engineering Advisory Council . Please welcome to the stage , Thomas Curian and Alexander Wang . This is a super exciting conversation . Thanks for being here , Thomas ."] + + +# Define the input text for summarization +text = sample_chunks[1] + +inputs = tokenizer(text, truncation=True, padding="longest", return_tensors="pt").to(device) + +# Generate the summary +summary_ids = model.generate( + inputs["input_ids"], + attention_mask=inputs["attention_mask"], + max_length=200, + num_beams=4, + length_penalty=2.0, + early_stopping=True, +) + +# Decode and print the summary +summary = tokenizer.decode(summary_ids[0], skip_special_tokens=True) +print("Summary:", summary) diff --git a/trials/t5.py b/trials/t5.py new file mode 100644 index 00000000..0c366ac6 --- /dev/null +++ b/trials/t5.py @@ -0,0 +1,27 @@ +from transformers import T5ForConditionalGeneration, T5Tokenizer +import torch +# Load the T5 model and tokenizer +model_name = "t5-base" +model = T5ForConditionalGeneration.from_pretrained(model_name) +tokenizer = T5Tokenizer.from_pretrained(model_name) + +# Set the device to use +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model.to(device) + +sample_chunks = ["You all just came off of your incredible Google Cloud next conference where you released a wide variety of functionality and features and new products across artisan television and also across the entire sort of cloud ecosystem . You want to just first by walking through , first start by walking through all the innovations that you sort of released and what you 're excited about when you come to Google Cloud ? Now our vision is super simple . If you look at what smartphones did for a consumer , you know they took a computer and internet browser , a communication device , and a camera , and made it so that it 's in everybody 's pocket , so it really brought computation to every person . We feel that , you know , our , what we 're trying to do is take all the technological innovation that Google 's doing , but make it super simple so that everyone can consume it . And so that includes our global data center footprint , all the new types of hardware and large-scale systems we work on , the software that we 're making available for people to do high-scale computation , tools for data processing , tools for cybersecurity , processing , tools for cyber security , tools for machine learning , but make it so simple that everyone can use it . And every step that we do to simplify things for people , we think adoption can grow . And so that 's a lot of what we 've done these last three , four years , and we made a number of announcements that next in machine learning and AI in particular , you know , we look at our work as four elements , how we take our large-scale compute systems that were building for AI and how we make that available to everybody . Second , what we 're doing with the software stacks and top of it , things like jacks and other things and how we 're making those available to everybody . Third is advances because different people have different levels of expertise . Some people say I need the hardware to build my own large language model or algorithm . Other people say , look , I really need to use a building block . You guys give me . So , 30s we 've done a lot with AutoML and we announce new capability for image , video , and translation to make it available to everybody . And then lastly , we 're also building completely packaged solutions for some areas and we announce some new stuff . ", + " We 're joined next by Thomas Curian , CEO of Google Cloud , and Alexander Wang , CEO and founder of Scale AI . Thomas joined Google in November 2018 as the CEO of Google Cloud . Prior to Google , Thomas spent 22 years at Oracle , where most recently he was president of product development . Before that , Thomas worked at McKinsey as a business analyst and engagement manager . His nearly 30 years of experience have given him a deep knowledge of engineering enterprise relationships and leadership of large organizations . Thomas 's degrees include an MBA in administration and management from Stanford University , as an RJ Miller scholar and a BSEE in electrical engineering and computer science from Princeton University , where he graduated suma cum laude . Thomas serves as a member of the Stanford graduate School of Business Advisory Council and Princeton University School of Engineering Advisory Council . Please welcome to the stage , Thomas Curian and Alexander Wang . This is a super exciting conversation . Thanks for being here , Thomas ."] + + +# Define the input text for summarization +text = "Summarize the following text in 3 key points. text : " + sample_chunks[1] + +# Tokenize the input text +inputs = tokenizer.encode(text, return_tensors="pt").to(device) + +# Generate the summary +summary_ids = model.generate(inputs, max_length=1000, num_beams=4, early_stopping=True) + +# Decode and print the summary +summary = tokenizer.decode(summary_ids.squeeze(), skip_special_tokens=True) +print("Summary:", summary) diff --git a/trials/vicuna.py b/trials/vicuna.py new file mode 100644 index 00000000..588869c0 --- /dev/null +++ b/trials/vicuna.py @@ -0,0 +1,44 @@ +from gpt4all import GPT4All + +model = GPT4All("/Users/gokulmohanarangan/Library/Application Support/nomic.ai/GPT4All/ggml-vicuna-13b-1.1-q4_2.bin") + +import spacy + + +def split_text_file(filename, token_count): + nlp = spacy.load('en_core_web_md') + + with open(filename, 'r') as file: + text = file.read() + + doc = nlp(text) + total_tokens = len(doc) + + parts = [] + start_index = 0 + + while start_index < total_tokens: + end_index = start_index + token_count + part_tokens = doc[start_index:end_index] + part = ' '.join(token.text for token in part_tokens) + parts.append(part) + start_index = end_index + + return parts + +parts = split_text_file("transcript.txt", 1800) +final_summary = [] +for part in parts: + prompt = f""" + ### Human: + Summarize the following text without missing any key points and action items. + + {part} + ### Assistant: + """ + output = model.generate(prompt) + final_summary.append(output) + + +with open("sum.txt", "w") as sum: + sum.write(" ".join(final_summary)) diff --git a/trials/youtube_scraping.py b/trials/youtube_scraping.py new file mode 100644 index 00000000..b0892f47 --- /dev/null +++ b/trials/youtube_scraping.py @@ -0,0 +1,98 @@ +import json +import yt_dlp as youtube_dl +from whisper_jax import FlaxWhisperPipline +import jax.numpy as jnp + +# Function to extract chapter information from a YouTube video URL +def get_youtube_chapters(video_id): + video_url = "https://www.youtube.com/watch?v=" + video_id + ydl_opts = { + 'extract_flat': 'in_playlist', + 'skip_download': True, + 'quiet': True, + } + + with youtube_dl.YoutubeDL(ydl_opts) as ydl: + video_info = ydl.extract_info(video_url, download=False) + + chapters = [] + + if 'chapters' in video_info: + for chapter in video_info['chapters']: + start_time = chapter['start_time'] + end_time = chapter['end_time'] + title = chapter['title'] + + chapters.append({ + 'start': start_time, + 'end': end_time, + 'title': title + }) + + return chapters + + +# Function to extract video transcription using yt_dlp +def get_youtube_transcription(video_id): + ydl_opts = { + 'format': 'bestaudio/best', + 'postprocessors': [{ + 'key': 'FFmpegExtractAudio', + 'preferredcodec': 'mp3', + 'preferredquality': '192', + }], + 'outtmpl': './artefacts/audio', # Specify output file path and name + } + + # Download the audio + with youtube_dl.YoutubeDL(ydl_opts) as ydl: + ydl.download(["https://www.youtube.com/watch?v=" + video_id]) + media_file = "./artefacts/audio.mp3" + + pipeline = FlaxWhisperPipline("openai/whisper-" + "tiny", + dtype=jnp.float16, + batch_size=16) + whisper_result = pipeline(media_file, return_timestamps=True) + return whisper_result["chunks"] + + + +# Function to scrape YouTube video transcripts and chapter information +def scrape_youtube_data(video_id): + transcript_text = get_youtube_transcription(video_id) + chapters = get_youtube_chapters(video_id) + print("transcript_text", transcript_text) + print("chapters", chapters) + return transcript_text, chapters + + +# Function to generate fine-tuning dataset from YouTube data +def generate_finetuning_dataset(video_ids): + prompt_completion_pairs = [] + for video_id in video_ids: + transcript_text, chapters = scrape_youtube_data(video_id) + if transcript_text is not None and chapters is not None: + for chapter in chapters: + start_time = chapter["start"] + end_time = chapter["end"] + chapter_text = chapter["title"] + + prompt = "" + for transcript in transcript_text: + if transcript["timestamp"][0] >= start_time and transcript["timestamp"][1] < end_time: + prompt += transcript["text"] + + if prompt is not None: + completion = chapter_text + prompt_completion_pairs.append({"prompt": prompt, "completion": completion}) + + return prompt_completion_pairs + + +# Add all the video ids here, the videos must have captions [chapters] +video_ids = ["yTnSEZIwnkU"] +dataset = generate_finetuning_dataset(video_ids) + +with open("finetuning_dataset.jsonl", "w") as f: + for example in dataset: + f.write(json.dumps(example) + "\n")