selfhostyourtech/reflector
2
0
Fork 0
mirror of https://github.com/Monadical-SAS/reflector.git synced 2025-12-21 12:49:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

code cleanup

Browse Source
This commit is contained in:
Gokul Mohanarangan
2023-07-11 12:09:30 +05:30
parent b7fbfb2a54
commit 8e9cd6c568
15 changed files with 249 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
utils/text_utilities.py
View File

@@ -6,14 +6,12 @@ from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from transformers import BartTokenizer, BartForConditionalGeneration
from transformers import BartForConditionalGeneration, BartTokenizer
from run_utils import config
from log_utils import logger
nltk.download('punkt', quiet=True)
config = configparser.ConfigParser()
config.read('config.ini')
def preprocess_sentence(sentence):
@@ -74,7 +72,7 @@ def remove_whisper_repetitive_hallucination(nonduplicate_sentences):
for sent in nonduplicate_sentences:
temp_result = ""
seen = {}
seen = { }
words = nltk.word_tokenize(sent)
n_gram_filter = 3
for i in range(len(words)):