App now retains original index following cleaning to allow for referring back to original data

Dockerfile

@@ -18,7 +18,8 @@ COPY requirements_aws.txt .
 RUN pip install torch==2.4.0+cpu --target=/install --index-url https://download.pytorch.org/whl/cpu \
 && pip install --no-cache-dir --target=/install sentence-transformers==3.0.1 --no-deps \
 && pip install --no-cache-dir --target=/install bertopic==0.16.2 --no-deps \
-&& pip install --no-cache-dir --target=/install -r requirements_aws.txt
+&& pip install --no-cache-dir --target=/install -r requirements_aws.txt \
+&& pip install --no-cache-dir --target=/install gradio=4.44.0
 
 # Add /install to the PYTHONPATH
 ENV PYTHONPATH="/install:${PYTHONPATH}"

README.md

@@ -4,7 +4,7 @@ emoji: 🚀
 colorFrom: red
 colorTo: yellow
 sdk: gradio
-sdk_version: 4.41.0
+sdk_version: 4.44.0
 app_file: app.py
 pinned: true
 license: apache-2.0

app.py

@@ -5,6 +5,7 @@ import numpy as np
 
 from funcs.topic_core_funcs import pre_clean, optimise_zero_shot, extract_topics, reduce_outliers, represent_topics, visualise_topics, save_as_pytorch_model, change_default_vis_col
 from funcs.helper_functions import initial_file_load, custom_regex_load, ensure_output_folder_exists, output_folder, get_connection_params, get_or_create_env_var
+from funcs.embeddings import make_or_load_embeddings
 from sklearn.feature_extraction.text import CountVectorizer
 from funcs.auth import authenticate_user, download_file_from_s3
 

funcs/anonymiser.py

@@ -42,61 +42,22 @@ from presidio_anonymizer.entities import OperatorConfig
 from typing import List
 
 # Function to Split Text and Create DataFrame using SpaCy
-def expand_sentences_spacy(df, colname, custom_delimiters:List[str]=[], nlp=nlp):
+def expand_sentences_spacy(df:pd.DataFrame, colname:str, custom_delimiters:List[str]=[], nlp=nlp):
+    '''
+    Expand passages into sentences using Spacy's built in NLP capabilities
+    '''
     expanded_data = []
 
-    # if not custom_delimiters:
-    #     custom_delimiters = []
-
     df = df.drop('index', axis = 1, errors="ignore").reset_index(names='index')
 
-    # sentencizer = Sentencizer()
-
-    # new_punct_chars = sentencizer.default_punct_chars
-    # new_punct_chars.extend(custom_delimiters)
-
-    # config = {"punct_chars": new_punct_chars}
-    # nlp.add_pipe("sentencizer", config=config)
-
     for index, row in df.iterrows():
         doc = nlp(row[colname])
         for sent in doc.sents:
-            expanded_data.append({'document_index': row['index'], colname: sent.text})
+            expanded_data.append({'original_index':row['original_index'],'document_index': row['index'], colname: sent.text})
     return pd.DataFrame(expanded_data)
 
-# def expand_sentences_spacy(df, colname, custom_delimiters:List[str]=[], nlp=nlp):
-
-#     #print("Custom delimiters:", custom_delimiters)
-
-#     expanded_data = []
-#     df = df.drop('index', axis = 1, errors="ignore").reset_index(names='index')
-
-#     sentencizer = Sentencizer()
-
-#     new_punct_chars = sentencizer.default_punct_chars
-#     if custom_delimiters:
-#         new_punct_chars.extend(custom_delimiters)
-
-#     pattern = "(" + "|".join(re.escape(punct) for punct in new_punct_chars) + ")"
-#     #print("Patterns:", pattern)
-#     split_list = []
-
-#     for idx, string in enumerate(df[colname]):
-#         new_split = re.split(pattern, string)
-#         for n, sentence in enumerate(new_split):
-#             if sentence:
-#                 # If there is a split delimiter in the 'sentence' after, add it to the previous sentence as it will be removed at a later step
-#                 if n + 1 < len(new_split):
-#                     if new_split[n + 1]:
-#                         # If the next split is in the list of split characters, then add it to this current sentence
-#                         if new_split[n + 1] in new_punct_chars:
-#                             split_list.append({'document_index': idx, colname: sentence + new_split[n + 1]})
-#                 else:
-#                     split_list.append({'document_index': idx, colname: sentence})
-    
-#     return pd.DataFrame(split_list)
 
-def anon_consistent_names(df):
+def anon_consistent_names(df:pd.DataFrame):
     # ## Pick out common names and replace them with the same person value
     df_dict = df.to_dict(orient="list")
 

funcs/embeddings.py

@@ -1,5 +1,6 @@
 import time
 import numpy as np
+import os
 from torch import cuda, backends, version
 
 # Check for torch cuda
@@ -12,7 +13,7 @@ if cuda.is_available():
     torch_device = "gpu"
     print("Cuda version installed is: ", version.cuda)
     high_quality_mode = "Yes"
-    #os.system("nvidia-smi")
+    os.system("nvidia-smi")
 else: 
     torch_device =  "cpu"
     high_quality_mode = "No"

funcs/helper_functions.py

@@ -5,6 +5,7 @@ import pandas as pd
 import gradio as gr
 import gzip
 import pickle
+import csv
 import numpy as np
 from bertopic import BERTopic
 from datetime import datetime
@@ -129,7 +130,7 @@ def read_file(filename):
     print("Loading in file")
 
     if file_type == 'csv':
-        file = pd.read_csv(filename, low_memory=False)#.reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
+        file = pd.read_csv(filename)#.reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
     elif file_type == 'xlsx':
         file = pd.read_excel(filename)#.reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
     elif file_type == 'parquet':

funcs/topic_core_funcs.py

@@ -93,6 +93,10 @@ def pre_clean(data: pd.DataFrame, in_colnames: list, data_file_name_no_ext: str,
 
     in_colnames_list_first = in_colnames[0]
 
+    # Reset original index to a new column so you can link it to data outputted from cleaning
+    if not "original_index" in data.columns:
+        data = data.reset_index(names="original_index")
+
     if clean_text == "Yes":
         clean_tic = time.perf_counter()
         print("Starting data clean.")
@@ -343,6 +347,7 @@ def extract_topics(
     if not candidate_topics:
         
         try:
+            # print("vectoriser_model:", vectoriser_model)
 
             topic_model = BERTopic( embedding_model=embedding_model,
                                     vectorizer_model=vectoriser_model,

requirements.txt

@@ -4,7 +4,7 @@ transformers==4.41.2
 accelerate==0.26.1
 torch==2.4.0
 bertopic==0.16.2
-spacy==3.7.4
+spacy==3.7.5
 en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1.tar.gz
 pyarrow==14.0.2
 openpyxl==3.1.2