def batch_iterate(lst, batch_size):
for i in range(0, len(lst), batch_size):
yield lst[i : i + batch_size]
class EmbedData:
...
def embed(self, contexts):
self.contexts = contexts
for batch_context in tqdm(batch_iterate(contexts, self.batch_size),
total=len(contexts)//self.batch_size,
desc="Embedding data in batches"):
batch_embeddings = self.generate_embedding(batch_context)
self.embeddings.extend(batch_embeddings)
Create batches
Iterate over each batch