mrow

PersephoneKarnstein · Jul 3, 2023 · d2189e2 · d2189e2
1 parent 4cb1db1
commit d2189e2
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 3 deletions.
diff --git a/pdf-texts/alexjones.txt b/pdf-texts/alexjones.txt
diff --git a/terfy/zip_texts.py b/terfy/zip_texts.py
@@ -25,9 +25,16 @@ def get_corpus_data():
 
 with open("valid.txt", 'w') as f:
     f.write("\n\n".join(text_list))
+
+with open("train.txt", 'w') as f:
+    with open("pdf-texts/alexjones.txt", 'r') as g:
+        data = g.read()
+        tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
+        sentences = tokenizer.tokenize(data)
+        f.write("\n\n".join(sentences))
 
 with tarfile.open("texts.tar.gz", "w:gz") as tarhandle:
-      for a in ["valid.txt"]:
+      for a in ["valid.txt","train.txt"]:
             tarhandle.add(a)
             os.remove(a)
 # with zipfile.ZipFile("texts.zip", "w") as f:

diff --git a/texts.tar.gz b/texts.tar.gz
diff --git a/training-texts/alexjones-decimated.txt b/training-texts/alexjones-decimated.txt