clean up

.gitignore

@@ -24,4 +24,5 @@ htmlcov/
 .coverage.*
 *,cover
 venv
-*_cache.json
+*_cache.json
+flask_session/

README.md

@@ -1,11 +1,9 @@
 ---
-title: News Clustering Observable
-emoji: 🚀
+title: Sentence Embeddings Visualization
+emoji: 📈
 colorFrom: green
 colorTo: indigo
 sdk: gradio
 app_file: app.py
 pinned: false
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces#reference

app.py

@@ -1,7 +1,8 @@
 from umap_reducer import UMAPReducer
 from embeddings_encoder import EmbeddingsEncoder
-from flask import Flask, request, render_template, jsonify, make_response
-from flask_cors import CORS
+from flask import Flask, request, render_template, jsonify, make_response, session
+from flask_session import Session
+from flask_cors import CORS, cross_origin
 import os
 from dotenv import load_dotenv
 import feedparser
@@ -10,14 +11,22 @@ from dateutil import parser
 import re
 import numpy as np
 import gzip
+import hashlib
 
 load_dotenv()
 
 
 app = Flask(__name__, static_url_path='/static')
+app.config["SECRET_KEY"] = os.environ.get("SECRET_KEY") 
+app.config["SESSION_PERMANENT"] = True
+app.config["SESSION_TYPE"] = "filesystem"
+app.config["SESSION_COOKIE_SAMESITE"] = "None"
+app.config["SESSION_COOKIE_SECURE"] = True
+Session(app)
+CORS(app)
+
 reducer = UMAPReducer()
 encoder = EmbeddingsEncoder()
-CORS(app)
 
 
 @app.route('/')
@@ -26,15 +35,27 @@ def index():
 
 
 @app.route('/run-umap', methods=['POST'])
+@cross_origin(supports_credentials=True)
 def run_umap():
     input_data = request.get_json()
     sentences = input_data['data']['sentences']
     umap_options = input_data['data']['umap_options']
     cluster_options = input_data['data']['cluster_options']
+    # create unique hash for input, avoid recalculating embeddings
+    sentences_input_hash = hashlib.sha256(
+        ''.join(sentences).encode("utf-8")).hexdigest()
 
-    print("input options:", umap_options, cluster_options)
+    print("input options:", sentences_input_hash,
+          umap_options, cluster_options, "\n\n")
     try:
-        embeddings = encoder.encode(sentences)
+        if not session.get(sentences_input_hash):
+            print("New input, calculating embeddings" "\n\n")
+            embeddings = encoder.encode(sentences)
+            session[sentences_input_hash] = embeddings.tolist()
+        else:
+            print("Input already calculated, using cached embeddings", "\n\n")
+            embeddings = session[sentences_input_hash]
+
         # UMAP embeddings
         reducer.setParams(umap_options, cluster_options)
         umap_embeddings = reducer.embed(embeddings)
@@ -51,7 +72,7 @@ def run_umap():
         response.headers['Content-Encoding'] = 'gzip'
         return response
     except Exception as e:
-        return jsonify({"error": str(e)}), 201
+        return jsonify({"error": str(e)}), 400
 
 
 if __name__ == '__main__':

main.py

@@ -1,3 +0,0 @@
-import subprocess
-
-subprocess.run(["make", "build-all"], shell=False)

requirements.txt

@@ -1,10 +1,11 @@
 feedparser==6.0.8
 Flask==2.0.3
 flask_cors==3.0.10
+flask_session==0.4.0
 hdbscan==0.8.28
 numpy==1.22.2
 python-dotenv==0.19.2
 python_dateutil==2.8.2
+torch==1.10.2
 transformers==4.16.2
-umap-learn==0.5.2
-torch
+umap-learn==0.5.2

templates/index.html

@@ -11,16 +11,117 @@
       rel="stylesheet"
       href="https://cdn.jsdelivr.net/npm/@observablehq/inspector@3/dist/inspector.css"
     />
+    <style>
+      @import url("https://fonts.googleapis.com/css2?family=Roboto&family=Source+Serif+4:wght@700&display=swap");
+    </style>
+    <style>
+      .mw8 {
+        max-width: 64rem;
+      }
+      .mx-auto {
+        margin-left: auto;
+        margin-right: auto;
+      }
+      .ph3 {
+        padding-left: 1rem;
+        padding-right: 1rem;
+      }
+      .measure-wide {
+        max-width: 34em;
+      }
+      * {
+        font-family: "Roboto", sans-serif;
+      }
+      h1,
+      h2,
+      h3,
+      h4,
+      h5,
+      h6 {
+        font-family: "Source Serif 4", serif;
+      }
+    </style>
   </head>
   <body>
-    <div id="observablehq-3f13b363"></div>
+    <article class="mw8 mx-auto ph3 sans-serif">
+      <div class="measure-wide" id="observablehq-intro-7dbb745c"></div>
+      <div id="observablehq-viewof-sentences-7dbb745c"></div>
+      <div id="observablehq-viewof-params-7dbb745c"></div>
+      <div id="observablehq-viewof-tryme-7dbb745c"></div>
+      <div id="observablehq-viewof-clear-7dbb745c"></div>
+      <div id="observablehq-dialog-7dbb745c"></div>
+      <div id="observablehq-viewof-scatter-7dbb745c"></div>
+      <div id="observablehq-umapoptions-7dbb745c"></div>
+      <div id="observablehq-viewof-umapOptions-7dbb745c"></div>
+      <div id="observablehq-umapbutton-7dbb745c"></div>
+      <div id="observablehq-hdbscanoptions-7dbb745c"></div>
+      <div id="observablehq-viewof-clusterOptions-7dbb745c"></div>
+      <div id="observablehq-hdbcscanbutton-7dbb745c"></div>
+    </article>
+
     <script type="module">
       import {
         Runtime,
         Inspector,
       } from "https://cdn.jsdelivr.net/npm/@observablehq/runtime@4/dist/runtime.js";
       import define from "https://api.observablehq.com/d/843a8bdf01fc2c8f.js?v=3";
-      new Runtime().module(define, Inspector.into("#observablehq-3f13b363"));
+      new Runtime().module(define, (name) => {
+        if (name === "intro")
+          return new Inspector(
+            document.querySelector("#observablehq-intro-7dbb745c")
+          );
+        if (name === "viewof sentences")
+          return new Inspector(
+            document.querySelector("#observablehq-viewof-sentences-7dbb745c")
+          );
+        if (name === "viewof params")
+          return new Inspector(
+            document.querySelector("#observablehq-viewof-params-7dbb745c")
+          );
+        if (name === "viewof tryme")
+          return new Inspector(
+            document.querySelector("#observablehq-viewof-tryme-7dbb745c")
+          );
+        if (name === "viewof clear")
+          return new Inspector(
+            document.querySelector("#observablehq-viewof-clear-7dbb745c")
+          );
+        if (name === "dialog")
+          return new Inspector(
+            document.querySelector("#observablehq-dialog-7dbb745c")
+          );
+        if (name === "viewof scatter")
+          return new Inspector(
+            document.querySelector("#observablehq-viewof-scatter-7dbb745c")
+          );
+        if (name === "umapoptions")
+          return new Inspector(
+            document.querySelector("#observablehq-umapoptions-7dbb745c")
+          );
+        if (name === "viewof umapOptions")
+          return new Inspector(
+            document.querySelector("#observablehq-viewof-umapOptions-7dbb745c")
+          );
+        if (name === "umapbutton")
+          return new Inspector(
+            document.querySelector("#observablehq-umapbutton-7dbb745c")
+          );
+        if (name === "hdbscanoptions")
+          return new Inspector(
+            document.querySelector("#observablehq-hdbscanoptions-7dbb745c")
+          );
+        if (name === "viewof clusterOptions")
+          return new Inspector(
+            document.querySelector(
+              "#observablehq-viewof-clusterOptions-7dbb745c"
+            )
+          );
+        if (name === "hdbcscanbutton")
+          return new Inspector(
+            document.querySelector("#observablehq-hdbcscanbutton-7dbb745c")
+          );
+        return ["update", "data", "colorScale"].includes(name);
+      });
     </script>
   </body>
 </html>