From bfa65a6de475742159dc54c0a7db9812df426619 Mon Sep 17 00:00:00 2001 From: Eric Ihli Date: Mon, 2 Nov 2020 14:17:47 -0800 Subject: [PATCH] Add kv store and worker queue --- .gitignore | 3 ++- db/README.org | 2 +- kv/Dockerfile | 3 +++ kv/README.org | 20 ++++++++++++++ kv/redis.conf | 1 + kv/run.sh | 10 +++++++ kv/start.sh | 1 + web/deps.edn | 1 + web/dev/user.clj | 24 ++++++++++++++--- web/resources/server/config.edn | 4 +++ .../com/darklimericks/server/limericks.clj | 16 ++++++++++-- web/src/com/darklimericks/server/system.clj | 26 +++++++++++++++++-- 12 files changed, 101 insertions(+), 10 deletions(-) create mode 100644 kv/Dockerfile create mode 100644 kv/README.org create mode 100644 kv/redis.conf create mode 100755 kv/run.sh create mode 100755 kv/start.sh diff --git a/.gitignore b/.gitignore index 9c76f18..23bcc1a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ -/db/data/dev +/db/data .nrepl-port .cpcache +/kv/data diff --git a/db/README.org b/db/README.org index f283e74..4459e06 100644 --- a/db/README.org +++ b/db/README.org @@ -27,7 +27,7 @@ docker build -t db . #+END_SRC #+BEGIN_SRC sh :tangle run.sh :tangle-mode (identity #o755) -#!/usr/bin/env bash +#!/usr/bin/env sh set -euo pipefail PGDATA=${PGDATA:-"$(pwd)/data/dev"} diff --git a/kv/Dockerfile b/kv/Dockerfile new file mode 100644 index 0000000..bd09a8b --- /dev/null +++ b/kv/Dockerfile @@ -0,0 +1,3 @@ +FROM redis +COPY redis.conf /usr/local/etc/redis/redis.conf +CMD [ "redis-server", "/usr/local/etc/redis/redis.conf" ] diff --git a/kv/README.org b/kv/README.org new file mode 100644 index 0000000..ad65c74 --- /dev/null +++ b/kv/README.org @@ -0,0 +1,20 @@ +#+TITLE: Persistent KV Store + +There's really nothing to build right now. + +#+BEGIN_SRC sh :tangle run.sh :tangle-mode (identity #o755) +#!/usr/bin/env sh +set -euo pipefail + +KVDATA=${KVDATA:-"$(pwd)/data/dev"} + +docker run \ + -p 127.0.0.1:6379:6379 \ + -v $KVDATA:/data \ + --name darklimericks-kv \ + redis --appendonly yes +#+END_SRC + +#+BEGIN_SRC sh :tangle start.sh :tangle-mode (identity #o755) +docker start darklimericks-kv +#+END_SRC diff --git a/kv/redis.conf b/kv/redis.conf new file mode 100644 index 0000000..77ad619 --- /dev/null +++ b/kv/redis.conf @@ -0,0 +1 @@ +requirepass dev diff --git a/kv/run.sh b/kv/run.sh new file mode 100755 index 0000000..d73d54d --- /dev/null +++ b/kv/run.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh +set -euo pipefail + +KVDATA=${KVDATA:-"$(pwd)/data/dev"} + +docker run \ + -p 127.0.0.1:6379:6379 \ + -v $KVDATA:/data \ + --name darklimericks-kv \ + redis --appendonly yes diff --git a/kv/start.sh b/kv/start.sh new file mode 100755 index 0000000..2d5051d --- /dev/null +++ b/kv/start.sh @@ -0,0 +1 @@ +docker start darklimericks-kv diff --git a/web/deps.edn b/web/deps.edn index 3d95fd2..8385a12 100644 --- a/web/deps.edn +++ b/web/deps.edn @@ -5,6 +5,7 @@ :sha "d492e97259c013ba401c5238842cd3445839d020"} hiccup {:mvn/version "1.0.5"} com.taoensso/timbre {:mvn/version "5.1.0"} + com.taoensso/carmine {:mvn/version "3.0.1"} http-kit {:mvn/version "2.5.0"} integrant {:mvn/version "0.8.0"} integrant/repl {:mvn/version "0.3.2"} diff --git a/web/dev/user.clj b/web/dev/user.clj index 7f7f2a0..f438720 100644 --- a/web/dev/user.clj +++ b/web/dev/user.clj @@ -2,6 +2,8 @@ (:require [clojure.tools.deps.alpha.repl :refer [add-lib]] [clojure.tools.namespace.repl :refer [set-refresh-dirs]] [integrant.repl :as repl] + [taoensso.carmine :as car] + [taoensso.carmine.message-queue :as car-mq] [integrant.repl.state :as state] [migratus.core :as migratus] [com.darklimericks.server.handlers :as handlers] @@ -59,11 +61,25 @@ (comment (init) (auto-reset) + (-> state/system :worker/limerick-gen) + (car/wcar + (-> state/system :database.kv/connection) + (car-mq/enqueue "limericks" '((A 9) (A 9) (B 5) (B 5) (A 9)))) + + (car/wcar + (-> state/system :database.kv/connection) + (car/ping) + (car/set "foo" "bar") + (car/set "baz" "buzz") + (car/get "baz")) (limericks/get-artist-and-album-for-new-limerick (-> state/system :database.sql/connection)) - (let [handler (handlers/limerick-generation-post-handler - (-> state/system :database.sql/connection) - (-> state/system :app/cache))] - (handler {:params {:scheme "A9 A9 B5 B5 A9" #_'((A 9) (A 9) (B 5) (B 5) (A 9))}})) + (repeatedly + 50 + (fn [] + (let [handler (handlers/limerick-generation-post-handler + (-> state/system :database.sql/connection) + (-> state/system :app/cache))] + (handler {:params {:scheme "A9 A9 B5 B5 A9" #_'((A 9) (A 9) (B 5) (B 5) (A 9))}})))) (reitit/match-by-path (-> state/system :app/router) diff --git a/web/resources/server/config.edn b/web/resources/server/config.edn index 040f623..7239024 100644 --- a/web/resources/server/config.edn +++ b/web/resources/server/config.edn @@ -1,6 +1,10 @@ {:app/logging {:min-level :debug} :app/server {:port 8000 :handler #ig/ref :app/handler} :app/cache {} + :worker/limerick-gen {:kv #ig/ref :database.kv/connection + :db #ig/ref :database.sql/connection} + :database.kv/connection {:pool {} + :spec {:uri ""}} :database.sql/connection {} :app/router {:db #ig/ref :database.sql/connection :cache #ig/ref :app/cache} :app/handler {:router #ig/ref :app/router}} diff --git a/web/src/com/darklimericks/server/limericks.clj b/web/src/com/darklimericks/server/limericks.clj index be14561..692f982 100644 --- a/web/src/com/darklimericks/server/limericks.clj +++ b/web/src/com/darklimericks/server/limericks.clj @@ -4,7 +4,7 @@ [reitit.core :as reitit] [com.darklimericks.db.artists :as artists] [com.darklimericks.db.albums :as albums] - [com.darklimericks.db.limericks :as limericks] + [com.darklimericks.db.limericks :as db.limericks] [com.darklimericks.linguistics.core :as linguistics] [com.owoga.prhyme.limerick :as limerick] [com.owoga.prhyme.data.dictionary :as dict] @@ -60,7 +60,7 @@ (defn get-artist-and-album-for-new-limerick [db] (let [artist (artists/most-recent-artist db) albums (albums/artist-albums db (:artist/id artist)) - limericks (limericks/album-limericks db (:album/id (first albums)))] + limericks (db.limericks/album-limericks db (:album/id (first albums)))] (cond (< (count limericks) 10) [(:artist/id artist) (:album/id (first albums))] @@ -102,3 +102,15 @@ (println "Exception" e) (swap! db update-in [:tasks task-id] assoc :status :failed) (swap! db assoc-in [:tasks task-id] {:text (.getMessage e)})))))) + +(defn generate-limerick-worker [db scheme] + (let [limerick (limerick/rhyme-from-scheme + dict/prhyme-dict + darklyrics-markov-2 + scheme) + [artist-id album-id] (get-artist-and-album-for-new-limerick db)] + (db.limericks/insert-limerick + db + (get-limerick-name limerick) + (string/join "\n" limerick) + album-id))) diff --git a/web/src/com/darklimericks/server/system.clj b/web/src/com/darklimericks/server/system.clj index 867a074..7b1cf24 100644 --- a/web/src/com/darklimericks/server/system.clj +++ b/web/src/com/darklimericks/server/system.clj @@ -1,20 +1,42 @@ (ns com.darklimericks.server.system (:require [integrant.core :as ig] - [clojure.java.io :as io] [taoensso.timbre :as timbre] [org.httpkit.server :as kit] [next.jdbc :as jdbc] [environ.core :refer [env]] + [taoensso.carmine.message-queue :as car-mq] [reitit.http :as http] [reitit.ring :as ring] [reitit.coercion.spec] - [reitit.ring.coercion :as coercion] [reitit.interceptor.sieppari :as sieppari] [reitit.http.interceptors.parameters :refer [parameters-interceptor]] [com.darklimericks.server.handlers :as handlers] + [com.darklimericks.server.limericks :as limericks] [com.darklimericks.server.interceptors :as interceptors] [com.darklimericks.server.db :as db])) +(defmethod ig/init-key :worker/limerick-gen [_ {:keys [db kv]}] + (car-mq/worker + kv + "limericks" + {:handler + (fn [{:keys [message attempt]}] + (timbre/info "Received" message) + (limericks/generate-limerick-worker db message))})) + +(defmethod ig/prep-key :database.kv/connection [_ config] + (let [pass (or (env :redis-pass) "dev")] + (assoc-in + config + [:spec :uri] + (format "redis://localhost:6379/")))) + +;; Point of this? We aren't and maintaining a connection here. +;; I guess carmine does that behind the scenes with it's "pool" +;; option. But it's nice to have all boundary config in one place. +(defmethod ig/init-key :database.kv/connection [_ config] + config) + (defmethod ig/prep-key :database.sql/connection [_ _] {:jdbcUrl (str "jdbc:postgresql://localhost:5432/?user=" (or (env :postgres-user)