Add kv store and worker queue

main
Eric Ihli 4 years ago
parent 86681ec999
commit bfa65a6de4

3
.gitignore vendored

@ -1,3 +1,4 @@
/db/data/dev
/db/data
.nrepl-port
.cpcache
/kv/data

@ -27,7 +27,7 @@ docker build -t db .
#+END_SRC
#+BEGIN_SRC sh :tangle run.sh :tangle-mode (identity #o755)
#!/usr/bin/env bash
#!/usr/bin/env sh
set -euo pipefail
PGDATA=${PGDATA:-"$(pwd)/data/dev"}

@ -0,0 +1,3 @@
FROM redis
COPY redis.conf /usr/local/etc/redis/redis.conf
CMD [ "redis-server", "/usr/local/etc/redis/redis.conf" ]

@ -0,0 +1,20 @@
#+TITLE: Persistent KV Store
There's really nothing to build right now.
#+BEGIN_SRC sh :tangle run.sh :tangle-mode (identity #o755)
#!/usr/bin/env sh
set -euo pipefail
KVDATA=${KVDATA:-"$(pwd)/data/dev"}
docker run \
-p 127.0.0.1:6379:6379 \
-v $KVDATA:/data \
--name darklimericks-kv \
redis --appendonly yes
#+END_SRC
#+BEGIN_SRC sh :tangle start.sh :tangle-mode (identity #o755)
docker start darklimericks-kv
#+END_SRC

@ -0,0 +1 @@
requirepass dev

@ -0,0 +1,10 @@
#!/usr/bin/env sh
set -euo pipefail
KVDATA=${KVDATA:-"$(pwd)/data/dev"}
docker run \
-p 127.0.0.1:6379:6379 \
-v $KVDATA:/data \
--name darklimericks-kv \
redis --appendonly yes

@ -0,0 +1 @@
docker start darklimericks-kv

@ -5,6 +5,7 @@
:sha "d492e97259c013ba401c5238842cd3445839d020"}
hiccup {:mvn/version "1.0.5"}
com.taoensso/timbre {:mvn/version "5.1.0"}
com.taoensso/carmine {:mvn/version "3.0.1"}
http-kit {:mvn/version "2.5.0"}
integrant {:mvn/version "0.8.0"}
integrant/repl {:mvn/version "0.3.2"}

@ -2,6 +2,8 @@
(:require [clojure.tools.deps.alpha.repl :refer [add-lib]]
[clojure.tools.namespace.repl :refer [set-refresh-dirs]]
[integrant.repl :as repl]
[taoensso.carmine :as car]
[taoensso.carmine.message-queue :as car-mq]
[integrant.repl.state :as state]
[migratus.core :as migratus]
[com.darklimericks.server.handlers :as handlers]
@ -59,11 +61,25 @@
(comment
(init)
(auto-reset)
(-> state/system :worker/limerick-gen)
(car/wcar
(-> state/system :database.kv/connection)
(car-mq/enqueue "limericks" '((A 9) (A 9) (B 5) (B 5) (A 9))))
(car/wcar
(-> state/system :database.kv/connection)
(car/ping)
(car/set "foo" "bar")
(car/set "baz" "buzz")
(car/get "baz"))
(limericks/get-artist-and-album-for-new-limerick (-> state/system :database.sql/connection))
(let [handler (handlers/limerick-generation-post-handler
(-> state/system :database.sql/connection)
(-> state/system :app/cache))]
(handler {:params {:scheme "A9 A9 B5 B5 A9" #_'((A 9) (A 9) (B 5) (B 5) (A 9))}}))
(repeatedly
50
(fn []
(let [handler (handlers/limerick-generation-post-handler
(-> state/system :database.sql/connection)
(-> state/system :app/cache))]
(handler {:params {:scheme "A9 A9 B5 B5 A9" #_'((A 9) (A 9) (B 5) (B 5) (A 9))}}))))
(reitit/match-by-path
(-> state/system :app/router)

@ -1,6 +1,10 @@
{:app/logging {:min-level :debug}
:app/server {:port 8000 :handler #ig/ref :app/handler}
:app/cache {}
:worker/limerick-gen {:kv #ig/ref :database.kv/connection
:db #ig/ref :database.sql/connection}
:database.kv/connection {:pool {}
:spec {:uri ""}}
:database.sql/connection {}
:app/router {:db #ig/ref :database.sql/connection :cache #ig/ref :app/cache}
:app/handler {:router #ig/ref :app/router}}

@ -4,7 +4,7 @@
[reitit.core :as reitit]
[com.darklimericks.db.artists :as artists]
[com.darklimericks.db.albums :as albums]
[com.darklimericks.db.limericks :as limericks]
[com.darklimericks.db.limericks :as db.limericks]
[com.darklimericks.linguistics.core :as linguistics]
[com.owoga.prhyme.limerick :as limerick]
[com.owoga.prhyme.data.dictionary :as dict]
@ -60,7 +60,7 @@
(defn get-artist-and-album-for-new-limerick [db]
(let [artist (artists/most-recent-artist db)
albums (albums/artist-albums db (:artist/id artist))
limericks (limericks/album-limericks db (:album/id (first albums)))]
limericks (db.limericks/album-limericks db (:album/id (first albums)))]
(cond
(< (count limericks) 10)
[(:artist/id artist) (:album/id (first albums))]
@ -102,3 +102,15 @@
(println "Exception" e)
(swap! db update-in [:tasks task-id] assoc :status :failed)
(swap! db assoc-in [:tasks task-id] {:text (.getMessage e)}))))))
(defn generate-limerick-worker [db scheme]
(let [limerick (limerick/rhyme-from-scheme
dict/prhyme-dict
darklyrics-markov-2
scheme)
[artist-id album-id] (get-artist-and-album-for-new-limerick db)]
(db.limericks/insert-limerick
db
(get-limerick-name limerick)
(string/join "\n" limerick)
album-id)))

@ -1,20 +1,42 @@
(ns com.darklimericks.server.system
(:require [integrant.core :as ig]
[clojure.java.io :as io]
[taoensso.timbre :as timbre]
[org.httpkit.server :as kit]
[next.jdbc :as jdbc]
[environ.core :refer [env]]
[taoensso.carmine.message-queue :as car-mq]
[reitit.http :as http]
[reitit.ring :as ring]
[reitit.coercion.spec]
[reitit.ring.coercion :as coercion]
[reitit.interceptor.sieppari :as sieppari]
[reitit.http.interceptors.parameters :refer [parameters-interceptor]]
[com.darklimericks.server.handlers :as handlers]
[com.darklimericks.server.limericks :as limericks]
[com.darklimericks.server.interceptors :as interceptors]
[com.darklimericks.server.db :as db]))
(defmethod ig/init-key :worker/limerick-gen [_ {:keys [db kv]}]
(car-mq/worker
kv
"limericks"
{:handler
(fn [{:keys [message attempt]}]
(timbre/info "Received" message)
(limericks/generate-limerick-worker db message))}))
(defmethod ig/prep-key :database.kv/connection [_ config]
(let [pass (or (env :redis-pass) "dev")]
(assoc-in
config
[:spec :uri]
(format "redis://localhost:6379/"))))
;; Point of this? We aren't and maintaining a connection here.
;; I guess carmine does that behind the scenes with it's "pool"
;; option. But it's nice to have all boundary config in one place.
(defmethod ig/init-key :database.kv/connection [_ config]
config)
(defmethod ig/prep-key :database.sql/connection [_ _]
{:jdbcUrl (str "jdbc:postgresql://localhost:5432/?user="
(or (env :postgres-user)

Loading…
Cancel
Save