Add larger models and use new rhyme api

README_WGU.org

@@ -45,3 +45,7 @@ See wordcloud
 ** Quick Start
 
 •  a quick start guide summarizing the steps necessary to install and use the product
+
+* Notes
+
+http-kit doesn't support https so no need to bother with keystore stuff like you would with jetty. Just proxy from haproxy.

web/deps.edn

@@ -18,7 +18,8 @@
   migratus {:mvn/version "1.3.2"}
   ring/ring-core {:mvn/version "1.8.2"}
   environ {:mvn/version "1.2.0"}
-  prhyme {:local/root "/home/eihli/src/prhyme"}}
+  prhyme {:local/root "/home/eihli/src/prhyme"}
+  com.owoga/phonetics {:mvn/version "0.1.3"}}
  :paths ["src" "resources"]
  :aliases {:dev {:extra-paths ["dev"]
                  :extra-deps {hawk {:mvn/version "0.2.11"}

web/resources/models/.gitattributes

@@ -0,0 +1,4 @@
+rhyme-trie-unstressed-vowels-and-trailing-consonants.bin filter=lfs diff=lfs merge=lfs -text
+markov-database-4-gram-backwards.bin filter=lfs diff=lfs merge=lfs -text
+markov-tightly-packed-trie-4-gram-backwards.bin filter=lfs diff=lfs merge=lfs -text
+rhyme-trie-primary-stressed-vowels-and-trailing-consonants.bin filter=lfs diff=lfs merge=lfs -text

web/src/com/darklimericks/server/limericks.clj

@@ -11,6 +11,8 @@
             [com.owoga.trie :as trie]
             [com.owoga.tightly-packed-trie :as tpt]
             [com.owoga.corpus.markov :as markov]
+            [com.owoga.prhyme.core :as prhyme]
+            [com.owoga.phonetics :as phonetics]
             [com.owoga.prhyme.data-transform :as data-transform]
             [com.owoga.tightly-packed-trie.encoding :as encoding]
             [taoensso.timbre :as timbre]))
@@ -116,14 +118,57 @@
 
   )
 
+(def rhyme-trie-unstressed-trailing-consonants
+  (markov/->RhymeTrie
+   models/rhyme-trie
+   (fn [phones]
+     (->> phones
+          prhyme/take-vowels-and-tail-consonants
+          prhyme/remove-all-stress))
+   (fn [phones choices]
+     (every? phonetics/consonant (butlast phones)))))
+
+
+(comment
+  (let [result {'[a 8]
+                [[[["P" "AE1" "S"] "pass"]
+                  [["P" "ER0" "EH1" "N" "IY0" "AH0" "L" "IY0"] "perennially"]
+                  [["Y" "UW1"] "you"]
+                  [["T" "UW1"] "to"]]
+                 [[["OW1" "V" "ER0" "P" "AE2" "S"] "overpass"]
+                  [["AH0" "N"] "an"]
+                  [["AO1" "N"] "on"]
+                  [["M" "AH0" "N" "IH2" "P" "Y" "AH0" "L" "EY1" "SH" "AH0" "N"]
+                   "manipulation"]]
+                 [[["M" "IH1" "D" "AH0" "L" "K" "L" "AE1" "S"] "middle-class"]
+                  [["HH" "AY1" "D" "IH0" "NG"] "hiding"]
+                  [["M" "AA1" "N" "S" "T" "ER0"] "monster"]
+                  [["K" "R" "UW1" "AH0" "L"] "cruel"]]],
+                '[b 5]
+                [[[["R" "EY1" "S"] "race"]
+                  [["M" "AH0" "T" "IH1" "R" "IY0" "AH0" "L"] "material"]]
+                 [[["B" "AO1" "R" "G" "EY0" "S"] "borges"]
+                  [["IY2" "K" "W" "AH0" "L" "IH1" "B" "R" "IY0" "AH0" "M"] "equilibrium"]]]}
+        [[a1 a2 a2] [b1 b2]] (vals result)]
+    (->> [a1 a2 b1]
+         (map reverse)
+         (map (partial map second))))
+
+  )
+
 (defn generate-limerick-worker [db message]
   (timbre/info "Begin generate limerick worker.")
   (let [{:keys [scheme session-id]} message
-        limerick (->> (markov/rhyme-from-scheme
+
-                       scheme
+        [[a1 a2 a3] [b1 b2]]
-                       models/database
+        (vals
-                       models/markov-trie
+         (markov/rhyme-from-scheme-v2
-                       models/rhyme-trie)
+          scheme
+          models/database
+          models/markov-trie
+          rhyme-trie-unstressed-trailing-consonants))
+
+        limerick (->> [a1 a2 b1 b2 a3]
                       (map reverse)
                       (map (partial map second))
                       (map data-transform/untokenize))

web/src/com/darklimericks/server/models.clj

@@ -5,8 +5,13 @@
             [clojure.java.io :as io]
             [com.owoga.corpus.markov :as markov]))
 
-(def database (nippy/thaw-from-resource "models/database.bin"))
+(def database (nippy/thaw-from-resource
-(def rhyme-trie (into (trie/make-trie) (nippy/thaw-from-resource "models/rhyme-trie.bin")))
+               "models/markov-database-4-gram-backwards.bin"))
+
+(def rhyme-trie (into (trie/make-trie)
+                      (nippy/thaw-from-resource
+                       "models/rhyme-trie-unstressed-vowels-and-trailing-consonants.bin")))
+
 (def markov-trie (tpt/load-tightly-packed-trie-from-file
-                  (io/resource "models/tpt.bin")
+                  (io/resource "models/markov-tightly-packed-trie-4-gram-backwards.bin")
                   (markov/decode-fn database)))