Add sentence validity detection

main
Eric Ihli 4 years ago
parent b6bdba58ad
commit 731ac8bd03

@ -5,6 +5,7 @@
org.clojure/core.async {:mvn/version "1.2.603"} org.clojure/core.async {:mvn/version "1.2.603"}
inflections {:mvn/version "0.13.2"} inflections {:mvn/version "0.13.2"}
com.taoensso/tufte {:mvn/version "2.2.0"} com.taoensso/tufte {:mvn/version "2.2.0"}
clojure-opennlp {:mvn/version "0.5.0"}
enlive {:mvn/version "1.1.6"} enlive {:mvn/version "1.1.6"}
com.taoensso/timbre {:mvn/version "4.10.0"}} com.taoensso/timbre {:mvn/version "4.10.0"}}
:aliases {:dev {:extra-paths ["test"] :aliases {:dev {:extra-paths ["test"]

@ -3,6 +3,7 @@
[clojure.string :as string] [clojure.string :as string]
[com.owoga.prhyme.util.weighted-rand :as wr] [com.owoga.prhyme.util.weighted-rand :as wr]
[com.owoga.prhyme.core :as prhyme] [com.owoga.prhyme.core :as prhyme]
[com.owoga.prhyme.util.nlp :as nlp]
[taoensso.tufte :as tufte :refer [defnp p profiled profile]] [taoensso.tufte :as tufte :refer [defnp p profiled profile]]
[com.owoga.prhyme.frp :as frp] [com.owoga.prhyme.frp :as frp]
[clojure.java.io :as io] [clojure.java.io :as io]
@ -437,7 +438,7 @@
weight-non-markovs (apply + (map :weight non-markovs)) weight-non-markovs (apply + (map :weight non-markovs))
target-weight-markovs (* ratio weight-non-markovs) target-weight-markovs (* ratio weight-non-markovs)
count-markovs (count markovs) count-markovs (count markovs)
adjustment-markovs (/ target-weight-markovs count-markovs)] adjustment-markovs (if (= 0 count-markovs) 1 (/ target-weight-markovs count-markovs))]
(concat (concat
(map (map
(fn [markov] (fn [markov]
@ -460,11 +461,11 @@
(> sentinel 5)) (> sentinel 5))
result result
(let [markov-options (markov (list (:norm-word (first result)))) (let [markov-options (markov (list (:norm-word (first result))))
markov-adjuster (adjust-for-markov-1 markov-options 0.8) markov-adjuster (adjust-for-markov-1 markov-options 0.9)
syllable-count-adjuster (adjust-for-over-syllables target) syllable-count-adjuster (adjust-for-over-syllables target)
rhyme-adjuster (adjust-for-rhymes-1 target 0.8) rhyme-adjuster (adjust-for-rhymes-1 target 0.9)
lovecraft-set (into #{} (map (comp first first) lovecraft-markov)) lovecraft-set (into #{} (map (comp first first) lovecraft-markov))
lovecraft-filter (adjust-for-membership-1 lovecraft-set 0.8) lovecraft-filter (adjust-for-membership-1 lovecraft-set 0.9)
adjust (comp rhyme-adjuster adjust (comp rhyme-adjuster
syllable-count-adjuster syllable-count-adjuster
markov-adjuster markov-adjuster
@ -502,7 +503,38 @@
(apply + (map :syllable-count result))))))) (apply + (map :syllable-count result)))))))
poem-lines)) poem-lines))
(defn rhymer [words markov target stop]
(cons (e-prhyme
words
markov
target
stop)
(lazy-seq (rhymer words markov target stop))))
(defn stop [target]
(fn [inner-target result]
(<= (count (:syllables target))
(apply + (map :syllable-count result)))))
(defn sentence-stop [target]
(fn [inner-target result]
(let [result-sentence (string/join " " (map :norm-word result))]
(when-not (empty? result)
(or (nlp/valid-sentence? result-sentence)
(< (:syllable-count target)
(apply + (map :syllable-count result)))
(< 5 (count result)))))))
(comment (comment
(let [phrase (frp/phrase->word frp/words "i solemnly swear i am up to no good")
r (rhymer
frp/popular
lovecraft-markov
phrase
(sentence-stop phrase))]
(take 2 (map #(string/join " " (map :norm-word %))
(filter #(nlp/valid-sentence? (string/join " " (map :norm-word %))) r))))
(let [poem-lines ["mister sandman" (let [poem-lines ["mister sandman"
"give me a dream" "give me a dream"
"make him the cutest" "make him the cutest"

@ -0,0 +1,19 @@
(ns com.owoga.prhyme.util.nlp
(:require [opennlp.nlp :as nlp]
[opennlp.treebank :as tb]
[clojure.string :as string]))
(def tokenize (nlp/make-tokenizer "models/en-token.bin"))
(def get-sentences (nlp/make-sentence-detector "models/en-sent.bin"))
(def parse (tb/make-treebank-parser "en-parser-chunking.bin"))
(defn valid-sentence? [phrase]
(->> phrase
tokenize
(string/join " ")
vector
parse
first
tb/make-tree
(#(= 'S (:tag (first (:chunk %)))))))
Loading…
Cancel
Save