Add comments to nlp/g code
Eric Ihli
4 years ago
parent
c0ad5acefc
commit
0c7da21610
@ -1,210 +0,0 @@
|
|||||||
(ns com.owoga.prhyme.util.nlp
|
|
||||||
(:require [opennlp.nlp :as nlp]
|
|
||||||
[opennlp.treebank :as tb]
|
|
||||||
[clojure.string :as string]
|
|
||||||
[clojure.java.io :as io]
|
|
||||||
[clojure.zip :as zip]
|
|
||||||
[com.owoga.prhyme.nlp.tag-sets.treebank-ii :as tb2])
|
|
||||||
(:import (opennlp.tools.postag POSModel POSTaggerME)))
|
|
||||||
|
|
||||||
(def tokenize (nlp/make-tokenizer (io/resource "models/en-token.bin")))
|
|
||||||
(def get-sentences (nlp/make-sentence-detector (io/resource "models/en-sent.bin")))
|
|
||||||
(def parse (tb/make-treebank-parser (io/resource "models/en-parser-chunking.bin")))
|
|
||||||
(def pos-tagger (nlp/make-pos-tagger (io/resource "models/en-pos-maxent.bin")))
|
|
||||||
|
|
||||||
;;;; The tagger that onennlp.nlp gives us doesn't provide access
|
|
||||||
;;;; to the probabilities of all tags. It gives us the probability of the
|
|
||||||
;;;; top tag through some metadata. But to get probs for all tags, we
|
|
||||||
;;;; need to implement our own tagger.
|
|
||||||
(defprotocol Tagger
|
|
||||||
(tags [this sent])
|
|
||||||
(probs [this])
|
|
||||||
(top-k-sequences [this sent]))
|
|
||||||
|
|
||||||
(defn make-pos-tagger
|
|
||||||
[modelfile]
|
|
||||||
(let [model (with-open [model-stream (io/input-stream modelfile)]
|
|
||||||
(POSModel. model-stream))
|
|
||||||
tagger (POSTaggerME. model)]
|
|
||||||
(reify Tagger
|
|
||||||
(tags [_ tokens]
|
|
||||||
(let [token-array (into-array String tokens)]
|
|
||||||
(map vector tokens (.tag tagger #^"[Ljava.lang.String;" token-array))))
|
|
||||||
(probs [_] (seq (.probs tagger)))
|
|
||||||
(top-k-sequences [_ tokens]
|
|
||||||
(let [token-array (into-array String tokens)]
|
|
||||||
(.topKSequences tagger #^"[Ljava.lang.String;" token-array))))))
|
|
||||||
|
|
||||||
(def prhyme-pos-tagger (make-pos-tagger (io/resource "models/en-pos-maxent.bin")))
|
|
||||||
|
|
||||||
(comment
|
|
||||||
(let [phrase "The feeling hurts."]
|
|
||||||
(map (juxt #(.getOutcomes %)
|
|
||||||
#(map float (.getProbs %)))
|
|
||||||
(top-k-sequences prhyme-pos-tagger (tokenize phrase))))
|
|
||||||
;; => ([["DT" "NN" "VBZ" "."] (0.9758878 0.93964833 0.7375927 0.95285994)]
|
|
||||||
;; [["DT" "VBG" "VBZ" "."] (0.9758878 0.03690145 0.27251 0.9286113)])
|
|
||||||
)
|
|
||||||
|
|
||||||
(defn valid-sentence?
|
|
||||||
"Tokenizes and parses the phrase using OpenNLP models from
|
|
||||||
http://opennlp.sourceforge.net/models-1.5/
|
|
||||||
|
|
||||||
If the parse tree has an clause as the top-level tag, then
|
|
||||||
we consider it a valid English sentence."
|
|
||||||
[phrase]
|
|
||||||
(->> phrase
|
|
||||||
tokenize
|
|
||||||
(string/join " ")
|
|
||||||
vector
|
|
||||||
parse
|
|
||||||
first
|
|
||||||
tb/make-tree
|
|
||||||
:chunk
|
|
||||||
first
|
|
||||||
:tag
|
|
||||||
tb2/clauses
|
|
||||||
boolean))
|
|
||||||
|
|
||||||
(defn unmake-tree
|
|
||||||
"Tokenizing and then parsing a sentence returns a string
|
|
||||||
representation of the parse tree. This is a helper function
|
|
||||||
to make working with the parse tree more convenient. We
|
|
||||||
can use `opennlp.treebank/make-tree` to make a clojure map
|
|
||||||
representation of the tree, then we can `unmake` the tree
|
|
||||||
to turn it into a list representation of the tree that
|
|
||||||
we can easily use in a clojure zipper. (read-string almost works,
|
|
||||||
but falls apart when reading things like commas)."
|
|
||||||
[node]
|
|
||||||
(cond
|
|
||||||
(string? node) node
|
|
||||||
(map? node) (list (:tag node) (unmake-tree (:chunk node)))
|
|
||||||
:else (map unmake-tree node)))
|
|
||||||
|
|
||||||
(comment
|
|
||||||
(let [phrase "Hello, Eric"]
|
|
||||||
(->> phrase
|
|
||||||
tokenize
|
|
||||||
(string/join " ")
|
|
||||||
vector
|
|
||||||
parse
|
|
||||||
(map tb/make-tree)
|
|
||||||
unmake-tree))
|
|
||||||
;; => ((TOP ((S ((INTJ ((UH ("Hello")))) (, (",")) (. ("Eric")))))))
|
|
||||||
)
|
|
||||||
|
|
||||||
(defn treebank-zipper
|
|
||||||
"Turns a bit of text into a parse tree into a zipper."
|
|
||||||
[text]
|
|
||||||
(let [tree (->> text
|
|
||||||
tokenize
|
|
||||||
(string/join " ")
|
|
||||||
vector
|
|
||||||
parse
|
|
||||||
(map tb/make-tree)
|
|
||||||
unmake-tree)]
|
|
||||||
(zip/zipper seq? seq (fn [_ c] c) tree)))
|
|
||||||
|
|
||||||
(defn node-constituents
|
|
||||||
"Given a node of a parse tree, like ('NP (('PRP$ (\"my\" 'NN (\"name\"))))),
|
|
||||||
returns a list of the top-level node tag and its first-level child tags.
|
|
||||||
"
|
|
||||||
[node]
|
|
||||||
(list
|
|
||||||
(first node)
|
|
||||||
(if (every? string? (map first (rest node)))
|
|
||||||
nil
|
|
||||||
(map first (first (rest node))))))
|
|
||||||
|
|
||||||
(defn phrase-constituents
|
|
||||||
"Given a bit of text that can be parsed into a treebank tree,
|
|
||||||
Get a sequence of the tags and their chunks.
|
|
||||||
For example:
|
|
||||||
My name is Eric.
|
|
||||||
Returns the sequence:
|
|
||||||
At the TOP tag, we have a 'S part-of-speech (a clause).
|
|
||||||
At the 'S tag, we have a 'NP, 'VP, '. (noun-phrase + verb-phrase + period)
|
|
||||||
At the 'NP tag, we have a 'PRP$, 'NN (personal-pronoun + singular-noun)
|
|
||||||
...
|
|
||||||
"
|
|
||||||
[text]
|
|
||||||
(->> (treebank-zipper text)
|
|
||||||
(iterate zip/next)
|
|
||||||
(take-while (complement zip/end?))
|
|
||||||
(filter (complement zip/branch?))
|
|
||||||
(map zip/path)
|
|
||||||
(map last)
|
|
||||||
(map node-constituents)
|
|
||||||
(remove #(string? (first %)))))
|
|
||||||
|
|
||||||
(comment
|
|
||||||
(phrase-constituents "My name is Eric.")
|
|
||||||
;; => ((TOP (S)) (S (NP VP .)) (NP (PRP$ NN)) (VP (VBZ NP)) (NP (NNP)))
|
|
||||||
(phrase-constituents "How are you?")
|
|
||||||
;; => ((TOP (SBARQ)) (SBARQ (WHADVP SQ .)) (WHADVP (WRB)) (SQ (VBP NP)) (NP (PRP)))
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
(defn pos-constituent-frequencies
|
|
||||||
"Frequencies of the parts of speech that make up phrases.
|
|
||||||
Example:
|
|
||||||
Clauses are made up of:
|
|
||||||
NounPhrase + VerbPhrase 2 times
|
|
||||||
Clause + CoordinatingConjuction + Clause 1 times
|
|
||||||
NounPhrases are made up of:
|
|
||||||
ProperNouns 2 times
|
|
||||||
PersonalPronoun + SingularNoun 3 times
|
|
||||||
|
|
||||||
Does not include frequencies for leaf words. By that I mean: A SingularNoun might
|
|
||||||
appear 5 times all together, 3 times as part of a PersonalPronoun + SingularNoun pair
|
|
||||||
and 2 times as part of an Adjective + SingularNoun pair, but the data structure returned
|
|
||||||
by this function won't include that 5 anywhere. This is due to the (remove #(nil? (second %)))
|
|
||||||
line. This data structure is used as a kind of markov selection process and we don't really
|
|
||||||
care how often the leafs are used. We just care about the ratio at which we should pick each
|
|
||||||
leaf from a given parent.
|
|
||||||
"
|
|
||||||
[texts]
|
|
||||||
(reduce
|
|
||||||
(fn [acc text]
|
|
||||||
(let [constituents (->> text
|
|
||||||
phrase-constituents
|
|
||||||
(remove #(nil? (second %))))]
|
|
||||||
(reduce
|
|
||||||
(fn [acc constituent]
|
|
||||||
(let [k1 (first constituent)
|
|
||||||
k2 (second constituent)]
|
|
||||||
(update-in acc [k1 k2] (fnil inc 0))))
|
|
||||||
acc
|
|
||||||
constituents)))
|
|
||||||
{}
|
|
||||||
texts))
|
|
||||||
|
|
||||||
(comment
|
|
||||||
(pos-constituent-frequencies
|
|
||||||
["My name is Eric."
|
|
||||||
"My hat is blue and I like cake."
|
|
||||||
"Your name is Taylor."
|
|
||||||
"How are you?"])
|
|
||||||
;; => {TOP {(S) 3, (SBARQ) 1},
|
|
||||||
;; S {(NP VP .) 2, (S CC S .) 1, (NP VP) 2},
|
|
||||||
;; NP {(PRP$ NN) 3, (NNP) 2, (PRP) 2, (NN) 1},
|
|
||||||
;; VP {(VBZ NP) 2, (VBZ ADJP) 1, (VBP NP) 1},
|
|
||||||
;; ADJP {(JJ) 1},
|
|
||||||
;; SBARQ {(WHADVP SQ .) 1},
|
|
||||||
;; WHADVP {(WRB) 1},
|
|
||||||
;; SQ {(VBP NP) 1}}
|
|
||||||
|
|
||||||
(let [phrase "How are you today?"]
|
|
||||||
(->> phrase
|
|
||||||
tokenize
|
|
||||||
(string/join " ")
|
|
||||||
vector
|
|
||||||
parse
|
|
||||||
(map tb/make-tree)))
|
|
||||||
|
|
||||||
(let [phrase "I gave the cake to John at the store."]
|
|
||||||
(parse (tokenize phrase)))
|
|
||||||
|
|
||||||
(let [phrase "I've got a good feeling"]
|
|
||||||
(pos-tagger (tokenize phrase)))
|
|
||||||
)
|
|
||||||
Loading…
Reference in New Issue