From 33b2b018abc1adf8019991f230790a3781d1fc43 Mon Sep 17 00:00:00 2001 From: Andrej Karpathy Date: Wed, 8 Dec 2021 00:17:37 -0800 Subject: [PATCH] kind of big changes here: we can now inspect & see the most positive and negative words for a trained svm, to help tune the value C. then there is also the UI for setting value C in the SVM. Finally the value of C I adjusted to default to 0.01 (was 0.1 before) because the results and the weights look more sensible based on manual inspection. We need some dataset of people libraries in order to potentially cross-validate a good value C automatically. For now there are not enough active users of the site that such a thing could be attempted and succeed. Doing my best now just by eyeballing --- serve.py | 40 ++++++++++++++++++++++++++++------------ static/paper_detail.js | 27 --------------------------- static/paper_list.js | 4 ++++ static/style.css | 22 ++++++++++++++++++++++ static/word_list.js | 30 ++++++++++++++++++++++++++++++ templates/index.html | 14 +++++++++++++- templates/inspect.html | 2 ++ 7 files changed, 99 insertions(+), 40 deletions(-) create mode 100644 static/word_list.js diff --git a/serve.py b/serve.py index 5ec22df..4a9fca9 100644 --- a/serve.py +++ b/serve.py @@ -114,12 +114,12 @@ def time_rank(): scores = [(tnow - v['_time'])/60/60/24 for k, v in ms] # time delta in days return pids, scores -def svm_rank(tags: str = '', pid: str = '', svm_c: str = ''): +def svm_rank(tags: str = '', pid: str = '', C: float = 0.01): # tag can be one tag or a few comma-separated tags or 'all' for all tags we have in db # pid can be a specific paper id to set as positive for a kind of nearest neighbor search if not (tags or pid): - return [], [] + return [], [], [] # load all of the features features = load_features() @@ -143,15 +143,9 @@ def svm_rank(tags: str = '', pid: str = '', svm_c: str = ''): y[ptoi[pid]] = 1.0 if y.sum() == 0: - return [], [] # there are no positives? + return [], [], [] # there are no positives? # classify - C = 0.1 - if svm_c: # if a desired C is provided attempt to use it as a float - try: - C = float(svm_c) - except ValueError: - C = 1.0 clf = svm.LinearSVC(class_weight='balanced', verbose=False, max_iter=10000, tol=1e-6, C=C) clf.fit(x, y) s = clf.decision_function(x) @@ -159,7 +153,18 @@ def svm_rank(tags: str = '', pid: str = '', svm_c: str = ''): pids = [itop[ix] for ix in sortix] scores = [100*float(s[ix]) for ix in sortix] - return pids, scores + # get the words that score most positively and most negatively for the svm + ivocab = {v:k for k,v in features['vocab'].items()} # index to word mapping + weights = clf.coef_[0] # (n_features,) weights of the trained svm + sortix = np.argsort(-weights) + words = [] + for ix in list(sortix[:40]) + list(sortix[-20:]): + words.append({ + 'word': ivocab[ix], + 'weight': weights[ix], + }) + + return pids, scores, words def search_rank(q: str = ''): if not q: @@ -209,13 +214,20 @@ def main(): if opt_q: opt_rank = 'search' + # try to parse opt_svm_c into something sensible (a float) + try: + C = float(opt_svm_c) + except ValueError: + C = 0.01 # sensible default, i think + # rank papers: by tags, by time, by random + words = [] # only populated in the case of svm rank if opt_rank == 'search': pids, scores = search_rank(q=opt_q) elif opt_rank == 'tags': - pids, scores = svm_rank(tags=opt_tags, svm_c=opt_svm_c) + pids, scores, words = svm_rank(tags=opt_tags, C=C) elif opt_rank == 'pid': - pids, scores = svm_rank(pid=opt_pid, svm_c=opt_svm_c) + pids, scores, words = svm_rank(pid=opt_pid, C=C) elif opt_rank == 'time': pids, scores = time_rank() elif opt_rank == 'random': @@ -257,6 +269,8 @@ def main(): context = default_context() context['papers'] = papers context['tags'] = rtags + context['words'] = words + context['words_desc'] = "Here are the top 40 most positive and bottom 20 most negative weights of the SVM. If they don't look great then try tuning the regularization strength hyperparameter of the SVM, svm_c, above. Lower C is higher regularization." context['gvars'] = {} context['gvars']['rank'] = opt_rank context['gvars']['tags'] = opt_tags @@ -264,6 +278,7 @@ def main(): context['gvars']['time_filter'] = opt_time_filter context['gvars']['skip_have'] = opt_skip_have context['gvars']['search_query'] = opt_q + context['gvars']['svm_c'] = str(C) return render_template('index.html', **context) @app.route('/inspect', methods=['GET']) @@ -296,6 +311,7 @@ def inspect(): context = default_context() context['paper'] = paper context['words'] = words + context['words_desc'] = "The following are the tokens and their (tfidf) weight in the paper vector. This is the actual summary that feeds into the SVM to power recommendations, so hopefully it is good and representative!" return render_template('inspect.html', **context) @app.route('/profile') diff --git a/static/paper_detail.js b/static/paper_detail.js index 306c627..44b0457 100644 --- a/static/paper_detail.js +++ b/static/paper_detail.js @@ -13,31 +13,4 @@ const PaperLite = props => { ) } - -const Word = props => { - const p = props.word; - // word, weight, idf - return ( -
-
{p.weight.toFixed(2)}
- {/*
{p.idf.toFixed(2)}
*/} -
{p.word}
-
- ) -} - -const WordList = props => { - const lst = props.words; - const wlst = lst.map((jword, ix) => ); - return ( -
-
The following are the tokens and their (tfidf) weight in the paper vector. This is the actual summary that feeds into the SVM to power recommendations, so hopefully it is good and representative!
-
- {wlst} -
-
- ) -} - ReactDOM.render(, document.getElementById('wrap')) -ReactDOM.render(, document.getElementById('wordwrap')) diff --git a/static/paper_list.js b/static/paper_list.js index 61c8504..7434911 100644 --- a/static/paper_list.js +++ b/static/paper_list.js @@ -80,12 +80,16 @@ const TagList = props => { const tlst = lst.map((jtag, ix) => ); const deleter = () => fetch("/del/" + prompt("delete tag name:")) .then(response => console.log(response.text())); + // show the #wordwrap element if the user clicks inspect + const show_inspect = () => { document.getElementById("wordwrap").style.display = "block"; }; + const inspect_elt = words.length > 0 ?
inspect
: null; return (
-
{tlst}
+ {inspect_elt}
) } diff --git a/static/style.css b/static/style.css index e4cc133..dac83a2 100644 --- a/static/style.css +++ b/static/style.css @@ -253,3 +253,25 @@ body { margin: 10px 40px 0 40px; font-size: 16px; } +#tags_field { + width: 100px; +} +#pid_field { + width: 60px; +} +#time_filter_field { + width: 20px; +} +#svm_c_field { + width: 30px; +} +#tagList { + display: inline-block; +} +#inspect_svm { + display: inline-block; + font-size: 16px; + cursor: pointer; + text-decoration: underline; + color: #009; +} \ No newline at end of file diff --git a/static/word_list.js b/static/word_list.js new file mode 100644 index 0000000..9ba9531 --- /dev/null +++ b/static/word_list.js @@ -0,0 +1,30 @@ +'use strict'; + +const Word = props => { + const p = props.word; + // word, weight, idf + return ( +
+
{p.weight.toFixed(2)}
+ {/*
{p.idf.toFixed(2)}
*/} +
{p.word}
+
+ ) +} + +const WordList = props => { + const lst = props.words; + const words_desc = props.words_desc; + const wlst = lst.map((jword, ix) => ); + return ( +
+
{words_desc}
+
+ {wlst} +
+
+ ) +} + +ReactDOM.render(, + document.getElementById('wordwrap')); diff --git a/templates/index.html b/templates/index.html index 0f4058c..1d726ba 100644 --- a/templates/index.html +++ b/templates/index.html @@ -4,6 +4,8 @@ {% endblock %} @@ -43,9 +45,13 @@ var gvars = {{ gvars | tojson }}; - + + + + +