diff --git a/serve.py b/serve.py
index c3373e4..6aecb8e 100644
--- a/serve.py
+++ b/serve.py
@@ -87,6 +87,8 @@ def render_pid(pid):
# render a single paper with just the information we need for the UI
pdb = get_papers()
tags = get_tags()
+ thumb_path = 'static/thumb/' + pid + '.jpg'
+ thumb_url = thumb_path if os.path.isfile(thumb_path) else ''
d = pdb[pid]
return dict(
weight = 0.0,
@@ -97,6 +99,7 @@ def render_pid(pid):
tags = ', '.join(t['term'] for t in d['tags']),
utags = [t for t, pids in tags.items() if pid in pids],
summary = d['summary'],
+ thumb_url = thumb_url,
)
def random_rank():
diff --git a/static/paper_list.js b/static/paper_list.js
index 7434911..e325ffa 100644
--- a/static/paper_list.js
+++ b/static/paper_list.js
@@ -22,7 +22,7 @@ const Paper = props => {
const utags = p.utags.map((utxt, ix) => );
const similar_url = "/?rank=pid&pid=" + p.id;
const inspect_url = "/inspect?pid=" + p.id;
-
+ const thumb_img = p.thumb_url === '' ? null :
;
// if the user is logged in then we can show add/sub buttons
let utag_controls = null;
if(user) {
@@ -43,6 +43,7 @@ const Paper = props => {
{p.time}
{p.tags}
{utag_controls}
+ {thumb_img}
{p.summary}
diff --git a/static/style.css b/static/style.css
index 3ac3ff6..29efaa3 100644
--- a/static/style.css
+++ b/static/style.css
@@ -3,6 +3,7 @@ body {
padding: 0;
font-family: sans-serif;
line-height: 1.2;
+ background-color: #eee;
}
#header {
height: 24px;
@@ -50,7 +51,7 @@ body {
.rel_paper {
margin-bottom: 10px;
padding: 10px;
- background-color: #eee;
+ background-color: white;
border-radius: 5px;
}
diff --git a/thumb_daemon.py b/thumb_daemon.py
new file mode 100644
index 0000000..2a7b010
--- /dev/null
+++ b/thumb_daemon.py
@@ -0,0 +1,104 @@
+"""
+Iterates over the current database and makes best effort to download the papers,
+convert them to thumbnail images and save them to disk, for display in the UI.
+Atm only runs the most recent 5K papers. Intended to be run as a cron job daily
+or something like that.
+"""
+
+import os
+import time
+import random
+import requests
+from subprocess import Popen
+from aslite.db import get_papers_db, get_metas_db
+
+# create the tmp directory if it does not exist, where we will do temporary work
+TMP_DIR = 'tmp'
+if not os.path.exists(TMP_DIR):
+ os.makedirs(TMP_DIR)
+# create the thumb directory, where we will store the paper thumbnails
+THUMB_DIR = os.path.join('static', 'thumb')
+if not os.path.exists(THUMB_DIR):
+ os.makedirs(THUMB_DIR)
+
+# open the database, determine which papers we'll try to get thumbs for
+pdb = get_papers_db()
+n = len(pdb)
+mdb = get_metas_db()
+metas = list(mdb.items())
+metas.sort(key=lambda kv: kv[1]['_time'], reverse=True) # most recent papers first
+keys = [k for k,v in metas[:5000]] # only the most recent papers
+
+for i, key in enumerate(keys):
+ time.sleep(0.01) # for safety
+
+ # the path where we would store the thumbnail for this key
+ thumb_path = os.path.join(THUMB_DIR, key + '.jpg')
+ if os.path.exists(thumb_path):
+ continue
+
+ # fetch the paper
+ p = pdb[key]
+ print("%d/%d: paper to process: %s" % (i, n, key))
+
+ # get the link to the pdf
+ url = p['link'].replace('abs', 'pdf')
+
+ # attempt to download the pdf
+ print("attempting to download pdf from: ", url)
+ try:
+ x = requests.get(url, timeout=10, allow_redirects=True)
+ with open(os.path.join(TMP_DIR, 'paper.pdf'), 'wb') as f:
+ f.write(x.content)
+ print("OK")
+ except Exception as e:
+ print("error downloading the pdf at url", url)
+ print(e)
+ continue
+ time.sleep(5 + random.uniform(0, 5)) # take a breather
+
+ # mv away the previous temporary files if they exist
+ if os.path.isfile(os.path.join(TMP_DIR, 'thumb-0.png')):
+ for i in range(8):
+ f1 = os.path.join(TMP_DIR, 'thumb-%d.png' % (i,))
+ f2 = os.path.join(TMP_DIR, 'thumbbuf-%d.png' % (i,))
+ if os.path.isfile(f1):
+ cmd = 'mv %s %s' % (f1, f2)
+ os.system(cmd)
+
+ # convert pdf to png images per page. spawn async because convert can unfortunately enter an infinite loop, have to handle this.
+ # this command will generate 8 independent images thumb-0.png ... thumb-7.png of the thumbnails
+ print("converting the pdf to png images")
+ pp = Popen(['convert', '%s[0-7]' % ('tmp/paper.pdf', ), '-thumbnail', 'x156', os.path.join(TMP_DIR, 'thumb.png')])
+ t0 = time.time()
+ while time.time() - t0 < 20: # give it 20 seconds deadline
+ ret = pp.poll()
+ if not (ret is None):
+ # process terminated
+ break
+ time.sleep(0.1)
+ ret = pp.poll()
+ if ret is None:
+ print("convert command did not terminate in 20 seconds, terminating.")
+ pp.terminate() # give up
+ continue
+
+ if not os.path.isfile(os.path.join(TMP_DIR, 'thumb-0.png')):
+ # failed to render pdf, replace with missing image
+ #missing_thumb_path = os.path.join('static', 'missing.jpg')
+ #os.system('cp %s %s' % (missing_thumb_path, thumb_path))
+ #print("could not render pdf, creating a missing image placeholder")
+ print("could not render pdf, skipping")
+ continue
+ else:
+ # otherwise concatenate the 8 images into one
+ cmd = "montage -mode concatenate -quality 80 -tile x1 %s %s" \
+ % (os.path.join(TMP_DIR, 'thumb-*.png'), thumb_path)
+ print(cmd)
+ os.system(cmd)
+
+ # remove the temporary paper.pdf file
+ tmp_pdf = os.path.join(TMP_DIR, 'paper.pdf')
+ if os.path.isfile(tmp_pdf):
+ os.remove(tmp_pdf)
+