diff --git a/arxiv_daemon.py b/arxiv_daemon.py index de71089..53e93a3 100644 --- a/arxiv_daemon.py +++ b/arxiv_daemon.py @@ -18,10 +18,16 @@ if __name__ == '__main__': logging.basicConfig(level=logging.INFO, format='%(name)s %(levelname)s %(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') parser = argparse.ArgumentParser(description='Arxiv Daemon') - parser.add_argument('-n', '--num', type=int, default=100, help='how many papers to fetch') + parser.add_argument('-n', '--num', type=int, default=100, help='up to how many papers to fetch') parser.add_argument('-s', '--start', type=int, default=0, help='start at what index') + parser.add_argument('-b', '--break-after', type=int, default=3, help='how many 0 new papers in a row would cause us to stop early? or 0 to disable.') args = parser.parse_args() print(args) + """ + Quick note on the break_after argument: In a typical setting where one wants to update + the papers database you'd choose a slightly higher num, but then break out early in case + we've reached older papers that are already part of the database, to spare the arxiv API. + """ # query string of papers to look for q = 'cat:cs.CV+OR+cat:cs.LG+OR+cat:cs.CL+OR+cat:cs.AI+OR+cat:cs.NE+OR+cat:cs.RO' @@ -35,6 +41,7 @@ if __name__ == '__main__': mdb[p['_id']] = {'_time': p['_time']} # fetch the latest papers + zero_updates_in_a_row = 0 for k in range(args.start, args.start + args.num, 100): logging.info('querying arxiv api for query %s at start_index %d' % (q, k)) @@ -79,5 +86,14 @@ if __name__ == '__main__': logging.info("k=%d, out of %d: had %d, replaced %d, new %d. now have: %d" % (k, len(papers), nhad, nreplace, nnew, prevn)) + # early termination criteria + if nnew == 0: + zero_updates_in_a_row += 1 + if args.break_after > 0 and zero_updates_in_a_row >= args.break_after: + logging.info("breaking out early, no new papers %d times in a row" % (args.break_after, )) + break + else: + zero_updates_in_a_row = 0 + # zzz time.sleep(1 + random.uniform(0, 3))