From b2b590e4c9010ba307fa1f350fedd18b0068a7ee Mon Sep 17 00:00:00 2001 From: Niles Rogoff Date: Fri, 29 Jul 2016 19:28:39 -0700 Subject: [PATCH] Added status --- index.pyhtml | 20 ++++++++++++++++++-- scraper.py | 11 +++++++++++ style.css | 15 +++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/index.pyhtml b/index.pyhtml index 1a82799..5908839 100644 --- a/index.pyhtml +++ b/index.pyhtml @@ -15,9 +15,21 @@ +

IPFS Scraper

+
${open("/dev/shm/lainchan-scraper-status", "r").read()}
% for post in dumped[::-1]:
@@ -25,11 +37,15 @@ public local - ${post["match"]} - ${time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(post["time"]))} + ${post["time"]}
${post["text"].replace(u"\u03bb", "lam").replace('href="/' + post["board"], 'href="https://lainchan.org/' + post["board"])}
% endfor +
+ Note that only /tech/ and /λ/ are scraped at this time.
+ The source can be found on gitla.in/nilesrogoff/lain-ipfs-scraper
+ Hypothetically it should be compatable with any vichan compatable bbs.
- + diff --git a/scraper.py b/scraper.py index baf1d84..87a5b81 100644 --- a/scraper.py +++ b/scraper.py @@ -1,6 +1,9 @@ import urllib.request, json, BTEdb, re, time db = BTEdb.Database("/dev/shm/lainchan-scraper.json") db.BeginTransaction(False) +status = open("/dev/shm/lainchan-scraper-status", "w") +status.write("Update in progress") +status.close() boards = ['lam', "tech"] regex = re.compile(r"\b[A-Za-z0-9]{46}\b") if db.TableExists("table"): @@ -8,6 +11,10 @@ if db.TableExists("table"): else: db.CreateTable("table") for board in boards: + # DEBUG + # time.sleep(5) + # continue + # END DEBUG threads = json.loads(urllib.request.urlopen("https://lainchan.org/"+board+"/threads.json").read().decode("utf-8")) for page in threads: for thread in page["threads"]: @@ -27,3 +34,7 @@ for board in boards: i+= 1 db.CommitTransaction() db.Destroy() +import time +status = open("/dev/shm/lainchan-scraper-status", "w") +status.write("Last scrape at " + str(int(time.time())) + "
Next scrape at " + str(3600 - (int(time.time()) % 3600) + int(time.time())) + "") +status.close() diff --git a/style.css b/style.css index fa41db3..20a32f4 100644 --- a/style.css +++ b/style.css @@ -43,3 +43,18 @@ a:hover { padding: 2px; z-index: 100; } +#status { + position: fixed; + padding: 5px; + top: 24px; + right: 1%; + width: 400px; + border-radius: 6px; + border: 2px dotted black; +} +#footer a, #footer a:hover, #footer a:visited, #footer a:active { + color: inherit; + text-decoration: none; + cursor: text; +} +