nilesrogoff
/
lain-ipfs-scraper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added status

This commit is contained in:
Niles Rogoff 2016-07-29 19:28:39 -07:00
parent b86682931f
commit b2b590e4c9
3 changed files with 44 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
index.pyhtml
View File

@ -15,9 +15,21 @@
<script type="text/javascript" src="https://lainchan.org/js/jquery.min.js"></script>
<script type="text/javascript" src="https://lainchan.org/js/prettify.js"></script>
<script type="text/javascript" src="https://lainchan.org/js/hilight.js"></script>
<script>
$( document ).ready(function() {
var s = document.getElementsByClassName("date");
for (var i = 0; i < s.length; i++) {
var el = s[i];
d = new Date(parseInt(el.innerHTML + "000")); // uses milliseconds, hence the extra 000
/*el.innerHTML = d.toLocaleDateString() + " " + d.toLocaleTimeString();*/
el.innerHTML = d.toLocaleTimeString() + " " + d.toLocaleDateString();
}
});
</script>
</head>
<body>
<h1>IPFS Scraper</h1>
<div id="status">${open("/dev/shm/lainchan-scraper-status", "r").read()}</div>
<div id="content">
% for post in dumped[::-1]:
<div class="post">
@ -25,11 +37,15 @@
<a class="ipfslink-public" href="https://ipfs.io/ipfs/${post["match"]}">public</a>
<a class="ipfslink-local" href="http://localhost:8080/ipfs/${post["match"]}">local</a> -
${post["match"]}
<span class="date">${time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(post["time"]))}</span>
<span class="date">${post["time"]}</span>
<div class="embedded">${post["text"].replace(u"\u03bb", "lam").replace('href="/' + post["board"], 'href="https://lainchan.org/' + post["board"])}</div>
</div>
% endfor
<br />
Note that only /tech/ and /&lambda;/ are scraped at this time.<br />
The source can be found on <a href="https://gitla.in/nilesrogoff/lain-ipfs-scraper/">gitla.in/nilesrogoff/lain-ipfs-scraper</a><br />
Hypothetically it should be compatable with any vichan compatable bbs.
</div>
<div id="footer">Niles Rogoff 2016</div>
<div id="footer">Niles Rogoff 2016<br /><a href="https://pgp.mit.edu/pks/lookup?op=vindex&search=0xB78B908F23430F80">2048R/23430F80</a></div>
</body>
</html>

11
scraper.py
View File

@ -1,6 +1,9 @@
import urllib.request, json, BTEdb, re, time
db = BTEdb.Database("/dev/shm/lainchan-scraper.json")
db.BeginTransaction(False)
status = open("/dev/shm/lainchan-scraper-status", "w")
status.write("Update in progress")
status.close()
boards = ['lam', "tech"]
regex = re.compile(r"\b[A-Za-z0-9]{46}\b")
if db.TableExists("table"):
@ -8,6 +11,10 @@ if db.TableExists("table"):
else:
db.CreateTable("table")
for board in boards:
# DEBUG
# time.sleep(5)
# continue
# END DEBUG
threads = json.loads(urllib.request.urlopen("https://lainchan.org/"+board+"/threads.json").read().decode("utf-8"))
for page in threads:
for thread in page["threads"]:
@ -27,3 +34,7 @@ for board in boards:
i+= 1
db.CommitTransaction()
db.Destroy()
import time
status = open("/dev/shm/lainchan-scraper-status", "w")
status.write("Last scrape at <span class=\"date\">" + str(int(time.time())) + "</span><br /> Next scrape at <span class=\"date\">" + str(3600 - (int(time.time()) % 3600) + int(time.time())) + "</span>")
status.close()

15
style.css
View File

@ -43,3 +43,18 @@ a:hover {
padding: 2px;
z-index: 100;
}
#status {
position: fixed;
padding: 5px;
top: 24px;
right: 1%;
width: 400px;
border-radius: 6px;
border: 2px dotted black;
}
#footer a, #footer a:hover, #footer a:visited, #footer a:active {
color: inherit;
text-decoration: none;
cursor: text;
}