This commit is contained in:
Stefan Rupp 2018-04-01 18:37:01 +02:00
commit f45afb3035
3 changed files with 102 additions and 18 deletions

View File

@ -1,18 +1,25 @@
#!/usr/bin/python3 #!/usr/bin/python3
#/* vim:set ts=2 set noexpandtab */ #/* vim:set ts=2 set noexpandtab */
import json, uuid, hashlib, sqlite3, base64
from hashlib import sha1
import bencoder
import requests
from flask import Flask, render_template, url_for, request, send_file, redirect from flask import Flask, render_template, url_for, request, send_file, redirect
from flask_babel import Babel, gettext as _, lazy_gettext from flask_babel import Babel, gettext as _, lazy_gettext
from werkzeug import secure_filename from werkzeug import secure_filename
from hurry.filesize import size from hurry.filesize import size
from hashlib import sha1
import threading
import binascii
import bencoder
import requests
import hashlib
import sqlite3
import base64
import urllib
import json
import uuid
import time
app = Flask(__name__) app = Flask(__name__)
babel = Babel(app) babel = Babel(app)
LANGUAGES = ['en', 'de'] LANGUAGES = ['en', 'de']
settings = None settings = None
@ -38,14 +45,74 @@ class Categories():
return (cat_name, sub_name) return (cat_name, sub_name)
class ScrapeState():
stats = {}
def __init__(self):
pass
def update(self):
self._statedump()
self._tpbs()
self._fullscrape()
def _statedump(self):
url = settings["scrape_url"]
statedump_url = url + "/stats"
params = { "mode" : "statedump" }
req = requests.get(statedump_url, params=params)
dump = req.text.strip()
dump = dump.split("\n")
for entry in dump:
entry = entry.split(":")
key = entry[0].lower()
if not key in self.stats.keys():
self.stats.update({ key : {}})
self.stats.get(key).update({ "base" : entry[1], "unsure_downloaded" : entry[2] })
def _tpbs(self):
url = settings["scrape_url"]
tpbs_url = url + "/stats"
params = { "mode" : "tpbs", "format" : "ben" }
req = requests.get(tpbs_url, params=params)
decoded = bencoder.decode(req.content)
for torrent in decoded[b"files"]:
info_hash = binascii.b2a_hex(torrent)
stats = decoded[b"files"][torrent]
key = info_hash.decode("utf-8").lower()
self.stats.get(key).update({ "seeds" : stats[b"complete"], "peers" : stats[b"incomplete"], "complete" : stats[b"downloaded"] })
def _fullscrape(self):
connection = sqlite3.connect("torrentdb.sqlite")
c = connection.cursor()
c.execute("SELECT fileid FROM torrents")
all_hashes = c.fetchall()
for info_hash in all_hashes:
info_hash = info_hash[0]
url_param = binascii.a2b_hex(info_hash.encode())
url = settings["scrape_url"]
req = requests.get(url + "/scrape", params={"info_hash" : url_param})
decoded = bencoder.decode(req.content)
info = decoded[b"files"]
try:
ugly_hash, stats = info.popitem()
key = info_hash.lower()
self.stats.get(key).update({ "seeds" : stats[b"complete"], "peers" : stats[b"incomplete"], "complete" : stats[b"downloaded"] })
except KeyError:
print("No stats found for {}".format(info_hash))
@app.route("/") @app.route("/")
def index(): def index():
return render_template("search.html", categories=categories.categories) return render_template("search.html", categories=categories.categories)
@app.route("/categories") @app.route("/categories")
def categorys(): def categorys():
return render_template("categories.html", categories=categories.categories) return render_template("categories.html", categories=categories.categories)
@app.route("/create", methods=['GET','POST']) @app.route("/create", methods=['GET','POST'])
def create(): def create():
if request.method == "GET": if request.method == "GET":
@ -58,6 +125,7 @@ def create():
else: else:
return render_template("create.html", categories=categories.categories, errors=newTorrent.errors) return render_template("create.html", categories=categories.categories, errors=newTorrent.errors)
@app.route("/download/<filename>") @app.route("/download/<filename>")
def download(filename): def download(filename):
connection = sqlite3.connect("torrentdb.sqlite") connection = sqlite3.connect("torrentdb.sqlite")
@ -66,6 +134,7 @@ def download(filename):
name = c.fetchone()[0] name = c.fetchone()[0]
return send_file("torrentFiles/" + filename, as_attachment=True, attachment_filename=name + ".torrent", conditional=True) return send_file("torrentFiles/" + filename, as_attachment=True, attachment_filename=name + ".torrent", conditional=True)
@app.route("/search", methods=['GET']) @app.route("/search", methods=['GET'])
def search(): def search():
connection = sqlite3.connect("torrentdb.sqlite") connection = sqlite3.connect("torrentdb.sqlite")
@ -101,7 +170,8 @@ def search():
r = row[0:2] + (size(float(row[2])) , ) + row[3:] r = row[0:2] + (size(float(row[2])) , ) + row[3:]
results.append(r) results.append(r)
return render_template("result.html", results=results, categories=categories.categories) return render_template("result.html", results=results, categories=categories.categories, stats=scrapeState.stats)
@app.route("/details", methods=['GET']) @app.route("/details", methods=['GET'])
def details(): def details():
@ -110,11 +180,6 @@ def details():
tf.fromDb() tf.fromDb()
return render_template("details.html", categories=categories.categories, torrent=tf) return render_template("details.html", categories=categories.categories, torrent=tf)
def scrapeAll():
TRACKER_URL = ""
statedump = requests.get(TRACKER_URL + "stats" + "?mode=statedump")
return
def init(): def init():
global settings global settings
@ -123,7 +188,19 @@ def init():
initDb() initDb()
global categories global categories
categories = Categories() categories = Categories()
#scrapeAll() global scrapeState
scrapeState = ScrapeState()
scrape = threading.Thread(target=scraper)
scrape.start()
def scraper():
while True:
print("Start scraping")
scrapeState.update()
print("Scraping done")
time.sleep(60)
def initDb(): def initDb():
connection = sqlite3.connect("torrentdb.sqlite") connection = sqlite3.connect("torrentdb.sqlite")
@ -133,6 +210,7 @@ def initDb():
connection.commit() connection.commit()
connection.close() connection.close()
def createNewTorrent(reuqest): def createNewTorrent(reuqest):
uploadfile = request.files["torrentFile"] uploadfile = request.files["torrentFile"]
filename = secure_filename(uploadfile.filename) filename = secure_filename(uploadfile.filename)
@ -204,6 +282,7 @@ def createNewTorrent(reuqest):
return newTFile return newTFile
class Metadata(): class Metadata():
def __init__(self, fileid): def __init__(self, fileid):
try: try:
@ -214,7 +293,7 @@ class Metadata():
self.fileid = fileid self.fileid = fileid
self.bcoded = bencoder.decode(torrent) self.bcoded = bencoder.decode(torrent)
self.created_by = self.bcoded.get(b'created by', b"").decode("utf-8", "ignore") self.created_by = self.bcoded.get(b'created by', b"").decode("utf-8", "ignore")
self.creation_date = self.bcoded.get(b'creation date', b"").decode("utf-8", "ignore") self.creation_date = self.bcoded.get(b'creation date', 0)
self.announce_url = self.bcoded.get(b'info', dict()).get(b'', "") self.announce_url = self.bcoded.get(b'info', dict()).get(b'', "")
self.source = self.bcoded.get(b'info', dict()).get(b'source', b"") self.source = self.bcoded.get(b'info', dict()).get(b'source', b"")
self.torrentsize = ((len(self.bcoded.get(b'info', dict()).get(b'pieces', "")) / 20) * self.bcoded.get(b'info', dict()).get(b'piece length')) self.torrentsize = ((len(self.bcoded.get(b'info', dict()).get(b'pieces', "")) / 20) * self.bcoded.get(b'info', dict()).get(b'piece length'))
@ -230,6 +309,7 @@ class Metadata():
b64name = base64.b64encode(self.name) b64name = base64.b64encode(self.name)
c.execute("INSERT INTO metadata(fileid, created_by, creation_date, announce_url, source, torrentsize, name, private) VALUES(:fileid, :created_by, :creation_date, :announce_url, :source, :torrentsize, :name, :private)", { 'fileid' : self.fileid, 'created_by' : b64created_by, 'creation_date' : self.creation_date, 'announce_url' : b64announce_url, 'source' : b64source , 'torrentsize' : self.torrentsize, 'name' : b64name, 'private' : self.private}) c.execute("INSERT INTO metadata(fileid, created_by, creation_date, announce_url, source, torrentsize, name, private) VALUES(:fileid, :created_by, :creation_date, :announce_url, :source, :torrentsize, :name, :private)", { 'fileid' : self.fileid, 'created_by' : b64created_by, 'creation_date' : self.creation_date, 'announce_url' : b64announce_url, 'source' : b64source , 'torrentsize' : self.torrentsize, 'name' : b64name, 'private' : self.private})
class TorrentFile(): class TorrentFile():
errors = [] errors = []
fileid = None fileid = None
@ -280,10 +360,12 @@ class TorrentFile():
self.videoquality_description = (base64.b64decode(res["videoquality_description"])).decode() self.videoquality_description = (base64.b64decode(res["videoquality_description"])).decode()
self.metadata = Metadata(self.fileid) self.metadata = Metadata(self.fileid)
@babel.localeselector @babel.localeselector
def get_locale(): def get_locale():
return request.accept_languages.best_match(LANGUAGES) return request.accept_languages.best_match(LANGUAGES)
if __name__ == "__main__": if __name__ == "__main__":
init() init()
app.jinja_env.globals.update(json=json) app.jinja_env.globals.update(json=json)

View File

@ -189,6 +189,8 @@
"valid_tracker" : [ "valid_tracker" : [
"udp://tracker.lootbox.cf:6969/announce", "udp://tracker.lootbox.cf:6969/announce",
"udp://tracker.lootbox.cf:6969/announce/" "udp://tracker.lootbox.cf:6969/announce/"
] ],
"scrape_url" : "http://tracker.lootbox.gq:6969"
} }

View File

@ -19,9 +19,9 @@ vim: ts=2 noexpandtab
<tr> <tr>
<td><a href="/details?h={{ result[0] }}">{{ result[1] }}</a></td> <td><a href="/details?h={{ result[0] }}">{{ result[1] }}</a></td>
<td>{{ result[2] }}</td> <td>{{ result[2] }}</td>
<td>{{ _("N/A") }}</td> <td>{{ stats.get(result[0], {}).get("complete", _("N/A")) }}</td>
<td>{{ _("N/A") }}</td> <td>{{ stats.get(result[0], {}).get("seeds", _("N/A")) }}</td>
<td>{{ _("N/A") }}</td> <td>{{ stats.get(result[0], {}).get("peers", _("N/A")) }}</td>
</tr> </tr>
{% endfor %} {% endfor %}
</table> </table>