From 0e417ee30a3d060b905ed568829878422b6b1092 Mon Sep 17 00:00:00 2001 From: sqozz Date: Sun, 1 Apr 2018 18:14:22 +0200 Subject: [PATCH] Add torrent statistics --- indexer.py | 110 ++++++++++++++++++++++++++++++++++++------ settings.json | 4 +- templates/result.html | 6 +-- 3 files changed, 102 insertions(+), 18 deletions(-) diff --git a/indexer.py b/indexer.py index 066113c..84d7f92 100644 --- a/indexer.py +++ b/indexer.py @@ -1,18 +1,25 @@ #!/usr/bin/python3 #/* vim:set ts=2 set noexpandtab */ -import json, uuid, hashlib, sqlite3, base64 -from hashlib import sha1 -import bencoder -import requests from flask import Flask, render_template, url_for, request, send_file, redirect from flask_babel import Babel, gettext as _, lazy_gettext from werkzeug import secure_filename from hurry.filesize import size +from hashlib import sha1 +import threading +import binascii +import bencoder +import requests +import hashlib +import sqlite3 +import base64 +import urllib +import json +import uuid +import time + app = Flask(__name__) babel = Babel(app) - LANGUAGES = ['en', 'de'] - settings = None @@ -38,14 +45,74 @@ class Categories(): return (cat_name, sub_name) +class ScrapeState(): + stats = {} + + def __init__(self): + pass + + def update(self): + self._statedump() + self._tpbs() + self._fullscrape() + + + def _statedump(self): + url = settings["scrape_url"] + statedump_url = url + "/stats" + params = { "mode" : "statedump" } + req = requests.get(statedump_url, params=params) + dump = req.text.strip() + dump = dump.split("\n") + for entry in dump: + entry = entry.split(":") + key = entry[0].lower() + if not key in self.stats.keys(): + self.stats.update({ key : {}}) + self.stats.get(key).update({ "base" : entry[1], "unsure_downloaded" : entry[2] }) + + def _tpbs(self): + url = settings["scrape_url"] + tpbs_url = url + "/stats" + params = { "mode" : "tpbs", "format" : "ben" } + req = requests.get(tpbs_url, params=params) + decoded = bencoder.decode(req.content) + for torrent in decoded[b"files"]: + info_hash = binascii.b2a_hex(torrent) + stats = decoded[b"files"][torrent] + key = info_hash.decode("utf-8").lower() + self.stats.get(key).update({ "seeds" : stats[b"complete"], "peers" : stats[b"incomplete"], "complete" : stats[b"downloaded"] }) + + def _fullscrape(self): + connection = sqlite3.connect("torrentdb.sqlite") + c = connection.cursor() + c.execute("SELECT fileid FROM torrents") + all_hashes = c.fetchall() + for info_hash in all_hashes: + info_hash = info_hash[0] + url_param = binascii.a2b_hex(info_hash.encode()) + url = settings["scrape_url"] + req = requests.get(url + "/scrape", params={"info_hash" : url_param}) + decoded = bencoder.decode(req.content) + info = decoded[b"files"] + try: + ugly_hash, stats = info.popitem() + key = info_hash.lower() + self.stats.get(key).update({ "seeds" : stats[b"complete"], "peers" : stats[b"incomplete"], "complete" : stats[b"downloaded"] }) + except KeyError: + print("No stats found for {}".format(info_hash)) + + @app.route("/") def index(): return render_template("search.html", categories=categories.categories) + @app.route("/categories") def categorys(): return render_template("categories.html", categories=categories.categories) + @app.route("/create", methods=['GET','POST']) def create(): if request.method == "GET": @@ -58,6 +125,7 @@ def create(): else: return render_template("create.html", categories=categories.categories, errors=newTorrent.errors) + @app.route("/download/") def download(filename): connection = sqlite3.connect("torrentdb.sqlite") @@ -66,6 +134,7 @@ def download(filename): name = c.fetchone()[0] return send_file("torrentFiles/" + filename, as_attachment=True, attachment_filename=name + ".torrent", conditional=True) + @app.route("/search", methods=['GET']) def search(): connection = sqlite3.connect("torrentdb.sqlite") @@ -101,7 +170,8 @@ def search(): r = row[0:2] + (size(float(row[2])) , ) + row[3:] results.append(r) - return render_template("result.html", results=results, categories=categories.categories) + return render_template("result.html", results=results, categories=categories.categories, stats=scrapeState.stats) + @app.route("/details", methods=['GET']) def details(): @@ -110,11 +180,6 @@ def details(): tf.fromDb() return render_template("details.html", categories=categories.categories, torrent=tf) -def scrapeAll(): - TRACKER_URL = "" - statedump = requests.get(TRACKER_URL + "stats" + "?mode=statedump") - - return def init(): global settings @@ -123,7 +188,19 @@ def init(): initDb() global categories categories = Categories() - #scrapeAll() + global scrapeState + scrapeState = ScrapeState() + scrape = threading.Thread(target=scraper) + scrape.start() + + +def scraper(): + while True: + print("Start scraping") + scrapeState.update() + print("Scraping done") + time.sleep(60) + def initDb(): connection = sqlite3.connect("torrentdb.sqlite") @@ -133,6 +210,7 @@ def initDb(): connection.commit() connection.close() + def createNewTorrent(reuqest): uploadfile = request.files["torrentFile"] filename = secure_filename(uploadfile.filename) @@ -201,6 +279,7 @@ def createNewTorrent(reuqest): return newTFile + class Metadata(): def __init__(self, fileid): try: @@ -211,7 +290,7 @@ class Metadata(): self.fileid = fileid self.bcoded = bencoder.decode(torrent) self.created_by = self.bcoded.get(b'created by', b"").decode("utf-8", "ignore") - self.creation_date = self.bcoded.get(b'creation date', b"").decode("utf-8", "ignore") + self.creation_date = self.bcoded.get(b'creation date', 0) self.announce_url = self.bcoded.get(b'info', dict()).get(b'', "") self.source = self.bcoded.get(b'info', dict()).get(b'source', b"") self.torrentsize = ((len(self.bcoded.get(b'info', dict()).get(b'pieces', "")) / 20) * self.bcoded.get(b'info', dict()).get(b'piece length')) @@ -227,6 +306,7 @@ class Metadata(): b64name = base64.b64encode(self.name) c.execute("INSERT INTO metadata(fileid, created_by, creation_date, announce_url, source, torrentsize, name, private) VALUES(:fileid, :created_by, :creation_date, :announce_url, :source, :torrentsize, :name, :private)", { 'fileid' : self.fileid, 'created_by' : b64created_by, 'creation_date' : self.creation_date, 'announce_url' : b64announce_url, 'source' : b64source , 'torrentsize' : self.torrentsize, 'name' : b64name, 'private' : self.private}) + class TorrentFile(): errors = [] fileid = None @@ -277,10 +357,12 @@ class TorrentFile(): self.videoquality_description = (base64.b64decode(res["videoquality_description"])).decode() self.metadata = Metadata(self.fileid) + @babel.localeselector def get_locale(): return request.accept_languages.best_match(LANGUAGES) + if __name__ == "__main__": init() app.jinja_env.globals.update(json=json) diff --git a/settings.json b/settings.json index 6997ccb..3821c97 100644 --- a/settings.json +++ b/settings.json @@ -189,6 +189,8 @@ "valid_tracker" : [ "udp://tracker.lootbox.cf:6969/announce", "udp://tracker.lootbox.cf:6969/announce/" - ] + ], + + "scrape_url" : "http://tracker.lootbox.gq:6969" } diff --git a/templates/result.html b/templates/result.html index 4e2df37..04255a5 100644 --- a/templates/result.html +++ b/templates/result.html @@ -19,9 +19,9 @@ vim: ts=2 noexpandtab {{ result[1] }} {{ result[2] }} - {{ _("N/A") }} - {{ _("N/A") }} - {{ _("N/A") }} + {{ stats.get(result[0], {}).get("complete", _("N/A")) }} + {{ stats.get(result[0], {}).get("seeds", _("N/A")) }} + {{ stats.get(result[0], {}).get("peers", _("N/A")) }} {% endfor %}