Merge branch 'master' of https://git.geekify.de/sqozz/TorrentIndexer

indexer.py

@@ -1,18 +1,25 @@
 #!/usr/bin/python3
 #/* vim:set ts=2 set noexpandtab */
-import json, uuid, hashlib, sqlite3, base64
-from hashlib import sha1
-import bencoder
-import requests
 from flask import Flask, render_template, url_for, request, send_file, redirect
 from flask_babel import Babel, gettext as _, lazy_gettext
 from werkzeug import secure_filename
 from hurry.filesize import size
+from hashlib import sha1
+import threading
+import binascii
+import bencoder
+import requests
+import hashlib
+import sqlite3
+import base64
+import urllib
+import json
+import uuid
+import time
+
 app = Flask(__name__)
 babel = Babel(app)
-
 LANGUAGES = ['en', 'de']
-
 settings = None
 
 
@@ -38,14 +45,74 @@ class Categories():
 		return (cat_name, sub_name)
 
 
+class ScrapeState():
+	stats = {}
+
+	def __init__(self):
+		pass
+
+	def update(self):
+		self._statedump()
+		self._tpbs()
+		self._fullscrape()
+
+
+	def _statedump(self):
+		url = settings["scrape_url"]
+		statedump_url = url + "/stats"
+		params = { "mode" : "statedump" }
+		req = requests.get(statedump_url, params=params)
+		dump = req.text.strip()
+		dump = dump.split("\n")
+		for entry in dump:
+			entry = entry.split(":")
+			key = entry[0].lower()
+			if not key in self.stats.keys():
+				self.stats.update({ key : {}})
+			self.stats.get(key).update({ "base" : entry[1], "unsure_downloaded" : entry[2] })
+
+	def _tpbs(self):
+		url = settings["scrape_url"]
+		tpbs_url = url + "/stats"
+		params = { "mode" : "tpbs", "format" : "ben" }
+		req = requests.get(tpbs_url, params=params)
+		decoded = bencoder.decode(req.content)
+		for torrent in decoded[b"files"]:
+			info_hash = binascii.b2a_hex(torrent)
+			stats = decoded[b"files"][torrent]
+			key = info_hash.decode("utf-8").lower()
+			self.stats.get(key).update({ "seeds" : stats[b"complete"], "peers" : stats[b"incomplete"], "complete" : stats[b"downloaded"] })
+
+	def _fullscrape(self):
+		connection = sqlite3.connect("torrentdb.sqlite")
+		c = connection.cursor()
+		c.execute("SELECT fileid FROM torrents")
+		all_hashes = c.fetchall()
+		for info_hash in all_hashes:
+			info_hash = info_hash[0]
+			url_param = binascii.a2b_hex(info_hash.encode())
+			url = settings["scrape_url"]
+			req = requests.get(url + "/scrape", params={"info_hash" : url_param})
+			decoded = bencoder.decode(req.content)
+			info = decoded[b"files"]
+			try:
+				ugly_hash, stats = info.popitem()
+				key = info_hash.lower()
+				self.stats.get(key).update({ "seeds" : stats[b"complete"], "peers" : stats[b"incomplete"], "complete" : stats[b"downloaded"] })
+			except KeyError:
+				print("No stats found for {}".format(info_hash))
+
+
 @app.route("/")
 def index():
 	return render_template("search.html", categories=categories.categories)
 
+
 @app.route("/categories")
 def categorys():
 	return render_template("categories.html", categories=categories.categories)
 
+
 @app.route("/create", methods=['GET','POST'])
 def create():
 	if request.method == "GET":
@@ -58,6 +125,7 @@ def create():
 		else:
 			return render_template("create.html", categories=categories.categories, errors=newTorrent.errors)
 
+
 @app.route("/download/<filename>")
 def download(filename):
 	connection = sqlite3.connect("torrentdb.sqlite")
@@ -66,6 +134,7 @@ def download(filename):
 	name = c.fetchone()[0]
 	return send_file("torrentFiles/" + filename, as_attachment=True, attachment_filename=name + ".torrent", conditional=True)
 
+
 @app.route("/search", methods=['GET'])
 def search():
 	connection = sqlite3.connect("torrentdb.sqlite")
@@ -101,7 +170,8 @@ def search():
 		r = row[0:2] + (size(float(row[2])) , )  + row[3:]
 		results.append(r)
 
-	return render_template("result.html", results=results, categories=categories.categories)
+	return render_template("result.html", results=results, categories=categories.categories, stats=scrapeState.stats)
+
 
 @app.route("/details", methods=['GET'])
 def details():
@@ -110,11 +180,6 @@ def details():
 	tf.fromDb()
 	return render_template("details.html", categories=categories.categories, torrent=tf)
 
-def scrapeAll():
-	TRACKER_URL = ""
-	statedump = requests.get(TRACKER_URL + "stats" + "?mode=statedump")
-
-	return
 
 def init():
 	global settings
@@ -123,7 +188,19 @@ def init():
 	initDb()
 	global categories
 	categories = Categories()
-	#scrapeAll()
+	global scrapeState
+	scrapeState = ScrapeState()
+	scrape = threading.Thread(target=scraper)
+	scrape.start()
+
+
+def scraper():
+	while True:
+		print("Start scraping")
+		scrapeState.update()
+		print("Scraping done")
+		time.sleep(60)
+
 
 def initDb():
 	connection = sqlite3.connect("torrentdb.sqlite")
@@ -133,6 +210,7 @@ def initDb():
 	connection.commit()
 	connection.close()
 
+
 def createNewTorrent(reuqest):
 	uploadfile = request.files["torrentFile"]
 	filename = secure_filename(uploadfile.filename)
@@ -204,6 +282,7 @@ def createNewTorrent(reuqest):
 
 	return newTFile
 
+
 class Metadata():
 	def __init__(self, fileid):
 		try:
@@ -214,7 +293,7 @@ class Metadata():
 		self.fileid = fileid
 		self.bcoded = bencoder.decode(torrent)
 		self.created_by = self.bcoded.get(b'created by', b"").decode("utf-8", "ignore")
-		self.creation_date = self.bcoded.get(b'creation date', b"").decode("utf-8", "ignore")
+		self.creation_date = self.bcoded.get(b'creation date', 0)
 		self.announce_url = self.bcoded.get(b'info', dict()).get(b'', "")
 		self.source = self.bcoded.get(b'info', dict()).get(b'source', b"")
 		self.torrentsize = ((len(self.bcoded.get(b'info', dict()).get(b'pieces', "")) / 20) * self.bcoded.get(b'info', dict()).get(b'piece length'))
@@ -230,6 +309,7 @@ class Metadata():
 		b64name = base64.b64encode(self.name)
 		c.execute("INSERT INTO metadata(fileid, created_by, creation_date, announce_url, source, torrentsize, name, private) VALUES(:fileid, :created_by, :creation_date, :announce_url, :source, :torrentsize, :name, :private)", { 'fileid' : self.fileid, 'created_by' : b64created_by, 'creation_date' : self.creation_date, 'announce_url' : b64announce_url, 'source' : b64source , 'torrentsize' : self.torrentsize, 'name' : b64name, 'private' : self.private})
 
+
 class TorrentFile():
 	errors = []
 	fileid = None
@@ -280,10 +360,12 @@ class TorrentFile():
 		self.videoquality_description = (base64.b64decode(res["videoquality_description"])).decode()
 		self.metadata = Metadata(self.fileid)
 
+
 @babel.localeselector
 def get_locale():
 	return request.accept_languages.best_match(LANGUAGES)
 
+
 if __name__ == "__main__":
 	init()
 	app.jinja_env.globals.update(json=json)

settings.json

@@ -189,6 +189,8 @@
 	"valid_tracker" : [
 		"udp://tracker.lootbox.cf:6969/announce",
 		"udp://tracker.lootbox.cf:6969/announce/"
-	]
+	],
+
+	"scrape_url" : "http://tracker.lootbox.gq:6969"
 }
 

templates/result.html

@@ -19,9 +19,9 @@ vim: ts=2 noexpandtab
 		<tr>
 			<td><a href="/details?h={{ result[0] }}">{{ result[1] }}</a></td>
 			<td>{{ result[2] }}</td>
-			<td>{{ _("N/A") }}</td>
+			<td>{{ stats.get(result[0], {}).get("complete", _("N/A")) }}</td>
-			<td>{{ _("N/A") }}</td>
+			<td>{{ stats.get(result[0], {}).get("seeds", _("N/A")) }}</td>
-			<td>{{ _("N/A") }}</td>
+			<td>{{ stats.get(result[0], {}).get("peers", _("N/A")) }}</td>
 		</tr>
 		{% endfor %}
 	</table>