TorrentIndexer/indexer.py

#!/usr/bin/python3
#/* vim:set ts=2 set noexpandtab */
from flask import Flask, render_template, url_for, request, send_file, redirect
from flask_babel import Babel, gettext as _, lazy_gettext
from werkzeug import secure_filename
from hurry.filesize import size
from hashlib import sha1
import threading
import binascii
import bencoder
import requests
import hashlib
import sqlite3
import base64
import urllib
import json
import uuid
import time

app = Flask(__name__)
babel = Babel(app)
LANGUAGES = ['en', 'de']
settings = None


class Categories():
	def __init__(self):

		self.categories = settings["categories"]
		for c in self.categories:
			c["label"] = str(lazy_gettext(c["label"]))
			for s in c["subcategories"]:
				s["label"] = str(lazy_gettext(s["label"]))

	def find(self, category, subcategory = None):
		cat_name = ""
		sub_name = ""
		for cat in self.categories:
			if cat["id"] == category:
				cat_name = cat["label"]
			if subcategory != None:
				for sub in cat["subcategories"]:
					if sub["id"] == subcategory:
						sub_name = sub["label"]
		return (cat_name, sub_name)


class ScrapeState():
	stats = {}

	def __init__(self):
		pass

	def update(self):
		self._statedump()
		self._tpbs()
		self._fullscrape()


	def _statedump(self):
		url = settings["scrape_url"]
		statedump_url = url + "/stats"
		params = { "mode" : "statedump" }
		req = requests.get(statedump_url, params=params)
		dump = req.text.strip()
		dump = dump.split("\n")
		for entry in dump:
			entry = entry.split(":")
			key = entry[0].lower()
			if not key in self.stats.keys():
				self.stats.update({ key : {}})
			self.stats.get(key).update({ "base" : entry[1], "unsure_downloaded" : entry[2] })

	def _tpbs(self):
		url = settings["scrape_url"]
		tpbs_url = url + "/stats"
		params = { "mode" : "tpbs", "format" : "ben" }
		req = requests.get(tpbs_url, params=params)
		decoded = bencoder.decode(req.content)
		for torrent in decoded[b"files"]:
			info_hash = binascii.b2a_hex(torrent)
			stats = decoded[b"files"][torrent]
			key = info_hash.decode("utf-8").lower()
			self.stats.get(key).update({ "seeds" : stats[b"complete"], "peers" : stats[b"incomplete"], "complete" : stats[b"downloaded"] })

	def _fullscrape(self):
		connection = sqlite3.connect("torrentdb.sqlite")
		c = connection.cursor()
		c.execute("SELECT fileid FROM torrents")
		all_hashes = c.fetchall()
		connection.close()
		for info_hash in all_hashes:
			info_hash = info_hash[0]
			url_param = binascii.a2b_hex(info_hash.encode())
			url = settings["scrape_url"]
			req = requests.get(url + "/scrape", params={"info_hash" : url_param})
			decoded = bencoder.decode(req.content)
			info = decoded[b"files"]
			try:
				ugly_hash, stats = info.popitem()
				key = info_hash.lower()
				self.stats.get(key).update({ "seeds" : stats[b"complete"], "peers" : stats[b"incomplete"], "complete" : stats[b"downloaded"] })
			except KeyError:
				print("No stats found for {}".format(info_hash))


@app.route("/")
def index():
	return render_template("search.html", categories=categories.categories)


@app.route("/categories")
def categorys():
	return render_template("categories.html", categories=categories.categories)


@app.route("/create", methods=['GET','POST'])
def create():
	if request.method == "GET":
		return render_template("create.html", categories=categories.categories, errors=None, tracker=settings["valid_tracker"])
	elif request.method == "POST":
		newTorrent = createNewTorrent(request)
		if len(newTorrent.errors) == 0:
			message = _("Successfully created torrent <a href=\"/search?h={}\">{}</a>").format(newTorrent.fileid, newTorrent.fileid[:-20])
			return render_template("create.html", categories=categories.categories, messages=[message], tracker=settings["valid_tracker"])
		else:
			return render_template("create.html", categories=categories.categories, errors=newTorrent.errors, tracker=settings["valid_tracker"]), 409


@app.route("/download/<filename>")
def download(filename):
	connection = sqlite3.connect("torrentdb.sqlite")
	c = connection.cursor()
	c.execute("SELECT name FROM torrents WHERE fileid = :fileid", { 'fileid' : filename})
	name = c.fetchone()[0]
	connection.close()
	return send_file("torrentFiles/" + filename, as_attachment=True, attachment_filename=name + ".torrent", conditional=True)


@app.route("/search", methods=['GET'])
def search():
	connection = sqlite3.connect("torrentdb.sqlite")
	c = connection.cursor()

	search_params = []
	search = ""
	fields = list(request.args.keys())
	for field in fields:
		query_list = request.args.getlist(field)
		for query in query_list:
			if len(search) > 0:
				search += " AND "

			if field is "q":
				names = query.split(" ")
				search_params += list(map(lambda x: "%" + x + "%", names))
				search += " AND ".join(["torrents.name LIKE (?)"] * len(query.split(" ")))
			elif field is "c":
				search_params += query.split(" ")
				search += " AND ".join(["torrents.category LIKE (?)"] * len(query.split(" ")))
			elif field is "s":
				search_params += query.split(" ")
				search += " AND ".join(["torrents.subcategory LIKE (?)"] * len(query.split(" ")))
			elif field is "h":
				hashes = query.split(" ")
				search_params += list(map(lambda x: x + "%", hashes))
				search += " AND ".join(["torrents.fileid LIKE (?)"] * len(query.split(" ")))

	print(search)
	results = list()
	for row in c.execute("SELECT torrents.fileid, torrents.name, metadata.torrentsize FROM torrents LEFT JOIN metadata on metadata.fileid = torrents.fileid WHERE " + search, search_params):
		r = row[0:2] + (size(float(row[2])) , )  + row[3:]
		results.append(r)

	connection.close()
	return render_template("result.html", results=results, categories=categories.categories, stats=scrapeState.stats)


@app.route("/details", methods=['GET'])
def details():
	info_hash = request.args["h"]
	tf = TorrentFile(fileid=info_hash)
	tf.fromDb()
	return render_template("details.html", categories=categories.categories, torrent=tf)


def init():
	global settings
	with open("settings.json") as settingsJson:
		settings = json.load(settingsJson)
	initDb()
	global categories
	categories = Categories()
	global scrapeState
	scrapeState = ScrapeState()
	scrape = threading.Thread(target=scraper)
	scrape.start()


def scraper():
	while True:
		print("Start scraping")
		scrapeState.update()
		print("Scraping done")
		time.sleep(60)


def initDb():
	connection = sqlite3.connect("torrentdb.sqlite")
	c = connection.cursor()
	c.execute('CREATE TABLE IF NOT EXISTS torrents (fileid TEXT PRIMARY KEY NOT NULL, name TEXT NOT NULL, category TEXT NOT NULL, subcategory TEXT NOT NULL, description TEXT NOT NULL, audioquality_description TEXT NOT NULL, videoquality_description TEXT NOT NULL);')
	c.execute('CREATE TABLE IF NOT EXISTS metadata (fileid TEXT PRIMARY KEY NOT NULL, created_by TEXT, creation_date TEXT, announce_url TEXT NOT NULL, source TEXT, torrentsize TEXT NOT NULL, name TEXT NOT NULL, private TEXT NOT NULL)')
	connection.commit()
	connection.close()


def createNewTorrent(reuqest):
	uploadfile = request.files["torrentFile"]
	filename = secure_filename(uploadfile.filename)

	content = request.files["torrentFile"].stream.read()
	bcoded = bencoder.decode(content)
	info_hash = sha1(bencoder.encode(bcoded[b'info'])).hexdigest()

	#TODO: Validate the input serverside before writing it to the database
	name = request.form["name"]
	category = request.form["category"]
	subcategory = request.form["subcategory"]
	description = request.form["description"]
	audioquality_description = request.form["audioquality_description"]
	videoquality_description = request.form["videoquality_description"]
	newTFile = TorrentFile(info_hash, name, category, subcategory, description, audioquality_description, videoquality_description)
	try:
		announce = bcoded[b'announce-list']
	except KeyError:
		try:
			announce = (bcoded[b'announce'], )
		except KeyError:
			announce = []

	is_ours = False
	for a in announce:
		a = a.decode("utf-8", "ignore")
		if a in settings["valid_tracker"]:
			is_ours = True
			break

	if not is_ours:
		newTFile.errors = ["Rejecting torrent <a href=\"/search?h={}\">{}</a>, as it does not use our tracker".format(info_hash, info_hash[:-20])]
		return newTFile
	
	
	with open("torrentFiles/" + info_hash, "wb") as torrent_file:
		torrent_file.write(content)

	bcoded = bencoder.decode(content)
	size = ((len(bcoded[b'info'][b'pieces']) / 20) * bcoded[b'info'][b'piece length']) / 1024 / 1024

	print("=== CREATE NEW TORRENT FILE ===")
	print( "Name:         " + request.form["name"] )
	print( "Torrent file: " + info_hash )
	print( "Category:     " + request.form["category"] )
	print( "Subcategory:  " + request.form["subcategory"] )
	print( "Description:  " + request.form["description"] )

	#TODO: Validate the input serverside before writing it to the database
	name = request.form["name"]
	category = request.form["category"]
	subcategory = request.form["subcategory"]
	description = request.form["description"]
	audioquality_description = request.form["audioquality_description"]
	videoquality_description = request.form["videoquality_description"]
	newTFile = TorrentFile(info_hash, name, category, subcategory, description, audioquality_description, videoquality_description)
	try:
		connection = sqlite3.connect("torrentdb.sqlite")
		newTFile.writeToDb(connection.cursor())
		newTFile.metadata.writeToDb(connection.cursor())
		connection.commit()
		connection.close()
	except sqlite3.IntegrityError as e:
		print(e)
		newTFile.errors = [_("Torrent <a href=\"/search?h={}\">{}</a> already exists").format(info_hash, info_hash[:-20])]
	except Exception as e:
		newTFile.errors = [_("Unknown error in creation")]

	return newTFile


class Metadata():
	def __init__(self, fileid):
		try:
			with open("torrentFiles/" + fileid, "rb") as f:
				torrent = f.read()
		except FileNotFoundError:
			return
		self.fileid = fileid
		self.bcoded = bencoder.decode(torrent)
		self.created_by = self.bcoded.get(b'created by', b"")
		self.creation_date = self.bcoded.get(b'creation date', 0)
		self.announce_url = self.bcoded.get(b'info', dict()).get(b'', "")
		self.source = self.bcoded.get(b'info', dict()).get(b'source', b"")
		self.torrentsize = ((len(self.bcoded.get(b'info', dict()).get(b'pieces', "")) / 20) * self.bcoded.get(b'info', dict()).get(b'piece length'))
		self.torrentsize_human = size(self.torrentsize)
		self.name = self.bcoded.get(b'info', dict()).get(b'name', b"")
		self.private = self.bcoded.get(b'info', dict()).get(b'private', b"")

	def writeToDb(self, cursor):
		c = cursor
		b64created_by = base64.b64encode(self.created_by) if self.created_by else ""
		b64announce_url = base64.b64encode(self.announce_url.decode()) if self.announce_url else ""
		b64source = base64.b64encode(self.source) if self.source else ""
		b64name = base64.b64encode(self.name)
		c.execute("INSERT INTO metadata(fileid, created_by, creation_date, announce_url, source, torrentsize, name, private) VALUES(:fileid, :created_by, :creation_date, :announce_url, :source, :torrentsize, :name, :private)", { 'fileid' : self.fileid, 'created_by' : b64created_by, 'creation_date' : self.creation_date, 'announce_url' : b64announce_url, 'source' : b64source , 'torrentsize' : self.torrentsize, 'name' : b64name, 'private' : self.private})


class TorrentFile():
	errors = []
	fileid = None
	name = None
	category = None
	subcategory = None
	description = None
	audioquality_description = None
	videoquality_description = None

	def __init__(self, fileid=fileid, name=name, category=category, subcategory=subcategory, description=description, audioquality_description=audioquality_description, videoquality_description=videoquality_description):
		self.fileid = fileid
		self.name = name
		self.category = category
		self.subcategory = subcategory
		self.description = description
		self.audioquality_description = audioquality_description
		self.videoquality_description = videoquality_description
		if self.fileid:
			self.metadata = Metadata(fileid)

	def writeToDb(self, cursor):
		c = cursor
		b64description = base64.b64encode(self.description.encode())
		b64audioquality_description = base64.b64encode(self.audioquality_description.encode())
		b64videoquality_description = base64.b64encode(self.videoquality_description.encode())
		c.execute("INSERT INTO torrents(fileid, name, category, subcategory, description, audioquality_description, videoquality_description) VALUES(:fileid, :name, :category, :subcategory, :description, :audioquality_description, :videoquality_description)", { 'fileid' : self.fileid, 'name' : self.name, 'category' : self.category, 'subcategory' : self.subcategory, 'description' : b64description , 'audioquality_description' : b64audioquality_description, 'videoquality_description' : b64videoquality_description})

	def fromDb(self):
		def dict_factory(cursor, row):
			d = {}
			for idx, col in enumerate(cursor.description):
					d[col[0]] = row[idx]
			return d

		con = sqlite3.connect("torrentdb.sqlite")
		con.row_factory = dict_factory
		c = con.cursor()
		res = c.execute("SELECT torrents.*, metadata.* FROM torrents LEFT JOIN metadata on metadata.fileid = torrents.fileid WHERE torrents.fileid LIKE :fileid", { "fileid" : self.fileid })
		res = res.fetchone()
		self.fileid = res["fileid"]
		self.name = (base64.b64decode(res["name"])).decode()
		self.category = res["category"]
		self.subcategory = res["subcategory"]
		self.category_string, self.subcategory_string = categories.find(int(self.category), int(self.subcategory))
		self.description = (base64.b64decode(res["description"])).decode()
		self.audioquality_description = (base64.b64decode(res["audioquality_description"])).decode()
		self.videoquality_description = (base64.b64decode(res["videoquality_description"])).decode()
		self.metadata = Metadata(self.fileid)


@babel.localeselector
def get_locale():
	return request.accept_languages.best_match(LANGUAGES)


if __name__ == "__main__":
	init()
	app.jinja_env.globals.update(json=json)
	app.jinja_env.globals.update(sorted=sorted)
	app.run(debug=False, host="127.0.0.1")

# vim: set ts=2 sts=2 sw=2 noexpandtab: