Merge branch 'master' of https://git.geekify.de/sqozz/TorrentIndexer
This commit is contained in:
Stefan Rupp
commit
f45afb3035
110
indexer.py
110
indexer.py
|
|
@ -1,18 +1,25 @@
|
|||
#!/usr/bin/python3
|
||||
#/* vim:set ts=2 set noexpandtab */
|
||||
import json, uuid, hashlib, sqlite3, base64
|
||||
from hashlib import sha1
|
||||
import bencoder
|
||||
import requests
|
||||
from flask import Flask, render_template, url_for, request, send_file, redirect
|
||||
from flask_babel import Babel, gettext as _, lazy_gettext
|
||||
from werkzeug import secure_filename
|
||||
from hurry.filesize import size
|
||||
from hashlib import sha1
|
||||
import threading
|
||||
import binascii
|
||||
import bencoder
|
||||
import requests
|
||||
import hashlib
|
||||
import sqlite3
|
||||
import base64
|
||||
import urllib
|
||||
import json
|
||||
import uuid
|
||||
import time
|
||||
|
||||
app = Flask(__name__)
|
||||
babel = Babel(app)
|
||||
|
||||
LANGUAGES = ['en', 'de']
|
||||
|
||||
settings = None
|
||||
|
||||
|
||||
|
|
@ -38,14 +45,74 @@ class Categories():
|
|||
return (cat_name, sub_name)
|
||||
|
||||
|
||||
class ScrapeState():
|
||||
stats = {}
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def update(self):
|
||||
self._statedump()
|
||||
self._tpbs()
|
||||
self._fullscrape()
|
||||
|
||||
|
||||
def _statedump(self):
|
||||
url = settings["scrape_url"]
|
||||
statedump_url = url + "/stats"
|
||||
params = { "mode" : "statedump" }
|
||||
req = requests.get(statedump_url, params=params)
|
||||
dump = req.text.strip()
|
||||
dump = dump.split("\n")
|
||||
for entry in dump:
|
||||
entry = entry.split(":")
|
||||
key = entry[0].lower()
|
||||
if not key in self.stats.keys():
|
||||
self.stats.update({ key : {}})
|
||||
self.stats.get(key).update({ "base" : entry[1], "unsure_downloaded" : entry[2] })
|
||||
|
||||
def _tpbs(self):
|
||||
url = settings["scrape_url"]
|
||||
tpbs_url = url + "/stats"
|
||||
params = { "mode" : "tpbs", "format" : "ben" }
|
||||
req = requests.get(tpbs_url, params=params)
|
||||
decoded = bencoder.decode(req.content)
|
||||
for torrent in decoded[b"files"]:
|
||||
info_hash = binascii.b2a_hex(torrent)
|
||||
stats = decoded[b"files"][torrent]
|
||||
key = info_hash.decode("utf-8").lower()
|
||||
self.stats.get(key).update({ "seeds" : stats[b"complete"], "peers" : stats[b"incomplete"], "complete" : stats[b"downloaded"] })
|
||||
|
||||
def _fullscrape(self):
|
||||
connection = sqlite3.connect("torrentdb.sqlite")
|
||||
c = connection.cursor()
|
||||
c.execute("SELECT fileid FROM torrents")
|
||||
all_hashes = c.fetchall()
|
||||
for info_hash in all_hashes:
|
||||
info_hash = info_hash[0]
|
||||
url_param = binascii.a2b_hex(info_hash.encode())
|
||||
url = settings["scrape_url"]
|
||||
req = requests.get(url + "/scrape", params={"info_hash" : url_param})
|
||||
decoded = bencoder.decode(req.content)
|
||||
info = decoded[b"files"]
|
||||
try:
|
||||
ugly_hash, stats = info.popitem()
|
||||
key = info_hash.lower()
|
||||
self.stats.get(key).update({ "seeds" : stats[b"complete"], "peers" : stats[b"incomplete"], "complete" : stats[b"downloaded"] })
|
||||
except KeyError:
|
||||
print("No stats found for {}".format(info_hash))
|
||||
|
||||
|
||||
@app.route("/")
|
||||
def index():
|
||||
return render_template("search.html", categories=categories.categories)
|
||||
|
||||
|
||||
@app.route("/categories")
|
||||
def categorys():
|
||||
return render_template("categories.html", categories=categories.categories)
|
||||
|
||||
|
||||
@app.route("/create", methods=['GET','POST'])
|
||||
def create():
|
||||
if request.method == "GET":
|
||||
|
|
@ -58,6 +125,7 @@ def create():
|
|||
else:
|
||||
return render_template("create.html", categories=categories.categories, errors=newTorrent.errors)
|
||||
|
||||
|
||||
@app.route("/download/<filename>")
|
||||
def download(filename):
|
||||
connection = sqlite3.connect("torrentdb.sqlite")
|
||||
|
|
@ -66,6 +134,7 @@ def download(filename):
|
|||
name = c.fetchone()[0]
|
||||
return send_file("torrentFiles/" + filename, as_attachment=True, attachment_filename=name + ".torrent", conditional=True)
|
||||
|
||||
|
||||
@app.route("/search", methods=['GET'])
|
||||
def search():
|
||||
connection = sqlite3.connect("torrentdb.sqlite")
|
||||
|
|
@ -101,7 +170,8 @@ def search():
|
|||
r = row[0:2] + (size(float(row[2])) , ) + row[3:]
|
||||
results.append(r)
|
||||
|
||||
return render_template("result.html", results=results, categories=categories.categories)
|
||||
return render_template("result.html", results=results, categories=categories.categories, stats=scrapeState.stats)
|
||||
|
||||
|
||||
@app.route("/details", methods=['GET'])
|
||||
def details():
|
||||
|
|
@ -110,11 +180,6 @@ def details():
|
|||
tf.fromDb()
|
||||
return render_template("details.html", categories=categories.categories, torrent=tf)
|
||||
|
||||
def scrapeAll():
|
||||
TRACKER_URL = ""
|
||||
statedump = requests.get(TRACKER_URL + "stats" + "?mode=statedump")
|
||||
|
||||
return
|
||||
|
||||
def init():
|
||||
global settings
|
||||
|
|
@ -123,7 +188,19 @@ def init():
|
|||
initDb()
|
||||
global categories
|
||||
categories = Categories()
|
||||
#scrapeAll()
|
||||
global scrapeState
|
||||
scrapeState = ScrapeState()
|
||||
scrape = threading.Thread(target=scraper)
|
||||
scrape.start()
|
||||
|
||||
|
||||
def scraper():
|
||||
while True:
|
||||
print("Start scraping")
|
||||
scrapeState.update()
|
||||
print("Scraping done")
|
||||
time.sleep(60)
|
||||
|
||||
|
||||
def initDb():
|
||||
connection = sqlite3.connect("torrentdb.sqlite")
|
||||
|
|
@ -133,6 +210,7 @@ def initDb():
|
|||
connection.commit()
|
||||
connection.close()
|
||||
|
||||
|
||||
def createNewTorrent(reuqest):
|
||||
uploadfile = request.files["torrentFile"]
|
||||
filename = secure_filename(uploadfile.filename)
|
||||
|
|
@ -204,6 +282,7 @@ def createNewTorrent(reuqest):
|
|||
|
||||
return newTFile
|
||||
|
||||
|
||||
class Metadata():
|
||||
def __init__(self, fileid):
|
||||
try:
|
||||
|
|
@ -214,7 +293,7 @@ class Metadata():
|
|||
self.fileid = fileid
|
||||
self.bcoded = bencoder.decode(torrent)
|
||||
self.created_by = self.bcoded.get(b'created by', b"").decode("utf-8", "ignore")
|
||||
self.creation_date = self.bcoded.get(b'creation date', b"").decode("utf-8", "ignore")
|
||||
self.creation_date = self.bcoded.get(b'creation date', 0)
|
||||
self.announce_url = self.bcoded.get(b'info', dict()).get(b'', "")
|
||||
self.source = self.bcoded.get(b'info', dict()).get(b'source', b"")
|
||||
self.torrentsize = ((len(self.bcoded.get(b'info', dict()).get(b'pieces', "")) / 20) * self.bcoded.get(b'info', dict()).get(b'piece length'))
|
||||
|
|
@ -230,6 +309,7 @@ class Metadata():
|
|||
b64name = base64.b64encode(self.name)
|
||||
c.execute("INSERT INTO metadata(fileid, created_by, creation_date, announce_url, source, torrentsize, name, private) VALUES(:fileid, :created_by, :creation_date, :announce_url, :source, :torrentsize, :name, :private)", { 'fileid' : self.fileid, 'created_by' : b64created_by, 'creation_date' : self.creation_date, 'announce_url' : b64announce_url, 'source' : b64source , 'torrentsize' : self.torrentsize, 'name' : b64name, 'private' : self.private})
|
||||
|
||||
|
||||
class TorrentFile():
|
||||
errors = []
|
||||
fileid = None
|
||||
|
|
@ -280,10 +360,12 @@ class TorrentFile():
|
|||
self.videoquality_description = (base64.b64decode(res["videoquality_description"])).decode()
|
||||
self.metadata = Metadata(self.fileid)
|
||||
|
||||
|
||||
@babel.localeselector
|
||||
def get_locale():
|
||||
return request.accept_languages.best_match(LANGUAGES)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
init()
|
||||
app.jinja_env.globals.update(json=json)
|
||||
|
|
|
|||
|
|
@ -189,6 +189,8 @@
|
|||
"valid_tracker" : [
|
||||
"udp://tracker.lootbox.cf:6969/announce",
|
||||
"udp://tracker.lootbox.cf:6969/announce/"
|
||||
]
|
||||
],
|
||||
|
||||
"scrape_url" : "http://tracker.lootbox.gq:6969"
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,9 +19,9 @@ vim: ts=2 noexpandtab
|
|||
<tr>
|
||||
<td><a href="/details?h={{ result[0] }}">{{ result[1] }}</a></td>
|
||||
<td>{{ result[2] }}</td>
|
||||
<td>{{ _("N/A") }}</td>
|
||||
<td>{{ _("N/A") }}</td>
|
||||
<td>{{ _("N/A") }}</td>
|
||||
<td>{{ stats.get(result[0], {}).get("complete", _("N/A")) }}</td>
|
||||
<td>{{ stats.get(result[0], {}).get("seeds", _("N/A")) }}</td>
|
||||
<td>{{ stats.get(result[0], {}).get("peers", _("N/A")) }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
|
|
|
|||
Loading…
Reference in a new issue