Browse Source

Fixes #40 and fixes #39

master
Abrax 3 years ago
parent
commit
d7ff84117f
3 changed files with 68 additions and 52 deletions
  1. 27
    26
      lizard/__init__.py
  2. 8
    6
      lizard/db_methods.py
  3. 33
    20
      lizard/web_methods.py

+ 27
- 26
lizard/__init__.py View File

@@ -28,23 +28,23 @@ Usage: lizard [command] [options]
Available commands:
la - Lists all threads in database. Legend: ! - new replies, x - 404'd, Y - >(You).
l - Lists only threads that have activity.
r - Refresh threads which are likely to receive new replies (to reduce unnecessary network activity).
ra - Refresh all threads and update database.
rl - Conservative refresh and list threads.
ro - Conservative refresh and open threads with new replies in browsed.
r - Refresh threads which are likely to receive new replies (to reduce unnecessary network activity). Use tr for
onion link.
ra - Refresh all threads and update database. Use tra for onion link.
rl - Conservative refresh and list threads. Use trl for onion link.
ro - Conservative refresh and open threads with new replies in browsed. Use tro for onion link.
o - Open threads with new replies in the default browser. Will also update the "new replies"
field accordingly.
oa - Open all threads.
to - Open threads with new replies, using the onion URL. Use this if you are using Tor.
field accordingly. Use to for onion link.
oa - Open all threads. Use toa for onion link.
tor - Same as to but also opens the Tor CAPTCHA page for convenience.
p - Purge (remove) 404'd threads from database.
b - Backup current database.
e - Export list of threads. This will create a list of lizard add commands for all threads currently in the
database. Useful for migrating between database versions.
database. Useful for migrating between database versions.
a - Add thread to database. Requires option (URL of the thread). Will also attempt to download
the thread.
the thread. Supports onion links.
d - Remove all instances of a thread (board & number) from the database.
Example: "lizard d b 123" will remove thread >>>/b/123
Example: "lizard d b 123" will remove thread >>>/b/123
create - Create new database.
""".format(version))
exit()
@@ -52,8 +52,12 @@ Available commands:

def main():
n = len(sys.argv)
if n < 2: help_info()
if n < 2:
help_info()

command = sys.argv[1]
onion = (command[0] == 't') # The tor version of each command starts with t

if n == 2:
if command == 'la':
db_methods.check_database()
@@ -61,29 +65,26 @@ def main():
elif command == 'l':
db_methods.check_database()
db_methods.list_threads(interesting_only=True)
elif command == 'ra':
elif (command == 'r') or (command == 'tr'):
db_methods.check_database()
db_methods.refresh_all_threads()
elif command == 'r':
db_methods.refresh_all_threads(conservative=True, use_onion_link=onion)
elif (command == 'ra') or (command == 'tra'):
db_methods.check_database()
db_methods.refresh_all_threads(conservative=True)
elif command == 'rl':
db_methods.refresh_all_threads(use_onion_link=onion)
elif (command == 'rl') or (command == 'trl'):
db_methods.check_database()
db_methods.refresh_all_threads(conservative=True)
db_methods.refresh_all_threads(conservative=True, use_onion_link=onion)
db_methods.list_threads(interesting_only=True)
elif command == 'ro':
elif (command == 'ro') or (command == 'tro'):
db_methods.check_database()
db_methods.refresh_all_threads(conservative=True)
db_methods.refresh_all_threads(conservative=True, use_onion_link=onion)
db_methods.open_threads(only_threads_with_new_replies=True)
elif command == 'o':
elif (command == 'o') and (command == 'to'):
db_methods.check_database()
db_methods.open_threads(only_threads_with_new_replies=True)
elif command == 'oa':
db_methods.check_database()
db_methods.open_threads(only_threads_with_new_replies=False)
elif command == 'to':
db_methods.open_threads(only_threads_with_new_replies=True, use_onion_link=onion)
elif (command == 'oa') and (command == 'toa'):
db_methods.check_database()
db_methods.open_threads(only_threads_with_new_replies=True, use_onion_link=True)
db_methods.open_threads(only_threads_with_new_replies=False, use_onion_link=onion)
elif command == 'tor':
db_methods.check_database()
db_methods.open_threads(only_threads_with_new_replies=True, use_onion_link=True)

+ 8
- 6
lizard/db_methods.py View File

@@ -78,8 +78,10 @@ def add_thread_to_db(url):
last_reply_time=last_reply_time
)

dump_thread_html(board, thread_no)
dump_thread_files(board, thread_no, json)
onion = url.startswith('http://oxwugzccvk3dk6tj.onion')

dump_thread_html(board, thread_no, use_onion_link=onion)
dump_thread_files(board, thread_no, json, use_onion_link=onion)
else:
print("Thread didn't load, doing nothing.")

@@ -129,7 +131,7 @@ def list_threads(interesting_only=False):
print(s)


def refresh_all_threads(conservative=False):
def refresh_all_threads(conservative=False, use_onion_link=False):
print('Getting list of threads from database...')
threads = Thread.select().where(Thread.is_up)

@@ -143,7 +145,7 @@ def refresh_all_threads(conservative=False):

print('Refreshing {} threads...'.format(len(threads)))
for t in threads:
json = fetch_and_parse_thread_json(t.board, t.thread_no)
json = fetch_and_parse_thread_json(t.board, t.thread_no, use_onion_link=use_onion_link)
if json:
t.last_reply_time = time_of_last_reply(json)
if t.last_reply_time > t.last_refreshed:
@@ -162,8 +164,8 @@ def refresh_all_threads(conservative=False):
t.oldest_new_reply = get_oldest_reply(new_replies)
t.replies_to_anchored = len(replies_to_anchored)

dump_thread_html(t.board, t.thread_no)
dump_thread_files(t.board, t.thread_no, json)
dump_thread_html(t.board, t.thread_no, use_onion_link=use_onion_link)
dump_thread_files(t.board, t.thread_no, json, use_onion_link=use_onion_link)
else:
print("/{}/{} has 404'd.".format(t.board, t.thread_no))
t.is_up = 0

+ 33
- 20
lizard/web_methods.py View File

@@ -11,20 +11,20 @@ version = pkg_resources.require("lizard")[0].version
user_agent = 'Lizard v{}: https://git.teknik.io/abrax/lizard'.format(version)


def fetch_and_parse_thread_json(board, thread_no):
url = generate_thread_json_url(board, thread_no)
def fetch_and_parse_thread_json(board, thread_no, use_onion_link=False):
url = generate_thread_json_url(board, thread_no, use_onion_link=use_onion_link)
p = generate_json_path(board, thread_no)

if not download_file(url, p):
if not download_file(url, p, use_onion_link=use_onion_link):
return None
else:
return json.load(open(p))


def dump_thread_html(board, thread_no):
def dump_thread_html(board, thread_no, use_onion_link=False):
print('Downloading HTML for /{}/{}...'.format(board, thread_no))

url = generate_thread_html_url(board, thread_no)
url = generate_thread_html_url(board, thread_no, use_onion_link=use_onion_link)
p = generate_html_path(board, thread_no)

print('Downloading html to {}'.format(p))
@@ -32,7 +32,7 @@ def dump_thread_html(board, thread_no):
download_file(url, p)


def dump_thread_files(board, thread_no, thread_json):
def dump_thread_files(board, thread_no, thread_json, use_onion_link=False):
""" Downloads the files referenced in the supplied JSON. """

# Make a list of the files in the thread
@@ -47,13 +47,13 @@ def dump_thread_files(board, thread_no, thread_json):
normalized = normalize_filename(f)
print('\t{}'.format(normalized))

file_url = generate_file_url(f['hashed_name'])
file_url = generate_file_url(f['hashed_name'], use_onion_link=use_onion_link)
p = path_to_cached_file(board, thread_no, normalized)
if not download_file(file_url, p):
print('\t\tAttempt failed, trying alternate link.')

# Try alternate link
alternate_url = generate_alternate_file_url(board, f['hashed_name'])
alternate_url = generate_alternate_file_url(board, f['hashed_name'], use_onion_link=use_onion_link)
download_file(alternate_url, p)


@@ -81,34 +81,47 @@ def download_file(file_url, file_path):
"if that doesn't work submit an issue to the tracker.".format(r.status_code, file_url))


def generate_thread_json_url(board, thread_no):
url = 'https://8ch.net/{}/res/{}.json'.format(board, thread_no)
def thread_domain(use_onion_link=False):
if use_onion_link:
url = 'http://oxwugzccvk3dk6tj.onion'
else:
url = 'https://8ch.net'

return url


def generate_thread_json_url(board, thread_no, use_onion_link=False):
url = '{}/{}/res/{}.json'.format(thread_domain(use_onion_link), board, thread_no)
return url


def generate_thread_html_url(board, thread_no, use_onion_link=False):
url = '{}/{}/res/{}.html'.format(thread_domain(use_onion_link), board, thread_no)
return url


def generate_file_url(filename, use_onion_link=False):
if use_onion_link:
url = 'http://oxwugzccvk3dk6tj.onion/{}/res/{}.html'.format(board, thread_no)
url = 'http://oxwugzccvk3dk6tj.onion/file_store/{}'.format(filename)
else:
url = 'https://8ch.net/{}/res/{}.html'.format(board, thread_no)
url = 'https://media.8ch.net/file_store/{}'.format(filename)

return url


def generate_file_url(filename):
url = 'https://media.8ch.net/file_store/{}'.format(filename)
return url
def generate_alternate_file_url(board, filename, use_onion_link=False):
"""Some images, like the OP pic of the /tech/ sticky, use an alternate media2 URL. """

if use_onion_link:
url = 'http://oxwugzccvk3dk6tj.onion/{}/res/{}.html'.format(board, filename)
else:
url = 'https://media2.8ch.net/{}/src/{}'.format(board, filename)

def generate_alternate_file_url(board, filename):
"""Some images, like the OP pic of the /tech/ sticky, use an alternate media2 URL. """
url = 'https://media2.8ch.net/{}/src/{}'.format(board, filename)
return url


def parse_url(url):
"""Extracts the board name and thread no from a URL.
"""
""" Extracts the board name and thread no from a URL. """
parts = url.split('#', 1)

board, thread_no = re.findall('(\w+)\/res\/(\d+)', parts[0])[0]

Loading…
Cancel
Save