Browse Source

Fixes #40 and fixes #39

master
Abrax 6 years ago
parent
commit
d7ff84117f
  1. 53
      lizard/__init__.py
  2. 14
      lizard/db_methods.py
  3. 53
      lizard/web_methods.py

53
lizard/__init__.py

@ -28,23 +28,23 @@ Usage: lizard [command] [options] @@ -28,23 +28,23 @@ Usage: lizard [command] [options]
Available commands:
la - Lists all threads in database. Legend: ! - new replies, x - 404'd, Y - >(You).
l - Lists only threads that have activity.
r - Refresh threads which are likely to receive new replies (to reduce unnecessary network activity).
ra - Refresh all threads and update database.
rl - Conservative refresh and list threads.
ro - Conservative refresh and open threads with new replies in browsed.
r - Refresh threads which are likely to receive new replies (to reduce unnecessary network activity). Use tr for
onion link.
ra - Refresh all threads and update database. Use tra for onion link.
rl - Conservative refresh and list threads. Use trl for onion link.
ro - Conservative refresh and open threads with new replies in browsed. Use tro for onion link.
o - Open threads with new replies in the default browser. Will also update the "new replies"
field accordingly.
oa - Open all threads.
to - Open threads with new replies, using the onion URL. Use this if you are using Tor.
field accordingly. Use to for onion link.
oa - Open all threads. Use toa for onion link.
tor - Same as to but also opens the Tor CAPTCHA page for convenience.
p - Purge (remove) 404'd threads from database.
b - Backup current database.
e - Export list of threads. This will create a list of lizard add commands for all threads currently in the
database. Useful for migrating between database versions.
database. Useful for migrating between database versions.
a - Add thread to database. Requires option (URL of the thread). Will also attempt to download
the thread.
the thread. Supports onion links.
d - Remove all instances of a thread (board & number) from the database.
Example: "lizard d b 123" will remove thread >>>/b/123
Example: "lizard d b 123" will remove thread >>>/b/123
create - Create new database.
""".format(version))
exit()
@ -52,8 +52,12 @@ Available commands: @@ -52,8 +52,12 @@ Available commands:
def main():
n = len(sys.argv)
if n < 2: help_info()
if n < 2:
help_info()
command = sys.argv[1]
onion = (command[0] == 't') # The tor version of each command starts with t
if n == 2:
if command == 'la':
db_methods.check_database()
@ -61,29 +65,26 @@ def main(): @@ -61,29 +65,26 @@ def main():
elif command == 'l':
db_methods.check_database()
db_methods.list_threads(interesting_only=True)
elif command == 'ra':
elif (command == 'r') or (command == 'tr'):
db_methods.check_database()
db_methods.refresh_all_threads()
elif command == 'r':
db_methods.refresh_all_threads(conservative=True, use_onion_link=onion)
elif (command == 'ra') or (command == 'tra'):
db_methods.check_database()
db_methods.refresh_all_threads(conservative=True)
elif command == 'rl':
db_methods.refresh_all_threads(use_onion_link=onion)
elif (command == 'rl') or (command == 'trl'):
db_methods.check_database()
db_methods.refresh_all_threads(conservative=True)
db_methods.refresh_all_threads(conservative=True, use_onion_link=onion)
db_methods.list_threads(interesting_only=True)
elif command == 'ro':
elif (command == 'ro') or (command == 'tro'):
db_methods.check_database()
db_methods.refresh_all_threads(conservative=True)
db_methods.refresh_all_threads(conservative=True, use_onion_link=onion)
db_methods.open_threads(only_threads_with_new_replies=True)
elif command == 'o':
elif (command == 'o') and (command == 'to'):
db_methods.check_database()
db_methods.open_threads(only_threads_with_new_replies=True)
elif command == 'oa':
db_methods.check_database()
db_methods.open_threads(only_threads_with_new_replies=False)
elif command == 'to':
db_methods.open_threads(only_threads_with_new_replies=True, use_onion_link=onion)
elif (command == 'oa') and (command == 'toa'):
db_methods.check_database()
db_methods.open_threads(only_threads_with_new_replies=True, use_onion_link=True)
db_methods.open_threads(only_threads_with_new_replies=False, use_onion_link=onion)
elif command == 'tor':
db_methods.check_database()
db_methods.open_threads(only_threads_with_new_replies=True, use_onion_link=True)

14
lizard/db_methods.py

@ -78,8 +78,10 @@ def add_thread_to_db(url): @@ -78,8 +78,10 @@ def add_thread_to_db(url):
last_reply_time=last_reply_time
)
dump_thread_html(board, thread_no)
dump_thread_files(board, thread_no, json)
onion = url.startswith('http://oxwugzccvk3dk6tj.onion')
dump_thread_html(board, thread_no, use_onion_link=onion)
dump_thread_files(board, thread_no, json, use_onion_link=onion)
else:
print("Thread didn't load, doing nothing.")
@ -129,7 +131,7 @@ def list_threads(interesting_only=False): @@ -129,7 +131,7 @@ def list_threads(interesting_only=False):
print(s)
def refresh_all_threads(conservative=False):
def refresh_all_threads(conservative=False, use_onion_link=False):
print('Getting list of threads from database...')
threads = Thread.select().where(Thread.is_up)
@ -143,7 +145,7 @@ def refresh_all_threads(conservative=False): @@ -143,7 +145,7 @@ def refresh_all_threads(conservative=False):
print('Refreshing {} threads...'.format(len(threads)))
for t in threads:
json = fetch_and_parse_thread_json(t.board, t.thread_no)
json = fetch_and_parse_thread_json(t.board, t.thread_no, use_onion_link=use_onion_link)
if json:
t.last_reply_time = time_of_last_reply(json)
if t.last_reply_time > t.last_refreshed:
@ -162,8 +164,8 @@ def refresh_all_threads(conservative=False): @@ -162,8 +164,8 @@ def refresh_all_threads(conservative=False):
t.oldest_new_reply = get_oldest_reply(new_replies)
t.replies_to_anchored = len(replies_to_anchored)
dump_thread_html(t.board, t.thread_no)
dump_thread_files(t.board, t.thread_no, json)
dump_thread_html(t.board, t.thread_no, use_onion_link=use_onion_link)
dump_thread_files(t.board, t.thread_no, json, use_onion_link=use_onion_link)
else:
print("/{}/{} has 404'd.".format(t.board, t.thread_no))
t.is_up = 0

53
lizard/web_methods.py

@ -11,20 +11,20 @@ version = pkg_resources.require("lizard")[0].version @@ -11,20 +11,20 @@ version = pkg_resources.require("lizard")[0].version
user_agent = 'Lizard v{}: https://git.teknik.io/abrax/lizard'.format(version)
def fetch_and_parse_thread_json(board, thread_no):
url = generate_thread_json_url(board, thread_no)
def fetch_and_parse_thread_json(board, thread_no, use_onion_link=False):
url = generate_thread_json_url(board, thread_no, use_onion_link=use_onion_link)
p = generate_json_path(board, thread_no)
if not download_file(url, p):
if not download_file(url, p, use_onion_link=use_onion_link):
return None
else:
return json.load(open(p))
def dump_thread_html(board, thread_no):
def dump_thread_html(board, thread_no, use_onion_link=False):
print('Downloading HTML for /{}/{}...'.format(board, thread_no))
url = generate_thread_html_url(board, thread_no)
url = generate_thread_html_url(board, thread_no, use_onion_link=use_onion_link)
p = generate_html_path(board, thread_no)
print('Downloading html to {}'.format(p))
@ -32,7 +32,7 @@ def dump_thread_html(board, thread_no): @@ -32,7 +32,7 @@ def dump_thread_html(board, thread_no):
download_file(url, p)
def dump_thread_files(board, thread_no, thread_json):
def dump_thread_files(board, thread_no, thread_json, use_onion_link=False):
""" Downloads the files referenced in the supplied JSON. """
# Make a list of the files in the thread
@ -47,13 +47,13 @@ def dump_thread_files(board, thread_no, thread_json): @@ -47,13 +47,13 @@ def dump_thread_files(board, thread_no, thread_json):
normalized = normalize_filename(f)
print('\t{}'.format(normalized))
file_url = generate_file_url(f['hashed_name'])
file_url = generate_file_url(f['hashed_name'], use_onion_link=use_onion_link)
p = path_to_cached_file(board, thread_no, normalized)
if not download_file(file_url, p):
print('\t\tAttempt failed, trying alternate link.')
# Try alternate link
alternate_url = generate_alternate_file_url(board, f['hashed_name'])
alternate_url = generate_alternate_file_url(board, f['hashed_name'], use_onion_link=use_onion_link)
download_file(alternate_url, p)
@ -81,34 +81,47 @@ def download_file(file_url, file_path): @@ -81,34 +81,47 @@ def download_file(file_url, file_path):
"if that doesn't work submit an issue to the tracker.".format(r.status_code, file_url))
def generate_thread_json_url(board, thread_no):
url = 'https://8ch.net/{}/res/{}.json'.format(board, thread_no)
def thread_domain(use_onion_link=False):
if use_onion_link:
url = 'http://oxwugzccvk3dk6tj.onion'
else:
url = 'https://8ch.net'
return url
def generate_thread_json_url(board, thread_no, use_onion_link=False):
url = '{}/{}/res/{}.json'.format(thread_domain(use_onion_link), board, thread_no)
return url
def generate_thread_html_url(board, thread_no, use_onion_link=False):
url = '{}/{}/res/{}.html'.format(thread_domain(use_onion_link), board, thread_no)
return url
def generate_file_url(filename, use_onion_link=False):
if use_onion_link:
url = 'http://oxwugzccvk3dk6tj.onion/{}/res/{}.html'.format(board, thread_no)
url = 'http://oxwugzccvk3dk6tj.onion/file_store/{}'.format(filename)
else:
url = 'https://8ch.net/{}/res/{}.html'.format(board, thread_no)
url = 'https://media.8ch.net/file_store/{}'.format(filename)
return url
def generate_file_url(filename):
url = 'https://media.8ch.net/file_store/{}'.format(filename)
return url
def generate_alternate_file_url(board, filename, use_onion_link=False):
"""Some images, like the OP pic of the /tech/ sticky, use an alternate media2 URL. """
if use_onion_link:
url = 'http://oxwugzccvk3dk6tj.onion/{}/res/{}.html'.format(board, filename)
else:
url = 'https://media2.8ch.net/{}/src/{}'.format(board, filename)
def generate_alternate_file_url(board, filename):
"""Some images, like the OP pic of the /tech/ sticky, use an alternate media2 URL. """
url = 'https://media2.8ch.net/{}/src/{}'.format(board, filename)
return url
def parse_url(url):
"""Extracts the board name and thread no from a URL.
"""
""" Extracts the board name and thread no from a URL. """
parts = url.split('#', 1)
board, thread_no = re.findall('(\w+)\/res\/(\d+)', parts[0])[0]

Loading…
Cancel
Save