Refactored to make fetching and downloading posts seperate
This commit is contained in:
parent
8fb4da20ad
commit
bfaaa50117
@ -24,24 +24,33 @@ class downloader:
|
||||
username = None
|
||||
password = None
|
||||
tmp = None
|
||||
banned = None
|
||||
unallowed_extensions = (".zip",)
|
||||
|
||||
def __init__(self, banned = tuple(), username=None, password=None, tmp="/tmp"):
|
||||
def __init__(self, username=None, password=None, tmp="/tmp"):
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.tmp = tmp
|
||||
self.banned = banned
|
||||
|
||||
|
||||
def is_banned(self, tag_list):
|
||||
for tag in self.banned:
|
||||
if tag in tag_list:
|
||||
return True
|
||||
return False
|
||||
def download_post(self, post):
|
||||
file_url = post["file_url"]
|
||||
full_path = post["full_path"]
|
||||
|
||||
remote_image = requests.get(file_url)
|
||||
|
||||
if remote_image.status_code != 200:
|
||||
print("Remote image request returned:", remote_image.status_code)
|
||||
return None
|
||||
|
||||
for d in full_path:
|
||||
with open(d, "wb") as f:
|
||||
f.write(remote_image.content)
|
||||
|
||||
return post
|
||||
|
||||
|
||||
def download(self, profile):
|
||||
# Search ratings: s=safe, e=explicit
|
||||
def fetch_post(self, profile):
|
||||
# Search ratings: s=safe, e=nsfw
|
||||
# base_url = "https://danbooru.donmai.us/posts.json?random=true&tags={}&rating=e&limit=1"
|
||||
tags = profile["tags"]
|
||||
search_url = "https://danbooru.donmai.us/posts.json?random=true&limit=1"
|
||||
@ -50,7 +59,6 @@ class downloader:
|
||||
search_url = "{}&tags={}".format(search_url, search_tags)
|
||||
|
||||
search_request = None
|
||||
while True:
|
||||
if self.username and self.password:
|
||||
search_request = requests.get(search_url,
|
||||
auth=(self.username, self.password)
|
||||
@ -60,40 +68,25 @@ class downloader:
|
||||
|
||||
if search_request.status_code != 200:
|
||||
print("Search request returned:", search_request.status_code)
|
||||
continue
|
||||
elif "large_file_url" not in search_request.json()[0]:
|
||||
continue
|
||||
elif "tag_string" not in search_request.json()[0]:
|
||||
continue
|
||||
elif "tag_string_general" not in search_request.json()[0]:
|
||||
continue
|
||||
elif self.is_banned(search_request.json()[0]["tag_string"]):
|
||||
print("Banned Tag1:", search_request.json()[0]["tag_string"])
|
||||
continue
|
||||
elif self.is_banned(search_request.json()[0]["tag_string_general"]):
|
||||
print("Banned Tag2",search_request.json()[0]["tag_string_general"])
|
||||
continue
|
||||
break
|
||||
|
||||
import pprint
|
||||
pprint.pprint(search_request.json()[0]["tag_string"])
|
||||
|
||||
large_file_url = search_request.json()[0]["file_url"]
|
||||
|
||||
explicit = search_request.json()[0]["rating"]
|
||||
explicit = get_most_sever_rating(explicit)
|
||||
|
||||
remote_image = requests.get(large_file_url)
|
||||
|
||||
if remote_image.status_code != 200:
|
||||
print("Remote image request returned:", remote_image.status_code)
|
||||
return None
|
||||
response = search_request.json()[0]
|
||||
tag_response = []
|
||||
if "file_url" not in response:
|
||||
print("file_url is not in response")
|
||||
return None
|
||||
|
||||
basename = large_file_url.rsplit("/", 1)[1]
|
||||
full_path = os.path.join(self.tmp, basename)
|
||||
# Aggregate Tags
|
||||
for tag_type in "tag_string", "tag_string_general":
|
||||
if tag_type in response:
|
||||
tag_response.append(response[tag_type].strip())
|
||||
|
||||
with open(full_path, "wb") as f:
|
||||
f.write(remote_image.content)
|
||||
nsfw = search_request.json()[0]["rating"]
|
||||
nsfw = get_most_sever_rating(nsfw)
|
||||
|
||||
file_url = response["file_url"]
|
||||
|
||||
basename = file_url.rsplit("/", 1)[1]
|
||||
full_path = os.path.join(self.tmp, basename)
|
||||
|
||||
r = {
|
||||
# Add profile to dictioanry
|
||||
@ -101,13 +94,14 @@ class downloader:
|
||||
"backend": profile["backend"],
|
||||
"tags": profile["tags"],
|
||||
"message": profile["message"],
|
||||
"message_nsfw": profile["message_nsfw"],
|
||||
|
||||
# Query results
|
||||
"search_url": search_url,
|
||||
"large_file_url": large_file_url,
|
||||
"file_url": file_url,
|
||||
"full_path": [full_path],
|
||||
"explicit": explicit,
|
||||
"tag_response": " ".join(tag_response),
|
||||
"nsfw": nsfw
|
||||
}
|
||||
|
||||
|
||||
return r
|
||||
|
Reference in New Issue
Block a user