diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..4da0770 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +ignore=E501 \ No newline at end of file diff --git a/.gitignore b/.gitignore index d09ecc7..82adea9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ successful.txt successful2.txt img2txt.py tmpls/python.tmpl +.stances.json +img2txt.py \ No newline at end of file diff --git a/scrape.py b/scrape.py index a8e1417..881295d 100644 --- a/scrape.py +++ b/scrape.py @@ -1,6 +1,7 @@ from asyncio import run, sleep import json from aiohttp import ClientSession +import aiofiles from pathlib import Path from yarl import URL from enum import Enum @@ -32,8 +33,6 @@ class StanceType(Enum): self.display_name = self._name_.lower() -# https://maplestory.io/api/GMS/231/mob/ -# https://maplestory.io/api/GMS/231/mob/100004/ images = Path("./images/sorted/Mob/") collection = set() base_url = URL("https://maplestory.io/api/") @@ -53,10 +52,7 @@ with open("failed.txt", "r") as f: async def pre_list(): async with ClientSession() as session: for mob_id in to_grab: - async with session.request( - "GET", (base_url / f"GMS/231/mob/{mob_id}/") - ) as resp: - print(resp.status, mob_id) + async with session.request("GET", (base_url / "GMS" / "")) as resp: if resp.status != 200: continue mob_data = await resp.json() @@ -64,7 +60,7 @@ async def pre_list(): for k, v in mob_data["framebooks"].items(): STANCES[mob_id][k] = v if len(STANCES) % 200 == 0: - await sleep(5) + await sleep(3) for mob_id, stance in STANCES.items(): mob_folder = images / f"{mob_id}" @@ -80,8 +76,6 @@ async def pre_list(): "GET", (base_url / f"GMS/231/mob/{mob_id}/render/{k}/{i+1}"), ) as resp: - # try: - print(resp.status, mob_id) if resp.status != 200: failed.append(mob_id) continue @@ -91,7 +85,8 @@ async def pre_list(): if not file.exists(): image_bytes = await resp.read() file.touch() - file.write_bytes(image_bytes) + async with aiofiles.open(file, "wb") as imgfl: + await imgfl.write(image_bytes) successful.append(mob_id) @@ -129,31 +124,19 @@ async def main(): "GET", base_url / f"GMS/231/mob/{mob['mob_id']}/render/{k}/{i+1}", ) as resp: - # try: if resp.status == 200: failed.append(mob["mob_id"]) continue image_bytes = await resp.read() file = stance_folder / f"{k}.{i+1}.png" file.touch() - file.write_bytes(image_bytes) + async with aiofiles.open(file, "wb") as imgfl: + await imgfl.write(image_bytes) successful.append(mob["mob_id"]) - # except Exception: - # failed.append(mob["mob_id"]) - - with open("successful.txt", "a+") as f: - f.write(json.dumps(successful, indent=4)) - with open("failed.txt", "a+") as f: - f.write(json.dumps(failed, indent=4)) - try: run(pre_list()) finally: - with open("successful2.txt", "a+") as f: - f.write(json.dumps(successful, indent=4)) - with open("failed2.txt", "a+") as f: - f.write(json.dumps(failed, indent=4)) with open("stances.json", "a+") as f: f.write(json.dumps(STANCES)) diff --git a/server.py b/server.py index 7cddf62..55c42f5 100644 --- a/server.py +++ b/server.py @@ -1,25 +1,20 @@ from pathlib import Path -from img2txt import gen from random import choice from re import compile from asyncio import get_running_loop, Event, sleep from io import BytesIO from fastapi import FastAPI, Depends, Response -from fastapi.params import Path as APIPath -from fastapi.responses import HTMLResponse, FileResponse +from fastapi.responses import HTMLResponse from starlette.concurrency import run_in_threadpool from starlette.requests import Request from uvicorn import run from PIL import Image as _Image, UnidentifiedImageError from PIL.Image import Image +from PIL.PyAccess import _PyAccess32_4 from jinja2 import Environment, select_autoescape, FileSystemLoader -from aiohttp import ClientSession -from io import TextIOWrapper, BytesIO -from rich import print - -out = TextIOWrapper(buffer=BytesIO()) -print(file=out) +from aiohttp import ClientSession, ContentTypeError +import aiofiles try: @@ -77,7 +72,7 @@ class Mob: def render_html(img: Image, frame_id=None, link_id=None, scale=1): - pixels = img.load() + pixels: _PyAccess32_4 = img.load() # type: ignore width, height = img.size nodes = [] x, y = 0, 0 @@ -87,9 +82,8 @@ def render_html(img: Image, frame_id=None, link_id=None, scale=1): start_val = pixels[x_start, y] for xx in range(x_start + 1, width): - if ( - pixels[xx, y] == start_val - and pixels[xx, y][3] / 255 == start_val[3] / 255 + if (pixels[xx, y] == start_val) and ( + pixels[xx, y][3] / 255 == start_val[3] / 255 ): i += 1 else: @@ -127,7 +121,7 @@ def render_html(img: Image, frame_id=None, link_id=None, scale=1): return f'' -def render_page( +async def render_page( folder_path: Path, mob_id: None | int = None, stance: None | str = None ): if not mob_id: @@ -136,7 +130,8 @@ def render_page( stance = folder_path.name if (svg_file := (svgs / f"{stance}" / f"{mob_id}.svg")).exists(): - return svg_file.read_text() + async with aiofiles.open(svg_file, "r") as svg: + return await svg.read() frames = [] images = {} @@ -149,7 +144,8 @@ def render_page( v = int(version.group(1)) else: v = i - img = _Image.open(file) + async with aiofiles.open(file, "rb") as img_: + img = _Image.open(BytesIO(await img_.read())) widths.append(img.size[0]) heights.append(img.size[1]) images[v] = img @@ -196,19 +192,19 @@ def render_page( """ - svg_file.write_text(svg_cont) + async with aiofiles.open(svg_file, "w") as svg: + await svg.write(svg_cont) current.setdefault(mob_id, {}) current[mob_id][stance] = svg_file return svg_cont -def get_rand_page(): +async def get_rand_page(): while True: mob_id = int(choice(available_keys)) stance = choice(list(available[mob_id].keys())) @@ -216,7 +212,9 @@ def get_rand_page(): if not page_file: try: for f in available[mob_id][stance].glob("*.png"): - _Image.open(f) + async with aiofiles.open(f, "rb+") as img: + byt = BytesIO(await img.read()) + _Image.open(byt) break except UnidentifiedImageError: available[mob_id][stance].unlink() @@ -224,7 +222,7 @@ def get_rand_page(): continue return template.render( - frame=render_page( + frame=await render_page( available[mob_id][stance], mob_id=mob_id, stance=stance ), mob_id=mob_id, @@ -232,7 +230,8 @@ def get_rand_page(): ) break - return page_file.read_text() + async with aiofiles.open(page_file, "r") as ret_page: + return await ret_page.read() async def get_specific_mob(mob_id: int = 0, stance: str | None = None): @@ -241,14 +240,15 @@ async def get_specific_mob(mob_id: int = 0, stance: str | None = None): possibly_page = current.get(mob_id, {}).get(stance, None) if possibly_page and possibly_page.exists(): - return template.render( - frame=possibly_page.read_text(), mob_id=mob_id, stance=stance - ) + async with aiofiles.open(possibly_page, "r") as cont: + return template.render( + frame=await cont.read(), mob_id=mob_id, stance=stance + ) possibly_exists = available.get(mob_id, {}).get(stance, None) if possibly_exists and possibly_exists.exists(): return template.render( - frame=render_page(possibly_exists, mob_id=mob_id, stance=stance), + frame=await render_page(possibly_exists, mob_id=mob_id, stance=stance), mob_id=mob_id, stance=stance, ) @@ -290,30 +290,31 @@ async def export_mob_svg( mob_id = int(mob_id) possibly_page = current.get(mob_id, {}).get(stance, None) if possibly_page and possibly_page.exists(): - return Response( - possibly_page.read_text(), headers={"Content-Type": "image/svg+xml"} - ) + async with aiofiles.open(possibly_page, "r") as page: + return Response( + await page.read(), headers={"Content-Type": "image/svg+xml"} + ) possibly_exists = available.get(mob_id, {}).get(stance, None) if possibly_exists and possibly_exists.exists(): return Response( - render_page(possibly_exists, mob_id=mob_id, stance=stance), + await render_page(possibly_exists, mob_id=mob_id, stance=stance), headers={"Content-Type": "image/svg+xml"}, ) return HTMLResponse("No data") -async def gay(arg1: Request, arg2): +async def sumshit(arg1: Request, arg2): return Response( content=( "
" "" - "is this what they call friendship" + "but thats *my* junk