weeee have pixels

This commit is contained in:
rooba 2022-06-06 03:53:32 +00:00
commit 067e04f6db
7 changed files with 790 additions and 0 deletions

11
.gitignore vendored Normal file
View File

@ -0,0 +1,11 @@
**/svgs
**/pages
**/images
**/__pycache__
failed.txt
failed2.txt
run.sh
successful.txt
successful2.txt
img2txt.py
tmpls/python.tmpl

159
scrape.py Normal file
View File

@ -0,0 +1,159 @@
from asyncio import run, sleep
import json
from aiohttp import ClientSession
from pathlib import Path
from yarl import URL
from enum import Enum
class FolderType(Enum):
MOB = 1
NPC = 2
class StanceType(Enum):
ALL = (0, "")
STAND = (1, "stand")
HIT = (2, "hit1")
JUMP = (3, "jump")
MOVE = (4, "move")
FLY = (MOVE, "fly")
ATTACK = (5, "attack1")
DIE = (6, "die1")
ATTACK_BALL = (7, "attack.info.ball")
ATTACK_HIT = (8, "attack.info.hit")
ATTACK_EFFECT = (9, "attack.info.effect")
def __init__(self, value, image_naming=None):
self._value_ = image_naming
if not image_naming:
self.display_name = [t.display_name for t in self.__class__ if t != self]
else:
self.display_name = self._name_.lower()
# https://maplestory.io/api/GMS/231/mob/
# https://maplestory.io/api/GMS/231/mob/100004/
images = Path("./images/sorted/Mob/")
collection = set()
base_url = URL("https://maplestory.io/api/")
desired_stances = [StanceType.STAND, StanceType.ATTACK, StanceType.MOVE]
failed = []
successful = []
STANCES = {}
for folder in images.glob("./*/"):
collection.add(int(folder.name))
to_grab = []
with open("failed.txt", "r") as f:
to_grab = json.loads(f.read())
async def pre_list():
async with ClientSession() as session:
for mob_id in to_grab:
async with session.request(
"GET", (base_url / f"GMS/231/mob/{mob_id}/")
) as resp:
print(resp.status, mob_id)
if resp.status != 200:
continue
mob_data = await resp.json()
STANCES[mob_id] = {}
for k, v in mob_data["framebooks"].items():
STANCES[mob_id][k] = v
if len(STANCES) % 200 == 0:
await sleep(5)
for mob_id, stance in STANCES.items():
mob_folder = images / f"{mob_id}"
if not mob_folder.exists():
mob_folder.mkdir(exist_ok=False)
for k, v in stance.items():
stance_folder = mob_folder / f"{k}"
if not stance_folder.exists():
stance_folder.mkdir(exist_ok=False)
for i in range(v):
async with session.request(
"GET",
(base_url / f"GMS/231/mob/{mob_id}/render/{k}/{i+1}"),
) as resp:
# try:
print(resp.status, mob_id)
if resp.status != 200:
failed.append(mob_id)
continue
file = stance_folder / f"{k}.{i+1}.png"
if not file.exists():
image_bytes = await resp.read()
file.touch()
file.write_bytes(image_bytes)
successful.append(mob_id)
async def main():
mobs = set()
filtered_mobs = []
async with ClientSession() as session:
async with session.request("GET", base_url / "GMS/231/mob/") as resp:
all_mobs = await resp.json()
for mob in all_mobs:
if int(mob["id"]) in collection:
continue
mobs.add(int(mob["id"]))
for mob_id in mobs:
async with session.request(
"GET", base_url / f"GMS/231/mob/{mob_id}/"
) as resp:
mob_data = await resp.json()
cleaned = {"mob_id": mob_id, "frames": {}}
for k, v in mob_data.get("framebooks", {}).items():
if StanceType(k) in desired_stances:
cleaned["frames"][k] = mob_data["framebooks"][k]
filtered_mobs.append(cleaned)
for mob in filtered_mobs:
mob_folder = images / f"{mob['mob_id']}"
mob_folder.mkdir(exist_ok=False)
for k, v in mob["frames"].items():
stance_folder = mob_folder / f"{k}"
stance_folder.mkdir(exist_ok=False)
for i in range(v):
async with session.request(
"GET",
base_url / f"GMS/231/mob/{mob['mob_id']}/render/{k}/{i+1}",
) as resp:
# try:
if resp.status == 200:
failed.append(mob["mob_id"])
continue
image_bytes = await resp.read()
file = stance_folder / f"{k}.{i+1}.png"
file.touch()
file.write_bytes(image_bytes)
successful.append(mob["mob_id"])
# except Exception:
# failed.append(mob["mob_id"])
with open("successful.txt", "a+") as f:
f.write(json.dumps(successful, indent=4))
with open("failed.txt", "a+") as f:
f.write(json.dumps(failed, indent=4))
try:
run(pre_list())
finally:
with open("successful2.txt", "a+") as f:
f.write(json.dumps(successful, indent=4))
with open("failed2.txt", "a+") as f:
f.write(json.dumps(failed, indent=4))
with open("stances.json", "a+") as f:
f.write(json.dumps(STANCES))

383
server.py Normal file
View File

@ -0,0 +1,383 @@
from pathlib import Path
from img2txt import gen
from random import choice
from re import compile
from asyncio import get_running_loop, Event, sleep
from io import BytesIO
from fastapi import FastAPI, Depends, Response
from fastapi.params import Path as APIPath
from fastapi.responses import HTMLResponse, FileResponse
from starlette.concurrency import run_in_threadpool
from starlette.requests import Request
from uvicorn import run
from PIL import Image as _Image, UnidentifiedImageError
from PIL.Image import Image
from jinja2 import Environment, select_autoescape, FileSystemLoader
from aiohttp import ClientSession
from io import TextIOWrapper, BytesIO
from rich import print
out = TextIOWrapper(buffer=BytesIO())
print(file=out)
try:
from uvloop import install
install()
except ImportError:
...
env = Environment(
loader=FileSystemLoader("./tmpls/"),
autoescape=select_autoescape(),
trim_blocks=True,
)
DIGITS = compile(r"^[0-9]+$")
VERSION = compile(r"(?=\w\d?\.(\d+)\.png)")
images = Path("/media/secondary/http/images/sorted/Mob/")
pages = Path("/media/secondary/http/pages/")
svgs = Path("/media/secondary/http/svgs/")
page_paths: list[Path] = []
template = env.get_template("page.tmpl")
py_page = env.get_template("python.tmpl")
wsgi_app = FastAPI(title="ra.tcp.direct", description="ra.tcp.direct")
available = {}
current = {}
mob_data = {}
CACHING = Event()
for mob_images in images.glob("./*/*/"):
stance = mob_images.name
mob_id = int(mob_images.parent.name)
available.setdefault(mob_id, {})
available[mob_id][stance] = mob_images
for mob_stance in svgs.glob("./*/*.svg"):
mob_id = mob_stance.name.removesuffix(".svg")
stance = mob_stance.parent.name
current.setdefault(mob_id, {})
current[mob_id][stance] = mob_stance
available_keys = list(available.keys())
class Mob:
__slots__ = "id", "name", "stance"
def __init__(self, _id=0, name="", stance=None):
self.id = _id
self.name = name
if stance:
if len(stance) > 0:
self.stance = stance
def render_html(img: Image, frame_id=None, link_id=None, scale=1):
pixels = img.load()
width, height = img.size
nodes = []
x, y = 0, 0
def count_x(x_start, y):
i = x_start
start_val = pixels[x_start, y]
for xx in range(x_start + 1, width):
if (
pixels[xx, y] == start_val
and pixels[xx, y][3] / 255 == start_val[3] / 255
):
i += 1
else:
break
return i - x_start
while y < height:
if y >= height - 1:
break
len_x = count_x(x, y)
color = pixels[x, y]
if (color[0], color[1], color[2], color[3]) != (0, 0, 0, 0.0):
nodes.append(
'<path style="fill: '
f'rgba({color[0]}, {color[1]}, {color[2]}, {color[3] / 255})" '
f'd="M{x*scale} {y*scale} H{(x+(len_x)+1)*scale} '
f'V{(y+1)*scale} L{x*scale} {(y+1)*scale} Z"/>'
)
x += len_x + 1
if x >= width:
x = 0
y += 1
if frame_id is not None:
begin = f"l{link_id}l.end" if frame_id != 0 else f"0s;l{link_id}l.end"
animate = f"""<animate attributeType="XML" attributeName="visibility"
from="visible" to="visible" id="l{frame_id}l"
begin="{begin}" dur="0.1s"/>"""
else:
animate = ""
visible = 'visibility="hidden"' if link_id is not None else ""
return f'<svg {visible} height="{height*scale}" width="{width*scale}" >{animate}{"".join(nodes)}</svg>'
def render_page(
folder_path: Path, mob_id: None | int = None, stance: None | str = None
):
if not mob_id:
mob_id = int(folder_path.parent.name)
if not stance:
stance = folder_path.name
if (svg_file := (svgs / f"{stance}" / f"{mob_id}.svg")).exists():
return svg_file.read_text()
frames = []
images = {}
widths = []
heights = []
scale = 1
for i, file in enumerate(folder_path.glob("./*.png")):
version = VERSION.search(file.name)
if version:
v = int(version.group(1))
else:
v = i
img = _Image.open(file)
widths.append(img.size[0])
heights.append(img.size[1])
images[v] = img
keys = sorted(images.keys(), key=lambda x: x)
images = [images[k] for k in keys]
maxes = (max(widths), max(heights))
for a, image in enumerate(images):
box = _Image.new("RGBA", maxes)
box.paste(image, (maxes[0] - image.size[0], maxes[1] - image.size[1]))
img_bytes = BytesIO()
box.save(img_bytes, "PNG")
if a < len(images) or a == 0:
lid = a - 1
else:
lid = 0
if a == 0:
lid = len(images) - 1
frame_id = a if len(images) > 1 else None
else:
frame_id = a
frames.append(
render_html(
box,
frame_id=frame_id,
link_id=lid if len(images) > 1 else None,
scale=scale,
)
)
if len(frames) < 1:
return ""
svg_folder = svgs / f"{stance}"
if not svg_folder.exists():
svg_folder.mkdir()
svg_file = svg_folder / f"{mob_id}.svg"
svg_file.touch()
svg_cont = f"""<?xml version="1.0" encoding="utf-8"?>
<!--MapleStory Mob-->
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg xmlns="http://www.w3.org/2000/svg"
viewbox="0 0 {maxes[0]*scale} {maxes[1]*scale}"
width="{maxes[0]*scale}"
height="{maxes[1]*scale}">
{"".join(frames)}
</svg>"""
svg_file.write_text(svg_cont)
current.setdefault(mob_id, {})
current[mob_id][stance] = svg_file
return svg_cont
def get_rand_page():
while True:
mob_id = int(choice(available_keys))
stance = choice(list(available[mob_id].keys()))
page_file = current.get(mob_id, {}).get(stance, None)
if not page_file:
try:
for f in available[mob_id][stance].glob("*.png"):
_Image.open(f)
break
except UnidentifiedImageError:
available[mob_id][stance].unlink()
available[mob_id].pop(stance)
continue
return template.render(
frame=render_page(
available[mob_id][stance], mob_id=mob_id, stance=stance
),
mob_id=mob_id,
stance=stance,
)
break
return page_file.read_text()
async def get_specific_mob(mob_id: int = 0, stance: str | None = None):
if not stance:
stance = choice(list(available.get(mob_id, [])))
possibly_page = current.get(mob_id, {}).get(stance, None)
if possibly_page and possibly_page.exists():
return template.render(
frame=possibly_page.read_text(), mob_id=mob_id, stance=stance
)
possibly_exists = available.get(mob_id, {}).get(stance, None)
if possibly_exists and possibly_exists.exists():
return template.render(
frame=render_page(possibly_exists, mob_id=mob_id, stance=stance),
mob_id=mob_id,
stance=stance,
)
return "Does not Exist"
@wsgi_app.get("/", status_code=200)
async def rand_page(page: str = Depends(get_rand_page)):
return HTMLResponse(page)
@wsgi_app.get("/mob/", status_code=200)
async def view_mobs():
if not mob_data:
await fill_cache()
mob_page = env.get_template("browse.tmpl")
return HTMLResponse(mob_page.render(mobs=mob_data))
@wsgi_app.get("/mob/{mob_id}/", status_code=200)
async def view_mob(mob_id: int):
data = await get_specific_mob(mob_id)
return HTMLResponse(data)
@wsgi_app.get("/mob/{mob_id}/{stance}", status_code=200)
async def view_mob_stance(mob_id: int, stance: str):
data = await get_specific_mob(mob_id, stance)
return HTMLResponse(data)
@wsgi_app.get("/export/mob/{mob_id:int}_{stance:path}.svg", status_code=200)
async def export_mob_svg(
mob_id: int,
stance: str,
):
mob_id = int(mob_id)
possibly_page = current.get(mob_id, {}).get(stance, None)
if possibly_page and possibly_page.exists():
return Response(
possibly_page.read_text(), headers={"Content-Type": "image/svg+xml"}
)
possibly_exists = available.get(mob_id, {}).get(stance, None)
if possibly_exists and possibly_exists.exists():
return Response(
render_page(possibly_exists, mob_id=mob_id, stance=stance),
headers={"Content-Type": "image/svg+xml"},
)
return HTMLResponse("No data")
async def gay(arg1: Request, arg2):
return Response(
content=(
"<body style='display: grid;background: #151515; margin: 0px;"
"color: #C4C4C4;height: 100%;align-items: center;'>"
"<pre style='text-align: center; display:inline;'>"
"is this what they call friendship</pre></body>"
),
status_code=451,
headers={"you-look": "good today"},
)
async def update_lib(mob_data):
i = 0
async with ClientSession() as session:
for mob_id in mob_data:
if not available.get(int(mob_id), {}):
available.setdefault(int(mob_id), {})
async with session.request(
"GET", f"https://maplestory.io/api/GMS/232/mob/{mob_id}/"
) as resp2:
try:
jsn = await resp2.json()
except:
continue
for frm_nm, frm_ct in jsn.get("framebooks", {}).items():
if i > 20:
await sleep(10)
i = 0
img_fld = images / f"{mob_id}" / ""
if not img_fld.exists():
img_fld.mkdir()
stnc_fld = img_fld / f"{frm_nm}" / ""
if not stnc_fld.exists():
stnc_fld.mkdir()
for i in range(frm_ct):
img_fle = stnc_fld / f"{frm_nm}.{i+1}.png"
if not img_fle.exists():
img_fle.touch()
async with session.request(
"GET",
f"https://maplestory.io/api/GMS/232/mob/{mob_id}/render/{frm_nm}/{i+1}",
) as resp3:
dat = await resp3.read()
img_fle.write_bytes(dat)
available.setdefault(int(mob_id), {})
available[mob_id].setdefault(frm_nm, {})
available[mob_id][frm_nm] = stnc_fld
async def fill_cache():
async with ClientSession() as session:
async with session.request(
"GET", "https://maplestory.io/api/GMS/232/mob/"
) as resp:
data = await resp.json()
if not CACHING.is_set():
get_running_loop().create_task(update_lib(mob_data))
CACHING.set()
for mob in data:
mob_data[mob["id"]] = Mob(
_id=mob["id"],
name=mob["name"],
stance=available.get(int(mob["id"]), {}),
)
async def app():
await run_in_threadpool(lambda: run("server:wsgi_app"))
wsgi_app.add_exception_handler(404, handler=gay)

1
stances.json Normal file

File diff suppressed because one or more lines are too long

63
tmpls/browse.tmpl Normal file
View File

@ -0,0 +1,63 @@
{% extends 'head.tmpl' %}
{% block content %}
<style>
.hello {
position: relative;
display: grid;
height: 100%;
align-items: center;
}
.top {
display: inline-flex;
align-items: top;
display: inline-flex;
justify-content: center;
}
body {
background-color:#101010;
color:#948DB8;
font-family:monospace;
text-align:center;
justify-content: center;
display: flex;
}
.browse {
display: grid;
width: 50%;
}
.mob {
padding: 10px;
display: grid;
}
.mob-main-link {
color: #81C17F;
text-decoration: none;
}
.mob-title-no-data {
color: #C1857F;
}
.stance {
text-decoration: none;
color: white;
}
</style>
<div class="browse">
{% for mob_id, mob in mobs.items() -%}
{% if mob.stance %}
<div class="mob">
<span class="mob-title"><a class="mob-main-link" href="/mob/{{ mob.id }}">{{ mob.name }}</a></span>
{% for stance, frames in mob.stance.items() %}
<span><a class="stance" href="/mob/{{ mob.id }}/{{ stance }}">{{ stance }}</a></span>
{% endfor %}
</div>
{% else %}
<div class="mob">
<span class="mob-title-no-data"><a href="https://maplestory.io/api/GMS/232/mob/{{ mob.id }}/">{{ mob.name }} (No data available)</a></span>
</div>
{% endif %}
{%- endfor %}
</div>
</body>
</html>
{% endblock content %}

29
tmpls/head.tmpl Normal file
View File

@ -0,0 +1,29 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1" />
<link rel="icon" href="/favicon.ico" type="image/x-icon" />
{# <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200" /> #}
<title>ra.tcp</title>
<style>
.top {
display: flex;
align-items: top;
justify-content: center;
position: absolute;
top: 0;
}
body {
background-color: #101010;
color: #948DB8;
font-family: monospace;
text-align: center;
height: 95%;
width: 99%;
}
</style>
</head>
<body>
<div class="top"><a href="/mob/">View All Mobs</a>&nbsp;{% block export %}{% endblock export %}</div>
{% block content %}{% endblock content %}

144
tmpls/page.tmpl Normal file
View File

@ -0,0 +1,144 @@
{% extends 'head.tmpl' %}
{% block export %}<a href="/export/mob/{{mob_id}}_{{stance}}.svg">Export SVG</a>{% endblock export %}
{%block content %}
<style>
{% raw %}
.top {
display: inline-flex;
justify-content: center;
position: initial;
top: 0;
}
html {
width: 100%;
// overflow: clip;
height: 100%;
}
.hello {
position: relative;
display: grid;
height: 100%;
align-items: center;
}
.fading {
position: absolute;
justify-content: center;
display: flex;
width: 100%;
}
@keyframes fading {
35% {
opacity: 0.9;
}
45% {
opacity: 0.3;
}
65% {
opacity: 0.9;
}
}
@keyframes turn {
1% {
margin: 0 -3px;
}
41% {
margin: 0 -3px;
}
51% {
margin: 0 -12px;
}
90% {
margin: 0 -12px;
}
}
.fading>svg {
display: flex;
scale: 150%;
}
/* rect {
fill: none !important;
} */
svg g text {
fill: #27C9B2 !important;
}
.line {
height: 15px;
font-size: 0em;
}
/* .line svg {
animation: turn 6s running infinite;
margin: 0px;
}*/
{% endraw %}
</style>
<div class="hello">
<pre class="fading">{{ frame }}</pre>
</div>
</body>
</html>
{% endblock content %}
{# <script>
let frames = document.getElementsByClassName("frame");
let prev = null;
let curFrame = 0;
let frameSpan = (frames.length / 2.4) * 10;
function startCycle() {
function rotate() {
if (prev != null) {
prev.style.visibility = "hidden";
}
if (curFrame >= frames.length) {
curFrame = 0;
}
prev = frames[curFrame];
prev.style.visibility = "visible";
curFrame = curFrame + 1;
}
setInterval(rotate, 120);
}
if (frames.length > 1) {
startCycle();
}
</script> #}
{#
<style>
{% for frame in frames -%}{% if loop.length > 1 -%}
.swap-{{ loop.index }} {{'{'}}
animation: appear-{{ loop.index }} ease {{ loop.length / 15 }}s infinite;
{{'}'}}
@keyframes appear-{{ loop.index }} {{ '{' }}
{{ (loop.index0 / loop.length) * 100}}% {{ '{' }}
visibility: hidden;
{{ '}' }}
{{ ((loop.index0 / loop.length) * 100) + 0.1 }}% {{'{'}}
visibility: visible;
{{ '}' }}
{{ (( loop.index0 / loop.length ) * 100 ) + (( 1 / loop.length ) * 100 ) }}% {{ '{' }}
visibility: hidden;
{{ '}' }}
{{ '}' }}
{% else %}
.swap-{{loop.index}} {{ '{' }}
visibility: visible;
{{ '}' }}
{% endif %}
{%- endfor %}
</style>
#}
{#
@keyframes appear-{{ loop.index }} {{ '{' }}
{{ (100 // loop.length) * loop.index0}}% {{ '{' }}
visibility: hidden;
{{ '}' }}
{{ ((100 // loop.length) * loop.index0) + 1}}% {{ '{' }}
visibility: visible;
{{ '}' }}
{% if loop.last %}100{% else %}{{ ((100 // loop.length) * (loop.index0 + 1)) }}{% endif %}% {{ '{' }}
visibility: hidden;
{{ '}' }}
{{ '}' }}
#}