From 28656695c6abde20032c846cb3cf5b97beea4e9e Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Fri, 13 May 2022 08:35:04 +0900 Subject: [PATCH 01/50] Show control after reading at the end in paged mode --- public/js/reader.js | 18 +++++++++++++++--- src/views/reader.html.ecr | 4 ++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/public/js/reader.js b/public/js/reader.js index 2cb3a66..31bead0 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -135,7 +135,11 @@ const readerComponent = () => { const idx = parseInt(this.curItem.id); const newIdx = idx + (isNext ? 1 : -1); - if (newIdx <= 0 || newIdx > this.items.length) return; + if (newIdx <= 0) return; + if (newIdx > this.items.length) { + this.showControl(idx); + return; + } if (newIdx + this.preloadLookahead < this.items.length + 1) { this.preloadImage(this.items[newIdx + this.preloadLookahead - 1].url); @@ -253,12 +257,20 @@ const readerComponent = () => { }); }, /** - * Shows the control modal + * Handles clicked image * * @param {Event} event - The triggering event */ - showControl(event) { + clickImage(event) { const idx = event.currentTarget.id; + this.showControl(idx); + }, + /** + * Shows the control modal + * + * @param {number} idx - selected page index + */ + showControl(idx) { this.selectedIndex = idx; UIkit.modal($('#modal-sections')).show(); }, diff --git a/src/views/reader.html.ecr b/src/views/reader.html.ecr index 395de41..feac115 100644 --- a/src/views/reader.html.ecr +++ b/src/views/reader.html.ecr @@ -30,7 +30,7 @@ :height="item.height" :id="item.id" :style="`margin-top:${margin}px; margin-bottom:${margin}px`" - @click="showControl($event)" + @click="clickImage($event)" /> <%- if next_entry_url -%> @@ -46,7 +46,7 @@ 'uk-align-center': true, 'uk-animation-slide-left': flipAnimation === 'left', 'uk-animation-slide-right': flipAnimation === 'right' - }" :data-src="curItem.url" :width="curItem.width" :height="curItem.height" :id="curItem.id" @click="showControl($event)" :style="` + }" :data-src="curItem.url" :width="curItem.width" :height="curItem.height" :id="curItem.id" @click="clickImage($event)" :style="` width:${mode === 'width' ? '100vw' : 'auto'}; height:${mode === 'height' ? '100vh' : 'auto'}; margin-bottom:0; From 0e4169cb22bb6546dc1488dea999b09d4c85b538 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Fri, 13 May 2022 08:43:25 +0900 Subject: [PATCH 02/50] Fix preload bug cause index error --- public/js/reader.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/js/reader.js b/public/js/reader.js index 2cb3a66..4b0ff01 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -58,7 +58,7 @@ const readerComponent = () => { // Preload Images this.preloadLookahead = +(localStorage.getItem('preloadLookahead') ?? 3); - const limit = Math.min(page + this.preloadLookahead, this.items.length + 1); + const limit = Math.min(page + this.preloadLookahead, this.items.length); for (let idx = page + 1; idx <= limit; idx++) { this.preloadImage(this.items[idx - 1].url); } From ea6cbbd9cef2d03c3712a00b668148b2c72c6f38 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 14 May 2022 22:52:19 +0900 Subject: [PATCH 03/50] Split Entry and ZippedEntry, Fix to work anyway make Entry an abstract class --- src/library/entry.cr | 269 ++++++++++++++++++-------------- src/library/title.cr | 10 +- src/routes/api.cr | 6 +- src/routes/reader.cr | 1 + src/views/opds/title.xml.ecr | 2 +- src/views/reader-error.html.ecr | 2 +- src/views/reader.html.ecr | 2 +- 7 files changed, 165 insertions(+), 127 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index dd50ed3..9fb1ef9 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -1,62 +1,27 @@ require "image_size" require "yaml" -class Entry - include YAML::Serializable +abstract class Entry + getter id : String, book : Title, title : String, + size : String, pages : Int32, mtime : Time, + encoded_path : String, encoded_title : String, err_msg : String? - getter zip_path : String, book : Title, title : String, - size : String, pages : Int32, id : String, encoded_path : String, - encoded_title : String, mtime : Time, err_msg : String? + def initialize( + @id, @title, @book, + @size, @pages, @mtime, + @encoded_path, @encoded_title, @err_msg) + end - @[YAML::Field(ignore: true)] - @sort_title : String? - - def initialize(@zip_path, @book) - storage = Storage.default - @encoded_path = URI.encode @zip_path - @title = File.basename @zip_path, File.extname @zip_path - @encoded_title = URI.encode @title - @size = (File.size @zip_path).humanize_bytes - id = storage.get_entry_id @zip_path, File.signature(@zip_path) - if id.nil? - id = random_str - storage.insert_entry_id({ - path: @zip_path, - id: id, - signature: File.signature(@zip_path).to_s, - }) - end - @id = id - @mtime = File.info(@zip_path).modification_time - - unless File.readable? @zip_path - @err_msg = "File #{@zip_path} is not readable." - Logger.warn "#{@err_msg} Please make sure the " \ - "file permission is configured correctly." - return - end - - archive_exception = validate_archive @zip_path - unless archive_exception.nil? - @err_msg = "Archive error: #{archive_exception}" - Logger.warn "Unable to extract archive #{@zip_path}. " \ - "Ignoring it. #{@err_msg}" - return - end - - file = ArchiveFile.new @zip_path - @pages = file.entries.count do |e| - SUPPORTED_IMG_TYPES.includes? \ - MIME.from_filename? e.filename - end - file.close + def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + # TODO: check node? and select proper subclass + ZippedEntry.new ctx, node end def build_json(*, slim = false) JSON.build do |json| json.object do - {% for str in %w(zip_path title size id) %} - json.field {{str}}, @{{str.id}} + {% for str in %w(path title size id) %} + json.field {{str}}, {{str.id}} {% end %} if err_msg json.field "err_msg", err_msg @@ -74,6 +39,9 @@ class Entry end end + @[YAML::Field(ignore: true)] + @sort_title : String? + def sort_title sort_title_cached = @sort_title return sort_title_cached if sort_title_cached @@ -131,58 +99,6 @@ class Entry url end - private def sorted_archive_entries - ArchiveFile.open @zip_path do |file| - entries = file.entries - .select { |e| - SUPPORTED_IMG_TYPES.includes? \ - MIME.from_filename? e.filename - } - .sort! { |a, b| - compare_numerically a.filename, b.filename - } - yield file, entries - end - end - - def read_page(page_num) - raise "Unreadble archive. #{@err_msg}" if @err_msg - img = nil - begin - sorted_archive_entries do |file, entries| - page = entries[page_num - 1] - data = file.read_entry page - if data - img = Image.new data, MIME.from_filename(page.filename), - page.filename, data.size - end - end - rescue e - Logger.warn "Unable to read page #{page_num} of #{@zip_path}. Error: #{e}" - end - img - end - - def page_dimensions - sizes = [] of Hash(String, Int32) - sorted_archive_entries do |file, entries| - entries.each_with_index do |e, i| - begin - data = file.read_entry(e).not_nil! - size = ImageSize.get data - sizes << { - "width" => size.width, - "height" => size.height, - } - rescue e - Logger.warn "Failed to read page #{i} of entry #{zip_path}. #{e}" - sizes << {"width" => 1000_i32, "height" => 1000_i32} - end - end - end - sizes - end - def next_entry(username) entries = @book.sorted_entries username idx = entries.index self @@ -197,20 +113,6 @@ class Entry entries[idx - 1] end - def date_added - date_added = nil - TitleInfo.new @book.dir do |info| - info_da = info.date_added[@title]? - if info_da.nil? - date_added = info.date_added[@title] = ctime @zip_path - info.save - else - date_added = info_da - end - end - date_added.not_nil! # is it ok to set not_nil! here? - end - # For backward backward compatibility with v0.1.0, we save entry titles # instead of IDs in info.json def save_progress(username, page) @@ -290,7 +192,7 @@ class Entry end Storage.default.save_thumbnail @id, img rescue e - Logger.warn "Failed to generate thumbnail for file #{@zip_path}. #{e}" + Logger.warn "Failed to generate thumbnail for file #{path}. #{e}" end img @@ -299,4 +201,139 @@ class Entry def get_thumbnail : Image? Storage.default.get_thumbnail @id end + + def date_added : Time + date_added = nil + TitleInfo.new @book.dir do |info| + info_da = info.date_added[@title]? + if info_da.nil? + date_added = info.date_added[@title] = createtime + info.save + else + date_added = info_da + end + end + date_added.not_nil! # is it ok to set not_nil! here? + end + + abstract def path : String + + abstract def createtime : Time + + abstract def read_page(page_num) + + abstract def page_dimensions + + abstract def exists? : Bool? +end + +class ZippedEntry < Entry + include YAML::Serializable + + getter zip_path : String + + def initialize(@zip_path, @book) + storage = Storage.default + @encoded_path = URI.encode @zip_path + @title = File.basename @zip_path, File.extname @zip_path + @encoded_title = URI.encode @title + @size = (File.size @zip_path).humanize_bytes + id = storage.get_entry_id @zip_path, File.signature(@zip_path) + if id.nil? + id = random_str + storage.insert_entry_id({ + path: @zip_path, + id: id, + signature: File.signature(@zip_path).to_s, + }) + end + @id = id + @mtime = File.info(@zip_path).modification_time + + unless File.readable? @zip_path + @err_msg = "File #{@zip_path} is not readable." + Logger.warn "#{@err_msg} Please make sure the " \ + "file permission is configured correctly." + return + end + + archive_exception = validate_archive @zip_path + unless archive_exception.nil? + @err_msg = "Archive error: #{archive_exception}" + Logger.warn "Unable to extract archive #{@zip_path}. " \ + "Ignoring it. #{@err_msg}" + return + end + + file = ArchiveFile.new @zip_path + @pages = file.entries.count do |e| + SUPPORTED_IMG_TYPES.includes? \ + MIME.from_filename? e.filename + end + file.close + end + + def path : String + @zip_path + end + + def createtime : Time + ctime @zip_path + end + + private def sorted_archive_entries + ArchiveFile.open @zip_path do |file| + entries = file.entries + .select { |e| + SUPPORTED_IMG_TYPES.includes? \ + MIME.from_filename? e.filename + } + .sort! { |a, b| + compare_numerically a.filename, b.filename + } + yield file, entries + end + end + + def read_page(page_num) + raise "Unreadble archive. #{@err_msg}" if @err_msg + img = nil + begin + sorted_archive_entries do |file, entries| + page = entries[page_num - 1] + data = file.read_entry page + if data + img = Image.new data, MIME.from_filename(page.filename), + page.filename, data.size + end + end + rescue e + Logger.warn "Unable to read page #{page_num} of #{@zip_path}. Error: #{e}" + end + img + end + + def page_dimensions + sizes = [] of Hash(String, Int32) + sorted_archive_entries do |file, entries| + entries.each_with_index do |e, i| + begin + data = file.read_entry(e).not_nil! + size = ImageSize.get data + sizes << { + "width" => size.width, + "height" => size.height, + } + rescue e + Logger.warn "Failed to read page #{i} of entry #{zip_path}. #{e}" + sizes << {"width" => 1000_i32, "height" => 1000_i32} + end + end + end + sizes + end + + def exists? : Bool + File.exists? @zip_path + end end diff --git a/src/library/title.cr b/src/library/title.cr index e3d79d5..eeef8e7 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -55,7 +55,7 @@ class Title next end if is_supported_file path - entry = Entry.new path, self + entry = ZippedEntry.new path, self @entries << entry if entry.pages > 0 || entry.err_msg end end @@ -127,12 +127,12 @@ class Title previous_entries_size = @entries.size @entries.select! do |entry| - existence = File.exists? entry.zip_path + existence = entry.exists? Fiber.yield context["deleted_entry_ids"] << entry.id unless existence existence end - remained_entry_zip_paths = @entries.map &.zip_path + remained_entry_zip_paths = @entries.map &.path is_titles_added = false is_entries_added = false @@ -162,7 +162,7 @@ class Title end if is_supported_file path next if remained_entry_zip_paths.includes? path - entry = Entry.new path, self + entry = ZippedEntry.new path, self if entry.pages > 0 || entry.err_msg @entries << entry is_entries_added = true @@ -627,7 +627,7 @@ class Title @entries.each do |e| next if da.has_key? e.title - da[e.title] = ctime e.zip_path + da[e.title] = ctime e.path end TitleInfo.new @dir do |info| diff --git a/src/routes/api.cr b/src/routes/api.cr index e664b28..03d6e53 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -40,7 +40,7 @@ struct APIRouter Koa.schema "entry", { "pages" => Int32, "mtime" => Int64, - }.merge(s %w(zip_path title size id title_id display_name cover_url)), + }.merge(s %w(path title size id title_id display_name cover_url)), desc: "An entry in a book" Koa.schema "title", { @@ -1138,7 +1138,7 @@ struct APIRouter entry = title.get_entry eid raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil? - file_hash = Digest::SHA1.hexdigest (entry.zip_path + entry.mtime.to_s) + file_hash = Digest::SHA1.hexdigest (entry.path + entry.mtime.to_s) e_tag = "W/#{file_hash}" if e_tag == prev_e_tag env.response.status_code = 304 @@ -1172,7 +1172,7 @@ struct APIRouter title = (Library.default.get_title env.params.url["tid"]).not_nil! entry = (title.get_entry env.params.url["eid"]).not_nil! - send_attachment env, entry.zip_path + send_attachment env, entry.path rescue e Logger.error e env.response.status_code = 404 diff --git a/src/routes/reader.cr b/src/routes/reader.cr index 40b86aa..052e212 100644 --- a/src/routes/reader.cr +++ b/src/routes/reader.cr @@ -53,6 +53,7 @@ struct ReaderRouter render "src/views/reader.html.ecr" rescue e Logger.error e + puts e.backtrace? env.response.status_code = 404 end end diff --git a/src/views/opds/title.xml.ecr b/src/views/opds/title.xml.ecr index b159687..1d82490 100644 --- a/src/views/opds/title.xml.ecr +++ b/src/views/opds/title.xml.ecr @@ -29,7 +29,7 @@ - + diff --git a/src/views/reader-error.html.ecr b/src/views/reader-error.html.ecr index 62a80fc..ad3580f 100644 --- a/src/views/reader-error.html.ecr +++ b/src/views/reader-error.html.ecr @@ -5,7 +5,7 @@

Error

-

<%= entry.zip_path %>

+

<%= entry.path %>

<%= entry.err_msg %>

diff --git a/src/views/reader.html.ecr b/src/views/reader.html.ecr index feac115..19e2b19 100644 --- a/src/views/reader.html.ecr +++ b/src/views/reader.html.ecr @@ -67,7 +67,7 @@

<%= entry.display_name %>

-

<%= entry.zip_path %>

+

<%= entry.path %>

From 10587f48cbad42cb5ad9d544167a6b1eea4c060e Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 16:03:15 +0900 Subject: [PATCH 04/50] Implement is_supported_image_file --- src/library/types.cr | 10 ---------- src/util/util.cr | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/library/types.cr b/src/library/types.cr index 973aa5e..d6a014f 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -1,13 +1,3 @@ -SUPPORTED_IMG_TYPES = %w( - image/jpeg - image/png - image/webp - image/apng - image/avif - image/gif - image/svg+xml -) - enum SortMethod Auto Title diff --git a/src/util/util.cr b/src/util/util.cr index e7b1b1a..7d834fb 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -3,6 +3,16 @@ ENTRIES_IN_HOME_SECTIONS = 8 UPLOAD_URL_PREFIX = "/uploads" STATIC_DIRS = %w(/css /js /img /webfonts /favicon.ico /robots.txt) SUPPORTED_FILE_EXTNAMES = [".zip", ".cbz", ".rar", ".cbr"] +SUPPORTED_IMG_TYPES = %w( + image/jpeg + image/png + image/webp + image/apng + image/avif + image/gif + image/svg+xml +) + def random_str UUID.random.to_s.gsub "-", "" @@ -49,6 +59,10 @@ def is_supported_file(path) SUPPORTED_FILE_EXTNAMES.includes? File.extname(path).downcase end +def is_supported_image_file(path) + SUPPORTED_IMG_TYPES.includes? MIME.from_filename? path +end + struct Int def or(other : Int) if self == 0 From 55ccd928a2c27f6c94cb4bb5dbad12b5a2ad1ca7 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 14:00:18 +0900 Subject: [PATCH 05/50] Implement DirectoryEntry --- src/library/entry.cr | 142 ++++++++++++++++++++++++++++++++++++++++++ src/util/signature.cr | 20 +++++- 2 files changed, 161 insertions(+), 1 deletion(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 9fb1ef9..b43dd77 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -337,3 +337,145 @@ class ZippedEntry < Entry File.exists? @zip_path end end + +class DirectoryEntry < Entry + include YAML::Serializable + + getter dir_path : String + + @[YAML::Field(ignore: true)] + @sorted_files : Array(String)? + + @signature : String + + def initialize(@dir_path, @book) + storage = Storage.default + @encoded_path = URI.encode @dir_path + @title = File.basename @dir_path + @encoded_title = URI.encode @title + + unless File.readable? @dir_path + @err_msg = "Directory #{@dir_path} is not readable." + Logger.warn "#{@err_msg} Please make sure the " \ + "file permission is configured correctly." + return + end + + unless DirectoryEntry.validate_directory_entry @dir_path + @err_msg = "Directory #{@dir_path} is not valid directory entry." + Logger.warn "#{@err_msg} Please make sure the " \ + "directory has valid images." + return + end + + size_sum = 0 + sorted_files.each do |file_path| + size_sum += File.size file_path + end + @size = size_sum.humanize_bytes + + @signature = Dir.directory_entry_signature @dir_path + id = storage.get_entry_id @dir_path, @signature + if id.nil? + id = random_str + storage.insert_entry_id({ + path: @dir_path, + id: id, + signature: @signature, + }) + end + @id = id + + mtimes = sorted_files.map { |file_path| File.info(file_path).modification_time } + @mtime = mtimes.max + + @pages = sorted_files.size + end + + def path : String + @dir_path + end + + def createtime : Time + ctime @dir_path + end + + def read_page(page_num) + img = nil + begin + files = sorted_files + file_path = files[page_num - 1] + data = File.read(file_path).to_slice + if data + img = Image.new data, MIME.from_filename(file_path), + File.basename(file_path), data.size + end + rescue e + Logger.warn "Unable to read page #{page_num} of #{@dir_path}. Error: #{e}" + end + img + end + + def page_dimensions + sizes = [] of Hash(String, Int32) + sorted_files.each_with_index do |path, i| + data = File.read(path).to_slice + begin + data.not_nil! + size = ImageSize.get data + sizes << { + "width" => size.width, + "height" => size.height, + } + rescue e + Logger.warn "Failed to read page #{i} of entry #{@dir_path}. #{e}" + sizes << {"width" => 1000_i32, "height" => 1000_i32} + end + end + sizes + end + + def exists? : Bool + existence = File.exists? @dir_path + return false unless existence + files = DirectoryEntry.get_valid_files @dir_path + signature = Dir.directory_entry_signature @dir_path + existence = files.size > 0 && @signature == signature + @sorted_files = nil unless existence + + # For more efficient, + # Fix a directory instance with new property + # and return true + existence + end + + def sorted_files + cached_sorted_files = @sorted_files + return cached_sorted_files if cached_sorted_files + @sorted_files = DirectoryEntry.get_valid_files_sorted @dir_path + @sorted_files.not_nil! + end + + def self.validate_directory_entry(dir_path) + files = DirectoryEntry.get_valid_files dir_path + files.size > 0 + end + + def self.get_valid_files(dir_path) + files = [] of String + Dir.entries(dir_path).each do |fn| + next if fn.starts_with? "." + path = File.join dir_path, fn + next unless is_supported_image_file path + next if File.directory? path + next unless File.readable? path + files << path + end + files + end + + def self.get_valid_files_sorted(dir_path) + files = DirectoryEntry.get_valid_files dir_path + files.sort! { |a, b| compare_numerically a, b } + end +end diff --git a/src/util/signature.cr b/src/util/signature.cr index 5ca3e14..63d31ce 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -19,7 +19,7 @@ class File # information as long as the above changes do not happen together with # a file/folder rename, with no library scan in between. def self.signature(filename) : UInt64 - if is_supported_file filename + if is_supported_file(filename) || is_supported_image_file(filename) File.info(filename).inode else 0u64 @@ -64,6 +64,9 @@ class Dir path = File.join dirname, fn if File.directory? path signatures << Dir.contents_signature path, cache + if DirectoryEntry.validate_directory_entry path + signatures << Dir.directory_entry_signature path, cache + end else # Only add its signature value to `signatures` when it is a # supported file @@ -76,4 +79,19 @@ class Dir cache[dirname] = hash hash end + + def self.directory_entry_signature(dirname, cache = {} of String => String) + return cache[dirname + "?entry"] if cache[dirname + "?entry"]? + Fiber.yield + signatures = [] of String + image_files = DirectoryEntry.get_valid_files_sorted dirname + if image_files.size > 0 + image_files.each do |path| + signatures << File.signature(path).to_s + end + end + hash = Digest::SHA1.hexdigest(signatures.join) + cache[dirname + "?entry"] = hash + hash + end end From 3b3a0738e83d2e009500fbbb4f701480378e60b0 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 15:31:11 +0900 Subject: [PATCH 06/50] Scan DirectoryEntry when init and examine --- src/library/title.cr | 55 +++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/src/library/title.cr b/src/library/title.cr index eeef8e7..ad1672d 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -49,9 +49,14 @@ class Title path = File.join dir, fn if File.directory? path title = Title.new path, @id, cache - next if title.entries.size == 0 && title.titles.size == 0 - Library.default.title_hash[title.id] = title - @title_ids << title.id + unless title.entries.size == 0 && title.titles.size == 0 + Library.default.title_hash[title.id] = title + @title_ids << title.id + end + if DirectoryEntry.validate_directory_entry path + entry = DirectoryEntry.new path, self + @entries << entry if entry.pages > 0 || entry.err_msg + end next end if is_supported_file path @@ -132,7 +137,7 @@ class Title context["deleted_entry_ids"] << entry.id unless existence existence end - remained_entry_zip_paths = @entries.map &.path + remained_entry_paths = @entries.map &.path is_titles_added = false is_entries_added = false @@ -140,28 +145,42 @@ class Title next if fn.starts_with? "." path = File.join dir, fn if File.directory? path + unless remained_entry_paths.includes? path + if DirectoryEntry.validate_directory_entry path + entry = DirectoryEntry.new path, self + if entry.pages > 0 || entry.err_msg + @entries << entry + is_entries_added = true + context["deleted_entry_ids"].select! do |deleted_entry_id| + entry.id != deleted_entry_id + end + end + end + end + next if remained_title_dirs.includes? path title = Title.new path, @id, context["cached_contents_signature"] - next if title.entries.size == 0 && title.titles.size == 0 - Library.default.title_hash[title.id] = title - @title_ids << title.id - is_titles_added = true + unless title.entries.size == 0 && title.titles.size == 0 + Library.default.title_hash[title.id] = title + @title_ids << title.id + is_titles_added = true - # We think they are removed, but they are here! - # Cancel reserved jobs - revival_title_ids = [title.id] + title.deep_titles.map &.id - context["deleted_title_ids"].select! do |deleted_title_id| - !(revival_title_ids.includes? deleted_title_id) - end - revival_entry_ids = title.deep_entries.map &.id - context["deleted_entry_ids"].select! do |deleted_entry_id| - !(revival_entry_ids.includes? deleted_entry_id) + # We think they are removed, but they are here! + # Cancel reserved jobs + revival_title_ids = [title.id] + title.deep_titles.map &.id + context["deleted_title_ids"].select! do |deleted_title_id| + !(revival_title_ids.includes? deleted_title_id) + end + revival_entry_ids = title.deep_entries.map &.id + context["deleted_entry_ids"].select! do |deleted_entry_id| + !(revival_entry_ids.includes? deleted_entry_id) + end end next end if is_supported_file path - next if remained_entry_zip_paths.includes? path + next if remained_entry_paths.includes? path entry = ZippedEntry.new path, self if entry.pages > 0 || entry.err_msg @entries << entry From 137e84dfb671c1240a9d2d60ff3c9f22ef86b86c Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 15:31:33 +0900 Subject: [PATCH 07/50] Fix caching policy Before rendering it, the Mango reader should check the E-Tag of page or it renders wrong image when an image file is moved/removed/reordered --- src/routes/api.cr | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/routes/api.cr b/src/routes/api.cr index 03d6e53..840ce92 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -142,8 +142,13 @@ struct APIRouter env.response.status_code = 304 "" else + if entry.is_a? DirectoryEntry + cache_control = "no-cache, max-age=86400" + else + cache_control = "public, max-age=86400" + end env.response.headers["ETag"] = e_tag - env.response.headers["Cache-Control"] = "public, max-age=86400" + env.response.headers["Cache-Control"] = cache_control send_img env, img end rescue e @@ -1138,15 +1143,24 @@ struct APIRouter entry = title.get_entry eid raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil? - file_hash = Digest::SHA1.hexdigest (entry.path + entry.mtime.to_s) + if entry.is_a? DirectoryEntry + file_hash = Digest::SHA1.hexdigest (entry.path + entry.mtime.to_s + entry.size) + else + file_hash = Digest::SHA1.hexdigest (entry.path + entry.mtime.to_s) + end e_tag = "W/#{file_hash}" if e_tag == prev_e_tag env.response.status_code = 304 send_text env, "" else sizes = entry.page_dimensions + if entry.is_a? DirectoryEntry + cache_control = "no-cache, max-age=86400" + else + cache_control = "public, max-age=86400" + end env.response.headers["ETag"] = e_tag - env.response.headers["Cache-Control"] = "public, max-age=86400" + env.response.headers["Cache-Control"] = cache_control send_json env, { "success" => true, "dimensions" => sizes, From caf4cfb6cd721827a312c54e25c17f81c0c753b7 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 16:12:43 +0900 Subject: [PATCH 08/50] Fix Entry.new in YAML::Serializable to support DirectyEntry so hacky --- src/library/entry.cr | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index b43dd77..94e13fb 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -14,7 +14,11 @@ abstract class Entry def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) # TODO: check node? and select proper subclass - ZippedEntry.new ctx, node + begin + ZippedEntry.new ctx, node + rescue e + DirectoryEntry.new ctx, node + end end def build_json(*, slim = false) From 9f6be70995c065f8959ddedaad1b0854d57549d2 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 16:28:53 +0900 Subject: [PATCH 09/50] Rename Entry.exists? to Entry.examine --- src/library/entry.cr | 11 +++++------ src/library/title.cr | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 94e13fb..7399b0d 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -228,7 +228,7 @@ abstract class Entry abstract def page_dimensions - abstract def exists? : Bool? + abstract def examine : Bool? end class ZippedEntry < Entry @@ -337,7 +337,7 @@ class ZippedEntry < Entry sizes end - def exists? : Bool + def examine : Bool File.exists? @zip_path end end @@ -439,7 +439,7 @@ class DirectoryEntry < Entry sizes end - def exists? : Bool + def examine : Bool existence = File.exists? @dir_path return false unless existence files = DirectoryEntry.get_valid_files @dir_path @@ -447,9 +447,8 @@ class DirectoryEntry < Entry existence = files.size > 0 && @signature == signature @sorted_files = nil unless existence - # For more efficient, - # Fix a directory instance with new property - # and return true + # For more efficient, update a directory entry with new property + # and return true like Title.examine existence end diff --git a/src/library/title.cr b/src/library/title.cr index ad1672d..f3b4866 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -132,7 +132,7 @@ class Title previous_entries_size = @entries.size @entries.select! do |entry| - existence = entry.exists? + existence = entry.examine Fiber.yield context["deleted_entry_ids"] << entry.id unless existence existence From 3a60286c3aaffe46967d1e24e738f51e5892ce00 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 17:02:29 +0900 Subject: [PATCH 10/50] Run 'crystal tool format' --- src/library/cache.cr | 8 ++++---- src/library/entry.cr | 3 ++- src/routes/admin.cr | 2 +- src/routes/api.cr | 4 ++-- src/util/util.cr | 3 +-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index 10e4f60..f35af8b 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -76,8 +76,8 @@ class SortedEntriesCacheEntry < CacheEntry(Array(String), Array(Entry)) entries : Array(Entry), opt : SortOptions?) entries_sig = Digest::SHA1.hexdigest (entries.map &.id).to_s user_context = opt && opt.method == SortMethod::Progress ? username : "" - sig = Digest::SHA1.hexdigest (book_id + entries_sig + user_context + - (opt ? opt.to_tuple.to_s : "nil")) + sig = Digest::SHA1.hexdigest(book_id + entries_sig + user_context + + (opt ? opt.to_tuple.to_s : "nil")) "#{sig}:sorted_entries" end end @@ -101,8 +101,8 @@ class SortedTitlesCacheEntry < CacheEntry(Array(String), Array(Title)) def self.gen_key(username : String, titles : Array(Title), opt : SortOptions?) titles_sig = Digest::SHA1.hexdigest (titles.map &.id).to_s user_context = opt && opt.method == SortMethod::Progress ? username : "" - sig = Digest::SHA1.hexdigest (titles_sig + user_context + - (opt ? opt.to_tuple.to_s : "nil")) + sig = Digest::SHA1.hexdigest(titles_sig + user_context + + (opt ? opt.to_tuple.to_s : "nil")) "#{sig}:sorted_titles" end end diff --git a/src/library/entry.cr b/src/library/entry.cr index 7399b0d..627f4fc 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -9,7 +9,8 @@ abstract class Entry def initialize( @id, @title, @book, @size, @pages, @mtime, - @encoded_path, @encoded_title, @err_msg) + @encoded_path, @encoded_title, @err_msg + ) end def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) diff --git a/src/routes/admin.cr b/src/routes/admin.cr index 23481f9..6987a11 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -63,7 +63,7 @@ struct AdminRouter redirect_url = URI.new \ path: "/admin/user/edit", query: hash_to_query({"username" => original_username, \ - "admin" => admin, "error" => e.message}) + "admin" => admin, "error" => e.message}) redirect env, redirect_url.to_s end diff --git a/src/routes/api.cr b/src/routes/api.cr index 840ce92..bf2c4c9 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -1144,9 +1144,9 @@ struct APIRouter raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil? if entry.is_a? DirectoryEntry - file_hash = Digest::SHA1.hexdigest (entry.path + entry.mtime.to_s + entry.size) + file_hash = Digest::SHA1.hexdigest(entry.path + entry.mtime.to_s + entry.size) else - file_hash = Digest::SHA1.hexdigest (entry.path + entry.mtime.to_s) + file_hash = Digest::SHA1.hexdigest(entry.path + entry.mtime.to_s) end e_tag = "W/#{file_hash}" if e_tag == prev_e_tag diff --git a/src/util/util.cr b/src/util/util.cr index 7d834fb..e08bd9d 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -3,7 +3,7 @@ ENTRIES_IN_HOME_SECTIONS = 8 UPLOAD_URL_PREFIX = "/uploads" STATIC_DIRS = %w(/css /js /img /webfonts /favicon.ico /robots.txt) SUPPORTED_FILE_EXTNAMES = [".zip", ".cbz", ".rar", ".cbr"] -SUPPORTED_IMG_TYPES = %w( +SUPPORTED_IMG_TYPES = %w( image/jpeg image/png image/webp @@ -13,7 +13,6 @@ SUPPORTED_IMG_TYPES = %w( image/svg+xml ) - def random_str UUID.random.to_s.gsub "-", "" end From 3da5d9ba4ed56ba7e9af51c4d5080cc58647168f Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 17:36:57 +0900 Subject: [PATCH 11/50] Fix contents_signature --- src/util/signature.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/signature.cr b/src/util/signature.cr index 63d31ce..904d4e6 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -65,7 +65,7 @@ class Dir if File.directory? path signatures << Dir.contents_signature path, cache if DirectoryEntry.validate_directory_entry path - signatures << Dir.directory_entry_signature path, cache + signatures << fn end else # Only add its signature value to `signatures` when it is a From 0ed565519b2f88f81c794c2437cbe9042d9eac0f Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 17:38:21 +0900 Subject: [PATCH 12/50] Rollback crystal format --- src/routes/admin.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/admin.cr b/src/routes/admin.cr index 6987a11..23481f9 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -63,7 +63,7 @@ struct AdminRouter redirect_url = URI.new \ path: "/admin/user/edit", query: hash_to_query({"username" => original_username, \ - "admin" => admin, "error" => e.message}) + "admin" => admin, "error" => e.message}) redirect env, redirect_url.to_s end From f18f6a5418f6c1c0d576a96551980489e4f5075d Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Thu, 19 May 2022 12:41:07 +0000 Subject: [PATCH 13/50] Fix linter issues --- src/library/entry.cr | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 627f4fc..77af3e5 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -15,11 +15,9 @@ abstract class Entry def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) # TODO: check node? and select proper subclass - begin - ZippedEntry.new ctx, node - rescue e - DirectoryEntry.new ctx, node - end + ZippedEntry.new ctx, node + rescue e + DirectoryEntry.new ctx, node end def build_json(*, slim = false) @@ -391,9 +389,9 @@ class DirectoryEntry < Entry end @id = id - mtimes = sorted_files.map { |file_path| File.info(file_path).modification_time } - @mtime = mtimes.max - + @mtime = sorted_files.map do |file_path| + File.info(file_path).modification_time + end.max @pages = sorted_files.size end From 1f5aed64f7e42f131324c2e61d7fac774fcd0824 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Fri, 20 May 2022 09:51:56 +0900 Subject: [PATCH 14/50] Rename Entries to ArchiveEntry and DirEntry --- src/library/entry.cr | 18 +++++++++--------- src/library/title.cr | 12 ++++++------ src/routes/api.cr | 6 +++--- src/util/signature.cr | 4 ++-- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 77af3e5..ef0556d 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -15,9 +15,9 @@ abstract class Entry def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) # TODO: check node? and select proper subclass - ZippedEntry.new ctx, node + ArchiveEntry.new ctx, node rescue e - DirectoryEntry.new ctx, node + DirEntry.new ctx, node end def build_json(*, slim = false) @@ -230,7 +230,7 @@ abstract class Entry abstract def examine : Bool? end -class ZippedEntry < Entry +class ArchiveEntry < Entry include YAML::Serializable getter zip_path : String @@ -341,7 +341,7 @@ class ZippedEntry < Entry end end -class DirectoryEntry < Entry +class DirEntry < Entry include YAML::Serializable getter dir_path : String @@ -364,7 +364,7 @@ class DirectoryEntry < Entry return end - unless DirectoryEntry.validate_directory_entry @dir_path + unless DirEntry.validate_directory_entry @dir_path @err_msg = "Directory #{@dir_path} is not valid directory entry." Logger.warn "#{@err_msg} Please make sure the " \ "directory has valid images." @@ -441,7 +441,7 @@ class DirectoryEntry < Entry def examine : Bool existence = File.exists? @dir_path return false unless existence - files = DirectoryEntry.get_valid_files @dir_path + files = DirEntry.get_valid_files @dir_path signature = Dir.directory_entry_signature @dir_path existence = files.size > 0 && @signature == signature @sorted_files = nil unless existence @@ -454,12 +454,12 @@ class DirectoryEntry < Entry def sorted_files cached_sorted_files = @sorted_files return cached_sorted_files if cached_sorted_files - @sorted_files = DirectoryEntry.get_valid_files_sorted @dir_path + @sorted_files = DirEntry.get_valid_files_sorted @dir_path @sorted_files.not_nil! end def self.validate_directory_entry(dir_path) - files = DirectoryEntry.get_valid_files dir_path + files = DirEntry.get_valid_files dir_path files.size > 0 end @@ -477,7 +477,7 @@ class DirectoryEntry < Entry end def self.get_valid_files_sorted(dir_path) - files = DirectoryEntry.get_valid_files dir_path + files = DirEntry.get_valid_files dir_path files.sort! { |a, b| compare_numerically a, b } end end diff --git a/src/library/title.cr b/src/library/title.cr index f3b4866..3f30490 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -53,14 +53,14 @@ class Title Library.default.title_hash[title.id] = title @title_ids << title.id end - if DirectoryEntry.validate_directory_entry path - entry = DirectoryEntry.new path, self + if DirEntry.validate_directory_entry path + entry = DirEntry.new path, self @entries << entry if entry.pages > 0 || entry.err_msg end next end if is_supported_file path - entry = ZippedEntry.new path, self + entry = ArchiveEntry.new path, self @entries << entry if entry.pages > 0 || entry.err_msg end end @@ -146,8 +146,8 @@ class Title path = File.join dir, fn if File.directory? path unless remained_entry_paths.includes? path - if DirectoryEntry.validate_directory_entry path - entry = DirectoryEntry.new path, self + if DirEntry.validate_directory_entry path + entry = DirEntry.new path, self if entry.pages > 0 || entry.err_msg @entries << entry is_entries_added = true @@ -181,7 +181,7 @@ class Title end if is_supported_file path next if remained_entry_paths.includes? path - entry = ZippedEntry.new path, self + entry = ArchiveEntry.new path, self if entry.pages > 0 || entry.err_msg @entries << entry is_entries_added = true diff --git a/src/routes/api.cr b/src/routes/api.cr index bf2c4c9..509f13d 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -142,7 +142,7 @@ struct APIRouter env.response.status_code = 304 "" else - if entry.is_a? DirectoryEntry + if entry.is_a? DirEntry cache_control = "no-cache, max-age=86400" else cache_control = "public, max-age=86400" @@ -1143,7 +1143,7 @@ struct APIRouter entry = title.get_entry eid raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil? - if entry.is_a? DirectoryEntry + if entry.is_a? DirEntry file_hash = Digest::SHA1.hexdigest(entry.path + entry.mtime.to_s + entry.size) else file_hash = Digest::SHA1.hexdigest(entry.path + entry.mtime.to_s) @@ -1154,7 +1154,7 @@ struct APIRouter send_text env, "" else sizes = entry.page_dimensions - if entry.is_a? DirectoryEntry + if entry.is_a? DirEntry cache_control = "no-cache, max-age=86400" else cache_control = "public, max-age=86400" diff --git a/src/util/signature.cr b/src/util/signature.cr index 904d4e6..b5fe781 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -64,7 +64,7 @@ class Dir path = File.join dirname, fn if File.directory? path signatures << Dir.contents_signature path, cache - if DirectoryEntry.validate_directory_entry path + if DirEntry.validate_directory_entry path signatures << fn end else @@ -84,7 +84,7 @@ class Dir return cache[dirname + "?entry"] if cache[dirname + "?entry"]? Fiber.yield signatures = [] of String - image_files = DirectoryEntry.get_valid_files_sorted dirname + image_files = DirEntry.get_valid_files_sorted dirname if image_files.size > 0 image_files.each do |path| signatures << File.signature(path).to_s From 238539c27da6a1af01e29c09354965d7169fba08 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Fri, 20 May 2022 14:21:08 +0900 Subject: [PATCH 15/50] Split files --- src/library/archive_entry.cr | 114 ++++++++++++++++ src/library/dir_entry.cr | 144 ++++++++++++++++++++ src/library/entry.cr | 253 ----------------------------------- 3 files changed, 258 insertions(+), 253 deletions(-) create mode 100644 src/library/archive_entry.cr create mode 100644 src/library/dir_entry.cr diff --git a/src/library/archive_entry.cr b/src/library/archive_entry.cr new file mode 100644 index 0000000..4075590 --- /dev/null +++ b/src/library/archive_entry.cr @@ -0,0 +1,114 @@ +require "yaml" + +require "./entry" + +class ArchiveEntry < Entry + include YAML::Serializable + + getter zip_path : String + + def initialize(@zip_path, @book) + storage = Storage.default + @encoded_path = URI.encode @zip_path + @title = File.basename @zip_path, File.extname @zip_path + @encoded_title = URI.encode @title + @size = (File.size @zip_path).humanize_bytes + id = storage.get_entry_id @zip_path, File.signature(@zip_path) + if id.nil? + id = random_str + storage.insert_entry_id({ + path: @zip_path, + id: id, + signature: File.signature(@zip_path).to_s, + }) + end + @id = id + @mtime = File.info(@zip_path).modification_time + + unless File.readable? @zip_path + @err_msg = "File #{@zip_path} is not readable." + Logger.warn "#{@err_msg} Please make sure the " \ + "file permission is configured correctly." + return + end + + archive_exception = validate_archive @zip_path + unless archive_exception.nil? + @err_msg = "Archive error: #{archive_exception}" + Logger.warn "Unable to extract archive #{@zip_path}. " \ + "Ignoring it. #{@err_msg}" + return + end + + file = ArchiveFile.new @zip_path + @pages = file.entries.count do |e| + SUPPORTED_IMG_TYPES.includes? \ + MIME.from_filename? e.filename + end + file.close + end + + def path : String + @zip_path + end + + def createtime : Time + ctime @zip_path + end + + private def sorted_archive_entries + ArchiveFile.open @zip_path do |file| + entries = file.entries + .select { |e| + SUPPORTED_IMG_TYPES.includes? \ + MIME.from_filename? e.filename + } + .sort! { |a, b| + compare_numerically a.filename, b.filename + } + yield file, entries + end + end + + def read_page(page_num) + raise "Unreadble archive. #{@err_msg}" if @err_msg + img = nil + begin + sorted_archive_entries do |file, entries| + page = entries[page_num - 1] + data = file.read_entry page + if data + img = Image.new data, MIME.from_filename(page.filename), + page.filename, data.size + end + end + rescue e + Logger.warn "Unable to read page #{page_num} of #{@zip_path}. Error: #{e}" + end + img + end + + def page_dimensions + sizes = [] of Hash(String, Int32) + sorted_archive_entries do |file, entries| + entries.each_with_index do |e, i| + begin + data = file.read_entry(e).not_nil! + size = ImageSize.get data + sizes << { + "width" => size.width, + "height" => size.height, + } + rescue e + Logger.warn "Failed to read page #{i} of entry #{zip_path}. #{e}" + sizes << {"width" => 1000_i32, "height" => 1000_i32} + end + end + end + sizes + end + + def examine : Bool + File.exists? @zip_path + end +end diff --git a/src/library/dir_entry.cr b/src/library/dir_entry.cr new file mode 100644 index 0000000..a444cb0 --- /dev/null +++ b/src/library/dir_entry.cr @@ -0,0 +1,144 @@ +require "yaml" + +require "./entry" + +class DirEntry < Entry + include YAML::Serializable + + getter dir_path : String + + @[YAML::Field(ignore: true)] + @sorted_files : Array(String)? + + @signature : String + + def initialize(@dir_path, @book) + storage = Storage.default + @encoded_path = URI.encode @dir_path + @title = File.basename @dir_path + @encoded_title = URI.encode @title + + unless File.readable? @dir_path + @err_msg = "Directory #{@dir_path} is not readable." + Logger.warn "#{@err_msg} Please make sure the " \ + "file permission is configured correctly." + return + end + + unless DirEntry.validate_directory_entry @dir_path + @err_msg = "Directory #{@dir_path} is not valid directory entry." + Logger.warn "#{@err_msg} Please make sure the " \ + "directory has valid images." + return + end + + size_sum = 0 + sorted_files.each do |file_path| + size_sum += File.size file_path + end + @size = size_sum.humanize_bytes + + @signature = Dir.directory_entry_signature @dir_path + id = storage.get_entry_id @dir_path, @signature + if id.nil? + id = random_str + storage.insert_entry_id({ + path: @dir_path, + id: id, + signature: @signature, + }) + end + @id = id + + @mtime = sorted_files.map do |file_path| + File.info(file_path).modification_time + end.max + @pages = sorted_files.size + end + + def path : String + @dir_path + end + + def createtime : Time + ctime @dir_path + end + + def read_page(page_num) + img = nil + begin + files = sorted_files + file_path = files[page_num - 1] + data = File.read(file_path).to_slice + if data + img = Image.new data, MIME.from_filename(file_path), + File.basename(file_path), data.size + end + rescue e + Logger.warn "Unable to read page #{page_num} of #{@dir_path}. Error: #{e}" + end + img + end + + def page_dimensions + sizes = [] of Hash(String, Int32) + sorted_files.each_with_index do |path, i| + data = File.read(path).to_slice + begin + data.not_nil! + size = ImageSize.get data + sizes << { + "width" => size.width, + "height" => size.height, + } + rescue e + Logger.warn "Failed to read page #{i} of entry #{@dir_path}. #{e}" + sizes << {"width" => 1000_i32, "height" => 1000_i32} + end + end + sizes + end + + def examine : Bool + existence = File.exists? @dir_path + return false unless existence + files = DirEntry.get_valid_files @dir_path + signature = Dir.directory_entry_signature @dir_path + existence = files.size > 0 && @signature == signature + @sorted_files = nil unless existence + + # For more efficient, update a directory entry with new property + # and return true like Title.examine + existence + end + + def sorted_files + cached_sorted_files = @sorted_files + return cached_sorted_files if cached_sorted_files + @sorted_files = DirEntry.get_valid_files_sorted @dir_path + @sorted_files.not_nil! + end + + def self.validate_directory_entry(dir_path) + files = DirEntry.get_valid_files dir_path + files.size > 0 + end + + def self.get_valid_files(dir_path) + files = [] of String + Dir.entries(dir_path).each do |fn| + next if fn.starts_with? "." + path = File.join dir_path, fn + next unless is_supported_image_file path + next if File.directory? path + next unless File.readable? path + files << path + end + files + end + + def self.get_valid_files_sorted(dir_path) + files = DirEntry.get_valid_files dir_path + files.sort! { |a, b| compare_numerically a, b } + end +end diff --git a/src/library/entry.cr b/src/library/entry.cr index ef0556d..b718ed7 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -1,5 +1,4 @@ require "image_size" -require "yaml" abstract class Entry getter id : String, book : Title, title : String, @@ -229,255 +228,3 @@ abstract class Entry abstract def examine : Bool? end - -class ArchiveEntry < Entry - include YAML::Serializable - - getter zip_path : String - - def initialize(@zip_path, @book) - storage = Storage.default - @encoded_path = URI.encode @zip_path - @title = File.basename @zip_path, File.extname @zip_path - @encoded_title = URI.encode @title - @size = (File.size @zip_path).humanize_bytes - id = storage.get_entry_id @zip_path, File.signature(@zip_path) - if id.nil? - id = random_str - storage.insert_entry_id({ - path: @zip_path, - id: id, - signature: File.signature(@zip_path).to_s, - }) - end - @id = id - @mtime = File.info(@zip_path).modification_time - - unless File.readable? @zip_path - @err_msg = "File #{@zip_path} is not readable." - Logger.warn "#{@err_msg} Please make sure the " \ - "file permission is configured correctly." - return - end - - archive_exception = validate_archive @zip_path - unless archive_exception.nil? - @err_msg = "Archive error: #{archive_exception}" - Logger.warn "Unable to extract archive #{@zip_path}. " \ - "Ignoring it. #{@err_msg}" - return - end - - file = ArchiveFile.new @zip_path - @pages = file.entries.count do |e| - SUPPORTED_IMG_TYPES.includes? \ - MIME.from_filename? e.filename - end - file.close - end - - def path : String - @zip_path - end - - def createtime : Time - ctime @zip_path - end - - private def sorted_archive_entries - ArchiveFile.open @zip_path do |file| - entries = file.entries - .select { |e| - SUPPORTED_IMG_TYPES.includes? \ - MIME.from_filename? e.filename - } - .sort! { |a, b| - compare_numerically a.filename, b.filename - } - yield file, entries - end - end - - def read_page(page_num) - raise "Unreadble archive. #{@err_msg}" if @err_msg - img = nil - begin - sorted_archive_entries do |file, entries| - page = entries[page_num - 1] - data = file.read_entry page - if data - img = Image.new data, MIME.from_filename(page.filename), - page.filename, data.size - end - end - rescue e - Logger.warn "Unable to read page #{page_num} of #{@zip_path}. Error: #{e}" - end - img - end - - def page_dimensions - sizes = [] of Hash(String, Int32) - sorted_archive_entries do |file, entries| - entries.each_with_index do |e, i| - begin - data = file.read_entry(e).not_nil! - size = ImageSize.get data - sizes << { - "width" => size.width, - "height" => size.height, - } - rescue e - Logger.warn "Failed to read page #{i} of entry #{zip_path}. #{e}" - sizes << {"width" => 1000_i32, "height" => 1000_i32} - end - end - end - sizes - end - - def examine : Bool - File.exists? @zip_path - end -end - -class DirEntry < Entry - include YAML::Serializable - - getter dir_path : String - - @[YAML::Field(ignore: true)] - @sorted_files : Array(String)? - - @signature : String - - def initialize(@dir_path, @book) - storage = Storage.default - @encoded_path = URI.encode @dir_path - @title = File.basename @dir_path - @encoded_title = URI.encode @title - - unless File.readable? @dir_path - @err_msg = "Directory #{@dir_path} is not readable." - Logger.warn "#{@err_msg} Please make sure the " \ - "file permission is configured correctly." - return - end - - unless DirEntry.validate_directory_entry @dir_path - @err_msg = "Directory #{@dir_path} is not valid directory entry." - Logger.warn "#{@err_msg} Please make sure the " \ - "directory has valid images." - return - end - - size_sum = 0 - sorted_files.each do |file_path| - size_sum += File.size file_path - end - @size = size_sum.humanize_bytes - - @signature = Dir.directory_entry_signature @dir_path - id = storage.get_entry_id @dir_path, @signature - if id.nil? - id = random_str - storage.insert_entry_id({ - path: @dir_path, - id: id, - signature: @signature, - }) - end - @id = id - - @mtime = sorted_files.map do |file_path| - File.info(file_path).modification_time - end.max - @pages = sorted_files.size - end - - def path : String - @dir_path - end - - def createtime : Time - ctime @dir_path - end - - def read_page(page_num) - img = nil - begin - files = sorted_files - file_path = files[page_num - 1] - data = File.read(file_path).to_slice - if data - img = Image.new data, MIME.from_filename(file_path), - File.basename(file_path), data.size - end - rescue e - Logger.warn "Unable to read page #{page_num} of #{@dir_path}. Error: #{e}" - end - img - end - - def page_dimensions - sizes = [] of Hash(String, Int32) - sorted_files.each_with_index do |path, i| - data = File.read(path).to_slice - begin - data.not_nil! - size = ImageSize.get data - sizes << { - "width" => size.width, - "height" => size.height, - } - rescue e - Logger.warn "Failed to read page #{i} of entry #{@dir_path}. #{e}" - sizes << {"width" => 1000_i32, "height" => 1000_i32} - end - end - sizes - end - - def examine : Bool - existence = File.exists? @dir_path - return false unless existence - files = DirEntry.get_valid_files @dir_path - signature = Dir.directory_entry_signature @dir_path - existence = files.size > 0 && @signature == signature - @sorted_files = nil unless existence - - # For more efficient, update a directory entry with new property - # and return true like Title.examine - existence - end - - def sorted_files - cached_sorted_files = @sorted_files - return cached_sorted_files if cached_sorted_files - @sorted_files = DirEntry.get_valid_files_sorted @dir_path - @sorted_files.not_nil! - end - - def self.validate_directory_entry(dir_path) - files = DirEntry.get_valid_files dir_path - files.size > 0 - end - - def self.get_valid_files(dir_path) - files = [] of String - Dir.entries(dir_path).each do |fn| - next if fn.starts_with? "." - path = File.join dir_path, fn - next unless is_supported_image_file path - next if File.directory? path - next unless File.readable? path - files << path - end - files - end - - def self.get_valid_files_sorted(dir_path) - files = DirEntry.get_valid_files dir_path - files.sort! { |a, b| compare_numerically a, b } - end -end From 648cdd772ccb9b279400b3cef66368ee69db70f8 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 22 May 2022 02:48:06 +0000 Subject: [PATCH 16/50] Add back `zip_path` for backward compatibility --- src/library/entry.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/library/entry.cr b/src/library/entry.cr index b718ed7..596d34d 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -28,6 +28,8 @@ abstract class Entry if err_msg json.field "err_msg", err_msg end + # for API backward compatability + json.field "zip_path", path json.field "title_id", @book.id json.field "title_title", @book.title json.field "sort_title", sort_title From ae503ae099545c23ff636d971068f8aacee9e337 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 22 May 2022 02:54:05 +0000 Subject: [PATCH 17/50] Remove unnecessary `createtime` method --- src/library/archive_entry.cr | 4 ---- src/library/dir_entry.cr | 4 ---- src/library/entry.cr | 4 +--- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/library/archive_entry.cr b/src/library/archive_entry.cr index 4075590..ab229af 100644 --- a/src/library/archive_entry.cr +++ b/src/library/archive_entry.cr @@ -52,10 +52,6 @@ class ArchiveEntry < Entry @zip_path end - def createtime : Time - ctime @zip_path - end - private def sorted_archive_entries ArchiveFile.open @zip_path do |file| entries = file.entries diff --git a/src/library/dir_entry.cr b/src/library/dir_entry.cr index a444cb0..601431b 100644 --- a/src/library/dir_entry.cr +++ b/src/library/dir_entry.cr @@ -60,10 +60,6 @@ class DirEntry < Entry @dir_path end - def createtime : Time - ctime @dir_path - end - def read_page(page_num) img = nil begin diff --git a/src/library/entry.cr b/src/library/entry.cr index 596d34d..5f19d77 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -211,7 +211,7 @@ abstract class Entry TitleInfo.new @book.dir do |info| info_da = info.date_added[@title]? if info_da.nil? - date_added = info.date_added[@title] = createtime + date_added = info.date_added[@title] = ctime path info.save else date_added = info_da @@ -222,8 +222,6 @@ abstract class Entry abstract def path : String - abstract def createtime : Time - abstract def read_page(page_num) abstract def page_dimensions From 82c60ccc1d1430467a61a6b688b9d4d777736876 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 22 May 2022 04:04:40 +0000 Subject: [PATCH 18/50] Replace puts with Logger.debug --- src/routes/reader.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/reader.cr b/src/routes/reader.cr index 052e212..f76dc2d 100644 --- a/src/routes/reader.cr +++ b/src/routes/reader.cr @@ -53,7 +53,7 @@ struct ReaderRouter render "src/views/reader.html.ecr" rescue e Logger.error e - puts e.backtrace? + Logger.debug e.backtrace? env.response.status_code = 404 end end From 872e6dc6d6730a3ebad0c5bfef05bf93e50f0b41 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 22 May 2022 04:20:14 +0000 Subject: [PATCH 19/50] Better method naming in DirEntry --- src/library/dir_entry.cr | 39 +++++++++++++++++---------------------- src/util/signature.cr | 2 +- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/src/library/dir_entry.cr b/src/library/dir_entry.cr index 601431b..59657af 100644 --- a/src/library/dir_entry.cr +++ b/src/library/dir_entry.cr @@ -98,7 +98,7 @@ class DirEntry < Entry def examine : Bool existence = File.exists? @dir_path return false unless existence - files = DirEntry.get_valid_files @dir_path + files = DirEntry.image_files @dir_path signature = Dir.directory_entry_signature @dir_path existence = files.size > 0 && @signature == signature @sorted_files = nil unless existence @@ -111,30 +111,25 @@ class DirEntry < Entry def sorted_files cached_sorted_files = @sorted_files return cached_sorted_files if cached_sorted_files - @sorted_files = DirEntry.get_valid_files_sorted @dir_path + @sorted_files = DirEntry.sorted_image_files @dir_path @sorted_files.not_nil! end + def self.image_files(dir_path) + Dir.entries(dir_path) + .reject(&.starts_with? ".") + .map { |fn| File.join dir_path, fn } + .select { |fn| is_supported_image_file fn } + .reject { |fn| File.directory? fn } + .select { |fn| File.readable? fn } + end + + def self.sorted_image_files(dir_path) + self.image_files(dir_path) + .sort { |a, b| compare_numerically a, b } + end + def self.validate_directory_entry(dir_path) - files = DirEntry.get_valid_files dir_path - files.size > 0 - end - - def self.get_valid_files(dir_path) - files = [] of String - Dir.entries(dir_path).each do |fn| - next if fn.starts_with? "." - path = File.join dir_path, fn - next unless is_supported_image_file path - next if File.directory? path - next unless File.readable? path - files << path - end - files - end - - def self.get_valid_files_sorted(dir_path) - files = DirEntry.get_valid_files dir_path - files.sort! { |a, b| compare_numerically a, b } + image_files(dir_path).size > 0 end end diff --git a/src/util/signature.cr b/src/util/signature.cr index b5fe781..6f67a58 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -84,7 +84,7 @@ class Dir return cache[dirname + "?entry"] if cache[dirname + "?entry"]? Fiber.yield signatures = [] of String - image_files = DirEntry.get_valid_files_sorted dirname + image_files = DirEntry.sorted_image_files dirname if image_files.size > 0 image_files.each do |path| signatures << File.signature(path).to_s From e6dbeb623b8e4e7c6580ae0b32d4f1837349b001 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 22 May 2022 05:12:43 +0000 Subject: [PATCH 20/50] Use `is_valid?` --- src/library/archive_entry.cr | 4 ++++ src/library/dir_entry.cr | 6 +++--- src/library/entry.cr | 13 +++++++++++-- src/library/title.cr | 4 ++-- src/util/signature.cr | 2 +- 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/library/archive_entry.cr b/src/library/archive_entry.cr index ab229af..da313cd 100644 --- a/src/library/archive_entry.cr +++ b/src/library/archive_entry.cr @@ -107,4 +107,8 @@ class ArchiveEntry < Entry def examine : Bool File.exists? @zip_path end + + def self.is_valid?(path : String) : Bool + is_supported_file path + end end diff --git a/src/library/dir_entry.cr b/src/library/dir_entry.cr index 59657af..82b7ccc 100644 --- a/src/library/dir_entry.cr +++ b/src/library/dir_entry.cr @@ -25,7 +25,7 @@ class DirEntry < Entry return end - unless DirEntry.validate_directory_entry @dir_path + unless DirEntry.is_valid? @dir_path @err_msg = "Directory #{@dir_path} is not valid directory entry." Logger.warn "#{@err_msg} Please make sure the " \ "directory has valid images." @@ -129,7 +129,7 @@ class DirEntry < Entry .sort { |a, b| compare_numerically a, b } end - def self.validate_directory_entry(dir_path) - image_files(dir_path).size > 0 + def self.is_valid?(path : String) : Bool + image_files(path).size > 0 end end diff --git a/src/library/entry.cr b/src/library/entry.cr index 5f19d77..131e489 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -28,8 +28,7 @@ abstract class Entry if err_msg json.field "err_msg", err_msg end - # for API backward compatability - json.field "zip_path", path + json.field "zip_path", path # for API backward compatability json.field "title_id", @book.id json.field "title_title", @book.title json.field "sort_title", sort_title @@ -220,6 +219,16 @@ abstract class Entry date_added.not_nil! # is it ok to set not_nil! here? end + # Hack to have abstract class methods + # https://github.com/crystal-lang/crystal/issues/5956 + private module ClassMethods + abstract def is_valid?(path : String) : Bool + end + + macro inherited + extend ClassMethods + end + abstract def path : String abstract def read_page(page_num) diff --git a/src/library/title.cr b/src/library/title.cr index 3f30490..e9873f2 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -53,7 +53,7 @@ class Title Library.default.title_hash[title.id] = title @title_ids << title.id end - if DirEntry.validate_directory_entry path + if DirEntry.is_valid? path entry = DirEntry.new path, self @entries << entry if entry.pages > 0 || entry.err_msg end @@ -146,7 +146,7 @@ class Title path = File.join dir, fn if File.directory? path unless remained_entry_paths.includes? path - if DirEntry.validate_directory_entry path + if DirEntry.is_valid? path entry = DirEntry.new path, self if entry.pages > 0 || entry.err_msg @entries << entry diff --git a/src/util/signature.cr b/src/util/signature.cr index 6f67a58..f0f68bd 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -64,7 +64,7 @@ class Dir path = File.join dirname, fn if File.directory? path signatures << Dir.contents_signature path, cache - if DirEntry.validate_directory_entry path + if DirEntry.is_valid? path signatures << fn end else From 5b23a112b2ebbb236fed2826b08d67cf935c22f9 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 22 May 2022 05:17:05 +0000 Subject: [PATCH 21/50] Remove unnecessary `path` method --- src/library/archive_entry.cr | 5 +---- src/library/dir_entry.cr | 5 +---- src/library/entry.cr | 6 ++---- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/library/archive_entry.cr b/src/library/archive_entry.cr index da313cd..cd63f17 100644 --- a/src/library/archive_entry.cr +++ b/src/library/archive_entry.cr @@ -9,6 +9,7 @@ class ArchiveEntry < Entry def initialize(@zip_path, @book) storage = Storage.default + @path = @zip_path @encoded_path = URI.encode @zip_path @title = File.basename @zip_path, File.extname @zip_path @encoded_title = URI.encode @title @@ -48,10 +49,6 @@ class ArchiveEntry < Entry file.close end - def path : String - @zip_path - end - private def sorted_archive_entries ArchiveFile.open @zip_path do |file| entries = file.entries diff --git a/src/library/dir_entry.cr b/src/library/dir_entry.cr index 82b7ccc..0ce4e71 100644 --- a/src/library/dir_entry.cr +++ b/src/library/dir_entry.cr @@ -14,6 +14,7 @@ class DirEntry < Entry def initialize(@dir_path, @book) storage = Storage.default + @path = @dir_path @encoded_path = URI.encode @dir_path @title = File.basename @dir_path @encoded_title = URI.encode @title @@ -56,10 +57,6 @@ class DirEntry < Entry @pages = sorted_files.size end - def path : String - @dir_path - end - def read_page(page_num) img = nil begin diff --git a/src/library/entry.cr b/src/library/entry.cr index 131e489..2cb4562 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -1,12 +1,12 @@ require "image_size" abstract class Entry - getter id : String, book : Title, title : String, + getter id : String, book : Title, title : String, path : String, size : String, pages : Int32, mtime : Time, encoded_path : String, encoded_title : String, err_msg : String? def initialize( - @id, @title, @book, + @id, @title, @book, @path, @size, @pages, @mtime, @encoded_path, @encoded_title, @err_msg ) @@ -229,8 +229,6 @@ abstract class Entry extend ClassMethods end - abstract def path : String - abstract def read_page(page_num) abstract def page_dimensions From 2fb620211d3db39465b73e6a81634af767015c2c Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 29 May 2022 05:24:41 +0000 Subject: [PATCH 22/50] Choose correct subclass based on YAML node --- src/library/entry.cr | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 2cb4562..58d1b26 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -1,5 +1,15 @@ require "image_size" +private def node_has_key(node : YAML::Nodes::Mapping, key : String) + node.nodes + .map_with_index { |n, i| {n, i} } + .select(&.[1].even?) + .map(&.[0]) + .select(&.is_a?(YAML::Nodes::Scalar)) + .map(&.as(YAML::Nodes::Scalar).value) + .includes? key +end + abstract class Entry getter id : String, book : Title, title : String, path : String, size : String, pages : Int32, mtime : Time, @@ -13,10 +23,20 @@ abstract class Entry end def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) - # TODO: check node? and select proper subclass - ArchiveEntry.new ctx, node - rescue e - DirEntry.new ctx, node + unless node.is_a? YAML::Nodes::Mapping + raise "Unexpected node type in YAML" + end + # Doing YAML::Any.new(ctx, node) here causes a weird error, so + # instead we are using a more hacky approach (see `node_has_key`). + # TODO: Use a more elegant approach + if node_has_key node, "zip_path" + ArchiveEntry.new ctx, node + elsif node_has_key node, "dir_path" + DirEntry.new ctx, node + else + raise "Unknown entry found in YAML cache. Try deleting the " \ + "`library.yml.gz` file" + end end def build_json(*, slim = false) From df618704ea50b9c450f1cfa89043acdda60f77d6 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 29 May 2022 05:28:50 +0000 Subject: [PATCH 23/50] Fix linter --- src/library/entry.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 58d1b26..82c1e04 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -5,7 +5,7 @@ private def node_has_key(node : YAML::Nodes::Mapping, key : String) .map_with_index { |n, i| {n, i} } .select(&.[1].even?) .map(&.[0]) - .select(&.is_a?(YAML::Nodes::Scalar)) + .select(YAML::Nodes::Scalar) .map(&.as(YAML::Nodes::Scalar).value) .includes? key end From 39a331c87937358986caaeb478e33e46f9fe3fcc Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 29 May 2022 05:44:11 +0000 Subject: [PATCH 24/50] Avoid not_nil in date_added --- src/library/entry.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 82c1e04..3a4748c 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -226,7 +226,7 @@ abstract class Entry end def date_added : Time - date_added = nil + date_added = Time::UNIX_EPOCH TitleInfo.new @book.dir do |info| info_da = info.date_added[@title]? if info_da.nil? @@ -236,7 +236,7 @@ abstract class Entry date_added = info_da end end - date_added.not_nil! # is it ok to set not_nil! here? + date_added end # Hack to have abstract class methods From 8e4bb995d316513c6f8b6fdcbe523c2d49dbdf0b Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 4 Jun 2022 00:18:45 +0900 Subject: [PATCH 25/50] Add zip_path to API document, add path property --- src/library/entry.cr | 1 + src/routes/api.cr | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 3a4748c..16666ea 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -49,6 +49,7 @@ abstract class Entry json.field "err_msg", err_msg end json.field "zip_path", path # for API backward compatability + json.field "path", path json.field "title_id", @book.id json.field "title_title", @book.title json.field "sort_title", sort_title diff --git a/src/routes/api.cr b/src/routes/api.cr index 509f13d..89b4a30 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -40,7 +40,7 @@ struct APIRouter Koa.schema "entry", { "pages" => Int32, "mtime" => Int64, - }.merge(s %w(path title size id title_id display_name cover_url)), + }.merge(s %w(zip_path path title size id title_id display_name cover_url)), desc: "An entry in a book" Koa.schema "title", { From 9ce8e918f0f255f21a51fa0a19e188811800992e Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 4 Jun 2022 00:19:14 +0900 Subject: [PATCH 26/50] Replace to is_valid? --- src/util/signature.cr | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/util/signature.cr b/src/util/signature.cr index f0f68bd..8d2b961 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -19,7 +19,7 @@ class File # information as long as the above changes do not happen together with # a file/folder rename, with no library scan in between. def self.signature(filename) : UInt64 - if is_supported_file(filename) || is_supported_image_file(filename) + if ArchiveEntry.is_valid?(filename) || is_supported_image_file(filename) File.info(filename).inode else 0u64 @@ -64,13 +64,11 @@ class Dir path = File.join dirname, fn if File.directory? path signatures << Dir.contents_signature path, cache - if DirEntry.is_valid? path - signatures << fn - end + signatures << fn if DirEntry.is_valid? path else # Only add its signature value to `signatures` when it is a # supported file - signatures << fn if is_supported_file fn + signatures << fn if ArchiveEntry.is_valid? fn end Fiber.yield end From d9dce4a881dc1dcffe32d246746fc173d7cef159 Mon Sep 17 00:00:00 2001 From: torta Date: Sun, 5 Jun 2022 19:06:29 +0800 Subject: [PATCH 27/50] Fix Continue Reading not show missed reading chapter if the latest chapter mark as read --- src/library/title.cr | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/library/title.cr b/src/library/title.cr index e9873f2..e2a112c 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -632,6 +632,15 @@ class Title if last_read_entry && last_read_entry.finished? username last_read_entry = last_read_entry.next_entry username + # Get the last read entry in greedy + if last_read_entry.nil? + sorted_entries(username).each do |e| + unless e.finished? username + last_read_entry = e + break + end + end + end end last_read_entry From 30d5ad0c193f27e4f15bab414dfddd7a9db939c9 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 5 Jun 2022 12:33:26 +0000 Subject: [PATCH 28/50] Hide subscribe button when not subscribable --- public/js/plugin-download.js | 3 +++ src/logger.cr | 1 + src/plugin/plugin.cr | 13 +++++++++++++ src/routes/api.cr | 6 ++++-- src/views/plugin-download.html.ecr | 6 ++++-- 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/public/js/plugin-download.js b/public/js/plugin-download.js index 2e9d0a0..c31350c 100644 --- a/public/js/plugin-download.js +++ b/public/js/plugin-download.js @@ -1,6 +1,7 @@ const component = () => { return { plugins: [], + subscribable: false, info: undefined, pid: undefined, chapters: undefined, // undefined: not searched yet, []: empty @@ -60,6 +61,7 @@ const component = () => { .then((data) => { if (!data.success) throw new Error(data.error); this.info = data.info; + this.subscribable = data.subscribable; this.pid = pid; }) .catch((e) => { @@ -140,6 +142,7 @@ const component = () => { if (!query) return; this.manga = undefined; + this.mid = undefined; if (this.info.version === 1) { this.searchChapters(query); } else { diff --git a/src/logger.cr b/src/logger.cr index 040e5aa..3fc5643 100644 --- a/src/logger.cr +++ b/src/logger.cr @@ -38,6 +38,7 @@ class Logger Log.setup do |c| c.bind "*", @@severity, @backend c.bind "db.*", :error, @backend + c.bind "duktape", :none, @backend end end diff --git a/src/plugin/plugin.cr b/src/plugin/plugin.cr index 5175b3a..7f9fcc8 100644 --- a/src/plugin/plugin.cr +++ b/src/plugin/plugin.cr @@ -223,6 +223,10 @@ class Plugin raise Error.new "Missing required fields in the Page type" end + def can_subscribe? : Bool + info.version > 1 && eval_exists?("newChapters") + end + def search_manga(query : String) if info.version == 1 raise Error.new "Manga searching is only available for plugins " \ @@ -327,6 +331,15 @@ class Plugin JSON.parse eval(str).as String end + private def eval_exists?(str) : Bool + @rt.eval str + true + rescue e : Duktape::ReferenceError + false + rescue e : Duktape::Error + raise Error.new e.message + end + private def def_helper_functions(sbx) sbx.push_object diff --git a/src/routes/api.cr b/src/routes/api.cr index 89b4a30..8d31fea 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -871,13 +871,15 @@ struct APIRouter "version" => Int32, "settings" => {} of String => String, }, + "subscribable" => Bool, } get "/api/admin/plugin/info" do |env| begin plugin = Plugin.new env.params.query["plugin"].as String send_json env, { - "success" => true, - "info" => plugin.info, + "success" => true, + "info" => plugin.info, + "subscribable" => plugin.can_subscribe?, }.to_json rescue e Logger.error e diff --git a/src/views/plugin-download.html.ecr b/src/views/plugin-download.html.ecr index 7c3b4d5..3ff107b 100644 --- a/src/views/plugin-download.html.ecr +++ b/src/views/plugin-download.html.ecr @@ -133,8 +133,10 @@ - - + + + +

From 5b58d8ac59efc3364edbb1b074123ef263066ad3 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 5 Jun 2022 12:40:45 +0000 Subject: [PATCH 29/50] Clear page when switching plugins --- public/js/plugin-download.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/public/js/plugin-download.js b/public/js/plugin-download.js index c31350c..0f19738 100644 --- a/public/js/plugin-download.js +++ b/public/js/plugin-download.js @@ -72,6 +72,9 @@ const component = () => { }); }, pluginChanged() { + this.manga = undefined; + this.chapters = undefined; + this.mid = undefined; this.loadPlugin(this.pid); localStorage.setItem("plugin", this.pid); }, From ea35faee91e82bc6d7e79881eaf8427eb7feb23f Mon Sep 17 00:00:00 2001 From: tr7zw Date: Tue, 7 Jun 2022 00:28:41 +0200 Subject: [PATCH 30/50] Add jxl support --- src/util/util.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/util/util.cr b/src/util/util.cr index e08bd9d..cb53b46 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -11,6 +11,7 @@ SUPPORTED_IMG_TYPES = %w( image/avif image/gif image/svg+xml + image/jxl ) def random_str @@ -49,6 +50,7 @@ def register_mime_types # defiend by Crystal in `MIME.DEFAULT_TYPES` ".apng" => "image/apng", ".avif" => "image/avif", + ".jxl" => "image/jxl", }.each do |k, v| MIME.register k, v end From ae583cf2a9a2c12f099811c63c67886a651fe76d Mon Sep 17 00:00:00 2001 From: tr7zw Date: Tue, 7 Jun 2022 16:09:02 +0200 Subject: [PATCH 31/50] Workaround for "0 width/height" api responses This needs a more proper fix probably. --- public/js/reader.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/public/js/reader.js b/public/js/reader.js index a5d8e6b..11b3b49 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -29,14 +29,14 @@ const readerComponent = () => { return { id: i + 1, url: `${base_url}api/page/${tid}/${eid}/${i+1}`, - width: d.width, - height: d.height, + width: d.width == 0 ? "100%" : d.width, + height: d.height == 0 ? "100%" : d.height, }; }); - const avgRatio = this.items.reduce((acc, cur) => { + const avgRatio = dimensions.reduce((acc, cur) => { return acc + cur.height / cur.width - }, 0) / this.items.length; + }, 0) / dimensions.length; console.log(avgRatio); this.longPages = avgRatio > 2; From be46dd1f86afbf58b8cbe987445ab95ef8d6f7ea Mon Sep 17 00:00:00 2001 From: Chris Alexander Date: Wed, 15 Jun 2022 10:12:51 -0500 Subject: [PATCH 32/50] Allow config defaults to be sourced from ENV This allows the default config to source values from ENV variables if they are set. With this change we don't have to modify the docker CMD or edit the config.yml and then relaunch. --- spec/config_spec.cr | 21 +++++++++++++++++++-- src/config.cr | 46 ++++++++++++++++++++++----------------------- src/util/util.cr | 4 ++-- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/spec/config_spec.cr b/spec/config_spec.cr index e2d5fca..5ea6f89 100644 --- a/spec/config_spec.cr +++ b/spec/config_spec.cr @@ -1,14 +1,31 @@ require "./spec_helper" describe Config do - it "creates config if it does not exist" do - with_default_config do |_, path| + it "creates default config if it does not exist" do + with_default_config do |config, path| File.exists?(path).should be_true + config.port.should eq 9000 end end it "correctly loads config" do config = Config.load "spec/asset/test-config.yml" config.port.should eq 3000 + config.base_url.should eq "/" + end + + it "correctly reads config defaults from ENV" do + ENV["LOG_LEVEL"] = "debug" + config = Config.load "spec/asset/test-config.yml" + config.log_level.should eq "debug" + config.base_url.should eq "/" + end + + it "correctly handles ENV truthiness" do + ENV["CACHE_ENABLED"] = "false" + config = Config.load "spec/asset/test-config.yml" + config.cache_enabled.should be_false + config.cache_log_enabled.should be_true + config.disable_login.should be_false end end diff --git a/src/config.cr b/src/config.cr index 807a74c..dd0fbda 100644 --- a/src/config.cr +++ b/src/config.cr @@ -4,28 +4,28 @@ class Config include YAML::Serializable @[YAML::Field(ignore: true)] - property path = "" - property host = "0.0.0.0" - property port : Int32 = 9000 - property base_url = "/" - property session_secret = "mango-session-secret" - property library_path = "~/mango/library" - property library_cache_path = "~/mango/library.yml.gz" - property db_path = "~/mango/mango.db" - property queue_db_path = "~/mango/queue.db" - property scan_interval_minutes : Int32 = 5 - property thumbnail_generation_interval_hours : Int32 = 24 - property log_level = "info" - property upload_path = "~/mango/uploads" - property plugin_path = "~/mango/plugins" - property download_timeout_seconds : Int32 = 30 - property cache_enabled = true - property cache_size_mbs = 50 - property cache_log_enabled = true - property disable_login = false - property default_username = "" - property auth_proxy_header_name = "" - property plugin_update_interval_hours : Int32 = 24 + property path : String = "" + property host : String = (ENV["LISTEN_HOST"]? || "0.0.0.0") + property port : Int32 = (ENV["LISTEN_PORT"]? || 9000).to_i + property base_url : String = (ENV["BASE_URL"]? || "/") + property session_secret : String = (ENV["SESSION_SECRET"]? || "mango-session-secret") + property library_path : String = (ENV["LIBRARY_PATH"]? || "~/mango/library") + property library_cache_path : String = (ENV["LIBRARY_CACHE_PATH"]? || "~/mango/library.yml.gz") + property db_path : String = (ENV["DB_PATH"]? || "~/mango/mango.db") + property queue_db_path : String = (ENV["QUEUE_DB_PATH"]? || "~/mango/queue.db") + property scan_interval_minutes : Int32 = (ENV["SCAN_INTERVAL"]? || 5).to_i + property thumbnail_generation_interval_hours : Int32 = (ENV["THUMBNAIL_INTERVAL"]? || 24).to_i + property log_level : String = (ENV["LOG_LEVEL"]? || "info") + property upload_path : String = (ENV["UPLOAD_PATH"]? || "~/mango/uploads") + property plugin_path : String = (ENV["PLUGIN_PATH"]? || "~/mango/plugins") + property download_timeout_seconds : Int32 = (ENV["DOWNLOAD_TIMEOUT"]? || 30).to_i + property cache_enabled : Bool = env_is_true?("CACHE_ENABLED", true) + property cache_size_mbs : Int32 = (ENV["CACHE_SIZE"]? || 50).to_i + property cache_log_enabled : Bool = env_is_true?("CACHE_LOG_ENABLED", true) + property disable_login : Bool = env_is_true?("DISABLE_LOGIN", false) + property default_username : String = (ENV["DEFAULT_USERNAME"]? || "") + property auth_proxy_header_name : String = (ENV["AUTH_PROXY_HEADER"]? || "") + property plugin_update_interval_hours : Int32 = (ENV["PLUGIN_UPDATE_INTERVAL"]? || 24).to_i @@singlet : Config? @@ -38,7 +38,7 @@ class Config end def self.load(path : String?) - path = "~/.config/mango/config.yml" if path.nil? + path = (ENV["CONFIG_PATH"]? || "~/.config/mango/config.yml") if path.nil? cfg_path = File.expand_path path, home: true if File.exists? cfg_path config = self.from_yaml File.read cfg_path diff --git a/src/util/util.cr b/src/util/util.cr index e08bd9d..ed3dc46 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -93,9 +93,9 @@ class String end end -def env_is_true?(key : String) : Bool +def env_is_true?(key : String, default : Bool = false) : Bool val = ENV[key.upcase]? || ENV[key.downcase]? - return false unless val + return default unless val val.downcase.in? "1", "true" end From bbc0c2cbb70407afb3bfaa2499c76d567b0f3276 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 18 Jun 2022 17:43:26 +0900 Subject: [PATCH 33/50] Fix Dir.contents_signature to detect valid image files added --- src/util/signature.cr | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/util/signature.cr b/src/util/signature.cr index 8d2b961..bb16bed 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -64,11 +64,10 @@ class Dir path = File.join dirname, fn if File.directory? path signatures << Dir.contents_signature path, cache - signatures << fn if DirEntry.is_valid? path else # Only add its signature value to `signatures` when it is a # supported file - signatures << fn if ArchiveEntry.is_valid? fn + signatures << fn if ArchiveEntry.is_valid?(fn) || is_supported_image_file(fn) end Fiber.yield end From 17a9c8ecd3b9ee1c6ad1ccf30470e66708018e7b Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 18 Jun 2022 18:51:33 +0900 Subject: [PATCH 34/50] pass lint --- src/util/signature.cr | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/util/signature.cr b/src/util/signature.cr index bb16bed..74c8b8e 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -67,7 +67,9 @@ class Dir else # Only add its signature value to `signatures` when it is a # supported file - signatures << fn if ArchiveEntry.is_valid?(fn) || is_supported_image_file(fn) + if ArchiveEntry.is_valid?(fn) || is_supported_image_file(fn) + signatures << fn + end end Fiber.yield end From a639392ca05c0c19676222ae06d339479b978a72 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 18 Jun 2022 10:22:25 +0000 Subject: [PATCH 35/50] Update comment --- src/library/title.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/library/title.cr b/src/library/title.cr index e2a112c..9c3ad78 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -632,8 +632,9 @@ class Title if last_read_entry && last_read_entry.finished? username last_read_entry = last_read_entry.next_entry username - # Get the last read entry in greedy if last_read_entry.nil? + # The last entry is finished. Return the first unfinished entry + # (if any) sorted_entries(username).each do |e| unless e.finished? username last_read_entry = e From fe440d82d49bab2ea793b41046f710fd4fd335ee Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 18 Jun 2022 11:10:14 +0000 Subject: [PATCH 36/50] Fix linter issue --- src/util/util.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/util.cr b/src/util/util.cr index cb53b46..68e26c7 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -50,7 +50,7 @@ def register_mime_types # defiend by Crystal in `MIME.DEFAULT_TYPES` ".apng" => "image/apng", ".avif" => "image/avif", - ".jxl" => "image/jxl", + ".jxl" => "image/jxl", }.each do |k, v| MIME.register k, v end From 31df058f8135f8191c92583f7f06557c3f1ec4e1 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 18 Jun 2022 11:25:20 +0000 Subject: [PATCH 37/50] Comment about infinity average ratio --- public/js/reader.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/js/reader.js b/public/js/reader.js index 11b3b49..fa66b77 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -34,6 +34,8 @@ const readerComponent = () => { }; }); + // Note: for image types not supported by image_size.cr, the width and height will be 0, and so `avgRatio` will be `Infinity`. + // TODO: support more image types in image_size.cr const avgRatio = dimensions.reduce((acc, cur) => { return acc + cur.height / cur.width }, 0) / dimensions.length; From 2e91028ead3ade765a75ac1fe76781c9a82d596d Mon Sep 17 00:00:00 2001 From: Chris Alexander Date: Wed, 15 Jun 2022 10:12:51 -0500 Subject: [PATCH 38/50] Allow config defaults to be sourced from ENV This allows the default config to source values from ENV variables if they are set. With this change we don't have to modify the docker CMD or edit the config.yml and then relaunch. --- spec/config_spec.cr | 21 ++++++++++++++++-- src/config.cr | 52 +++++++++++++++++++++++++-------------------- src/util/util.cr | 4 ++-- 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/spec/config_spec.cr b/spec/config_spec.cr index e2d5fca..5ea6f89 100644 --- a/spec/config_spec.cr +++ b/spec/config_spec.cr @@ -1,14 +1,31 @@ require "./spec_helper" describe Config do - it "creates config if it does not exist" do - with_default_config do |_, path| + it "creates default config if it does not exist" do + with_default_config do |config, path| File.exists?(path).should be_true + config.port.should eq 9000 end end it "correctly loads config" do config = Config.load "spec/asset/test-config.yml" config.port.should eq 3000 + config.base_url.should eq "/" + end + + it "correctly reads config defaults from ENV" do + ENV["LOG_LEVEL"] = "debug" + config = Config.load "spec/asset/test-config.yml" + config.log_level.should eq "debug" + config.base_url.should eq "/" + end + + it "correctly handles ENV truthiness" do + ENV["CACHE_ENABLED"] = "false" + config = Config.load "spec/asset/test-config.yml" + config.cache_enabled.should be_false + config.cache_log_enabled.should be_true + config.disable_login.should be_false end end diff --git a/src/config.cr b/src/config.cr index 807a74c..f4a2cf3 100644 --- a/src/config.cr +++ b/src/config.cr @@ -4,28 +4,34 @@ class Config include YAML::Serializable @[YAML::Field(ignore: true)] - property path = "" - property host = "0.0.0.0" - property port : Int32 = 9000 - property base_url = "/" - property session_secret = "mango-session-secret" - property library_path = "~/mango/library" - property library_cache_path = "~/mango/library.yml.gz" - property db_path = "~/mango/mango.db" - property queue_db_path = "~/mango/queue.db" - property scan_interval_minutes : Int32 = 5 - property thumbnail_generation_interval_hours : Int32 = 24 - property log_level = "info" - property upload_path = "~/mango/uploads" - property plugin_path = "~/mango/plugins" - property download_timeout_seconds : Int32 = 30 - property cache_enabled = true - property cache_size_mbs = 50 - property cache_log_enabled = true - property disable_login = false - property default_username = "" - property auth_proxy_header_name = "" - property plugin_update_interval_hours : Int32 = 24 + property path : String = "" + property host : String = (ENV["LISTEN_HOST"]? || "0.0.0.0") + property port : Int32 = (ENV["LISTEN_PORT"]? || 9000).to_i + property base_url : String = (ENV["BASE_URL"]? || "/") + property session_secret : String = \ + (ENV["SESSION_SECRET"]? || "mango-session-secret") + property library_path : String = (ENV["LIBRARY_PATH"]? || "~/mango/library") + property library_cache_path : String = \ + (ENV["LIBRARY_CACHE_PATH"]? || "~/mango/library.yml.gz") + property db_path : String = (ENV["DB_PATH"]? || "~/mango/mango.db") + property queue_db_path : String = \ + (ENV["QUEUE_DB_PATH"]? || "~/mango/queue.db") + property scan_interval_minutes : Int32 = (ENV["SCAN_INTERVAL"]? || 5).to_i + property thumbnail_generation_interval_hours : Int32 = \ + (ENV["THUMBNAIL_INTERVAL"]? || 24).to_i + property log_level : String = (ENV["LOG_LEVEL"]? || "info") + property upload_path : String = (ENV["UPLOAD_PATH"]? || "~/mango/uploads") + property plugin_path : String = (ENV["PLUGIN_PATH"]? || "~/mango/plugins") + property download_timeout_seconds : Int32 = \ + (ENV["DOWNLOAD_TIMEOUT"]? || 30).to_i + property cache_enabled : Bool = env_is_true?("CACHE_ENABLED", true) + property cache_size_mbs : Int32 = (ENV["CACHE_SIZE"]? || 50).to_i + property cache_log_enabled : Bool = env_is_true?("CACHE_LOG_ENABLED", true) + property disable_login : Bool = env_is_true?("DISABLE_LOGIN", false) + property default_username : String = (ENV["DEFAULT_USERNAME"]? || "") + property auth_proxy_header_name : String = (ENV["AUTH_PROXY_HEADER"]? || "") + property plugin_update_interval_hours : Int32 = \ + (ENV["PLUGIN_UPDATE_INTERVAL"]? || 24).to_i @@singlet : Config? @@ -38,7 +44,7 @@ class Config end def self.load(path : String?) - path = "~/.config/mango/config.yml" if path.nil? + path = (ENV["CONFIG_PATH"]? || "~/.config/mango/config.yml") if path.nil? cfg_path = File.expand_path path, home: true if File.exists? cfg_path config = self.from_yaml File.read cfg_path diff --git a/src/util/util.cr b/src/util/util.cr index 68e26c7..ce658aa 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -95,9 +95,9 @@ class String end end -def env_is_true?(key : String) : Bool +def env_is_true?(key : String, default : Bool = false) : Bool val = ENV[key.upcase]? || ENV[key.downcase]? - return false unless val + return default unless val val.downcase.in? "1", "true" end From f3eb62a2714326489b6773c4ea0a0be07a767dbe Mon Sep 17 00:00:00 2001 From: Chris Alexander Date: Mon, 27 Jun 2022 09:19:12 -0500 Subject: [PATCH 39/50] Disable line length warnings --- src/config.cr | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/config.cr b/src/config.cr index f4a2cf3..867b8ab 100644 --- a/src/config.cr +++ b/src/config.cr @@ -8,30 +8,30 @@ class Config property host : String = (ENV["LISTEN_HOST"]? || "0.0.0.0") property port : Int32 = (ENV["LISTEN_PORT"]? || 9000).to_i property base_url : String = (ENV["BASE_URL"]? || "/") - property session_secret : String = \ - (ENV["SESSION_SECRET"]? || "mango-session-secret") + # ameba:disable Layout/LineLength + property session_secret : String = (ENV["SESSION_SECRET"]? || "mango-session-secret") property library_path : String = (ENV["LIBRARY_PATH"]? || "~/mango/library") - property library_cache_path : String = \ - (ENV["LIBRARY_CACHE_PATH"]? || "~/mango/library.yml.gz") + # ameba:disable Layout/LineLength + property library_cache_path : String = (ENV["LIBRARY_CACHE_PATH"]? || "~/mango/library.yml.gz") property db_path : String = (ENV["DB_PATH"]? || "~/mango/mango.db") - property queue_db_path : String = \ - (ENV["QUEUE_DB_PATH"]? || "~/mango/queue.db") + # ameba:disable Layout/LineLength + property queue_db_path : String = (ENV["QUEUE_DB_PATH"]? || "~/mango/queue.db") property scan_interval_minutes : Int32 = (ENV["SCAN_INTERVAL"]? || 5).to_i - property thumbnail_generation_interval_hours : Int32 = \ - (ENV["THUMBNAIL_INTERVAL"]? || 24).to_i + # ameba:disable Layout/LineLength + property thumbnail_generation_interval_hours : Int32 = (ENV["THUMBNAIL_INTERVAL"]? || 24).to_i property log_level : String = (ENV["LOG_LEVEL"]? || "info") property upload_path : String = (ENV["UPLOAD_PATH"]? || "~/mango/uploads") property plugin_path : String = (ENV["PLUGIN_PATH"]? || "~/mango/plugins") - property download_timeout_seconds : Int32 = \ - (ENV["DOWNLOAD_TIMEOUT"]? || 30).to_i + # ameba:disable Layout/LineLength + property download_timeout_seconds : Int32 = (ENV["DOWNLOAD_TIMEOUT"]? || 30).to_i property cache_enabled : Bool = env_is_true?("CACHE_ENABLED", true) property cache_size_mbs : Int32 = (ENV["CACHE_SIZE"]? || 50).to_i property cache_log_enabled : Bool = env_is_true?("CACHE_LOG_ENABLED", true) property disable_login : Bool = env_is_true?("DISABLE_LOGIN", false) property default_username : String = (ENV["DEFAULT_USERNAME"]? || "") property auth_proxy_header_name : String = (ENV["AUTH_PROXY_HEADER"]? || "") - property plugin_update_interval_hours : Int32 = \ - (ENV["PLUGIN_UPDATE_INTERVAL"]? || 24).to_i + # ameba:disable Layout/LineLength + property plugin_update_interval_hours : Int32 = (ENV["PLUGIN_UPDATE_INTERVAL"]? || 24).to_i @@singlet : Config? From f2d6d28a72cbc0c932623a643ec541ba14e94287 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 3 Jul 2022 07:24:33 +0000 Subject: [PATCH 40/50] Define properties with macro --- src/config.cr | 68 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/src/config.cr b/src/config.cr index 867b8ab..2384431 100644 --- a/src/config.cr +++ b/src/config.cr @@ -1,37 +1,51 @@ require "yaml" class Config + private OPTIONS = { + "host" => "0.0.0.0", + "port" => 9000, + "base_url" => "/", + "session_secret" => "mango-session-secret", + "library_path" => "~/mango/library", + "library_cache_path" => "~/mango/library.yml.gz", + "db_path" => "~/mango.db", + "queue_db_path" => "~/mango/queue.db", + "scan_interval_minutes" => 5, + "thumbnail_generation_interval_hours" => 24, + "log_level" => "info", + "upload_path" => "~/mango/uploads", + "plugin_path" => "~/mango/plugins", + "download_timeout_seconds" => 30, + "cache_enabled" => true, + "cache_size_mbs" => 50, + "cache_log_enabled" => true, + "disable_login" => false, + "default_username" => "", + "auth_proxy_header_name" => "", + "plugin_update_interval_hours" => 24, + } + include YAML::Serializable @[YAML::Field(ignore: true)] property path : String = "" - property host : String = (ENV["LISTEN_HOST"]? || "0.0.0.0") - property port : Int32 = (ENV["LISTEN_PORT"]? || 9000).to_i - property base_url : String = (ENV["BASE_URL"]? || "/") - # ameba:disable Layout/LineLength - property session_secret : String = (ENV["SESSION_SECRET"]? || "mango-session-secret") - property library_path : String = (ENV["LIBRARY_PATH"]? || "~/mango/library") - # ameba:disable Layout/LineLength - property library_cache_path : String = (ENV["LIBRARY_CACHE_PATH"]? || "~/mango/library.yml.gz") - property db_path : String = (ENV["DB_PATH"]? || "~/mango/mango.db") - # ameba:disable Layout/LineLength - property queue_db_path : String = (ENV["QUEUE_DB_PATH"]? || "~/mango/queue.db") - property scan_interval_minutes : Int32 = (ENV["SCAN_INTERVAL"]? || 5).to_i - # ameba:disable Layout/LineLength - property thumbnail_generation_interval_hours : Int32 = (ENV["THUMBNAIL_INTERVAL"]? || 24).to_i - property log_level : String = (ENV["LOG_LEVEL"]? || "info") - property upload_path : String = (ENV["UPLOAD_PATH"]? || "~/mango/uploads") - property plugin_path : String = (ENV["PLUGIN_PATH"]? || "~/mango/plugins") - # ameba:disable Layout/LineLength - property download_timeout_seconds : Int32 = (ENV["DOWNLOAD_TIMEOUT"]? || 30).to_i - property cache_enabled : Bool = env_is_true?("CACHE_ENABLED", true) - property cache_size_mbs : Int32 = (ENV["CACHE_SIZE"]? || 50).to_i - property cache_log_enabled : Bool = env_is_true?("CACHE_LOG_ENABLED", true) - property disable_login : Bool = env_is_true?("DISABLE_LOGIN", false) - property default_username : String = (ENV["DEFAULT_USERNAME"]? || "") - property auth_proxy_header_name : String = (ENV["AUTH_PROXY_HEADER"]? || "") - # ameba:disable Layout/LineLength - property plugin_update_interval_hours : Int32 = (ENV["PLUGIN_UPDATE_INTERVAL"]? || 24).to_i + + # Go through the options constant above and define them as properties. + # Allow setting the default values through environment variables. + # Overall precedence: config file > environment variable > default value + {% begin %} + {% for k, v in OPTIONS %} + {% if v.is_a? StringLiteral %} + property {{k.id}} : String = ENV[{{k.upcase}}]? || {{ v }} + {% elsif v.is_a? NumberLiteral %} + property {{k.id}} : Int32 = (ENV[{{k.upcase}}]? || {{ v.id }}).to_i + {% elsif v.is_a? BoolLiteral %} + property {{k.id}} : Bool = env_is_true? {{ k.upcase }}, {{ v.id }} + {% else %} + raise "Unknown type in config option: {{ v.class_name.id }}" + {% end %} + {% end %} + {% end %} @@singlet : Config? From 405b958deb4428b2aca0568c7ecf00225fbd0de8 Mon Sep 17 00:00:00 2001 From: Hiers Date: Tue, 5 Jul 2022 22:01:21 +0100 Subject: [PATCH 41/50] First draft of image fit. --- public/js/reader.js | 26 ++++++++++++++++++++++++++ src/views/reader.html.ecr | 21 ++++++++++++++++----- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/public/js/reader.js b/public/js/reader.js index fa66b77..6cacf78 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -14,6 +14,7 @@ const readerComponent = () => { margin: 30, preloadLookahead: 3, enableRightToLeft: false, + fitType: 'vert', /** * Initialize the component by fetching the page dimensions @@ -65,6 +66,10 @@ const readerComponent = () => { this.preloadImage(this.items[idx - 1].url); } + const savedFitType = localStorage.getItem('fitType'); + if(savedFitType){ + this.fitType = savedFitType; + } const savedFlipAnimation = localStorage.getItem('enableFlipAnimation'); this.enableFlipAnimation = savedFlipAnimation === null || savedFlipAnimation === 'true'; @@ -301,6 +306,21 @@ const readerComponent = () => { }); }); }, + /** + * Changes how the view should fit to the screen or if it should use the image's real size + * + * @param {string} fitType - ver, horz and real for fitting to height, width, + * and showing real size, respectively + */ + setFit(fitType){ + if (fitType === 'vert'){ + document.styleSheets[0].rules[21].style.maxWidth = '100%'; + } else if(fitType === 'horz'){ + document.styleSheets[0].rules[21].style.maxWidth = '100%'; + } else if (fitType === 'real'){ + document.styleSheets[0].rules[21].style.maxWidth = ''; + } + }, /** * Marks progress as 100% and jumps to the next entry * @@ -335,6 +355,12 @@ const readerComponent = () => { this.toPage(this.selectedIndex); }, + fitChanged(){ + this.fitType = $('#fit-select').val(); + this.setFit(this.fitType); + localStorage.setItem('fitType', this.fitType); + }, + preloadLookaheadChanged() { localStorage.setItem('preloadLookahead', this.preloadLookahead); }, diff --git a/src/views/reader.html.ecr b/src/views/reader.html.ecr index 19e2b19..6e4fc8a 100644 --- a/src/views/reader.html.ecr +++ b/src/views/reader.html.ecr @@ -40,18 +40,18 @@ <%- end -%>
-
+
@@ -94,6 +94,17 @@
+
+ +
+ +
+
+
From db5e99b3f0a202029c1862da6013eaf356102d0e Mon Sep 17 00:00:00 2001 From: Hiers Date: Tue, 5 Jul 2022 22:24:31 +0100 Subject: [PATCH 42/50] Fix in reader.html.ecr. --- src/views/reader.html.ecr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/views/reader.html.ecr b/src/views/reader.html.ecr index 6e4fc8a..dacd222 100644 --- a/src/views/reader.html.ecr +++ b/src/views/reader.html.ecr @@ -48,7 +48,7 @@ 'uk-animation-slide-right': flipAnimation === 'right' }" :data-src="curItem.url" :width="curItem.width" :height="curItem.height" :id="curItem.id" @click="clickImage($event)" :style="` width:${fitType === 'horz' ? '100vw' : 'auto'}; - height:${mode === 'vert' ? '100vh' : 'auto'}; + height:${fitType === 'vert' ? '100vh' : 'auto'}; margin-bottom:0; max-width:${fitType === 'horz' ? '100%' : ''}; max-height:${fitType === 'vert' ? '100%' : ''}; From 6ddbe8d43657867bf663d209fe1d8e027f25c6f1 Mon Sep 17 00:00:00 2001 From: Hiers Date: Thu, 7 Jul 2022 08:55:54 +0100 Subject: [PATCH 43/50] Changed setFit function to not have redundant ifs and a better comment explaining what it does. --- public/js/reader.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/public/js/reader.js b/public/js/reader.js index 6cacf78..df92b4f 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -307,18 +307,16 @@ const readerComponent = () => { }); }, /** - * Changes how the view should fit to the screen or if it should use the image's real size + * Sets the image to not be restricted to the size of its container if it's not being scaled * * @param {string} fitType - ver, horz and real for fitting to height, width, * and showing real size, respectively */ setFit(fitType){ - if (fitType === 'vert'){ - document.styleSheets[0].rules[21].style.maxWidth = '100%'; - } else if(fitType === 'horz'){ - document.styleSheets[0].rules[21].style.maxWidth = '100%'; - } else if (fitType === 'real'){ + if (fitType === 'real'){ document.styleSheets[0].rules[21].style.maxWidth = ''; + } else { + document.styleSheets[0].rules[21].style.maxWidth = '100%'; } }, /** From 624283643ccc19551db3c19cbb1c9bbedcb8b377 Mon Sep 17 00:00:00 2001 From: Hiers Date: Wed, 13 Jul 2022 14:20:43 +0100 Subject: [PATCH 44/50] Fixed right flip panel not being all the way on the right; changed real image size option to not be hard coded. --- public/js/reader.js | 17 ++--------------- src/views/reader.html.ecr | 8 ++++---- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/public/js/reader.js b/public/js/reader.js index df92b4f..63cef7f 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -67,8 +67,9 @@ const readerComponent = () => { } const savedFitType = localStorage.getItem('fitType'); - if(savedFitType){ + if (savedFitType) { this.fitType = savedFitType; + $('#fit-select').val(savedFitType); } const savedFlipAnimation = localStorage.getItem('enableFlipAnimation'); this.enableFlipAnimation = savedFlipAnimation === null || savedFlipAnimation === 'true'; @@ -306,19 +307,6 @@ const readerComponent = () => { }); }); }, - /** - * Sets the image to not be restricted to the size of its container if it's not being scaled - * - * @param {string} fitType - ver, horz and real for fitting to height, width, - * and showing real size, respectively - */ - setFit(fitType){ - if (fitType === 'real'){ - document.styleSheets[0].rules[21].style.maxWidth = ''; - } else { - document.styleSheets[0].rules[21].style.maxWidth = '100%'; - } - }, /** * Marks progress as 100% and jumps to the next entry * @@ -355,7 +343,6 @@ const readerComponent = () => { fitChanged(){ this.fitType = $('#fit-select').val(); - this.setFit(this.fitType); localStorage.setItem('fitType', this.fitType); }, diff --git a/src/views/reader.html.ecr b/src/views/reader.html.ecr index dacd222..f1e7236 100644 --- a/src/views/reader.html.ecr +++ b/src/views/reader.html.ecr @@ -19,7 +19,7 @@
+ :class="{'uk-container': true, 'uk-container-small': mode === 'continuous', 'uk-container-expand': mode !== 'continuous'}" style="width: fit-content;">