diff --git a/.github/workflows/docker_build.yml b/.github/workflows/docker_build.yml new file mode 100644 index 00000000..489770b5 --- /dev/null +++ b/.github/workflows/docker_build.yml @@ -0,0 +1,102 @@ +name: Build Containers +on: + push: + tags: + - 'v*' + workflow_dispatch: [] + +jobs: + build-client: + runs-on: [ubuntu-latest] + steps: + - + name: Checkout + uses: actions/checkout@v2.3.4 + - + name: Docker meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v2 + with: + images: | + ghcr.io/hydrusnetwork/hydrus + tags: | + type=ref,event=tag + labels: | + org.opencontainers.image.title=Hydrus Network + org.opencontainers.image.description=A personal booru-style media tagger that can import files and tags from your hard drive and popular websites. + org.opencontainers.image.vendor=hydrusnetwork + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + buildkitd-flags: "--debug" + - + name: Login to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + - + name: Build + uses: docker/build-push-action@v2 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + file: ./static/build_files/docker/client/Dockerfile + platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/386,linux/arm/v7 + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + + build-server: + runs-on: [ubuntu-latest] + steps: + - + name: Checkout + uses: actions/checkout@v2.3.4 + - + name: Docker meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v2 + with: + images: | + ghcr.io/hydrusnetwork/hydrus + tags: | + type=ref,event=tag,prefix=server- + type=raw,enable=true,prefix=,suffix=,value=server + flavor: | + latest=false + labels: | + org.opencontainers.image.title=Hydrus Network Server + org.opencontainers.image.description=A personal booru-style media tagger that can import files and tags from your hard drive and popular websites. + org.opencontainers.image.vendor=hydrusnetwork + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + buildkitd-flags: "--debug" + - + name: Login to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + - + name: Build + uses: docker/build-push-action@v2 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + file: ./static/build_files/docker/server/Dockerfile + platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/386,linux/arm/v7 + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..8dcbfdfc --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,270 @@ +name: Release +on: + push: + tags: + - 'v*' + workflow_dispatch: [] + +jobs: + build-macos: + runs-on: macos-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Setup FFMPEG + uses: FedericoCarboni/setup-ffmpeg@v1 + id: setup_ffmpeg + with: + token: ${{ secrets.GITHUB_TOKEN }} + - + name: Setup Rust Tollchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - + name: Cargo Cache + uses: actions/cache@v2 + with: + path: ~/.cargo + key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.toml') }} + restore-keys: | + ${{ runner.os }}-cargo-${{ hashFiles('Cargo.toml') }} + ${{ runner.os }}-cargo + - + name: Cargo Artifacts Cache + uses: actions/cache@v2 + with: + path: target + key: ${{ runner.os }}-cargo-target-${{ hashFiles('Cargo.toml') }} + restore-keys: | + ${{ runner.os }}-cargo-target-${{ hashFiles('Cargo.toml') }} + ${{ runner.os }}-cargo-target + - + name: Install PyOxidizer + uses: actions-rs/cargo@v1.0.3 + with: + command: install + args: pyoxidizer + - + name: Build Hydrus + run: | + cd $GITHUB_WORKSPACE + cp ${{ steps.setup_ffmpeg.outputs.ffmpeg-path }} bin/ + cp static/build_files/macos/pyoxidizer.bzl pyoxidizer.bzl + cp static/build_files/linux/requirements.txt requirements.txt + basename $(rustc --print sysroot) | sed -e "s/^stable-//" > triple.txt + pyoxidizer build --release + cd build/$(head -n 1 triple.txt)/release + mkdir -p "Hydrus Network.app/Contents/MacOS" + mkdir -p "Hydrus Network.app/Contents/Resources" + mkdir -p "Hydrus Network.app/Contents/Frameworks" + mv install/static/icon.icns "Hydrus Network.app/Contents/Resources/icon.icns" + cp install/static/build_files/macos/Info.plist "Hydrus Network.app/Contents/Info.plist" + cp install/static/build_files/macos/ReadMeFirst.rtf ./ReadMeFirst.rtf + cp install/static/build_files/macos/running_from_app "install/running_from_app" + ln -s /Applications ./Applications + mv install/* "Hydrus Network.app/Contents/MacOS/" + rm -rf install + cd $GITHUB_WORKSPACE + temp_dmg="$(mktemp).dmg" + hdiutil create "$temp_dmg" -ov -volname "HydrusNetwork" -fs HFS+ -srcfolder "$GITHUB_WORKSPACE/build/$(head -n 1 triple.txt)/release" + hdiutil convert "$temp_dmg" -format UDZO -o HydrusNetwork.dmg + - + name: Upload a Build Artifact + uses: actions/upload-artifact@v2.2.1 + with: + name: MacOS-DMG + path: HydrusNetwork.dmg + if-no-files-found: error + retention-days: 2 + + build-ubuntu: + runs-on: ubuntu-18.04 + steps: + - + name: Checkout + uses: actions/checkout@v2 + with: + path: hydrus + - + name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + architecture: x64 + #- name: Cache Qt + # id: cache-qt + # uses: actions/cache@v1 + # with: + # path: Qt + # key: ${{ runner.os }}-QtCache + #- + # name: Install Qt + # uses: jurplel/install-qt-action@v2 + # with: + # install-deps: true + # setup-python: 'false' + # modules: qtcharts qtwidgets qtgui qtcore + # cached: ${{ steps.cache-qt.outputs.cache-hit }} + - + name: APT Install + run: | + sudo apt-get update + sudo apt-get install -y libmpv1 + - + name: Pip Installer + uses: BSFishy/pip-action@v1 + with: + packages: pyinstaller + requirements: hydrus/static/build_files/linux/requirements.txt + - + name: Build Hydrus + run: | + cp hydrus/static/build_files/linux/client.spec client.spec + cp hydrus/static/build_files/linux/server.spec server.spec + pyinstaller server.spec + pyinstaller client.spec + - + name: Remove Chonk + run: | + find dist/client/ -type f -name "*.pyc" -delete + while read line; do find dist/client/ -type f -name "${line}" -delete ; done < hydrus/static/build_files/linux/files_to_delete.txt + - + name: Upload a Build Artifact + uses: actions/upload-artifact@v2 + with: + name: Ubuntu-Extract + path: dist/client + if-no-files-found: error + retention-days: 2 + + build-windows: + runs-on: [windows-latest] + steps: + - + name: Checkout + uses: actions/checkout@v2 + with: + path: hydrus + - + name: Setup FFMPEG + uses: FedericoCarboni/setup-ffmpeg@v1 + id: setup_ffmpeg + with: + token: ${{ secrets.GITHUB_TOKEN }} + - + name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + architecture: x64 + - + name: Cache Qt + id: cache_qt + uses: actions/cache@v1 + with: + path: ../Qt + key: ${{ runner.os }}-QtCache + - + name: Install Qt + uses: jurplel/install-qt-action@v2 + with: + install-deps: true + setup-python: 'false' + modules: qtcharts qtwidgets qtgui qtcore + cached: ${{ steps.cache_qt.outputs.cache-hit }} + - + name: PIP Install Packages + uses: BSFishy/pip-action@v1 + with: + packages: pyinstaller + requirements: hydrus\static\build_files\windows\requirements.txt + - + name: Download mpv-dev + uses: carlosperate/download-file-action@v1.0.3 + id: download_mpv + with: + file-url: 'https://sourceforge.net/projects/mpv-player-windows/files/libmpv/mpv-dev-x86_64-20210228-git-d1be8bb.7z' + file-name: 'mpv-dev-x86_64.7z' + location: '.' + - + name: Process mpv-dev + run: | + 7z x ${{ steps.download_mpv.outputs.file-path }} + move mpv-1.dll hydrus\ + - + name: Build Hydrus + run: | + move ${{ steps.setup_ffmpeg.outputs.ffmpeg-path }} hydrus\bin\ + move hydrus\static\build_files\windows\sqlite3.dll hydrus\ + move hydrus\static\build_files\windows\client-win.spec client-win.spec + move hydrus\static\build_files\windows\server-win.spec server-win.spec + pyinstaller server-win.spec + pyinstaller client-win.spec + dir -r + - + name: InnoSetup + run: | + move hydrus\static\build_files\windows\InnoSetup.iss InnoSetup.iss + ISCC.exe InnoSetup.iss + - + name: Upload a Build Artifact + uses: actions/upload-artifact@v2 + with: + name: Windows-Install + path: dist\HydrusInstaller.exe + if-no-files-found: error + retention-days: 2 + - + name: Upload a Build Artifact + uses: actions/upload-artifact@v2 + with: + name: Windows-Extract + path: dist\Hydrus Network + if-no-files-found: error + retention-days: 2 + + create-release: + name: Create Release Entry + runs-on: ubuntu-20.04 + needs: [build-windows, build-ubuntu, build-macos] + steps: + - + name: Checkout code + uses: actions/checkout@v2 + - + name: Get All Artifacts + uses: actions/download-artifact@v2 + - + name: Extract version metadata + id: meta + run: | + echo "::set-output name=version::${GITHUB_REF##*/}" + echo "::set-output name=version_short::${GITHUB_REF##*/v}" + - + name: Rename Files + run: | + mkdir ubuntu windows + mv MacOS-DMG/HydrusNetwork.dmg Hydrus.Network.${{ steps.meta.outputs.version_short }}.-.macOS.-.App.dmg + mv Windows-Install/HydrusInstaller.exe Hydrus.Network.${{ steps.meta.outputs.version_short }}.-.Windows.-.Installer.exe + mv Windows-Extract "windows/Hydrus Network" + mv Ubuntu-Extract "ubuntu/Hydrus Network" + - + name: Compress Directories + run: | + zip -9 -r Hydrus.Network.${{ steps.meta.outputs.version_short }}.-.Windows.-.Extract.only.zip "windows/Hydrus Network" + tar -czvf Hydrus.Network.${{ steps.meta.outputs.version_short }}.-.Linux.-.Executable.tar.gz "ubuntu/Hydrus Network" + - + name: Release new + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + files: | + Hydrus.Network.${{ steps.meta.outputs.version_short }}.-.Windows.-.Installer.exe + Hydrus.Network.${{ steps.meta.outputs.version_short }}.-.Windows.-.Extract.only.zip + Hydrus.Network.${{ steps.meta.outputs.version_short }}.-.Linux.-.Executable.tar.gz + Hydrus.Network.${{ steps.meta.outputs.version_short }}.-.macOS.-.App.dmg + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/static/build_files/docker/client/Dockerfile b/static/build_files/docker/client/Dockerfile new file mode 100644 index 00000000..a91a47a0 --- /dev/null +++ b/static/build_files/docker/client/Dockerfile @@ -0,0 +1,50 @@ +FROM ghcr.io/suika/opencv-video-minimal:4.5-py3.8 + +ARG UID +ARG GID + +HEALTHCHECK --interval=20s --timeout=10s --retries=3 --start-period=30s CMD ! supervisorctl status | grep -v RUNNING +ENTRYPOINT ["/bin/sh", "/opt/hydrus/static/build_files/docker/client/entrypoint.sh"] +LABEL git="https://github.com/hydrusnetwork/hydrus" + +RUN apk --no-cache add jq fvwm x11vnc xvfb supervisor py3-beautifulsoup4 py3-psutil py3-pysocks py3-requests py3-twisted py3-yaml qt5-qtcharts py3-lz4 ffmpeg py3-pillow py3-numpy py3-numpy py3-qt5 py3-openssl openssl mpv mpv-libs nodejs patch \ + && apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/community font-noto font-noto-emoji \ + && apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/community font-noto-cjk +RUN pip install qtpy Send2Trash html5lib twisted python-mpv cloudscrape cloudscraper pyparsing + +RUN set -xe \ + && mkdir -p /opt/hydrus \ + && addgroup -g 1000 hydrus \ + && adduser -h /opt/hydrus -u 1000 -H -S -G hydrus hydrus + +RUN mkdir -p /opt/noVNC/utils/websockify \ + && wget $(wget https://api.github.com/repos/novnc/noVNC/releases/latest -qO- | jq -r '.tarball_url') -qO- | tar xzf - --strip-components=1 -C /opt/noVNC \ + && wget $(wget https://api.github.com/repos/novnc/websockify/releases/latest -qO- | jq -r '.tarball_url') -qO- | tar xzf - --strip-components=1 -C /opt/noVNC/utils/websockify \ + && sed -i -- "s/ps -p/ps -o pid | grep/g" /opt/noVNC/utils/launch.sh \ + && chown hydrus:hydrus -R /opt/noVNC + +COPY --chown=hydrus . /opt/hydrus +COPY --chown=hydrus --from=suika/swftools:2013-04-09-1007 /swftools/swfrender /opt/hydrus/bin/swfrender_linux + +RUN mv /opt/hydrus/static/build_files/docker/client/supervisord.conf /etc/supervisord.conf && \ + mv /opt/hydrus/static/build_files/docker/client/novnc/index.html /opt/noVNC/index.html && \ + mv /opt/hydrus/static/build_files/docker/client/novnc/icon.png /opt/noVNC/app/images/icons/icon.png + +RUN ln -fs /usr/bin/python3 /usr/bin/python && ln -fs /usr/bin/pip3 /usr/bin/pip + +VOLUME /opt/hydrus/db + +ENV QT_SCALE_FACTOR=1.1 \ + VNC_PORT=5900 \ + NOVNC_PORT=5800 \ + SUPERVISOR_PORT=9001 \ + XVFBRES=1680x1050x24 \ + UID=${UID:-1000} \ + GID=${GID:-1000} \ + DB_DIR=/opt/hydrus/db \ + XVFB_EXTRA="" \ + VNC_EXTRA="" \ + NOVNC_EXTRA="" \ + HYDRUS_EXTRA="" + +EXPOSE 5800 5900 diff --git a/static/build_files/docker/client/entrypoint.sh b/static/build_files/docker/client/entrypoint.sh new file mode 100644 index 00000000..ebc1b7ae --- /dev/null +++ b/static/build_files/docker/client/entrypoint.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +USER_ID=${UID} +GROUP_ID=${GID} + +echo "Starting Hydrus with UID/GID : $USER_ID/$GROUP_ID" + +cd /opt/hydrus/ + +if [ -f "/opt/hydrus/static/build_files/docker/client/patch.patch" ]; then + echo "Patching Hydrus" + patch -f -p1 -i /opt/hydrus/static/build_files/docker/client/patch.patch +fi + +if [ -f "/opt/hydrus/static/build_files/docker/client/requests.patch" ]; then + cd /usr/lib/python3.8/site-packages/requests + echo "Patching Requests" + patch -f -p2 -i /opt/hydrus/static/build_files/docker/client/requests.patch + cd /opt/hydrus/ +fi + +#if [ $USER_ID != 0 ] && [ $GROUP_ID != 0 ]; then +# find /opt/hydrus/ -not -path "/opt/hydrus/db/*" -exec chown hydrus:hydrus "{}" \; +#fi + +exec supervisord -c /etc/supervisord.conf diff --git a/static/build_files/docker/client/novnc/icon.png b/static/build_files/docker/client/novnc/icon.png new file mode 100644 index 00000000..91431202 Binary files /dev/null and b/static/build_files/docker/client/novnc/icon.png differ diff --git a/static/build_files/docker/client/novnc/index.html b/static/build_files/docker/client/novnc/index.html new file mode 100644 index 00000000..da570754 --- /dev/null +++ b/static/build_files/docker/client/novnc/index.html @@ -0,0 +1,328 @@ + + + + + + Hydrus + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
noVNC encountered an error:
+
+
+
+
+ + +
+ +
+
+ +
+ +

no
VNC

+ + + + + +
+ +
+ + + +
+
+ + + + + + +
+
+ + + +
+
+
+ Power +
+ + + +
+
+ + + +
+
+
+ Clipboard +
+ +
+ +
+
+ + + + + + +
+
+
    +
  • + Settings +
    • +
  • + +
    • +
  • + +
    • +
    • +
  • + +
    • +
  • + + +
    • +
    • +
  • +
    Advanced
    +
      +
    • + + +
      • +
    • + + +
      • +
      • +
    • + + +
      • +
    • +
      WebSocket
      +
        +
      • + +
        • +
      • + + +
        • +
      • + + +
        • +
      • + + +
        • +
      +
      • +
      • +
    • + +
      • +
    • + + +
      • +
      • +
    • + +
      • +
      • + +
    • + +
      • +
    +
    • +
    • +
  • + Version: + +
    • +
+
+
+ + + + +
+
+ +
+ +
+ + +
+ + +
+
+ +
+ Connect +
+
+
+ + +
+
+
    +
  • + + +
    • +
  • + + +
    • +
  • + +
    • +
+
+
+ + +
+
+
+ +
+ +
+ + +
+ + +
+ + + + diff --git a/static/build_files/docker/client/requests.patch b/static/build_files/docker/client/requests.patch new file mode 100644 index 00000000..73549e8a --- /dev/null +++ b/static/build_files/docker/client/requests.patch @@ -0,0 +1,189 @@ +From 063f2ae0e67111467fc21a2498e426e42f8fae4b Mon Sep 17 00:00:00 2001 +From: suika <2320837+Suika@users.noreply.github.com> +Date: Thu, 24 Sep 2020 15:49:53 +0200 +Subject: [PATCH 1/4] Bypass proxy if no_proxy or no are set, merge proxy and + self.proxy + +Check if the proxy should be bypassed in case no_proxy or no match the host of the url. +Also, since proxy can be defined in the session and the reuquest itself, both of them were never merged. Now they self.proxies will be updated by proxies. +--- + requests/sessions.py | 22 +++++++++++++++++++++- + 1 file changed, 21 insertions(+), 1 deletion(-) + +diff --git a/requests/sessions.py b/requests/sessions.py +index fdf7e9fe35..bdaf515a65 100644 +--- a/requests/sessions.py ++++ b/requests/sessions.py +@@ -529,6 +529,14 @@ def request(self, method, url, + + proxies = proxies or {} + ++ # Update self.proxy with proxy and assing the result to proxies ++ if isinstance(proxies,dict): ++ slef_proxies_tmp = self.proxies.copy() ++ slef_proxies_tmp.update(proxies) ++ proxies = slef_proxies_tmp.copy() ++ else: ++ proxies = self.proxies.copy() ++ + settings = self.merge_environment_settings( + prep.url, proxies, stream, verify, cert + ) +@@ -705,6 +713,7 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + :rtype: dict + """ + # Gather clues from the surrounding environment. ++ bypass_proxy = False + if self.trust_env: + # Set environment's proxies. + no_proxy = proxies.get('no_proxy') if proxies is not None else None +@@ -712,6 +721,14 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + for (k, v) in env_proxies.items(): + proxies.setdefault(k, v) + ++ # Check for no_proxy and no since they could be loaded from environment ++ no_proxy = proxies.get('no_proxy') if proxies is not None else None ++ no = proxies.get('no') if proxies is not None else None ++ if any([no_proxy,no]): ++ no_proxy = ','.join(filter(None, (no_proxy, no))) ++ if should_bypass_proxies(url, no_proxy): ++ bypass_proxy = True ++ + # Look for requests environment configuration and be compatible + # with cURL. + if verify is True or verify is None: +@@ -719,7 +736,10 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + os.environ.get('CURL_CA_BUNDLE')) + + # Merge all the kwargs. +- proxies = merge_setting(proxies, self.proxies) ++ if bypass_proxy: ++ proxies = {} ++ else: ++ proxies = merge_setting(proxies, self.proxies) + stream = merge_setting(stream, self.stream) + verify = merge_setting(verify, self.verify) + cert = merge_setting(cert, self.cert) + +From a3afa6b55d7596ab6a06207a32a5c96436a66e8c Mon Sep 17 00:00:00 2001 +From: suika <2320837+Suika@users.noreply.github.com> +Date: Thu, 24 Sep 2020 16:00:30 +0200 +Subject: [PATCH 2/4] Rename bypass_proxy to bypass_proxies + +Make it match the variable being returned +--- + requests/sessions.py | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/requests/sessions.py b/requests/sessions.py +index bdaf515a65..2db637dd6e 100644 +--- a/requests/sessions.py ++++ b/requests/sessions.py +@@ -713,7 +713,7 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + :rtype: dict + """ + # Gather clues from the surrounding environment. +- bypass_proxy = False ++ bypass_proxies = False + if self.trust_env: + # Set environment's proxies. + no_proxy = proxies.get('no_proxy') if proxies is not None else None +@@ -727,7 +727,7 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + if any([no_proxy,no]): + no_proxy = ','.join(filter(None, (no_proxy, no))) + if should_bypass_proxies(url, no_proxy): +- bypass_proxy = True ++ bypass_proxies = True + + # Look for requests environment configuration and be compatible + # with cURL. +@@ -736,7 +736,7 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + os.environ.get('CURL_CA_BUNDLE')) + + # Merge all the kwargs. +- if bypass_proxy: ++ if bypass_proxies: + proxies = {} + else: + proxies = merge_setting(proxies, self.proxies) + +From 78682f9e21933bc6defca8f236b6c0bde5ac045f Mon Sep 17 00:00:00 2001 +From: suika <2320837+Suika@users.noreply.github.com> +Date: Thu, 24 Sep 2020 16:12:39 +0200 +Subject: [PATCH 3/4] Move no_proxy check outside trust_env + +It makes more sense to have the check be outside the trust_env. Since it has to be always executed. Because proxy configuration can be performed on the Sessions class. +--- + requests/sessions.py | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/requests/sessions.py b/requests/sessions.py +index 2db637dd6e..178ca7e9a7 100644 +--- a/requests/sessions.py ++++ b/requests/sessions.py +@@ -721,20 +721,20 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + for (k, v) in env_proxies.items(): + proxies.setdefault(k, v) + +- # Check for no_proxy and no since they could be loaded from environment +- no_proxy = proxies.get('no_proxy') if proxies is not None else None +- no = proxies.get('no') if proxies is not None else None +- if any([no_proxy,no]): +- no_proxy = ','.join(filter(None, (no_proxy, no))) +- if should_bypass_proxies(url, no_proxy): +- bypass_proxies = True +- + # Look for requests environment configuration and be compatible + # with cURL. + if verify is True or verify is None: + verify = (os.environ.get('REQUESTS_CA_BUNDLE') or + os.environ.get('CURL_CA_BUNDLE')) + ++ # Check for no_proxy and no since they could be loaded from environment ++ no_proxy = proxies.get('no_proxy') if proxies is not None else None ++ no = proxies.get('no') if proxies is not None else None ++ if any([no_proxy, no]): ++ no_proxy = ','.join(filter(None, (no_proxy, no))) ++ if should_bypass_proxies(url, no_proxy): ++ bypass_proxy = True ++ + # Merge all the kwargs. + if bypass_proxies: + proxies = {} + +From 0f6bd04349dc1bb2c0808f4de8583eace7c5aaa3 Mon Sep 17 00:00:00 2001 +From: suika <2320837+Suika@users.noreply.github.com> +Date: Thu, 24 Sep 2020 16:34:20 +0200 +Subject: [PATCH 4/4] Remove bypass_proxies var and only use + should_bypass_proxies + +That logic was left from previous tires fixing no_proxy and since it's quite compact now, it can be removed and should_bypass_proxies should be used instead of setting a var. +--- + requests/sessions.py | 5 +---- + 1 file changed, 1 insertion(+), 4 deletions(-) + +diff --git a/requests/sessions.py b/requests/sessions.py +index 178ca7e9a7..4ac01315cb 100644 +--- a/requests/sessions.py ++++ b/requests/sessions.py +@@ -713,7 +713,6 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + :rtype: dict + """ + # Gather clues from the surrounding environment. +- bypass_proxies = False + if self.trust_env: + # Set environment's proxies. + no_proxy = proxies.get('no_proxy') if proxies is not None else None +@@ -732,11 +731,9 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + no = proxies.get('no') if proxies is not None else None + if any([no_proxy, no]): + no_proxy = ','.join(filter(None, (no_proxy, no))) +- if should_bypass_proxies(url, no_proxy): +- bypass_proxy = True + + # Merge all the kwargs. +- if bypass_proxies: ++ if should_bypass_proxies(url, no_proxy): + proxies = {} + else: + proxies = merge_setting(proxies, self.proxies) \ No newline at end of file diff --git a/static/build_files/docker/client/supervisord.conf b/static/build_files/docker/client/supervisord.conf new file mode 100644 index 00000000..f44532a9 --- /dev/null +++ b/static/build_files/docker/client/supervisord.conf @@ -0,0 +1,58 @@ +[unix_http_server] +file=/run/supervisor.sock + +[inet_http_server] +port=127.0.0.1:%(ENV_SUPERVISOR_PORT)s + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///run/supervisor.sock + +[supervisord] +nodaemon=true +[program:xvfb] +command=Xvfb :89 -ac -listen tcp -screen 0 %(ENV_XVFBRES)s %(ENV_XVFB_EXTRA)s +startretries=89 +autostart=true +autorestart=true + +[program:fvwm] +command=fvwm -d :89 +startretries=89 +autostart=true +autorestart=true + +[program:vnc] +command=x11vnc -display :89 -forever -noxrecord -noxfixes -noxdamage -rfbport %(ENV_VNC_PORT)s %(ENV_VNC_EXTRA)s +startretries=89 +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stderr_logfile=/dev/stderr +stdout_logfile_maxbytes=0 +stderr_logfile_maxbytes=0 + +[program:novnc] +command=sh /opt/noVNC/utils/launch.sh --vnc localhost:%(ENV_VNC_PORT)s --listen %(ENV_NOVNC_PORT)s %(ENV_NOVNC_EXTRA)s +startretries=89 +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stderr_logfile=/dev/stderr +stdout_logfile_maxbytes=0 +stderr_logfile_maxbytes=0 + +[program:hydrus] +environment=DISPLAY=":89",HOME=/opt/hydrus +user=hydrus +directory=/opt/hydrus +command=python3 /opt/hydrus/client.py --db_dir %(ENV_DB_DIR)s %(ENV_HYDRUS_EXTRA)s +startretries=89 +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stderr_logfile=/dev/stderr +stdout_logfile_maxbytes=0 +stderr_logfile_maxbytes=0 diff --git a/static/build_files/docker/docker_build.yml b/static/build_files/docker/docker_build.yml new file mode 100644 index 00000000..489770b5 --- /dev/null +++ b/static/build_files/docker/docker_build.yml @@ -0,0 +1,102 @@ +name: Build Containers +on: + push: + tags: + - 'v*' + workflow_dispatch: [] + +jobs: + build-client: + runs-on: [ubuntu-latest] + steps: + - + name: Checkout + uses: actions/checkout@v2.3.4 + - + name: Docker meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v2 + with: + images: | + ghcr.io/hydrusnetwork/hydrus + tags: | + type=ref,event=tag + labels: | + org.opencontainers.image.title=Hydrus Network + org.opencontainers.image.description=A personal booru-style media tagger that can import files and tags from your hard drive and popular websites. + org.opencontainers.image.vendor=hydrusnetwork + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + buildkitd-flags: "--debug" + - + name: Login to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + - + name: Build + uses: docker/build-push-action@v2 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + file: ./static/build_files/docker/client/Dockerfile + platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/386,linux/arm/v7 + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + + build-server: + runs-on: [ubuntu-latest] + steps: + - + name: Checkout + uses: actions/checkout@v2.3.4 + - + name: Docker meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v2 + with: + images: | + ghcr.io/hydrusnetwork/hydrus + tags: | + type=ref,event=tag,prefix=server- + type=raw,enable=true,prefix=,suffix=,value=server + flavor: | + latest=false + labels: | + org.opencontainers.image.title=Hydrus Network Server + org.opencontainers.image.description=A personal booru-style media tagger that can import files and tags from your hard drive and popular websites. + org.opencontainers.image.vendor=hydrusnetwork + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + buildkitd-flags: "--debug" + - + name: Login to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + - + name: Build + uses: docker/build-push-action@v2 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + file: ./static/build_files/docker/server/Dockerfile + platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/386,linux/arm/v7 + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} \ No newline at end of file diff --git a/static/build_files/docker/server/Dockerfile b/static/build_files/docker/server/Dockerfile new file mode 100644 index 00000000..25424b43 --- /dev/null +++ b/static/build_files/docker/server/Dockerfile @@ -0,0 +1,29 @@ +FROM suika/opencv-video-minimal:4.2-py3.7.5 + +ARG UID +ARG GID + +RUN apk --no-cache add py3-beautifulsoup4 py3-psutil py3-pysocks py3-requests py3-twisted py3-yaml py3-lz4 ffmpeg py3-pillow py3-numpy py3-openssl py3-service_identity openssl su-exec +RUN pip install Send2Trash html5lib twisted cloudscrape + +RUN set -xe \ + && mkdir -p /opt/hydrus \ + && addgroup -g 1000 hydrus \ + && adduser -h /opt/hydrus -u 1000 -H -S -G hydrus hydrus + +COPY --chown=hydrus . /opt/hydrus +COPY --chown=hydrus --from=suika/swftools:2013-04-09-1007 /swftools/swfrender /opt/hydrus/bin/swfrender_linux + +VOLUME /opt/hydrus/db + +ENV UID=${UID:-1000} \ + GID=${GID:-1000} \ + MGMT_PORT=45870 + +EXPOSE ${MGMT_PORT} + +ENTRYPOINT ["/bin/sh", "/opt/hydrus/static/build_files/docker/server/entrypoint.sh"] + +HEALTHCHECK --interval=1m --timeout=10s --retries=3 --start-period=10s \ + CMD wget --quiet --tries=1 --no-check-certificate --spider \ + https://localhost:${MGMT_PORT} || exit 1 \ No newline at end of file diff --git a/static/build_files/docker/server/entrypoint.sh b/static/build_files/docker/server/entrypoint.sh new file mode 100644 index 00000000..cb9cbd79 --- /dev/null +++ b/static/build_files/docker/server/entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +USER_ID=${UID} +GROUP_ID=${GID} + +echo "Starting Hydrus with UID/GID : $USER_ID/$GROUP_ID" + +stop() { + python3 /opt/hydrus/server.py stop -d="/opt/hydrus/db" +} + +trap "stop" SIGTERM + +su-exec ${USER_ID}:${GROUP_ID} python3 /opt/hydrus/server.py -d="/opt/hydrus/db" --no_daemons & + +wait $! diff --git a/static/build_files/linux/client.spec b/static/build_files/linux/client.spec new file mode 100644 index 00000000..d45a6df2 --- /dev/null +++ b/static/build_files/linux/client.spec @@ -0,0 +1,55 @@ +# -*- mode: python -*- + +import cloudscraper +import os +cloudscraper_dir = os.path.dirname( cloudscraper.__file__ ) + +block_cipher = None + + +a = Analysis(['hydrus/client.py'], + pathex=['.'], + binaries=[], + datas=[ + ('hydrus/bin', 'bin'), + ('hydrus/help', 'help'), + ('hydrus/static', 'static'), + ('hydrus/client.pyw', '.'), + ('hydrus/client.py', '.'), + ('hydrus/server.py', '.'), + ('dist/server/server', '.'), + ('hydrus/license.txt', '.'), + ('hydrus/Readme.txt', '.'), + ('hydrus/help my client will not boot.txt', '.'), + ('hydrus/db', 'db'), + ('hydrus/hydrus', 'hydrus'), + (cloudscraper_dir, 'cloudscraper') + ], + hiddenimports=['hydrus/server.py'], + hookspath=[], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) +exe = EXE(pyz, + a.scripts, + [], + exclude_binaries=True, + name='client', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + console=False ) +coll = COLLECT(exe, + a.binaries, + a.zipfiles, + a.datas, + strip=False, + upx=True, + upx_exclude=[], + name='client') \ No newline at end of file diff --git a/static/build_files/linux/files_to_delete.txt b/static/build_files/linux/files_to_delete.txt new file mode 100644 index 00000000..45225623 --- /dev/null +++ b/static/build_files/linux/files_to_delete.txt @@ -0,0 +1,97 @@ +ld-linux.so.2 +ld-linux-x86-64.so.2 +libanl.so.1 +libasound.so.2 +libBrokenLocale.so.1 +libcidn.so.1 +libcom_err.so.2 +libc.so.6 +libdl.so.2 +libdrm.so.2 +libEGL.so.1 +libexpat.so.1 +libfontconfig.so.1 +libfreetype.so.6 +libfribidi.so.0 +libgbm.so.1 +libgcc_s.so.1 +libgdk_pixbuf-2.0.so.0 +libgio-2.0.so.0 +libglapi.so.0 +libGLdispatch.so.0 +libglib-2.0.so.0 +libGL.so.1 +libGLX.so.0 +libgobject-2.0.so.0 +libharfbuzz.so.0 +libICE.so.6 +libjack.so.0 +libm.so.6 +libmvec.so.1 +libnss_compat.so.2 +libnss_db.so.2 +libnss_dns.so.2 +libnss_files.so.2 +libnss_hesiod.so.2 +libnss_nisplus.so.2 +libnss_nis.so.2 +libp11-kit.so.0 +libpango-1.0.so.0 +libpangocairo-1.0.so.0 +libpangoft2-1.0.so.0 +libpthread.so.0 +libresolv.so.2 +librt.so.1 +libSM.so.6 +libstdc++.so.6 +libthai.so.0 +libthread_db.so.1 +libusb-1.0.so.0 +libutil.so.1 +libuuid.so.1 +libX11.so.6 +libxcb-dri2.so.0 +libxcb-dri3.so.0 +libxcb.so.1 +libz.so.1 +libQt53DAnimation.so* +libQt53DCore.so* +libQt53DExtras.so* +libQt53DInput.so* +libQt53DLogic.so* +libQt53DQuick.so* +libQt53DQuickAnimation.so* +libQt53DQuickExtras.so* +libQt53DQuickInput.so* +libQt53DQuickRender.so* +libQt53DQuickScene2D.so* +libQt53DRender.so* +libQt5Bluetooth.so* +libQt5Bodymovin.so* +libQt5Concurrent.so* +libQt5DataVisualization.so* +libQt5EglFSDeviceIntegration.so* +libQt5Gamepad.so* +libQt5Location.so* +libQt5Multimedia.so* +libQt5MultimediaQuick.so* +libQt5Nfc.so* +libQt5Positioning.so* +libQt5PositioningQuick.so* +libQt5Purchasing.so* +libQt5QuickControls2.so* +libQt5QuickParticles.so* +libQt5QuickShapes.so* +libQt5QuickTemplates2.so* +libQt5QuickTest.so* +libQt5RemoteObjects.so* +libQt5Scxml.so* +libQt5Sensors.so* +libQt5Sql.so* +libQt5Test.so* +libQt5WaylandClient.so* +libQt5WaylandCompositor.so* +libQt5WebChannel.so* +libQt5WebEngine.so* +libQt5WebEngineCore.so* +libQt5WebView.so* \ No newline at end of file diff --git a/static/build_files/linux/linux_build.yml b/static/build_files/linux/linux_build.yml new file mode 100644 index 00000000..31444d6b --- /dev/null +++ b/static/build_files/linux/linux_build.yml @@ -0,0 +1,66 @@ +name: Release +on: + push: + tags: + - 'v*' + +jobs: + build-ubuntu: + runs-on: ubuntu-18.04 + steps: + - + name: Checkout + uses: actions/checkout@v2 + with: + path: hydrus + - + name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + architecture: x64 + #- name: Cache Qt + # id: cache-qt + # uses: actions/cache@v1 + # with: + # path: Qt + # key: ${{ runner.os }}-QtCache + #- + # name: Install Qt + # uses: jurplel/install-qt-action@v2 + # with: + # install-deps: true + # setup-python: 'false' + # modules: qtcharts qtwidgets qtgui qtcore + # cached: ${{ steps.cache-qt.outputs.cache-hit }} + - + name: APT Install + run: | + sudo apt-get update + sudo apt-get install -y libmpv1 + - + name: Pip Installer + uses: BSFishy/pip-action@v1 + with: + packages: pyinstaller + requirements: hydrus/static/build_files/linux/requirements.txt + - + name: Build Hydrus + run: | + cp hydrus/static/build_files/linux/client.spec client.spec + cp hydrus/static/build_files/linux/server.spec server.spec + pyinstaller server.spec + pyinstaller client.spec + - + name: Remove Chonk + run: | + find dist/client/ -type f -name "*.pyc" -delete + while read line; do find dist/client/ -type f -name "${line}" -delete ; done < hydrus/static/build_files/linux/files_to_delete.txt + - + name: Upload a Build Artifact + uses: actions/upload-artifact@v2 + with: + name: Ubuntu-Extract + path: dist/client + if-no-files-found: error + retention-days: 2 \ No newline at end of file diff --git a/static/build_files/linux/requirements.txt b/static/build_files/linux/requirements.txt new file mode 100644 index 00000000..f1fff14c --- /dev/null +++ b/static/build_files/linux/requirements.txt @@ -0,0 +1,23 @@ +beautifulsoup4>=4.0.0 +chardet>=3.0.4 +cloudscraper>=1.2.33 +html5lib>=1.0.1 +lxml>=4.5.0 +lz4>=3.0.0 +nose>=1.3.0 +numpy>=1.16.0 +opencv-python-headless>=4.0.0 +Pillow>=6.0.0 +psutil>=5.0.0 +pylzma>=0.5.0 +pyOpenSSL>=19.1.0 +PySide2>=5.15.0 +PySocks>=1.7.0 +python-mpv==0.4.5 +PyYAML>=5.0.0 +QtPy>=1.9.0 +requests==2.23.0 +Send2Trash>=1.5.0 +service-identity>=18.1.0 +six>=1.14.0 +Twisted>=20.3.0 diff --git a/static/build_files/linux/server.spec b/static/build_files/linux/server.spec new file mode 100644 index 00000000..0804ec75 --- /dev/null +++ b/static/build_files/linux/server.spec @@ -0,0 +1,36 @@ +# -*- mode: python -*- + +block_cipher = None + + +a = Analysis(['hydrus/server.py'], + pathex=['.'], + binaries=[], + datas=[], + hiddenimports=[], + hookspath=[], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) +exe = EXE(pyz, + a.scripts, + [], + exclude_binaries=True, + name='server', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + console=True ) +coll = COLLECT(exe, + a.binaries, + a.zipfiles, + a.datas, + strip=False, + upx=True, + name='server') \ No newline at end of file diff --git a/static/build_files/macos/Info.plist b/static/build_files/macos/Info.plist new file mode 100644 index 00000000..b5e9e52c --- /dev/null +++ b/static/build_files/macos/Info.plist @@ -0,0 +1,23 @@ + + + + + CFBundleDisplayName + client + CFBundleExecutable + MacOS/client + CFBundleIconFile + icon.icns + CFBundleIdentifier + client + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + client + CFBundlePackageType + APPL + CFBundleShortVersionString + 0.0.0 + NSHighResolutionCapable + True + \ No newline at end of file diff --git a/static/build_files/ReadMeFirst.rtf b/static/build_files/macos/ReadMeFirst.rtf similarity index 100% rename from static/build_files/ReadMeFirst.rtf rename to static/build_files/macos/ReadMeFirst.rtf diff --git a/static/build_files/macos/macos_build.yml b/static/build_files/macos/macos_build.yml new file mode 100644 index 00000000..71ccebec --- /dev/null +++ b/static/build_files/macos/macos_build.yml @@ -0,0 +1,80 @@ +name: Release +on: + push: + tags: + - 'v*' + +jobs: + build-macos: + runs-on: macos-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Setup FFMPEG + uses: FedericoCarboni/setup-ffmpeg@v1 + id: setup_ffmpeg + with: + token: ${{ secrets.GITHUB_TOKEN }} + - + name: Setup Rust Tollchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - + name: Cargo Cache + uses: actions/cache@v2 + with: + path: ~/.cargo + key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.toml') }} + restore-keys: | + ${{ runner.os }}-cargo-${{ hashFiles('Cargo.toml') }} + ${{ runner.os }}-cargo + - + name: Cargo Artifacts Cache + uses: actions/cache@v2 + with: + path: target + key: ${{ runner.os }}-cargo-target-${{ hashFiles('Cargo.toml') }} + restore-keys: | + ${{ runner.os }}-cargo-target-${{ hashFiles('Cargo.toml') }} + ${{ runner.os }}-cargo-target + - + name: Install PyOxidizer + uses: actions-rs/cargo@v1.0.3 + with: + command: install + args: pyoxidizer + - + name: Build Hydrus + run: | + cd $GITHUB_WORKSPACE + cp ${{ steps.setup_ffmpeg.outputs.ffmpeg-path }} bin/ + cp static/build_files/macos/pyoxidizer.bzl pyoxidizer.bzl + cp static/build_files/linux/requirements.txt requirements.txt + basename $(rustc --print sysroot) | sed -e "s/^stable-//" > triple.txt + pyoxidizer build --release + cd build/$(head -n 1 triple.txt)/release + mkdir -p "Hydrus Network.app/Contents/MacOS" + mkdir -p "Hydrus Network.app/Contents/Resources" + mkdir -p "Hydrus Network.app/Contents/Frameworks" + mv install/static/icon.icns "Hydrus Network.app/Contents/Resources/icon.icns" + cp install/static/build_files/macos/Info.plist "Hydrus Network.app/Contents/Info.plist" + cp install/static/build_files/macos/ReadMeFirst.rtf ./ReadMeFirst.rtf + cp install/static/build_files/macos/running_from_app "install/running_from_app" + ln -s /Applications ./Applications + mv install/* "Hydrus Network.app/Contents/MacOS/" + rm -rf install + cd $GITHUB_WORKSPACE + temp_dmg="$(mktemp).dmg" + hdiutil create "$temp_dmg" -ov -volname "HydrusNetwork" -fs HFS+ -srcfolder "$GITHUB_WORKSPACE/build/$(head -n 1 triple.txt)/release" + hdiutil convert "$temp_dmg" -format UDZO -o HydrusNetwork.dmg + - + name: Upload a Build Artifact + uses: actions/upload-artifact@v2.2.1 + with: + name: MacOS-DMG + path: HydrusNetwork.dmg + if-no-files-found: error + retention-days: 2 \ No newline at end of file diff --git a/static/build_files/macos/pyoxidizer.bzl b/static/build_files/macos/pyoxidizer.bzl new file mode 100644 index 00000000..e5dc2f7a --- /dev/null +++ b/static/build_files/macos/pyoxidizer.bzl @@ -0,0 +1,63 @@ +def make_dist(): + return default_python_distribution() + +def make_packaging_policy(dist): + policy = dist.make_python_packaging_policy() + policy.set_resource_handling_mode("files") + policy.resources_location = "filesystem-relative:lib" + policy.allow_files = True + policy.bytecode_optimize_level_zero = True + + return policy + +def make_client(dist, policy): + python_config = dist.make_python_interpreter_config() + python_config.module_search_paths = ["$ORIGIN", "$ORIGIN/lib"] + python_config.filesystem_importer = True + python_config.sys_frozen = True + python_config.run_command = "import os; import sys; exec(open(os.path.join(os.path.split(sys.executable)[0], 'client.py')).read())" + + client = dist.to_python_executable( + name="client", + packaging_policy=policy, + config=python_config, + ) + + client.add_python_resources(client.pip_install(["--prefer-binary", "-r", "requirements.txt"])) + + return client + +def make_embedded_resources(client): + return client.to_embedded_resources() + +def make_install(client, resources): + files = FileManifest() + files.add_python_resource(".", client) + + static_resources = glob(["./*.py", "./*.md", "./*txt", "./bin/**/*", "./static/**/*", "./help/**/*"], strip_prefix="{}/".format(CWD)) + files.add_manifest(static_resources) + + hydrus_source = glob(["./hydrus/**/*.py"], strip_prefix="{}/".format(CWD)) + files.add_manifest(hydrus_source) + + return files + +print(BUILD_TARGET_TRIPLE) +print(CWD) + +# Tell PyOxidizer about the build targets defined above. +register_target("dist", make_dist) +register_target("policy", make_packaging_policy, depends=["dist"]) +register_target("client", make_client, depends=["dist", "policy"]) +register_target("resources", make_embedded_resources, depends=["client"], default_build_script=True) +register_target("install", make_install, depends=["client", "resources"], default=True) + +resolve_targets() + +# END OF COMMON USER-ADJUSTED SETTINGS. +# +# Everything below this is typically managed by PyOxidizer and doesn't need +# to be updated by people. + +PYOXIDIZER_VERSION = "0.10.3" +PYOXIDIZER_COMMIT = "UNKNOWN" \ No newline at end of file diff --git a/static/build_files/macos/requirements.txt b/static/build_files/macos/requirements.txt new file mode 100644 index 00000000..58dc7502 --- /dev/null +++ b/static/build_files/macos/requirements.txt @@ -0,0 +1,23 @@ +beautifulsoup4>=4.0.0 +chardet>=3.0.4 +cloudscraper>=1.2.33 +html5lib>=1.0.1 +lxml>=4.5.0 +lz4>=3.0.0 +nose>=1.3.0 +numpy>=1.16.0 +opencv-python-headless>=4.0.0 +Pillow>=6.0.0 +psutil>=5.0.0 +pylzma>=0.5.0 +pyOpenSSL>=19.1.0 +PySide2>=5.15.0 +PySocks>=1.7.0 +python-mpv>=0.4.5 +PyYAML>=5.0.0 +QtPy>=1.9.0 +requests==2.23.0 +Send2Trash>=1.5.0 +service-identity>=18.1.0 +six>=1.14.0 +Twisted>=20.3.0 diff --git a/static/build_files/running_from_app b/static/build_files/macos/running_from_app similarity index 100% rename from static/build_files/running_from_app rename to static/build_files/macos/running_from_app diff --git a/static/build_files/windows/InnoSetup.iss b/static/build_files/windows/InnoSetup.iss new file mode 100644 index 00000000..ff9c0296 --- /dev/null +++ b/static/build_files/windows/InnoSetup.iss @@ -0,0 +1,69 @@ +[Icons] +Name: {group}\hydrus client; Filename: {app}\client.exe; WorkingDir: {app}; Tasks: programgroupicons +Name: {group}\hydrus server; Filename: {app}\server.exe; WorkingDir: {app}; Tasks: programgroupicons +Name: {group}\help; Filename: {app}\help\index.html; WorkingDir: {app}; Tasks: programgroupicons +Name: {group}\uninstall hydrus network; Filename: {uninstallexe}; WorkingDir: {app}; Tasks: programgroupicons; IconFilename: {app}\static\cross.ico +Name: {userdesktop}\hydrus client; Filename: {app}\client.exe; WorkingDir: {app}; Tasks: desktopicons +Name: {userdesktop}\hydrus server; Filename: {app}\server.exe; WorkingDir: {app}; Tasks: desktopicons +[Setup] +InternalCompressLevel=ultra64 +OutputDir=dist +OutputBaseFilename=HydrusInstaller +Compression=lzma/ultra64 +AppName=Hydrus Network +AppVerName=Hydrus Network +DefaultDirName={sd}\Hydrus Network +DefaultGroupName=Hydrus Network +DisableProgramGroupPage=yes +DisableReadyPage=yes +DisableDirPage=no +ShowLanguageDialog=no +SetupIconFile=hydrus\static\hydrus.ico +Uninstallable=IsComponentSelected('install') +UninstallDisplayIcon={app}\static\hydrus.ico +[Tasks] +Name: desktopicons; Description: Create desktop icons; Flags: unchecked; Components: install +Name: programgroupicons; Description: Create program group icons; Components: install +[Messages] +SelectDirBrowseLabel=To continue, click Next. If you would like to select a different folder, click Browse. Databases will be created beneath the install dir, so make sure the hard drive has enough spare space for your purposes and your user has permission to write there! If you install to program files, you'll need to run the programs as administrator! +[Components] +Name: install; Description: Install; Types: install; Flags: fixed +[Types] +Name: install; Description: Install +Name: extract; Description: Extract only +[Run] +Filename: {app}\help\index.html; Description: Open help/getting started guide (highly recommended for new users); Flags: postinstall unchecked shellexec +Filename: {app}\client.exe; Description: Open the client; Flags: postinstall nowait unchecked +[Files] +Source: dist\Hydrus Network\* ; DestDir: {app}; Flags: ignoreversion recursesubdirs createallsubdirs +[InstallDelete] +Name: {app}\Crypto; Type: filesandordirs; Components: install +Name: {app}\tcl; Type: filesandordirs; Components: install +Name: {app}\tk; Type: filesandordirs; Components: install +Name: {app}\wx; Type: filesandordirs; Components: install +Name: {app}\lz4-3.0.2-py3.7.egg-info; Type: filesandordirs; Components: install +Name: {app}\lz4-2.1.6-py3.6.egg-info; Type: filesandordirs; Components: install +Name: {app}\cryptography-2.9-py3.7.egg-info; Type: filesandordirs; Components: install +Name: {app}\cryptography-2.4.2-py3.6.egg-info; Type: filesandordirs; Components: install +Name: {app}\lib2to3; Type: filesandordirs; Components: install +Name: {app}\mpl-data; Type: filesandordirs; Components: install +Name: {app}\matplotlib; Type: filesandordirs; Components: install +Name: {app}\cryptography; Type: filesandordirs; Components: install +Name: {app}\opencv_ffmpeg344_64.dll; Type: files; Components: install +Name: {app}\opencv_ffmpeg400_64.dll; Type: files; Components: install +Name: {app}\opencv_ffmpeg410_64.dll; Type: files; Components: install +Name: {app}\opencv_videoio_ffmpeg411_64.dll; Type: files; Components: install +Name: {app}\opencv_videoio_ffmpeg412_64.dll; Type: files; Components: install +Name: {app}\opencv_videoio_ffmpeg420_64.dll; Type: files; Components: install +Name: {app}\opencv_videoio_ffmpeg440_64.dll; Type: files; Components: install +Name: {app}\wxmsw30u_core_vc140_x64.dll; Type: files; Components: install +Name: {app}\wxmsw30u_adv_vc140_x64.dll; Type: files; Components: install +Name: {app}\wxbase30u_vc140_x64.dll; Type: files; Components: install +Name: {app}\wxbase30u_net_vc140_x64.dll; Type: files; Components: install +Name: {app}\tk86t.dll; Type: files; Components: install +Name: {app}\tcl86t.dll; Type: files; Components: install +Name: {app}\_tkinter.pyd; Type: files; Components: install +Name: {app}\_yaml.cp36-win_amd64.pyd; Type: files; Components: install +Name: {app}\_yaml.cp37-win_amd64.pyd; Type: files; Components: install +Name: {app}\_cffi_backend.cp36-win_amd64.pyd; Type: files; Components: install +Name: {app}\_distutils_findvs.pyd; Type: files; Components: install \ No newline at end of file diff --git a/static/build_files/windows/client-win.spec b/static/build_files/windows/client-win.spec new file mode 100644 index 00000000..4b1555ce --- /dev/null +++ b/static/build_files/windows/client-win.spec @@ -0,0 +1,62 @@ +# -*- mode: python ; coding: utf-8 -*- + +import cloudscraper +import cv2 +import os +import glob +cloudscraper_dir = os.path.dirname( cloudscraper.__file__ ) +cv2_ffmpeg_dll = glob.glob(os.path.dirname( cv2.__file__ )+"/*.dll")[0] + +block_cipher = None + + +a = Analysis(['hydrus\\client.pyw'], + pathex=['.'], + binaries=[], + datas=[ + ('hydrus\\bin', 'bin'), + ('hydrus\\help', 'help'), + ('hydrus\\static', 'static'), + ('hydrus\\client.pyw', '.'), + ('hydrus\\client.py', '.'), + ('hydrus\\server.py', '.'), + ('dist\\server\\server.exe*', '.'), + ('hydrus\\license.txt', '.'), + ('hydrus\\Readme.txt', '.'), + ('hydrus\\help my client will not boot.txt', '.'), + ('hydrus\\db', 'db'), + ('hydrus\\hydrus', 'hydrus'), + ('hydrus\\sqlite3.dll', '.'), + ('hydrus\\mpv-1.dll', '.'), + (cloudscraper_dir, 'cloudscraper'), + (cv2_ffmpeg_dll, '.') + ], + hiddenimports=['hydrus\\server.py', 'cloudscraper'], + hookspath=[], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) +exe = EXE(pyz, + a.scripts, + [], + exclude_binaries=True, + name='client', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + console=False, + icon='hydrus\\static\\hydrus.ico' ) +coll = COLLECT(exe, + a.binaries, + a.zipfiles, + a.datas, + strip=False, + upx=False, + upx_exclude=[], + name='Hydrus Network') diff --git a/static/build_files/windows/requirements.txt b/static/build_files/windows/requirements.txt new file mode 100644 index 00000000..b7f443eb --- /dev/null +++ b/static/build_files/windows/requirements.txt @@ -0,0 +1,27 @@ +PyWin32 +pypiwin32 +pywin32-ctypes +pefile +beautifulsoup4>=4.0.0 +chardet>=3.0.4 +cloudscraper>=1.2.33 +html5lib>=1.0.1 +lxml>=4.5.0 +lz4>=3.0.0 +nose>=1.3.0 +numpy>=1.16.0 +opencv-python-headless>=4.0.0 +Pillow>=6.0.0 +psutil>=5.0.0 +pylzma>=0.5.0 +pyOpenSSL>=19.1.0 +PySide2>=5.15.0 +PySocks>=1.7.0 +python-mpv>=0.4.5 +PyYAML>=5.0.0 +QtPy>=1.9.0 +requests==2.23.0 +Send2Trash>=1.5.0 +service-identity>=18.1.0 +six>=1.14.0 +Twisted>=20.3.0 diff --git a/static/build_files/windows/server-win.spec b/static/build_files/windows/server-win.spec new file mode 100644 index 00000000..6146dd7a --- /dev/null +++ b/static/build_files/windows/server-win.spec @@ -0,0 +1,38 @@ +# -*- mode: python ; coding: utf-8 -*- + +block_cipher = None + + +a = Analysis(['hydrus\\server.py'], + pathex=['.'], + binaries=[], + datas=[], + hiddenimports=[], + hookspath=[], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) +exe = EXE(pyz, + a.scripts, + [], + exclude_binaries=True, + name='server', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=False, + console=False, + icon='hydrus\\static\\hydrus.ico' ) +coll = COLLECT(exe, + a.binaries, + a.zipfiles, + a.datas, + strip=False, + upx=False, + upx_exclude=[], + name='server') diff --git a/static/build_files/windows/sqlite3.dll b/static/build_files/windows/sqlite3.dll new file mode 100644 index 00000000..96f319bc Binary files /dev/null and b/static/build_files/windows/sqlite3.dll differ diff --git a/static/build_files/windows/windows_build.yml b/static/build_files/windows/windows_build.yml new file mode 100644 index 00000000..cbda23ce --- /dev/null +++ b/static/build_files/windows/windows_build.yml @@ -0,0 +1,92 @@ +name: Release +on: + push: + tags: + - 'v*' + +jobs: + build-windows: + runs-on: [windows-latest] + steps: + - + name: Checkout + uses: actions/checkout@v2 + with: + path: hydrus + - + name: Setup FFMPEG + uses: FedericoCarboni/setup-ffmpeg@v1 + id: setup_ffmpeg + with: + token: ${{ secrets.GITHUB_TOKEN }} + - + name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + architecture: x64 + - + name: Cache Qt + id: cache_qt + uses: actions/cache@v1 + with: + path: ../Qt + key: ${{ runner.os }}-QtCache + - + name: Install Qt + uses: jurplel/install-qt-action@v2 + with: + install-deps: true + setup-python: 'false' + modules: qtcharts qtwidgets qtgui qtcore + cached: ${{ steps.cache_qt.outputs.cache-hit }} + - + name: PIP Install Packages + uses: BSFishy/pip-action@v1 + with: + packages: pyinstaller + requirements: hydrus\static\build_files\windows\requirements.txt + - + name: Download mpv-dev + uses: carlosperate/download-file-action@v1.0.3 + id: download_mpv + with: + file-url: 'https://sourceforge.net/projects/mpv-player-windows/files/libmpv/mpv-dev-x86_64-20210228-git-d1be8bb.7z' + file-name: 'mpv-dev-x86_64.7z' + location: '.' + - + name: Process mpv-dev + run: | + 7z x ${{ steps.download_mpv.outputs.file-path }} + move mpv-1.dll hydrus\ + - + name: Build Hydrus + run: | + move ${{ steps.setup_ffmpeg.outputs.ffmpeg-path }} hydrus\bin\ + move hydrus\static\build_files\windows\sqlite3.dll hydrus\ + move hydrus\static\build_files\windows\client-win.spec client-win.spec + move hydrus\static\build_files\windows\server-win.spec server-win.spec + pyinstaller server-win.spec + pyinstaller client-win.spec + dir -r + - + name: InnoSetup + run: | + move hydrus\static\build_files\windows\InnoSetup.iss InnoSetup.iss + ISCC.exe InnoSetup.iss + - + name: Upload a Build Artifact + uses: actions/upload-artifact@v2 + with: + name: Windows-Install + path: dist\HydrusInstaller.exe + if-no-files-found: error + retention-days: 2 + - + name: Upload a Build Artifact + uses: actions/upload-artifact@v2 + with: + name: Windows-Extract + path: dist\Hydrus Network + if-no-files-found: error + retention-days: 2 \ No newline at end of file