Refactor build (#493)

* Add lint/format checks to build
* Make travis get full repo to get tags
* Run packr2 once in cross-compile
* Fix quotes in package.json
* Fix linting issues
* Formatting
* Fix vet issue
* Fix go lint issues
* Show start of each platform compilation
* Add validate target
* Set gitattributes for go fmt and mod vendor
* Fix tag name
* Add fmt-ui target

.gitattributes

@@ -1,4 +1,6 @@
 go.mod text eol=lf
 go.sum text eol=lf
+*.go text eol=lf
+vendor/** -text
 ui/v2.5/**/*.ts* text eol=lf
 ui/v2.5/**/*.scss text eol=lf

.travis.yml

@@ -1,6 +1,8 @@
-if: tag != develop_latest # dont build for the latest_develop tagged version
+if: tag != latest_develop # dont build for the latest_develop tagged version
 
 dist: xenial
+git:
+  depth: false
 language: go
 go:
 - 1.11.x
@@ -14,12 +16,12 @@ before_install:
 - nvm install 12
 - travis_retry yarn --cwd ui/v2.5 install --frozen-lockfile
 - make generate
-- CI=false yarn --cwd ui/v2.5 build # TODO: Fix warnings
+- CI=false yarn --cwd ui/v2.5 build-ci
 #- go get -v github.com/mgechev/revive
 script:
+# left lint off to avoid getting extra dependency
 #- make lint
-#- make vet
-- make it
+- make fmt-check vet it
 after_success:
 - docker pull stashapp/compiler:develop
 - sh ./scripts/cross-compile.sh

Makefile

@@ -28,6 +28,11 @@ generate:
 fmt:
 	go fmt ./...
 
+# Ensures that changed files have had gofmt run on them
+.PHONY: fmt-check
+fmt-check:
+	sh ./scripts/check-gofmt.sh
+
 # Runs go vet on the project's source code.
 .PHONY: vet
 vet:
@@ -47,7 +52,31 @@ test:
 it:
 	go test -mod=vendor -tags=integration ./...
 
+# installs UI dependencies. Run when first cloning repository, or if UI 
+# dependencies have changed
+.PHONY: pre-ui
+pre-ui:
+	cd ui/v2.5 && yarn install --frozen-lockfile
+
 .PHONY: ui
 ui:
 	cd ui/v2.5 && yarn build
 	packr2
+
+fmt-ui:
+	cd ui/v2.5 && yarn format
+
+# runs tests and checks on the UI and builds it
+.PHONY: ui-validate
+ui-validate:
+	cd ui/v2.5 && yarn run validate
+
+# just repacks the packr files - use when updating migrations and packed files without 
+# rebuilding the UI
+.PHONY: packr
+packr:
+	packr2
+
+# runs all of the tests and checks required for a PR to be accepted
+.PHONY: validate
+validate: ui-validate fmt-check vet lint it

README.md

@@ -92,11 +92,18 @@ NOTE: The `make` command in Windows will be `mingw32-make` with MingW.
 
 ## Commands
 
-* `make generate` - Generate Go GraphQL and packr2 files
+* `make generate` - Generate Go and UI GraphQL files
 * `make build` - Builds the binary (make sure to build the UI as well... see below)
-* `make ui` - Builds the frontend
+* `make pre-ui` - Installs the UI dependencies. Only needs to be run once before building the UI for the first time, or if the dependencies are updated
+* `make fmt-ui` - Formats the UI source code.
+* `make ui` - Builds the frontend and the packr2 files
+* `make packr` - Generate packr2 files (sub-target of `ui`. Use to regenerate packr2 files without rebuilding UI)
 * `make vet` - Run `go vet`
 * `make lint` - Run the linter
+* `make fmt` - Run `go fmt`
+* `make fmt-check` - Ensure changed files are formatted correctly
+* `make it` - Run the unit and integration tests
+* `make validate` - Run all of the tests and checks required to submit a PR
 
 ## Building a release
 

pkg/api/check_version.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -18,6 +19,10 @@ const apiTags string = "https://api.github.com/repos/stashapp/stash/tags"
 const apiAcceptHeader string = "application/vnd.github.v3+json"
 const developmentTag string = "latest_develop"
 
+// ErrNoVersion indicates that no version information has been embedded in the
+// stash binary
+var ErrNoVersion = errors.New("no stash version")
+
 var stashReleases = func() map[string]string {
 	return map[string]string{
 		"windows/amd64": "stash-win.exe",
@@ -140,7 +145,7 @@ func GetLatestVersion(shortHash bool) (latestVersion string, latestRelease strin
 
 	version, _, _ := GetVersion()
 	if version == "" {
-		return "", "", fmt.Errorf("Stash doesn't have a version. Version check not supported.")
+		return "", "", ErrNoVersion
 	}
 
 	// if the version is suffixed with -x-xxxx, then we are running a development build

pkg/api/server.go

@@ -30,9 +30,9 @@ import (
 	"github.com/stashapp/stash/pkg/utils"
 )
 
-var version string = ""
-var buildstamp string = ""
-var githash string = ""
+var version string
+var buildstamp string
+var githash string
 
 var uiBox *packr.Box
 

pkg/ffmpeg/encoder.go

@@ -18,8 +18,8 @@ type Encoder struct {
 }
 
 var (
-	runningEncoders      map[string][]*os.Process = make(map[string][]*os.Process)
-	runningEncodersMutex                          = sync.RWMutex{}
+	runningEncoders      = make(map[string][]*os.Process)
+	runningEncodersMutex = sync.RWMutex{}
 )
 
 func NewEncoder(ffmpegPath string) Encoder {

pkg/manager/manager_tasks.go

@@ -556,7 +556,7 @@ func (s *singleton) returnToIdleState() {
 }
 
 func (s *singleton) neededScan(paths []string) int64 {
-	var neededScans int64 = 0
+	var neededScans int64
 
 	for _, path := range paths {
 		task := ScanTask{FilePath: path}
@@ -577,14 +577,14 @@ type totalsGenerate struct {
 func (s *singleton) neededGenerate(scenes []*models.Scene, sprites, previews, markers, transcodes bool) *totalsGenerate {
 
 	var totals totalsGenerate
-	const timeoutSecs = 90 * time.Second
+	const timeout = 90 * time.Second
 
 	// create a control channel through which to signal the counting loop when the timeout is reached
 	chTimeout := make(chan struct{})
 
 	//run the timeout function in a separate thread
 	go func() {
-		time.Sleep(timeoutSecs)
+		time.Sleep(timeout)
 		chTimeout <- struct{}{}
 	}()
 

pkg/models/model_movie.go

@@ -40,4 +40,4 @@ type MoviePartial struct {
 	UpdatedAt  *SQLiteTimestamp `db:"updated_at" json:"updated_at"`
 }
 
-var DefaultMovieImage string = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAYAAABw4pVUAAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4wgVBQsJl1CMZAAAASJJREFUeNrt3N0JwyAYhlEj3cj9R3Cm5rbkqtAP+qrnGaCYHPwJpLlaa++mmLpbAERAgAgIEAEBIiBABERAgAgIEAEBIiBABERAgAgIEAHZuVflj40x4i94zhk9vqsVvEq6AsQqMP1EjORx20OACAgQRRx7T+zzcFBxcjNDfoB4ntQqTm5Awo7MlqywZxcgYQ+RlqywJ3ozJAQCSBiEJSsQA0gYBpDAgAARECACAkRAgAgIEAERECACAmSjUv6eAOSB8m8YIGGzBUjYbAESBgMkbBkDEjZbgITBAClcxiqQvEoatreYIWEBASIgJ4Gkf11ntXH3nS9uxfGWfJ5J9hAgAgJEQAQEiIAAERAgAgJEQAQEiIAAERAgAgJEQAQEiL7qBuc6RKLHxr0CAAAAAElFTkSuQmCC"
+var DefaultMovieImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAYAAABw4pVUAAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4wgVBQsJl1CMZAAAASJJREFUeNrt3N0JwyAYhlEj3cj9R3Cm5rbkqtAP+qrnGaCYHPwJpLlaa++mmLpbAERAgAgIEAEBIiBABERAgAgIEAEBIiBABERAgAgIEAHZuVflj40x4i94zhk9vqsVvEq6AsQqMP1EjORx20OACAgQRRx7T+zzcFBxcjNDfoB4ntQqTm5Awo7MlqywZxcgYQ+RlqywJ3ozJAQCSBiEJSsQA0gYBpDAgAARECACAkRAgAgIEAERECACAmSjUv6eAOSB8m8YIGGzBUjYbAESBgMkbBkDEjZbgITBAClcxiqQvEoatreYIWEBASIgJ4Gkf11ntXH3nS9uxfGWfJ5J9hAgAgJEQAQEiIAAERAgAgJEQAQEiIAAERAgAgJEQAQEiL7qBuc6RKLHxr0CAAAAAElFTkSuQmCC"

pkg/models/model_studio.go

@@ -14,4 +14,4 @@ type Studio struct {
 	UpdatedAt SQLiteTimestamp `db:"updated_at" json:"updated_at"`
 }
 
-var DefaultStudioImage string = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAYAAABw4pVUAAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4wgVBQsJl1CMZAAAASJJREFUeNrt3N0JwyAYhlEj3cj9R3Cm5rbkqtAP+qrnGaCYHPwJpLlaa++mmLpbAERAgAgIEAEBIiBABERAgAgIEAEBIiBABERAgAgIEAHZuVflj40x4i94zhk9vqsVvEq6AsQqMP1EjORx20OACAgQRRx7T+zzcFBxcjNDfoB4ntQqTm5Awo7MlqywZxcgYQ+RlqywJ3ozJAQCSBiEJSsQA0gYBpDAgAARECACAkRAgAgIEAERECACAmSjUv6eAOSB8m8YIGGzBUjYbAESBgMkbBkDEjZbgITBAClcxiqQvEoatreYIWEBASIgJ4Gkf11ntXH3nS9uxfGWfJ5J9hAgAgJEQAQEiIAAERAgAgJEQAQEiIAAERAgAgJEQAQEiL7qBuc6RKLHxr0CAAAAAElFTkSuQmCC"
+var DefaultStudioImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAYAAABw4pVUAAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4wgVBQsJl1CMZAAAASJJREFUeNrt3N0JwyAYhlEj3cj9R3Cm5rbkqtAP+qrnGaCYHPwJpLlaa++mmLpbAERAgAgIEAEBIiBABERAgAgIEAEBIiBABERAgAgIEAHZuVflj40x4i94zhk9vqsVvEq6AsQqMP1EjORx20OACAgQRRx7T+zzcFBxcjNDfoB4ntQqTm5Awo7MlqywZxcgYQ+RlqywJ3ozJAQCSBiEJSsQA0gYBpDAgAARECACAkRAgAgIEAERECACAmSjUv6eAOSB8m8YIGGzBUjYbAESBgMkbBkDEjZbgITBAClcxiqQvEoatreYIWEBASIgJ4Gkf11ntXH3nS9uxfGWfJ5J9hAgAgJEQAQEiIAAERAgAgJEQAQEiIAAERAgAgJEQAQEiL7qBuc6RKLHxr0CAAAAAElFTkSuQmCC"

pkg/scraper/stash.go

@@ -17,7 +17,7 @@ func getStashClient(c scraperTypeConfig) *graphql.Client {
 
 type stashFindPerformerNamePerformer struct {
 	ID   string `json:"id" graphql:"id"`
-	Name string `json:"id" graphql:"name"`
+	Name string `json:"name" graphql:"name"`
 }
 
 func (p stashFindPerformerNamePerformer) toPerformer() *models.ScrapedPerformer {

scripts/check-gofmt.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+# Copyright (c) 2012 The Go Authors. All rights reserved.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+gofiles=$(git diff --name-only --diff-filter=ACM develop -- '*.go' ':!vendor')
+[ -z "$gofiles" ] && exit 0
+
+unformatted=$(gofmt -l $gofiles)
+[ -z "$unformatted" ] && exit 0
+
+# Some files are not gofmt'd. Print message and fail.
+
+echo >&2 "Go files must be formatted with gofmt. Please run:"
+for fn in $unformatted; do
+	echo >&2 "  gofmt -w $PWD/$fn"
+done
+
+exit 1

scripts/cross-compile.sh

@@ -4,11 +4,11 @@ DATE=`go run -mod=vendor scripts/getDate.go`
 GITHASH=`git rev-parse --short HEAD`
 STASH_VERSION=`git describe --tags --exclude latest_develop`
 VERSION_FLAGS="-X 'github.com/stashapp/stash/pkg/api.version=$STASH_VERSION' -X 'github.com/stashapp/stash/pkg/api.buildstamp=$DATE' -X 'github.com/stashapp/stash/pkg/api.githash=$GITHASH'"
-SETUP="export GO111MODULE=on; export CGO_ENABLED=1;"
-WINDOWS="GOOS=windows GOARCH=amd64 CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ packr2 build -o dist/stash-win.exe -ldflags \"-extldflags '-static' $VERSION_FLAGS\" -tags extended -v -mod=vendor;"
-DARWIN="GOOS=darwin GOARCH=amd64 CC=o64-clang CXX=o64-clang++ packr2 build -o dist/stash-osx -ldflags \"$VERSION_FLAGS\" -tags extended -v -mod=vendor;"
-LINUX="packr2 build -o dist/stash-linux -ldflags \"$VERSION_FLAGS\" -v -mod=vendor;"
-RASPPI="GOOS=linux GOARCH=arm GOARM=5 CC=arm-linux-gnueabi-gcc packr2 build -o dist/stash-pi -ldflags \"$VERSION_FLAGS\" -v -mod=vendor;"
+SETUP="export GO111MODULE=on; export CGO_ENABLED=1; packr2;"
+WINDOWS="echo '=== Building Windows binary ==='; GOOS=windows GOARCH=amd64 CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ go build -o dist/stash-win.exe -ldflags \"-extldflags '-static' $VERSION_FLAGS\" -tags extended -v -mod=vendor;"
+DARWIN="echo '=== Building OSX binary ==='; GOOS=darwin GOARCH=amd64 CC=o64-clang CXX=o64-clang++ go build -o dist/stash-osx -ldflags \"$VERSION_FLAGS\" -tags extended -v -mod=vendor;"
+LINUX="echo '=== Building Linux binary ==='; go build -o dist/stash-linux -ldflags \"$VERSION_FLAGS\" -v -mod=vendor;"
+RASPPI="echo '=== Building Raspberry Pi binary ==='; GOOS=linux GOARCH=arm GOARM=5 CC=arm-linux-gnueabi-gcc go build -o dist/stash-pi -ldflags \"$VERSION_FLAGS\" -v -mod=vendor;"
 
 COMMAND="$SETUP $WINDOWS $DARWIN $LINUX $RASPPI"
 

ui/v2.5/package.json

@@ -8,10 +8,13 @@
     "build": "react-scripts build",
     "test": "react-scripts test",
     "eject": "react-scripts eject",
+    "build-ci": "yarn validate && yarn build",
+    "validate": "yarn lint && yarn format-check",
     "lint": "yarn lint:css && yarn lint:js",
     "lint:js": "eslint --cache src/**/*.{ts,tsx}",
-    "lint:css": "stylelint 'src/**/*.scss'",
+    "lint:css": "stylelint \"src/**/*.scss\"",
     "format": "prettier --write \"src/**/!(generated-graphql).{js,jsx,ts,tsx,scss}\"",
+    "format-check": "prettier --check \"src/**/!(generated-graphql).{js,jsx,ts,tsx,scss}\"",
     "gqlgen": "gql-gen --config codegen.yml",
     "extract": "NODE_ENV=development extract-messages -l=en,de -o src/locale -d en --flat false 'src/**/!(*.test).tsx'"
   },

ui/v2.5/src/components/List/Pagination.tsx

@@ -13,7 +13,6 @@ interface IPaginationIndexProps {
   itemsPerPage: number;
   currentPage: number;
   totalItems: number;
-  onClick?: () => void;
 }
 
 export const Pagination: React.FC<IPaginationProps> = ({
@@ -112,14 +111,25 @@ export const PaginationIndex: React.FC<IPaginationIndexProps> = ({
   itemsPerPage,
   currentPage,
   totalItems,
-  onClick
 }) => {
   const intl = useIntl();
 
   // Build the pagination index string
-  const firstItemCount:number = Math.min((currentPage-1)*itemsPerPage+1, totalItems);
-  const lastItemCount:number = Math.min(firstItemCount+(itemsPerPage-1), totalItems);
-  const indexText:string = `${intl.formatNumber(firstItemCount)}-${intl.formatNumber(lastItemCount)} of ${intl.formatNumber(totalItems)}`;
+  const firstItemCount: number = Math.min(
+    (currentPage - 1) * itemsPerPage + 1,
+    totalItems
+  );
+  const lastItemCount: number = Math.min(
+    firstItemCount + (itemsPerPage - 1),
+    totalItems
+  );
+  const indexText: string = `${intl.formatNumber(
+    firstItemCount
+  )}-${intl.formatNumber(lastItemCount)} of ${intl.formatNumber(totalItems)}`;
 
-  return <span className="filter-container text-muted paginationIndex" onClick={onClick}>{indexText}</span>
-};
+  return (
+    <span className="filter-container text-muted paginationIndex">
+      {indexText}
+    </span>
+  );
+};

ui/v2.5/src/hooks/ListHook.tsx

@@ -364,7 +364,6 @@ const useList = <QueryResult extends IQueryResult, QueryData extends IDataItem>(
           itemsPerPage={filter.itemsPerPage}
           currentPage={filter.currentPage}
           totalItems={totalCount}
-          onClick={() => {}}
         />
       );
     }

ui/v2.5/src/index.scss

@@ -196,6 +196,14 @@ div.dropdown-menu {
 
   .dropdown-item {
     display: flex;
+
+    & > :not(:last-child) {
+      margin-right: 7px;
+    }
+
+    & > :last-child {
+      margin-right: 0;
+    }
   }
 }