2020-08-08 02:05:35 +00:00
|
|
|
package plugin
|
|
|
|
|
|
|
|
import (
|
2022-03-23 22:22:41 +00:00
|
|
|
"context"
|
2020-08-08 02:05:35 +00:00
|
|
|
"encoding/json"
|
|
|
|
"errors"
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"os/exec"
|
|
|
|
"sync"
|
|
|
|
|
2022-03-17 00:33:59 +00:00
|
|
|
stashExec "github.com/stashapp/stash/pkg/exec"
|
2020-08-08 02:05:35 +00:00
|
|
|
"github.com/stashapp/stash/pkg/logger"
|
|
|
|
"github.com/stashapp/stash/pkg/plugin/common"
|
2022-03-23 22:22:41 +00:00
|
|
|
"github.com/stashapp/stash/pkg/python"
|
2020-08-08 02:05:35 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type rawTaskBuilder struct{}
|
|
|
|
|
|
|
|
func (*rawTaskBuilder) build(task pluginTask) Task {
|
|
|
|
return &rawPluginTask{
|
|
|
|
pluginTask: task,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
type rawPluginTask struct {
|
|
|
|
pluginTask
|
|
|
|
|
|
|
|
started bool
|
|
|
|
waitGroup sync.WaitGroup
|
|
|
|
cmd *exec.Cmd
|
|
|
|
done chan bool
|
|
|
|
}
|
|
|
|
|
|
|
|
func (t *rawPluginTask) Start() error {
|
|
|
|
if t.started {
|
|
|
|
return errors.New("task already started")
|
|
|
|
}
|
|
|
|
|
|
|
|
command := t.plugin.getExecCommand(t.operation)
|
|
|
|
if len(command) == 0 {
|
|
|
|
return fmt.Errorf("empty exec value in operation %s", t.operation.Name)
|
|
|
|
}
|
|
|
|
|
2022-03-23 22:22:41 +00:00
|
|
|
var cmd *exec.Cmd
|
|
|
|
if python.IsPythonCommand(command[0]) {
|
|
|
|
pythonPath := t.serverConfig.GetPythonPath()
|
|
|
|
var p *python.Python
|
|
|
|
if pythonPath != "" {
|
|
|
|
p = python.New(pythonPath)
|
|
|
|
} else {
|
|
|
|
p, _ = python.Resolve()
|
|
|
|
}
|
|
|
|
|
|
|
|
if p != nil {
|
|
|
|
cmd = p.Command(context.TODO(), command[1:])
|
2021-11-11 22:21:52 +00:00
|
|
|
}
|
2022-03-23 22:22:41 +00:00
|
|
|
|
|
|
|
// if could not find python, just use the command args as-is
|
2021-11-11 22:21:52 +00:00
|
|
|
}
|
|
|
|
|
2022-03-23 22:22:41 +00:00
|
|
|
if cmd == nil {
|
|
|
|
cmd = stashExec.Command(command[0], command[1:]...)
|
|
|
|
}
|
2020-08-08 02:05:35 +00:00
|
|
|
|
|
|
|
stdin, err := cmd.StdinPipe()
|
|
|
|
if err != nil {
|
Errorlint sweep + minor linter tweaks (#1796)
* Replace error assertions with Go 1.13 style
Use `errors.As(..)` over type assertions. This enables better use of
wrapped errors in the future, and lets us pass some errorlint checks
in the process.
The rewrite is entirely mechanical, and uses a standard idiom for
doing so.
* Use Go 1.13's errors.Is(..)
Rather than directly checking for error equality, use errors.Is(..).
This protects against error wrapping issues in the future.
Even though something like sql.ErrNoRows doesn't need the wrapping, do
so anyway, for the sake of consistency throughout the code base.
The change almost lets us pass the `errorlint` Go checker except for
a missing case in `js.go` which is to be handled separately; it isn't
mechanical, like these changes are.
* Remove goconst
goconst isn't a useful linter in many cases, because it's false positive
rate is high. It's 100% for the current code base.
* Avoid direct comparison of errors in recover()
Assert that we are catching an error from recover(). If we are,
check that the error caught matches errStop.
* Enable the "errorlint" checker
Configure the checker to avoid checking for errorf wraps. These are
often false positives since the suggestion is to blanket wrap errors
with %w, and that exposes the underlying API which you might not want
to do.
The other warnings are good however, and with the current patch stack,
the code base passes all these checks as well.
* Configure rowserrcheck
The project uses sqlx. Configure rowserrcheck to include said package.
* Mechanically rewrite a large set of errors
Mechanically search for errors that look like
fmt.Errorf("...%s", err.Error())
and rewrite those into
fmt.Errorf("...%v", err)
The `fmt` package is error-aware and knows how to call err.Error()
itself.
The rationale is that this is more idiomatic Go; it paves the
way for using error wrapping later with %w in some sites.
This patch only addresses the entirely mechanical rewriting caught by
a project-side search/replace. There are more individual sites not
addressed by this patch.
2021-10-12 03:03:08 +00:00
|
|
|
return fmt.Errorf("error getting plugin process stdin: %v", err)
|
2020-08-08 02:05:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
defer stdin.Close()
|
|
|
|
|
Hoist context, enable errchkjson (#2488)
* Make the script scraper context-aware
Connect the context to the command execution. This means command
execution can be aborted if the context is canceled. The context is
usually bound to user-interaction, i.e., a scraper operation issued
by the user. Hence, it seems correct to abort a command if the user
aborts.
* Enable errchkjson
Some json marshal calls are *safe* in that they can never fail. This is
conditional on the types of the the data being encoded. errchkjson finds
those calls which are unsafe, and also not checked for errors.
Add logging warnings to the place where unsafe encodings might happen.
This can help uncover usage bugs early in stash if they are tripped,
making debugging easier.
While here, keep the checker enabled in the linter to capture future
uses of json marshalling.
* Pass the context for zip file scanning.
* Pass the context in scanning
* Pass context, replace context.TODO()
Where applicable, pass the context down toward the lower functions in
the call stack. Replace uses of context.TODO() with the passed context.
This makes the code more context-aware, and you can rely on aborting
contexts to clean up subsystems to a far greater extent now.
I've left the cases where there is a context in a struct. My gut feeling
is that they have solutions that are nice, but they require more deep
thinking to unveil how to handle it.
* Remove context from task-structs
As a rule, contexts are better passed explicitly to functions than they
are passed implicitly via structs. In the case of tasks, we already
have a valid context in scope when creating the struct, so remove ctx
from the struct and use the scoped context instead.
With this change it is clear that the scanning functions are under a
context, and the task-starting caller has jurisdiction over the context
and its lifetime. A reader of the code don't have to figure out where
the context are coming from anymore.
While here, connect context.TODO() to the newly scoped context in most
of the scan code.
* Remove context from autotag struct too
* Make more context-passing explicit
In all of these cases, there is an applicable context which is close
in the call-tree. Hook up to this context.
* Simplify context passing in manager
The managers context handling generally wants to use an outer context
if applicable. However, the code doesn't pass it explicitly, but stores
it in a struct. Pull out the context from the struct and use it to
explicitly pass it.
At a later point in time, we probably want to handle this by handing
over the job to a different (program-lifetime) context for background
jobs, but this will do for a start.
2022-04-15 01:34:53 +00:00
|
|
|
inBytes, err := json.Marshal(t.input)
|
|
|
|
if err != nil {
|
|
|
|
logger.Warnf("error marshalling raw command input")
|
|
|
|
}
|
2021-09-20 23:34:25 +00:00
|
|
|
if k, err := io.WriteString(stdin, string(inBytes)); err != nil {
|
|
|
|
logger.Warnf("error writing input to plugins stdin (wrote %v bytes out of %v): %v", k, len(string(inBytes)), err)
|
|
|
|
}
|
2020-08-08 02:05:35 +00:00
|
|
|
}()
|
|
|
|
|
|
|
|
stderr, err := cmd.StderrPipe()
|
|
|
|
if err != nil {
|
2021-09-08 01:23:10 +00:00
|
|
|
logger.Error("plugin stderr not available: " + err.Error())
|
2020-08-08 02:05:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
stdout, err := cmd.StdoutPipe()
|
|
|
|
if nil != err {
|
2021-09-08 01:23:10 +00:00
|
|
|
logger.Error("plugin stdout not available: " + err.Error())
|
2020-08-08 02:05:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
t.waitGroup.Add(1)
|
|
|
|
t.done = make(chan bool, 1)
|
|
|
|
if err = cmd.Start(); err != nil {
|
Errorlint sweep + minor linter tweaks (#1796)
* Replace error assertions with Go 1.13 style
Use `errors.As(..)` over type assertions. This enables better use of
wrapped errors in the future, and lets us pass some errorlint checks
in the process.
The rewrite is entirely mechanical, and uses a standard idiom for
doing so.
* Use Go 1.13's errors.Is(..)
Rather than directly checking for error equality, use errors.Is(..).
This protects against error wrapping issues in the future.
Even though something like sql.ErrNoRows doesn't need the wrapping, do
so anyway, for the sake of consistency throughout the code base.
The change almost lets us pass the `errorlint` Go checker except for
a missing case in `js.go` which is to be handled separately; it isn't
mechanical, like these changes are.
* Remove goconst
goconst isn't a useful linter in many cases, because it's false positive
rate is high. It's 100% for the current code base.
* Avoid direct comparison of errors in recover()
Assert that we are catching an error from recover(). If we are,
check that the error caught matches errStop.
* Enable the "errorlint" checker
Configure the checker to avoid checking for errorf wraps. These are
often false positives since the suggestion is to blanket wrap errors
with %w, and that exposes the underlying API which you might not want
to do.
The other warnings are good however, and with the current patch stack,
the code base passes all these checks as well.
* Configure rowserrcheck
The project uses sqlx. Configure rowserrcheck to include said package.
* Mechanically rewrite a large set of errors
Mechanically search for errors that look like
fmt.Errorf("...%s", err.Error())
and rewrite those into
fmt.Errorf("...%v", err)
The `fmt` package is error-aware and knows how to call err.Error()
itself.
The rationale is that this is more idiomatic Go; it paves the
way for using error wrapping later with %w in some sites.
This patch only addresses the entirely mechanical rewriting caught by
a project-side search/replace. There are more individual sites not
addressed by this patch.
2021-10-12 03:03:08 +00:00
|
|
|
return fmt.Errorf("error running plugin: %v", err)
|
2020-08-08 02:05:35 +00:00
|
|
|
}
|
|
|
|
|
2021-09-19 00:06:34 +00:00
|
|
|
go t.handlePluginStderr(t.plugin.Name, stderr)
|
2020-08-08 02:05:35 +00:00
|
|
|
t.cmd = cmd
|
|
|
|
|
|
|
|
// send the stdout to the plugin output
|
|
|
|
go func() {
|
|
|
|
defer t.waitGroup.Done()
|
|
|
|
defer close(t.done)
|
2021-09-27 00:55:23 +00:00
|
|
|
stdoutData, _ := io.ReadAll(stdout)
|
2020-08-08 02:05:35 +00:00
|
|
|
stdoutString := string(stdoutData)
|
|
|
|
|
|
|
|
output := t.getOutput(stdoutString)
|
|
|
|
|
|
|
|
err := cmd.Wait()
|
|
|
|
if err != nil && output.Error == nil {
|
|
|
|
errStr := err.Error()
|
|
|
|
output.Error = &errStr
|
|
|
|
}
|
|
|
|
|
|
|
|
t.result = &output
|
|
|
|
}()
|
|
|
|
|
|
|
|
t.started = true
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (t *rawPluginTask) getOutput(output string) common.PluginOutput {
|
|
|
|
// try to parse the output as a PluginOutput json. If it fails just
|
|
|
|
// get the raw output
|
|
|
|
ret := common.PluginOutput{}
|
|
|
|
decodeErr := json.Unmarshal([]byte(output), &ret)
|
|
|
|
|
|
|
|
if decodeErr != nil {
|
|
|
|
ret.Output = &output
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret
|
|
|
|
}
|
|
|
|
|
|
|
|
func (t *rawPluginTask) Wait() {
|
|
|
|
t.waitGroup.Wait()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (t *rawPluginTask) Stop() error {
|
|
|
|
if t.cmd == nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return t.cmd.Process.Kill()
|
|
|
|
}
|