2019-02-09 12:30:49 +00:00
|
|
|
package utils
|
|
|
|
|
|
|
|
import (
|
Toward better context handling (#1835)
* Use the request context
The code uses context.Background() in a flow where there is a
http.Request. Use the requests context instead.
* Use a true context in the plugin example
Let AddTag/RemoveTag take a context and use that context throughout
the example.
* Avoid the use of context.Background
Prefer context.TODO over context.Background deep in the call chain.
This marks the site as something which we need to context-handle
later, and also makes it clear to the reader that the context is
sort-of temporary in the code base.
While here, be consistent in handling the `act` variable in each
branch of the if .. { .. } .. check.
* Prefer context.TODO over context.Background
For the different scraping operations here, there is a context
higher up the call chain, which we ought to use. Mark the call-sites
as TODO for now, so we can come back later on a sweep of which parts
can be context-lifted.
* Thread context upwards
Initialization requires context for transactions. Thread the context
upward the call chain.
At the intialization call, add a context.TODO since we can't break this
yet. The singleton assumption prevents us from pulling it up into main for
now.
* make tasks context-aware
Change the task interface to understand contexts.
Pass the context down in some of the branches where it is needed.
* Make QueryStashBoxScene context-aware
This call naturally sits inside the request-context. Use it.
* Introduce a context in the JS plugin code
This allows us to use a context for HTTP calls inside the system.
Mark the context with a TODO at top level for now.
* Nitpick error formatting
Use %v rather than %s for error interfaces.
Do not begin an error strong with a capital letter.
* Avoid the use of http.Get in FFMPEG download chain
Since http.Get has no context, it isn't possible to break out or have
policy induced. The call will block until the GET completes. Rewrite
to use a http Request and provide a context.
Thread the context through the call chain for now. provide
context.TODO() at the top level of the initialization chain.
* Make getRemoteCDPWSAddress aware of contexts
Eliminate a call to http.Get and replace it with a context-aware
variant.
Push the context upwards in the call chain, but plug it before the
scraper interface so we don't have to rewrite said interface yet.
Plugged with context.TODO()
* Scraper: make the getImage function context-aware
Use a context, and pass it upwards. Plug it with context.TODO()
up the chain before the rewrite gets too much out of hand for now.
Minor tweaks along the way, remove a call to context.Background()
deep in the call chain.
* Make NOTIFY request context-aware
The call sits inside a Request-handler. So it's natural to use the
requests context as the context for the outgoing HTTP request.
* Use a context in the url scraper code
We are sitting in code which has a context, so utilize it for the
request as well.
* Use a context when checking versions
When we check the version of stash on Github, use a context. Thread
the context up to the initialization routine of the HTTP/GraphQL
server and plug it with a context.TODO() for now.
This paves the way for providing a context to the HTTP server code in a
future patch.
* Make utils func ReadImage context-aware
In almost all of the cases, there is a context in the call chain which
is a natural use. This is true for all the GraphQL mutations.
The exception is in task_stash_box_tag, so plug that task with
context.TODO() for now.
* Make stash-box get context-aware
Thread a context through the call chain until we hit the Client API.
Plug it with context.TODO() there for now.
* Enable the noctx linter
The code is now free of any uncontexted HTTP request. This means we
pass the noctx linter, and we can enable it in the code base.
2021-10-14 04:32:41 +00:00
|
|
|
"context"
|
2020-06-22 23:19:19 +00:00
|
|
|
"crypto/md5"
|
2021-03-23 21:12:11 +00:00
|
|
|
"crypto/tls"
|
2019-02-09 12:30:49 +00:00
|
|
|
"encoding/base64"
|
2022-09-19 05:01:40 +00:00
|
|
|
"errors"
|
2019-02-09 12:30:49 +00:00
|
|
|
"fmt"
|
2021-09-27 00:55:23 +00:00
|
|
|
"io"
|
2020-06-22 23:19:19 +00:00
|
|
|
"net/http"
|
2019-02-09 12:30:49 +00:00
|
|
|
"regexp"
|
2020-06-22 23:19:19 +00:00
|
|
|
"strings"
|
2022-09-19 05:01:40 +00:00
|
|
|
"syscall"
|
2021-03-11 01:56:34 +00:00
|
|
|
"time"
|
2019-02-09 12:30:49 +00:00
|
|
|
)
|
|
|
|
|
2021-03-11 01:56:34 +00:00
|
|
|
// Timeout to get the image. Includes transfer time. May want to make this
|
|
|
|
// configurable at some point.
|
|
|
|
const imageGetTimeout = time.Second * 60
|
|
|
|
|
|
|
|
const base64RE = `^data:.+\/(.+);base64,(.*)$`
|
|
|
|
|
|
|
|
// ProcessImageInput transforms an image string either from a base64 encoded
|
|
|
|
// string, or from a URL, and returns the image as a byte slice
|
Toward better context handling (#1835)
* Use the request context
The code uses context.Background() in a flow where there is a
http.Request. Use the requests context instead.
* Use a true context in the plugin example
Let AddTag/RemoveTag take a context and use that context throughout
the example.
* Avoid the use of context.Background
Prefer context.TODO over context.Background deep in the call chain.
This marks the site as something which we need to context-handle
later, and also makes it clear to the reader that the context is
sort-of temporary in the code base.
While here, be consistent in handling the `act` variable in each
branch of the if .. { .. } .. check.
* Prefer context.TODO over context.Background
For the different scraping operations here, there is a context
higher up the call chain, which we ought to use. Mark the call-sites
as TODO for now, so we can come back later on a sweep of which parts
can be context-lifted.
* Thread context upwards
Initialization requires context for transactions. Thread the context
upward the call chain.
At the intialization call, add a context.TODO since we can't break this
yet. The singleton assumption prevents us from pulling it up into main for
now.
* make tasks context-aware
Change the task interface to understand contexts.
Pass the context down in some of the branches where it is needed.
* Make QueryStashBoxScene context-aware
This call naturally sits inside the request-context. Use it.
* Introduce a context in the JS plugin code
This allows us to use a context for HTTP calls inside the system.
Mark the context with a TODO at top level for now.
* Nitpick error formatting
Use %v rather than %s for error interfaces.
Do not begin an error strong with a capital letter.
* Avoid the use of http.Get in FFMPEG download chain
Since http.Get has no context, it isn't possible to break out or have
policy induced. The call will block until the GET completes. Rewrite
to use a http Request and provide a context.
Thread the context through the call chain for now. provide
context.TODO() at the top level of the initialization chain.
* Make getRemoteCDPWSAddress aware of contexts
Eliminate a call to http.Get and replace it with a context-aware
variant.
Push the context upwards in the call chain, but plug it before the
scraper interface so we don't have to rewrite said interface yet.
Plugged with context.TODO()
* Scraper: make the getImage function context-aware
Use a context, and pass it upwards. Plug it with context.TODO()
up the chain before the rewrite gets too much out of hand for now.
Minor tweaks along the way, remove a call to context.Background()
deep in the call chain.
* Make NOTIFY request context-aware
The call sits inside a Request-handler. So it's natural to use the
requests context as the context for the outgoing HTTP request.
* Use a context in the url scraper code
We are sitting in code which has a context, so utilize it for the
request as well.
* Use a context when checking versions
When we check the version of stash on Github, use a context. Thread
the context up to the initialization routine of the HTTP/GraphQL
server and plug it with a context.TODO() for now.
This paves the way for providing a context to the HTTP server code in a
future patch.
* Make utils func ReadImage context-aware
In almost all of the cases, there is a context in the call chain which
is a natural use. This is true for all the GraphQL mutations.
The exception is in task_stash_box_tag, so plug that task with
context.TODO() for now.
* Make stash-box get context-aware
Thread a context through the call chain until we hit the Client API.
Plug it with context.TODO() there for now.
* Enable the noctx linter
The code is now free of any uncontexted HTTP request. This means we
pass the noctx linter, and we can enable it in the code base.
2021-10-14 04:32:41 +00:00
|
|
|
func ProcessImageInput(ctx context.Context, imageInput string) ([]byte, error) {
|
2021-03-11 01:56:34 +00:00
|
|
|
regex := regexp.MustCompile(base64RE)
|
|
|
|
if regex.MatchString(imageInput) {
|
2022-03-17 00:33:59 +00:00
|
|
|
d, err := ProcessBase64Image(imageInput)
|
2021-03-11 01:56:34 +00:00
|
|
|
return d, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// assume input is a URL. Read it.
|
Toward better context handling (#1835)
* Use the request context
The code uses context.Background() in a flow where there is a
http.Request. Use the requests context instead.
* Use a true context in the plugin example
Let AddTag/RemoveTag take a context and use that context throughout
the example.
* Avoid the use of context.Background
Prefer context.TODO over context.Background deep in the call chain.
This marks the site as something which we need to context-handle
later, and also makes it clear to the reader that the context is
sort-of temporary in the code base.
While here, be consistent in handling the `act` variable in each
branch of the if .. { .. } .. check.
* Prefer context.TODO over context.Background
For the different scraping operations here, there is a context
higher up the call chain, which we ought to use. Mark the call-sites
as TODO for now, so we can come back later on a sweep of which parts
can be context-lifted.
* Thread context upwards
Initialization requires context for transactions. Thread the context
upward the call chain.
At the intialization call, add a context.TODO since we can't break this
yet. The singleton assumption prevents us from pulling it up into main for
now.
* make tasks context-aware
Change the task interface to understand contexts.
Pass the context down in some of the branches where it is needed.
* Make QueryStashBoxScene context-aware
This call naturally sits inside the request-context. Use it.
* Introduce a context in the JS plugin code
This allows us to use a context for HTTP calls inside the system.
Mark the context with a TODO at top level for now.
* Nitpick error formatting
Use %v rather than %s for error interfaces.
Do not begin an error strong with a capital letter.
* Avoid the use of http.Get in FFMPEG download chain
Since http.Get has no context, it isn't possible to break out or have
policy induced. The call will block until the GET completes. Rewrite
to use a http Request and provide a context.
Thread the context through the call chain for now. provide
context.TODO() at the top level of the initialization chain.
* Make getRemoteCDPWSAddress aware of contexts
Eliminate a call to http.Get and replace it with a context-aware
variant.
Push the context upwards in the call chain, but plug it before the
scraper interface so we don't have to rewrite said interface yet.
Plugged with context.TODO()
* Scraper: make the getImage function context-aware
Use a context, and pass it upwards. Plug it with context.TODO()
up the chain before the rewrite gets too much out of hand for now.
Minor tweaks along the way, remove a call to context.Background()
deep in the call chain.
* Make NOTIFY request context-aware
The call sits inside a Request-handler. So it's natural to use the
requests context as the context for the outgoing HTTP request.
* Use a context in the url scraper code
We are sitting in code which has a context, so utilize it for the
request as well.
* Use a context when checking versions
When we check the version of stash on Github, use a context. Thread
the context up to the initialization routine of the HTTP/GraphQL
server and plug it with a context.TODO() for now.
This paves the way for providing a context to the HTTP server code in a
future patch.
* Make utils func ReadImage context-aware
In almost all of the cases, there is a context in the call chain which
is a natural use. This is true for all the GraphQL mutations.
The exception is in task_stash_box_tag, so plug that task with
context.TODO() for now.
* Make stash-box get context-aware
Thread a context through the call chain until we hit the Client API.
Plug it with context.TODO() there for now.
* Enable the noctx linter
The code is now free of any uncontexted HTTP request. This means we
pass the noctx linter, and we can enable it in the code base.
2021-10-14 04:32:41 +00:00
|
|
|
return ReadImageFromURL(ctx, imageInput)
|
2021-03-11 01:56:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// ReadImageFromURL returns image data from a URL
|
Toward better context handling (#1835)
* Use the request context
The code uses context.Background() in a flow where there is a
http.Request. Use the requests context instead.
* Use a true context in the plugin example
Let AddTag/RemoveTag take a context and use that context throughout
the example.
* Avoid the use of context.Background
Prefer context.TODO over context.Background deep in the call chain.
This marks the site as something which we need to context-handle
later, and also makes it clear to the reader that the context is
sort-of temporary in the code base.
While here, be consistent in handling the `act` variable in each
branch of the if .. { .. } .. check.
* Prefer context.TODO over context.Background
For the different scraping operations here, there is a context
higher up the call chain, which we ought to use. Mark the call-sites
as TODO for now, so we can come back later on a sweep of which parts
can be context-lifted.
* Thread context upwards
Initialization requires context for transactions. Thread the context
upward the call chain.
At the intialization call, add a context.TODO since we can't break this
yet. The singleton assumption prevents us from pulling it up into main for
now.
* make tasks context-aware
Change the task interface to understand contexts.
Pass the context down in some of the branches where it is needed.
* Make QueryStashBoxScene context-aware
This call naturally sits inside the request-context. Use it.
* Introduce a context in the JS plugin code
This allows us to use a context for HTTP calls inside the system.
Mark the context with a TODO at top level for now.
* Nitpick error formatting
Use %v rather than %s for error interfaces.
Do not begin an error strong with a capital letter.
* Avoid the use of http.Get in FFMPEG download chain
Since http.Get has no context, it isn't possible to break out or have
policy induced. The call will block until the GET completes. Rewrite
to use a http Request and provide a context.
Thread the context through the call chain for now. provide
context.TODO() at the top level of the initialization chain.
* Make getRemoteCDPWSAddress aware of contexts
Eliminate a call to http.Get and replace it with a context-aware
variant.
Push the context upwards in the call chain, but plug it before the
scraper interface so we don't have to rewrite said interface yet.
Plugged with context.TODO()
* Scraper: make the getImage function context-aware
Use a context, and pass it upwards. Plug it with context.TODO()
up the chain before the rewrite gets too much out of hand for now.
Minor tweaks along the way, remove a call to context.Background()
deep in the call chain.
* Make NOTIFY request context-aware
The call sits inside a Request-handler. So it's natural to use the
requests context as the context for the outgoing HTTP request.
* Use a context in the url scraper code
We are sitting in code which has a context, so utilize it for the
request as well.
* Use a context when checking versions
When we check the version of stash on Github, use a context. Thread
the context up to the initialization routine of the HTTP/GraphQL
server and plug it with a context.TODO() for now.
This paves the way for providing a context to the HTTP server code in a
future patch.
* Make utils func ReadImage context-aware
In almost all of the cases, there is a context in the call chain which
is a natural use. This is true for all the GraphQL mutations.
The exception is in task_stash_box_tag, so plug that task with
context.TODO() for now.
* Make stash-box get context-aware
Thread a context through the call chain until we hit the Client API.
Plug it with context.TODO() there for now.
* Enable the noctx linter
The code is now free of any uncontexted HTTP request. This means we
pass the noctx linter, and we can enable it in the code base.
2021-10-14 04:32:41 +00:00
|
|
|
func ReadImageFromURL(ctx context.Context, url string) ([]byte, error) {
|
2021-03-11 01:56:34 +00:00
|
|
|
client := &http.Client{
|
2021-03-23 21:12:11 +00:00
|
|
|
Transport: &http.Transport{ // ignore insecure certificates
|
|
|
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
|
|
|
},
|
|
|
|
|
2021-03-11 01:56:34 +00:00
|
|
|
Timeout: imageGetTimeout,
|
|
|
|
}
|
|
|
|
|
Toward better context handling (#1835)
* Use the request context
The code uses context.Background() in a flow where there is a
http.Request. Use the requests context instead.
* Use a true context in the plugin example
Let AddTag/RemoveTag take a context and use that context throughout
the example.
* Avoid the use of context.Background
Prefer context.TODO over context.Background deep in the call chain.
This marks the site as something which we need to context-handle
later, and also makes it clear to the reader that the context is
sort-of temporary in the code base.
While here, be consistent in handling the `act` variable in each
branch of the if .. { .. } .. check.
* Prefer context.TODO over context.Background
For the different scraping operations here, there is a context
higher up the call chain, which we ought to use. Mark the call-sites
as TODO for now, so we can come back later on a sweep of which parts
can be context-lifted.
* Thread context upwards
Initialization requires context for transactions. Thread the context
upward the call chain.
At the intialization call, add a context.TODO since we can't break this
yet. The singleton assumption prevents us from pulling it up into main for
now.
* make tasks context-aware
Change the task interface to understand contexts.
Pass the context down in some of the branches where it is needed.
* Make QueryStashBoxScene context-aware
This call naturally sits inside the request-context. Use it.
* Introduce a context in the JS plugin code
This allows us to use a context for HTTP calls inside the system.
Mark the context with a TODO at top level for now.
* Nitpick error formatting
Use %v rather than %s for error interfaces.
Do not begin an error strong with a capital letter.
* Avoid the use of http.Get in FFMPEG download chain
Since http.Get has no context, it isn't possible to break out or have
policy induced. The call will block until the GET completes. Rewrite
to use a http Request and provide a context.
Thread the context through the call chain for now. provide
context.TODO() at the top level of the initialization chain.
* Make getRemoteCDPWSAddress aware of contexts
Eliminate a call to http.Get and replace it with a context-aware
variant.
Push the context upwards in the call chain, but plug it before the
scraper interface so we don't have to rewrite said interface yet.
Plugged with context.TODO()
* Scraper: make the getImage function context-aware
Use a context, and pass it upwards. Plug it with context.TODO()
up the chain before the rewrite gets too much out of hand for now.
Minor tweaks along the way, remove a call to context.Background()
deep in the call chain.
* Make NOTIFY request context-aware
The call sits inside a Request-handler. So it's natural to use the
requests context as the context for the outgoing HTTP request.
* Use a context in the url scraper code
We are sitting in code which has a context, so utilize it for the
request as well.
* Use a context when checking versions
When we check the version of stash on Github, use a context. Thread
the context up to the initialization routine of the HTTP/GraphQL
server and plug it with a context.TODO() for now.
This paves the way for providing a context to the HTTP server code in a
future patch.
* Make utils func ReadImage context-aware
In almost all of the cases, there is a context in the call chain which
is a natural use. This is true for all the GraphQL mutations.
The exception is in task_stash_box_tag, so plug that task with
context.TODO() for now.
* Make stash-box get context-aware
Thread a context through the call chain until we hit the Client API.
Plug it with context.TODO() there for now.
* Enable the noctx linter
The code is now free of any uncontexted HTTP request. This means we
pass the noctx linter, and we can enable it in the code base.
2021-10-14 04:32:41 +00:00
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
2021-03-11 01:56:34 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// assume is a URL for now
|
|
|
|
|
|
|
|
// set the host of the URL as the referer
|
|
|
|
if req.URL.Scheme != "" {
|
|
|
|
req.Header.Set("Referer", req.URL.Scheme+"://"+req.Host+"/")
|
|
|
|
}
|
2022-03-17 00:33:59 +00:00
|
|
|
req.Header.Set("User-Agent", getUserAgent())
|
2021-03-11 01:56:34 +00:00
|
|
|
|
|
|
|
resp, err := client.Do(req)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2021-03-23 21:12:11 +00:00
|
|
|
if resp.StatusCode >= 400 {
|
|
|
|
return nil, fmt.Errorf("http error %d", resp.StatusCode)
|
|
|
|
}
|
|
|
|
|
2021-03-11 01:56:34 +00:00
|
|
|
defer resp.Body.Close()
|
|
|
|
|
2021-09-27 00:55:23 +00:00
|
|
|
body, err := io.ReadAll(resp.Body)
|
2021-03-11 01:56:34 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return body, nil
|
|
|
|
}
|
|
|
|
|
2022-03-17 00:33:59 +00:00
|
|
|
// ProcessBase64Image transforms a base64 encoded string from a form post and
|
|
|
|
// returns the image itself as a byte slice.
|
|
|
|
func ProcessBase64Image(imageString string) ([]byte, error) {
|
2019-02-09 12:30:49 +00:00
|
|
|
if imageString == "" {
|
2022-03-17 00:33:59 +00:00
|
|
|
return nil, fmt.Errorf("empty image string")
|
2019-02-09 12:30:49 +00:00
|
|
|
}
|
|
|
|
|
2021-03-11 01:56:34 +00:00
|
|
|
regex := regexp.MustCompile(base64RE)
|
2019-02-09 12:30:49 +00:00
|
|
|
matches := regex.FindStringSubmatch(imageString)
|
|
|
|
var encodedString string
|
|
|
|
if len(matches) > 2 {
|
|
|
|
encodedString = regex.FindStringSubmatch(imageString)[2]
|
|
|
|
} else {
|
|
|
|
encodedString = imageString
|
|
|
|
}
|
|
|
|
imageData, err := GetDataFromBase64String(encodedString)
|
|
|
|
if err != nil {
|
2022-03-17 00:33:59 +00:00
|
|
|
return nil, err
|
2019-02-09 12:30:49 +00:00
|
|
|
}
|
|
|
|
|
2022-03-17 00:33:59 +00:00
|
|
|
return imageData, nil
|
2019-02-09 12:30:49 +00:00
|
|
|
}
|
|
|
|
|
2019-03-27 19:53:15 +00:00
|
|
|
// GetDataFromBase64String returns the given base64 encoded string as a byte slice
|
2019-02-09 12:30:49 +00:00
|
|
|
func GetDataFromBase64String(encodedString string) ([]byte, error) {
|
|
|
|
return base64.StdEncoding.DecodeString(encodedString)
|
|
|
|
}
|
|
|
|
|
2019-03-27 19:53:15 +00:00
|
|
|
// GetBase64StringFromData returns the given byte slice as a base64 encoded string
|
2019-02-09 12:30:49 +00:00
|
|
|
func GetBase64StringFromData(data []byte) string {
|
|
|
|
return base64.StdEncoding.EncodeToString(data)
|
2019-02-14 22:53:32 +00:00
|
|
|
}
|
2020-06-22 23:19:19 +00:00
|
|
|
|
|
|
|
func ServeImage(image []byte, w http.ResponseWriter, r *http.Request) error {
|
|
|
|
etag := fmt.Sprintf("%x", md5.Sum(image))
|
|
|
|
|
|
|
|
if match := r.Header.Get("If-None-Match"); match != "" {
|
|
|
|
if strings.Contains(match, etag) {
|
|
|
|
w.WriteHeader(http.StatusNotModified)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-07 00:35:43 +00:00
|
|
|
contentType := http.DetectContentType(image)
|
|
|
|
if contentType == "text/xml; charset=utf-8" || contentType == "text/plain; charset=utf-8" {
|
|
|
|
contentType = "image/svg+xml"
|
|
|
|
}
|
|
|
|
|
|
|
|
w.Header().Set("Content-Type", contentType)
|
2020-06-22 23:19:19 +00:00
|
|
|
w.Header().Add("Etag", etag)
|
2021-03-13 00:49:20 +00:00
|
|
|
w.Header().Set("Cache-Control", "public, max-age=604800, immutable")
|
2020-06-22 23:19:19 +00:00
|
|
|
_, err := w.Write(image)
|
2022-09-19 05:01:40 +00:00
|
|
|
// Broken pipe errors are common when serving images and the remote
|
|
|
|
// connection closes the connection. Filter them out of the error
|
|
|
|
// messages, as they are benign.
|
|
|
|
if errors.Is(err, syscall.EPIPE) {
|
|
|
|
return nil
|
|
|
|
}
|
2020-06-22 23:19:19 +00:00
|
|
|
return err
|
|
|
|
}
|