From 9ceea952b6e3ff79497d9c472a42b4f51ca89ebf Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Thu, 14 Mar 2024 11:03:40 +1100 Subject: [PATCH] Replace javascript module otto with goja (#4631) * Move plugin javascript to own package with goja * Use javascript package in scraper Remove otto --- go.mod | 6 +- go.sum | 30 ++++++++-- pkg/javascript/gql.go | 106 ++++++++++++++++++++++++++++++++++ pkg/javascript/log.go | 86 ++++++++++++++++++++++++++++ pkg/javascript/util.go | 25 ++++++++ pkg/javascript/vm.go | 64 +++++++++++++++++++++ pkg/plugin/common/msg.go | 8 +++ pkg/plugin/js.go | 92 +++++++++++++++++++----------- pkg/plugin/js/gql.go | 119 --------------------------------------- pkg/plugin/js/log.go | 96 ------------------------------- pkg/plugin/js/util.go | 29 ---------- pkg/scraper/mapped.go | 8 +-- 12 files changed, 381 insertions(+), 288 deletions(-) create mode 100644 pkg/javascript/gql.go create mode 100644 pkg/javascript/log.go create mode 100644 pkg/javascript/util.go create mode 100644 pkg/javascript/vm.go delete mode 100644 pkg/plugin/js/gql.go delete mode 100644 pkg/plugin/js/log.go delete mode 100644 pkg/plugin/js/util.go diff --git a/go.mod b/go.mod index ddc338bc1..2b54bf2e5 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/chromedp/chromedp v0.9.2 github.com/corona10/goimagehash v1.1.0 github.com/disintegration/imaging v1.6.2 + github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d github.com/doug-martin/goqu/v9 v9.18.0 github.com/go-chi/chi/v5 v5.0.10 github.com/go-chi/cors v1.2.1 @@ -35,7 +36,6 @@ require ( github.com/natefinch/pie v0.0.0-20170715172608-9a0d72014007 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 github.com/remeh/sizedwaitgroup v1.0.0 - github.com/robertkrimen/otto v0.0.0-20200922221731-ef014fd054ac github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f github.com/sirupsen/logrus v1.9.3 github.com/spf13/cast v1.5.1 @@ -67,11 +67,14 @@ require ( github.com/chromedp/sysutil v1.0.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dlclark/regexp2 v1.7.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.3.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect @@ -107,6 +110,5 @@ require ( golang.org/x/mod v0.12.0 // indirect golang.org/x/tools v0.13.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index a5f2fe67a..ba670c741 100644 --- a/go.sum +++ b/go.sum @@ -122,8 +122,11 @@ github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmt github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -146,6 +149,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -154,7 +158,15 @@ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= +github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= +github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d h1:wi6jN5LVt/ljaBG4ue79Ekzb12QfJ52L9Q98tl8SWhw= +github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= +github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= +github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/doug-martin/goqu/v9 v9.18.0 h1:/6bcuEtAe6nsSMVK/M+fOiXUNfyFF3yYtE07DBPFMYY= github.com/doug-martin/goqu/v9 v9.18.0/go.mod h1:nf0Wc2/hV3gYK9LiyqIrzBEVGlI8qW3GuDCEobC4wBQ= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -193,6 +205,8 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -285,6 +299,8 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= +github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -344,6 +360,7 @@ github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbc github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -374,10 +391,13 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -478,10 +498,9 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= -github.com/robertkrimen/otto v0.0.0-20200922221731-ef014fd054ac h1:kYPjbEN6YPYWWHI6ky1J813KzIq/8+Wg4TO4xU7A/KU= -github.com/robertkrimen/otto v0.0.0-20200922221731-ef014fd054ac/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -821,6 +840,7 @@ golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -848,6 +868,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -1075,8 +1096,9 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/guregu/null.v4 v4.0.0 h1:1Wm3S1WEA2I26Kq+6vcW+w0gcDo44YKYD7YIEJNHDjg= gopkg.in/guregu/null.v4 v4.0.0/go.mod h1:YoQhUrADuG3i9WqesrCmpNRwm1ypAgSHYqoOcTu/JrI= @@ -1084,8 +1106,6 @@ gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.3/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= -gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/javascript/gql.go b/pkg/javascript/gql.go new file mode 100644 index 000000000..c21fb082f --- /dev/null +++ b/pkg/javascript/gql.go @@ -0,0 +1,106 @@ +package javascript + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/dop251/goja" +) + +type responseWriter struct { + r strings.Builder + header http.Header + statusCode int +} + +func (w *responseWriter) Header() http.Header { + return w.header +} + +func (w *responseWriter) WriteHeader(statusCode int) { + w.statusCode = statusCode +} + +func (w *responseWriter) Write(b []byte) (int, error) { + return w.r.Write(b) +} + +type GQL struct { + Context context.Context + Cookie *http.Cookie + GQLHandler http.Handler +} + +func (g *GQL) gqlRequestFunc(vm *VM) func(query string, variables map[string]interface{}) (goja.Value, error) { + return func(query string, variables map[string]interface{}) (goja.Value, error) { + in := struct { + Query string `json:"query"` + Variables map[string]interface{} `json:"variables,omitempty"` + }{ + Query: query, + Variables: variables, + } + + var body bytes.Buffer + err := json.NewEncoder(&body).Encode(in) + if err != nil { + return nil, err + } + + r, err := http.NewRequestWithContext(g.Context, "POST", "/graphql", &body) + if err != nil { + return nil, fmt.Errorf("could not make request") + } + r.Header.Set("Content-Type", "application/json") + + if g.Cookie != nil { + r.AddCookie(g.Cookie) + } + + w := &responseWriter{ + header: make(http.Header), + } + + g.GQLHandler.ServeHTTP(w, r) + + if w.statusCode != http.StatusOK && w.statusCode != 0 { + vm.Throw(fmt.Errorf("graphQL query failed: %d - %s. Query: %s. Variables: %v", w.statusCode, w.r.String(), in.Query, in.Variables)) + } + + output := w.r.String() + // convert to JSON + var obj map[string]interface{} + if err = json.Unmarshal([]byte(output), &obj); err != nil { + vm.Throw(fmt.Errorf("could not unmarshal object %s: %s", output, err.Error())) + } + + retErr, hasErr := obj["errors"] + + if hasErr { + errOut, _ := json.Marshal(retErr) + vm.Throw(fmt.Errorf("graphql error: %s", string(errOut))) + } + + v := vm.ToValue(obj["data"]) + + return v, nil + } +} + +func (g *GQL) AddToVM(globalName string, vm *VM) error { + gql := vm.NewObject() + + if err := gql.Set("Do", g.gqlRequestFunc(vm)); err != nil { + return fmt.Errorf("unable to set GraphQL Do function: %w", err) + } + + if err := vm.Set(globalName, gql); err != nil { + return fmt.Errorf("unable to set gql: %w", err) + } + + return nil +} diff --git a/pkg/javascript/log.go b/pkg/javascript/log.go new file mode 100644 index 000000000..64333aa43 --- /dev/null +++ b/pkg/javascript/log.go @@ -0,0 +1,86 @@ +package javascript + +import ( + "encoding/json" + "fmt" + "math" + "reflect" + + "github.com/dop251/goja" + "github.com/stashapp/stash/pkg/logger" +) + +const pluginPrefix = "[Plugin] " + +type Log struct { + Progress chan float64 +} + +func (l *Log) argToString(call goja.FunctionCall) string { + arg := call.Argument(0) + var o map[string]interface{} + if arg.ExportType() == reflect.TypeOf(o) { + ii := arg.Export() + o = ii.(map[string]interface{}) + data, err := json.Marshal(o) + if err != nil { + logger.Warnf("Couldn't json encode object") + } + return string(data) + } + + return arg.String() +} + +func (l *Log) logTrace(call goja.FunctionCall) goja.Value { + logger.Trace(pluginPrefix + l.argToString(call)) + return nil +} + +func (l *Log) logDebug(call goja.FunctionCall) goja.Value { + logger.Debug(pluginPrefix + l.argToString(call)) + return nil +} + +func (l *Log) logInfo(call goja.FunctionCall) goja.Value { + logger.Info(pluginPrefix + l.argToString(call)) + return nil +} + +func (l *Log) logWarn(call goja.FunctionCall) goja.Value { + logger.Warn(pluginPrefix + l.argToString(call)) + return nil +} + +func (l *Log) logError(call goja.FunctionCall) goja.Value { + logger.Error(pluginPrefix + l.argToString(call)) + return nil +} + +// Progress logs the current progress value. The progress value should be +// between 0 and 1.0 inclusively, with 1 representing that the task is +// complete. Values outside of this range will be clamp to be within it. +func (l *Log) logProgress(value float64) { + value = math.Min(math.Max(0, value), 1) + l.Progress <- value +} + +func (l *Log) AddToVM(globalName string, vm *VM) error { + log := vm.NewObject() + if err := SetAll(log, + ObjectValueDef{"Trace", l.logTrace}, + ObjectValueDef{"Debug", l.logDebug}, + ObjectValueDef{"Info", l.logInfo}, + ObjectValueDef{"Warn", l.logWarn}, + ObjectValueDef{"Error", l.logError}, + ObjectValueDef{"Progress", l.logProgress}, + ); err != nil { + return err + } + + if err := vm.Set(globalName, log); err != nil { + return fmt.Errorf("unable to set log: %w", err) + } + + return nil +} diff --git a/pkg/javascript/util.go b/pkg/javascript/util.go new file mode 100644 index 000000000..f6f6d299f --- /dev/null +++ b/pkg/javascript/util.go @@ -0,0 +1,25 @@ +package javascript + +import ( + "fmt" + "time" +) + +type Util struct{} + +func (u *Util) sleepFunc(ms int64) { + time.Sleep(time.Millisecond * time.Duration(ms)) +} + +func (u *Util) AddToVM(globalName string, vm *VM) error { + util := vm.NewObject() + if err := util.Set("Sleep", u.sleepFunc); err != nil { + return fmt.Errorf("unable to set sleep func: %w", err) + } + + if err := vm.Set(globalName, util); err != nil { + return fmt.Errorf("unable to set util: %w", err) + } + + return nil +} diff --git a/pkg/javascript/vm.go b/pkg/javascript/vm.go new file mode 100644 index 000000000..cf75028df --- /dev/null +++ b/pkg/javascript/vm.go @@ -0,0 +1,64 @@ +package javascript + +import ( + "fmt" + "net/http" + "os" + + "github.com/dop251/goja" +) + +type VM struct { + *goja.Runtime + + Progress chan float64 + SessionCookie *http.Cookie + GQLHandler http.Handler +} + +func NewVM() *VM { + return &VM{Runtime: goja.New()} +} + +type APIAdder interface { + AddToVM(globalName string, vm *VM) error +} + +type ObjectValueDef struct { + Name string + Value interface{} +} + +type setter interface { + Set(name string, value interface{}) error +} + +func Compile(path string) (*goja.Program, error) { + js, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read file: %w", err) + } + return goja.Compile(path, string(js), true) +} + +func CompileScript(name, script string) (*goja.Program, error) { + return goja.Compile(name, string(script), true) +} + +func SetAll(s setter, defs ...ObjectValueDef) error { + for _, def := range defs { + if err := s.Set(def.Name, def.Value); err != nil { + return fmt.Errorf("failed to set %s: %w", def.Name, err) + } + } + return nil +} + +func (v *VM) Throw(err error) { + e, newErr := v.New(v.Get("Error"), v.ToValue(err)) + if newErr != nil { + panic(newErr) + } + + panic(e) +} diff --git a/pkg/plugin/common/msg.go b/pkg/plugin/common/msg.go index a7136517f..173f266ce 100644 --- a/pkg/plugin/common/msg.go +++ b/pkg/plugin/common/msg.go @@ -77,6 +77,14 @@ func (m ArgsMap) Float(key string) float64 { return ret } +func (m ArgsMap) ToMap() map[string]interface{} { + ret := make(map[string]interface{}) + for k, v := range m { + ret[k] = v + } + return ret +} + // PluginInput is the data structure that is sent to plugin instances when they // are spawned. type PluginInput struct { diff --git a/pkg/plugin/js.go b/pkg/plugin/js.go index e50376c54..b3b8c126b 100644 --- a/pkg/plugin/js.go +++ b/pkg/plugin/js.go @@ -7,9 +7,9 @@ import ( "path/filepath" "sync" - "github.com/robertkrimen/otto" + "github.com/dop251/goja" + "github.com/stashapp/stash/pkg/javascript" "github.com/stashapp/stash/pkg/plugin/common" - "github.com/stashapp/stash/pkg/plugin/js" ) var errStop = errors.New("stop") @@ -27,7 +27,7 @@ type jsPluginTask struct { started bool waitGroup sync.WaitGroup - vm *otto.Otto + vm *javascript.VM } func (t *jsPluginTask) onError(err error) { @@ -37,24 +37,67 @@ func (t *jsPluginTask) onError(err error) { } } -func (t *jsPluginTask) makeOutput(o otto.Value) { +func (t *jsPluginTask) makeOutput(o goja.Value) { t.result = &common.PluginOutput{} - asObj := o.Object() + asObj := o.ToObject(t.vm.Runtime) if asObj == nil { return } - output, _ := asObj.Get("Output") - t.result.Output, _ = output.Export() - - err, _ := asObj.Get("Error") - if !err.IsUndefined() { + t.result.Output = asObj.Get("Output") + err := asObj.Get("Error") + if !goja.IsNull(err) && !goja.IsUndefined(err) { errStr := err.String() t.result.Error = &errStr } } +func (t *jsPluginTask) initVM() error { + // converting the Args field to map[string]interface{} is required, otherwise + // it gets converted to an empty object + type pluginInput struct { + // Server details to connect to the stash server. + ServerConnection common.StashServerConnection + + // Arguments to the plugin operation. + Args map[string]interface{} + } + + input := pluginInput{ + ServerConnection: t.input.ServerConnection, + Args: t.input.Args.ToMap(), + } + + if err := t.vm.Set("input", input); err != nil { + return fmt.Errorf("error setting input: %w", err) + } + + log := &javascript.Log{ + Progress: t.progress, + } + + if err := log.AddToVM("log", t.vm); err != nil { + return fmt.Errorf("error adding log API: %w", err) + } + + util := &javascript.Util{} + if err := util.AddToVM("util", t.vm); err != nil { + return fmt.Errorf("error adding util API: %w", err) + } + + gql := &javascript.GQL{ + Context: context.TODO(), + Cookie: t.input.ServerConnection.SessionCookie, + GQLHandler: t.gqlHandler, + } + if err := gql.AddToVM("gql", t.vm); err != nil { + return fmt.Errorf("error adding GraphQL API: %w", err) + } + + return nil +} + func (t *jsPluginTask) Start() error { if t.started { return errors.New("task already started") @@ -68,31 +111,17 @@ func (t *jsPluginTask) Start() error { scriptFile := t.plugin.Exec[0] - t.vm = otto.New() + t.vm = javascript.NewVM() pluginPath := t.plugin.getConfigPath() - script, err := t.vm.Compile(filepath.Join(pluginPath, scriptFile), nil) + script, err := javascript.Compile(filepath.Join(pluginPath, scriptFile)) if err != nil { return err } - if err := t.vm.Set("input", t.input); err != nil { - return fmt.Errorf("error setting input: %w", err) + if err := t.initVM(); err != nil { + return err } - if err := js.AddLogAPI(t.vm, t.progress); err != nil { - return fmt.Errorf("error adding log API: %w", err) - } - - if err := js.AddUtilAPI(t.vm); err != nil { - return fmt.Errorf("error adding util API: %w", err) - } - - if err := js.AddGQLAPI(context.TODO(), t.vm, t.input.ServerConnection.SessionCookie, t.gqlHandler); err != nil { - return fmt.Errorf("error adding GraphQL API: %w", err) - } - - t.vm.Interrupt = make(chan func(), 1) - t.waitGroup.Add(1) go func() { @@ -107,7 +136,7 @@ func (t *jsPluginTask) Start() error { } }() - output, err := t.vm.Run(script) + output, err := t.vm.RunProgram(script) if err != nil { t.onError(err) @@ -124,9 +153,6 @@ func (t *jsPluginTask) Wait() { } func (t *jsPluginTask) Stop() error { - // TODO - need another way of doing this that doesn't require panic - t.vm.Interrupt <- func() { - panic(errStop) - } + t.vm.Interrupt(errStop) return nil } diff --git a/pkg/plugin/js/gql.go b/pkg/plugin/js/gql.go deleted file mode 100644 index 7a240fbb5..000000000 --- a/pkg/plugin/js/gql.go +++ /dev/null @@ -1,119 +0,0 @@ -package js - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "net/http" - "strings" - - "github.com/robertkrimen/otto" -) - -type responseWriter struct { - r strings.Builder - header http.Header - statusCode int -} - -func (w *responseWriter) Header() http.Header { - return w.header -} - -func (w *responseWriter) WriteHeader(statusCode int) { - w.statusCode = statusCode -} - -func (w *responseWriter) Write(b []byte) (int, error) { - return w.r.Write(b) -} - -func throw(vm *otto.Otto, str string) { - value, _ := vm.Call("new Error", nil, str) - panic(value) -} - -func gqlRequestFunc(ctx context.Context, vm *otto.Otto, cookie *http.Cookie, gqlHandler http.Handler) func(call otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - if len(call.ArgumentList) == 0 { - throw(vm, "missing argument") - } - - query := call.Argument(0) - vars := call.Argument(1) - var variables map[string]interface{} - if !vars.IsUndefined() { - exported, _ := vars.Export() - variables, _ = exported.(map[string]interface{}) - } - - in := struct { - Query string `json:"query"` - Variables map[string]interface{} `json:"variables,omitempty"` - }{ - Query: query.String(), - Variables: variables, - } - - var body bytes.Buffer - err := json.NewEncoder(&body).Encode(in) - if err != nil { - throw(vm, err.Error()) - } - - r, err := http.NewRequestWithContext(ctx, "POST", "/graphql", &body) - if err != nil { - throw(vm, "could not make request") - } - r.Header.Set("Content-Type", "application/json") - - if cookie != nil { - r.AddCookie(cookie) - } - - w := &responseWriter{ - header: make(http.Header), - } - - gqlHandler.ServeHTTP(w, r) - - if w.statusCode != http.StatusOK && w.statusCode != 0 { - throw(vm, fmt.Sprintf("graphQL query failed: %d - %s. Query: %s. Variables: %v", w.statusCode, w.r.String(), in.Query, in.Variables)) - } - - output := w.r.String() - // convert to JSON - var obj map[string]interface{} - if err = json.Unmarshal([]byte(output), &obj); err != nil { - throw(vm, fmt.Sprintf("could not unmarshal object %s: %s", output, err.Error())) - } - - retErr, hasErr := obj["errors"] - - if hasErr { - errOut, _ := json.Marshal(retErr) - throw(vm, fmt.Sprintf("graphql error: %s", string(errOut))) - } - - v, err := vm.ToValue(obj["data"]) - if err != nil { - throw(vm, fmt.Sprintf("could not create return value: %s", err.Error())) - } - - return v - } -} - -func AddGQLAPI(ctx context.Context, vm *otto.Otto, cookie *http.Cookie, gqlHandler http.Handler) error { - gql, _ := vm.Object("({})") - if err := gql.Set("Do", gqlRequestFunc(ctx, vm, cookie, gqlHandler)); err != nil { - return fmt.Errorf("unable to set GraphQL Do function: %w", err) - } - - if err := vm.Set("gql", gql); err != nil { - return fmt.Errorf("unable to set gql: %w", err) - } - - return nil -} diff --git a/pkg/plugin/js/log.go b/pkg/plugin/js/log.go deleted file mode 100644 index 5156d6599..000000000 --- a/pkg/plugin/js/log.go +++ /dev/null @@ -1,96 +0,0 @@ -package js - -import ( - "encoding/json" - "fmt" - "math" - - "github.com/robertkrimen/otto" - "github.com/stashapp/stash/pkg/logger" -) - -const pluginPrefix = "[Plugin] " - -func argToString(call otto.FunctionCall) string { - arg := call.Argument(0) - if arg.IsObject() { - o, _ := arg.Export() - data, err := json.Marshal(o) - if err != nil { - logger.Warnf("Couldn't json encode object") - } - return string(data) - } - - return arg.String() -} - -func logTrace(call otto.FunctionCall) otto.Value { - logger.Trace(pluginPrefix + argToString(call)) - return otto.UndefinedValue() -} - -func logDebug(call otto.FunctionCall) otto.Value { - logger.Debug(pluginPrefix + argToString(call)) - return otto.UndefinedValue() -} - -func logInfo(call otto.FunctionCall) otto.Value { - logger.Info(pluginPrefix + argToString(call)) - return otto.UndefinedValue() -} - -func logWarn(call otto.FunctionCall) otto.Value { - logger.Warn(pluginPrefix + argToString(call)) - return otto.UndefinedValue() -} - -func logError(call otto.FunctionCall) otto.Value { - logger.Error(pluginPrefix + argToString(call)) - return otto.UndefinedValue() -} - -// Progress logs the current progress value. The progress value should be -// between 0 and 1.0 inclusively, with 1 representing that the task is -// complete. Values outside of this range will be clamp to be within it. -func logProgressFunc(c chan float64) func(call otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - arg := call.Argument(0) - if !arg.IsNumber() { - return otto.UndefinedValue() - } - - progress, _ := arg.ToFloat() - progress = math.Min(math.Max(0, progress), 1) - c <- progress - - return otto.UndefinedValue() - } -} - -func AddLogAPI(vm *otto.Otto, progress chan float64) error { - log, _ := vm.Object("({})") - if err := log.Set("Trace", logTrace); err != nil { - return fmt.Errorf("error setting Trace: %w", err) - } - if err := log.Set("Debug", logDebug); err != nil { - return fmt.Errorf("error setting Debug: %w", err) - } - if err := log.Set("Info", logInfo); err != nil { - return fmt.Errorf("error setting Info: %w", err) - } - if err := log.Set("Warn", logWarn); err != nil { - return fmt.Errorf("error setting Warn: %w", err) - } - if err := log.Set("Error", logError); err != nil { - return fmt.Errorf("error setting Error: %w", err) - } - if err := log.Set("Progress", logProgressFunc(progress)); err != nil { - return fmt.Errorf("error setting Progress: %v", err) - } - if err := vm.Set("log", log); err != nil { - return fmt.Errorf("unable to set log: %w", err) - } - - return nil -} diff --git a/pkg/plugin/js/util.go b/pkg/plugin/js/util.go deleted file mode 100644 index 1fe5bb3ff..000000000 --- a/pkg/plugin/js/util.go +++ /dev/null @@ -1,29 +0,0 @@ -package js - -import ( - "fmt" - "time" - - "github.com/robertkrimen/otto" -) - -func sleepFunc(call otto.FunctionCall) otto.Value { - arg := call.Argument(0) - ms, _ := arg.ToInteger() - - time.Sleep(time.Millisecond * time.Duration(ms)) - return otto.UndefinedValue() -} - -func AddUtilAPI(vm *otto.Otto) error { - util, _ := vm.Object("({})") - if err := util.Set("Sleep", sleepFunc); err != nil { - return fmt.Errorf("unable to set sleep func: %w", err) - } - - if err := vm.Set("util", util); err != nil { - return fmt.Errorf("unable to set util: %w", err) - } - - return nil -} diff --git a/pkg/scraper/mapped.go b/pkg/scraper/mapped.go index 3d4ee4638..f8a096015 100644 --- a/pkg/scraper/mapped.go +++ b/pkg/scraper/mapped.go @@ -11,9 +11,9 @@ import ( "strings" "time" - "github.com/robertkrimen/otto" "gopkg.in/yaml.v2" + "github.com/stashapp/stash/pkg/javascript" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/sliceutil" @@ -528,19 +528,19 @@ func (p *postProcessLbToKg) Apply(ctx context.Context, value string, q mappedQue type postProcessJavascript string func (p *postProcessJavascript) Apply(ctx context.Context, value string, q mappedQuery) string { - vm := otto.New() + vm := javascript.NewVM() if err := vm.Set("value", value); err != nil { logger.Warnf("javascript failed to set value: %v", err) return value } - script, err := vm.Compile("", "(function() { "+string(*p)+"})()") + script, err := javascript.CompileScript("", "(function() { "+string(*p)+"})()") if err != nil { logger.Warnf("javascript failed to compile: %v", err) return value } - output, err := vm.Run(script) + output, err := vm.RunProgram(script) if err != nil { logger.Warnf("javascript failed to run: %v", err) return value