From 1ff29186026ddfe65146795f126785b8225c8457 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 31 Jul 2014 11:34:23 -0700 Subject: [PATCH] importers: add SupportsIncremental accessor If an importer is efficient at running regularly, it returns true. If it's not true, the UI won't allow turning on automatic runs. Flickr doesn't set it (yet). --- pkg/importer/dummy/dummy.go | 8 ++++++++ pkg/importer/feed/feed.go | 2 ++ pkg/importer/flickr/flickr.go | 2 ++ pkg/importer/foursquare/foursquare.go | 3 ++- pkg/importer/importer.go | 10 ++++++++++ pkg/importer/noop.go | 2 ++ pkg/importer/picasa/picasa.go | 20 ++++++++------------ pkg/importer/twitter/twitter.go | 3 ++- 8 files changed, 36 insertions(+), 14 deletions(-) diff --git a/pkg/importer/dummy/dummy.go b/pkg/importer/dummy/dummy.go index ef566e35a..dac3ba2c9 100644 --- a/pkg/importer/dummy/dummy.go +++ b/pkg/importer/dummy/dummy.go @@ -66,6 +66,14 @@ type imp struct { categoryRef map[string]blob.Ref // URL -> file schema ref } +func (*imp) SupportsIncremental() bool { + // SupportsIncremental signals to the importer host that this + // importer has been optimized to be run regularly (e.g. every 5 + // minutes or half hour). If it returns false, the user must + // manually start imports. + return false +} + func (*imp) NeedsAPIKey() bool { // This tells the importer framework that we our importer will // be calling the {RunContext,SetupContext}.Credentials method diff --git a/pkg/importer/feed/feed.go b/pkg/importer/feed/feed.go index 307814003..fc337d610 100644 --- a/pkg/importer/feed/feed.go +++ b/pkg/importer/feed/feed.go @@ -58,6 +58,8 @@ type imp struct { func (im *imp) NeedsAPIKey() bool { return false } +func (im *imp) SupportsIncremental() bool { return true } + func (im *imp) IsAccountReady(acctNode *importer.Object) (ok bool, err error) { if acctNode.Attr(acctAttrFeedURL) != "" { return true, nil diff --git a/pkg/importer/flickr/flickr.go b/pkg/importer/flickr/flickr.go index 4048c6da0..3175bff1b 100644 --- a/pkg/importer/flickr/flickr.go +++ b/pkg/importer/flickr/flickr.go @@ -59,6 +59,8 @@ type imp struct { func (imp) NeedsAPIKey() bool { return true } +func (imp) SupportsIncremental() bool { return false } + func (imp) IsAccountReady(acctNode *importer.Object) (ok bool, err error) { return acctNode.Attr(importer.AcctAttrUserName) != "" && acctNode.Attr(importer.AcctAttrAccessToken) != "", nil } diff --git a/pkg/importer/foursquare/foursquare.go b/pkg/importer/foursquare/foursquare.go index 46efb038b..e8f2ed758 100644 --- a/pkg/importer/foursquare/foursquare.go +++ b/pkg/importer/foursquare/foursquare.go @@ -84,7 +84,8 @@ type imp struct { importer.OAuth2 // for CallbackRequestAccount and CallbackURLParameters } -func (im *imp) NeedsAPIKey() bool { return true } +func (im *imp) NeedsAPIKey() bool { return true } +func (im *imp) SupportsIncremental() bool { return true } func (im *imp) IsAccountReady(acctNode *importer.Object) (ok bool, err error) { if acctNode.Attr(acctAttrUserId) != "" && acctNode.Attr(acctAttrAccessToken) != "" { diff --git a/pkg/importer/importer.go b/pkg/importer/importer.go index 7e958ccf6..265481144 100644 --- a/pkg/importer/importer.go +++ b/pkg/importer/importer.go @@ -70,6 +70,13 @@ type Importer interface { // can return false here. NeedsAPIKey() bool + // SupportsIncremental reports whether this importer has been optimized + // to run efficiently in regular incremental runs. (e.g. every 5 minutes + // or half hour). Eventually all importers might support this and we'll + // make it required, in which case we might delete this option. + // For now, some importers (e.g. Flickr) don't yet support this. + SupportsIncremental() bool + // IsAccountReady reports whether the provided account node // is configured. IsAccountReady(acctNode *Object) (ok bool, err error) @@ -862,6 +869,9 @@ func (ia *importerAcct) delete() error { func (ia *importerAcct) toggleAuto() error { old := ia.acct.Attr(attrImportAuto) + if old == "" && !ia.im.impl.SupportsIncremental() { + return fmt.Errorf("Importer %q doesn't support automatic mode.", ia.im.name) + } var new string if old == "" { new = "30m" // TODO: configurable? diff --git a/pkg/importer/noop.go b/pkg/importer/noop.go index fa0b9d6eb..bffc04cc7 100644 --- a/pkg/importer/noop.go +++ b/pkg/importer/noop.go @@ -30,6 +30,8 @@ type todoImp struct { func (todoImp) NeedsAPIKey() bool { return false } +func (todoImp) SupportsIncremental() bool { return false } + func (todoImp) Run(*RunContext) error { return errors.New("fake error from todo importer") } diff --git a/pkg/importer/picasa/picasa.go b/pkg/importer/picasa/picasa.go index cabc12c4b..9d6696eab 100644 --- a/pkg/importer/picasa/picasa.go +++ b/pkg/importer/picasa/picasa.go @@ -61,16 +61,14 @@ const ( attrPicasaId = "picasaId" ) -func init() { - importer.Register("picasa", newImporter()) -} - -var _ importer.ImporterSetupHTMLer = (*imp)(nil) +var _ importer.ImporterSetupHTMLer = imp{} type imp struct { extendedOAuth2 } +func (imp) SupportsIncremental() bool { return true } + var baseOAuthConfig = oauth.Config{ AuthURL: authURL, TokenURL: tokenURL, @@ -85,8 +83,8 @@ var baseOAuthConfig = oauth.Config{ ApprovalPrompt: "force", } -func newImporter() *imp { - return &imp{ +func init() { + importer.Register("picasa", imp{ newExtendedOAuth2( baseOAuthConfig, func(ctx *context.Context) (*userInfo, error) { @@ -105,10 +103,10 @@ func newImporter() *imp { LastName: lastName, }, nil }), - } + }) } -func (*imp) AccountSetupHTML(host *importer.Host) string { +func (imp) AccountSetupHTML(host *importer.Host) string { // Picasa doesn't allow a path in the origin. Remove it. origin := host.ImporterBaseURL() if u, err := url.Parse(origin); err == nil { @@ -135,7 +133,6 @@ and click "Create Project".

// A run is our state for a given run of the importer. type run struct { *importer.RunContext - im *imp incremental bool // whether we've completed a run in the past photoGate *syncutil.Gate @@ -152,7 +149,7 @@ func (r *run) errorf(format string, args ...interface{}) { var forceFullImport, _ = strconv.ParseBool(os.Getenv("CAMLI_PICASA_FULL_IMPORT")) -func (im *imp) Run(ctx *importer.RunContext) error { +func (imp) Run(ctx *importer.RunContext) error { clientId, secret, err := ctx.Credentials() if err != nil { return err @@ -180,7 +177,6 @@ func (im *imp) Run(ctx *importer.RunContext) error { r := &run{ RunContext: ctx, - im: im, incremental: !forceFullImport && acctNode.Attr(importer.AcctAttrCompletedVersion) == runCompleteVersion, photoGate: syncutil.NewGate(3), } diff --git a/pkg/importer/twitter/twitter.go b/pkg/importer/twitter/twitter.go index 6e4f7364d..4bb4c3275 100644 --- a/pkg/importer/twitter/twitter.go +++ b/pkg/importer/twitter/twitter.go @@ -92,7 +92,8 @@ type imp struct { importer.OAuth1 // for CallbackRequestAccount and CallbackURLParameters } -func (im *imp) NeedsAPIKey() bool { return true } +func (im *imp) NeedsAPIKey() bool { return true } +func (im *imp) SupportsIncremental() bool { return true } func (im *imp) IsAccountReady(acctNode *importer.Object) (ok bool, err error) { if acctNode.Attr(importer.AcctAttrUserID) != "" && acctNode.Attr(importer.AcctAttrAccessToken) != "" {