diff --git a/pkg/importer/importer.go b/pkg/importer/importer.go index 62a9f9cbf..44dfe2631 100644 --- a/pkg/importer/importer.go +++ b/pkg/importer/importer.go @@ -88,7 +88,6 @@ var importers = make(map[string]Importer) func init() { Register("flickr", TODOImporter) Register("picasa", TODOImporter) - Register("twitter", TODOImporter) } // Register registers a site-specific importer. It should only be called from init, @@ -759,12 +758,14 @@ func (ia *importerAcct) serveHTTPPost(w http.ResponseWriter, r *http.Request) { } func (ia *importerAcct) setup(w http.ResponseWriter, r *http.Request) { - ia.im.impl.ServeSetup(w, r, &SetupContext{ + if err := ia.im.impl.ServeSetup(w, r, &SetupContext{ Context: context.TODO(), Host: ia.im.host, AccountNode: ia.acct, ia: ia, - }) + }); err != nil { + log.Printf("%v", err) + } } func (ia *importerAcct) start() { diff --git a/pkg/importer/twitter/README b/pkg/importer/twitter/README index 4dda83d74..08d0a525a 100644 --- a/pkg/importer/twitter/README +++ b/pkg/importer/twitter/README @@ -3,16 +3,4 @@ Twitter Importer This is a Camlistore importer for Twitter. -To use: - -1) Visit https://apps.twitter.com/ and "Create a new app" to get an API key and - secret. Note that you *must* specify a callback URL in the application - settings UI. It doesn't matter what this URL is because Camlistore - overrides it at runtime. It just has to be some valid non-localhost URL. - -2) Start the devcam server with twitterapikey flag: - $ devcam server -twitterapikey=: - -3) Navigate to http:///importer-twitter/login - -4) Watch import progress on the command line +Go to http[s]://your_camlistore_server/importer/twitter diff --git a/pkg/importer/twitter/testdata/verify_credentials-res.json b/pkg/importer/twitter/testdata/verify_credentials-res.json new file mode 100755 index 000000000..8d06b4576 --- /dev/null +++ b/pkg/importer/twitter/testdata/verify_credentials-res.json @@ -0,0 +1,9 @@ +{ + "id":2325935334, + "id_str":"2325935334", + "name":"Mathieu Lonjaret", + "screen_name":"lejatorn", + "location":"", + "description":"potato, clever label, trendy word", + "url":"https:\/\/t.co\/TF5K7idMNj" +} diff --git a/pkg/importer/twitter/twitter.go b/pkg/importer/twitter/twitter.go index 7f8cf58d5..19b4d145d 100644 --- a/pkg/importer/twitter/twitter.go +++ b/pkg/importer/twitter/twitter.go @@ -30,81 +30,107 @@ import ( "camlistore.org/pkg/context" "camlistore.org/pkg/httputil" "camlistore.org/pkg/importer" - "camlistore.org/pkg/jsonconfig" "camlistore.org/pkg/schema" "camlistore.org/third_party/github.com/garyburd/go-oauth/oauth" ) const ( - apiURL = "https://api.twitter.com/1.1/" + apiURL = "https://api.twitter.com/1.1/" + temporaryCredentialRequestURL = "https://api.twitter.com/oauth/request_token" + resourceOwnerAuthorizationURL = "https://api.twitter.com/oauth/authorize" + tokenRequestURL = "https://api.twitter.com/oauth/access_token" + userInfoAPIPath = "account/verify_credentials.json" + + // Permanode attributes on account node: + acctAttrUserID = "twitterUserID" + acctAttrScreenName = "twitterScreenName" + acctAttrUserFirst = "twitterFirstName" + acctAttrUserLast = "twitterLastName" + acctAttrAccessToken = "oauthAccessToken" + tweetRequestLimit = 200 // max number of tweets we can get in a user_timeline request ) +func init() { + importer.Register("twitter", &imp{}) +} + +var _ importer.ImporterSetupHTMLer = (*imp)(nil) + var ( oauthClient = &oauth.Client{ - TemporaryCredentialRequestURI: "https://api.twitter.com/oauth/request_token", - ResourceOwnerAuthorizationURI: "https://api.twitter.com/oauth/authorize", - TokenRequestURI: "https://api.twitter.com/oauth/access_token", + TemporaryCredentialRequestURI: temporaryCredentialRequestURL, + ResourceOwnerAuthorizationURI: resourceOwnerAuthorizationURL, + TokenRequestURI: tokenRequestURL, } ) -func init() { - importer.Register("twitter", newFromConfig) -} - type imp struct { - host *importer.Host - userid string // empty if the user isn't authenticated - cred *oauth.Credentials + // cred are the various credentials passed around during OAUTH. First the temporary + // ones, then the access token and secret. + cred *oauth.Credentials } -func newFromConfig(cfg jsonconfig.Obj, host *importer.Host) (importer.Importer, error) { - apiKey := cfg.RequiredString("apiKey") - if err := cfg.Validate(); err != nil { - return nil, err - } - parts := strings.Split(apiKey, ":") - if len(parts) != 2 { - return nil, fmt.Errorf("Twitter importer: Invalid apiKey configuration: %q", apiKey) - } +func (im *imp) NeedsAPIKey() bool { return true } - oauthClient.Credentials = oauth.Credentials{ - Token: parts[0], - Secret: parts[1], +func (im *imp) IsAccountReady(acctNode *importer.Object) (ok bool, err error) { + if acctNode.Attr(acctAttrUserID) != "" && acctNode.Attr(acctAttrAccessToken) != "" { + return true, nil } - - return &imp{ - host: host, - cred: &oauthClient.Credentials, - }, nil + return false, nil } -func (im *imp) CanHandleURL(url string) bool { return false } -func (im *imp) ImportURL(url string) error { panic("unused") } - -func (im *imp) Prefix() string { - // This should only get called when we're importing, so it's OK to - // assume we're authenticated. - return fmt.Sprintf("twitter:%s", im.userid) -} - -func (im *imp) String() string { - // We use this in logging when we're not authenticated, so it should do - // something reasonable in that case. - userId := "" - if im.userid != "" { - userId = im.userid +func (im *imp) SummarizeAccount(acct *importer.Object) string { + ok, err := im.IsAccountReady(acct) + if err != nil { + return "Not configured; error = " + err.Error() } - return fmt.Sprintf("twitter:%s", userId) + if !ok { + return "Not configured" + } + if acct.Attr(acctAttrUserFirst) == "" && acct.Attr(acctAttrUserLast) == "" { + return fmt.Sprintf("@%s", acct.Attr(acctAttrScreenName)) + } + return fmt.Sprintf("@%s (%s %s)", acct.Attr(acctAttrScreenName), + acct.Attr(acctAttrUserFirst), acct.Attr(acctAttrUserLast)) } -func (im *imp) Run(ctx *context.Context) error { - log.Print("Twitter running...") +func (im *imp) AccountSetupHTML(host *importer.Host) string { + base := host.ImporterBaseURL() + "twitter" + return fmt.Sprintf(` +

Configuring Twitter

+

Visit https://apps.twitter.com/ and click "Create New App".

+

Use the following settings:

+
    +
  • Name: Does not matter. (camlistore-importer).
  • +
  • Description: Does not matter. (imports twitter data into camlistore).
  • +
  • Website: %s
  • +
  • Callback URL: %s
  • +
+

Click "Create your Twitter application".You should be redirected to the Application Management page of your newly created application. +
Go to the API Keys tab. Copy the "API key" and "API secret" into the "Client ID" and "Client Secret" boxes above.

+`, base, base+"/callback") +} - if err := im.importTweets(ctx); err != nil { +// A run is our state for a given run of the importer. +type run struct { + *importer.RunContext + im *imp +} + +func (im *imp) Run(ctx *importer.RunContext) error { + r := &run{ + RunContext: ctx, + im: im, + } + userID := ctx.AccountNode().Attr(acctAttrUserID) + if userID == "" { + return errors.New("UserID hasn't been set by account setup.") + } + + if err := r.importTweets(userID); err != nil { return err } - return nil } @@ -114,22 +140,25 @@ type tweetItem struct { CreatedAt string `json:"created_at"` } -func (im *imp) importTweets(ctx *context.Context) error { +func (r *run) importTweets(userID string) error { maxId := "" continueRequests := true for continueRequests { - if ctx.IsCanceled() { + if r.Context.IsCanceled() { log.Printf("Twitter importer: interrupted") return context.ErrCanceled } var resp []*tweetItem - if err := im.doAPI(&resp, "statuses/user_timeline.json", "count", strconv.Itoa(tweetRequestLimit), "max_id", maxId); err != nil { + if err := r.im.doAPI(r.Context, &resp, "statuses/user_timeline.json", + "user_id", userID, + "count", strconv.Itoa(tweetRequestLimit), + "max_id", maxId); err != nil { return err } - tweetsNode, err := im.getTopLevelNode("tweets", "Tweets") + tweetsNode, err := r.getTopLevelNode("tweets", "Tweets") if err != nil { return err } @@ -144,11 +173,11 @@ func (im *imp) importTweets(ctx *context.Context) error { } for _, tweet := range resp { - if ctx.IsCanceled() { + if r.Context.IsCanceled() { log.Printf("Twitter importer: interrupted") return context.ErrCanceled } - err = im.importTweet(tweetsNode, tweet) + err = r.importTweet(tweetsNode, tweet) if err != nil { log.Printf("Twitter importer: error importing tweet %s %v", tweet.Id, err) continue @@ -159,7 +188,7 @@ func (im *imp) importTweets(ctx *context.Context) error { return nil } -func (im *imp) importTweet(parent *importer.Object, tweet *tweetItem) error { +func (r *run) importTweet(parent *importer.Object, tweet *tweetItem) error { tweetNode, err := parent.ChildPathObject(tweet.Id) if err != nil { return err @@ -169,8 +198,7 @@ func (im *imp) importTweet(parent *importer.Object, tweet *tweetItem) error { createdTime, err := time.Parse(time.RubyDate, tweet.CreatedAt) if err != nil { - log.Printf("Twitter importer: error parsing time %s %v", tweet.Id, err) - return err + return fmt.Errorf("could not parse time %q: %v", tweet.CreatedAt, err) } // TODO: import photos referenced in tweets @@ -182,136 +210,149 @@ func (im *imp) importTweet(parent *importer.Object, tweet *tweetItem) error { "title", title) } -// utility - -func (im *imp) getTopLevelNode(path string, title string) (*importer.Object, error) { - root, err := im.getRootNode() +func (r *run) getTopLevelNode(path string, title string) (*importer.Object, error) { + tweets, err := r.RootNode().ChildPathObject(path) if err != nil { return nil, err } - - photos, err := root.ChildPathObject(path) - if err != nil { + if err := tweets.SetAttr("title", title); err != nil { return nil, err } - - if err := photos.SetAttr("title", title); err != nil { - return nil, err - } - return photos, nil + return tweets, nil } -func (im *imp) getRootNode() (*importer.Object, error) { - root, err := im.host.RootObject() - if err != nil { - return nil, err - } +// TODO(mpl): move to an api.go when we it gets bigger. - title := fmt.Sprintf("Twitter (%s)", im.userid) - if err := root.SetAttr("title", title); err != nil { - return nil, err - } - - return root, nil +type userInfo struct { + ID string `json:"id_str"` + ScreenName string `json:"screen_name"` + Name string `json:"name,omitempty"` } -// twitter api builders +func (im *imp) getUserInfo(ctx *context.Context) (userInfo, error) { + var ui userInfo + if err := im.doAPI(ctx, &ui, userInfoAPIPath); err != nil { + return ui, err + } + if ui.ID == "" { + return ui, fmt.Errorf("No userid returned") + } + return ui, nil +} -func (im *imp) doAPI(result interface{}, apiPath string, keyval ...string) error { +func (im *imp) doAPI(ctx *context.Context, result interface{}, apiPath string, keyval ...string) error { if len(keyval)%2 == 1 { - panic("Incorrect number of keyval arguments") + panic("Incorrect number of keyval arguments. must be even.") } if im.cred == nil { return fmt.Errorf("No authentication creds") } - if im.userid == "" { - return fmt.Errorf("No user id") - } - form := url.Values{} - form.Set("user_id", im.userid) for i := 0; i < len(keyval); i += 2 { if keyval[i+1] != "" { form.Set(keyval[i], keyval[i+1]) } } - res, err := im.doGet(apiURL+apiPath, form) + fullURL := apiURL + apiPath + res, err := im.doGet(ctx, fullURL, form) if err != nil { return err } err = httputil.DecodeJSON(res, result) if err != nil { - log.Printf("Error parsing response for %s: %s", apiURL, err) + return fmt.Errorf("could not parse response for %s: %v", fullURL, err) } - return err + return nil } -func (im *imp) doGet(url string, form url.Values) (*http.Response, error) { +func (im *imp) doGet(ctx *context.Context, url string, form url.Values) (*http.Response, error) { if im.cred == nil { - return nil, errors.New("Not logged in. Go to /importer-twitter/login.") + return nil, errors.New("No OAUTH credentials. Not logged in?") } - - res, err := oauthClient.Get(im.host.HTTPClient(), im.cred, url, form) + res, err := oauthClient.Get(ctx.HTTPClient(), im.cred, url, form) if err != nil { - return nil, err + return nil, fmt.Errorf("Error fetching %s: %v", url, err) } - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("Auth request failed with: %s", res.Status) + return nil, fmt.Errorf("Get request on %s failed with: %s", url, res.Status) } - return res, nil } -// auth endpoints - -func (im *imp) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if strings.HasSuffix(r.URL.Path, "/login") { - im.serveLogin(w, r) - } else if strings.HasSuffix(r.URL.Path, "/callback") { - im.serveCallback(w, r) - } else { - httputil.BadRequestError(w, "Unknown path: %s", r.URL.Path) +func auth(ctx *importer.SetupContext) (*oauth.Credentials, error) { + clientId, secret, err := ctx.Credentials() + if err != nil { + return nil, err } + return &oauth.Credentials{ + Token: clientId, + Secret: secret, + }, nil } -func (im *imp) serveLogin(w http.ResponseWriter, r *http.Request) { - callback := im.host.BaseURL + "callback" - tempCred, err := oauthClient.RequestTemporaryCredentials(im.host.HTTPClient(), callback, nil) +func (im *imp) ServeSetup(w http.ResponseWriter, r *http.Request, ctx *importer.SetupContext) error { + cred, err := auth(ctx) if err != nil { - httputil.ServeError(w, r, fmt.Errorf("Twitter importer: Error getting temp cred: %s", err)) - return + err = fmt.Errorf("Error getting API credentials: %v", err) + httputil.ServeError(w, r, err) + return err + } + oauthClient.Credentials = *cred + tempCred, err := oauthClient.RequestTemporaryCredentials(ctx.HTTPClient(), ctx.CallbackURL(), nil) + if err != nil { + err = fmt.Errorf("Error getting temp cred: %v", err) + httputil.ServeError(w, r, err) } - im.cred = tempCred authURL := oauthClient.AuthorizationURL(tempCred, nil) http.Redirect(w, r, authURL, 302) + return nil } -func (im *imp) serveCallback(w http.ResponseWriter, r *http.Request) { +func (im *imp) ServeCallback(w http.ResponseWriter, r *http.Request, ctx *importer.SetupContext) { if im.cred.Token != r.FormValue("oauth_token") { - httputil.BadRequestError(w, "Twitter importer: unexpected oauth_token") + log.Printf("unexpected oauth_token: got %v, want %v", r.FormValue("oauth_token"), im.cred.Token) + httputil.BadRequestError(w, "unexpected oauth_token") return } - tokenCred, vals, err := oauthClient.RequestToken(im.host.HTTPClient(), im.cred, r.FormValue("oauth_verifier")) + tokenCred, vals, err := oauthClient.RequestToken(ctx.Context.HTTPClient(), im.cred, r.FormValue("oauth_verifier")) if err != nil { - httputil.ServeError(w, r, fmt.Errorf("Twitter importer: error getting request token: %s ", err)) + httputil.ServeError(w, r, fmt.Errorf("Error getting request token: %v ", err)) + return + } + userid := vals.Get("user_id") + if userid == "" { + httputil.ServeError(w, r, fmt.Errorf("Couldn't get user id: %v", err)) return } im.cred = tokenCred - userid := vals.Get("user_id") - if userid == "" { - log.Printf("Couldn't get user id: %v", err) - http.Error(w, "can't get user id", 500) + u, err := im.getUserInfo(ctx.Context) + if err != nil { + httputil.ServeError(w, r, fmt.Errorf("Couldn't get user info: %v", err)) return } - im.userid = userid - - http.Redirect(w, r, im.host.BaseURL+"?mode=start", 302) + firstName, lastName := "", "" + if u.Name != "" { + if pieces := strings.Fields(u.Name); len(pieces) == 2 { + firstName = pieces[0] + lastName = pieces[1] + } + } + if err := ctx.AccountNode.SetAttrs( + acctAttrUserID, u.ID, + acctAttrUserFirst, firstName, + acctAttrUserLast, lastName, + acctAttrScreenName, u.ScreenName, + acctAttrAccessToken, tokenCred.Token, + ); err != nil { + httputil.ServeError(w, r, fmt.Errorf("Error setting attribute: %v", err)) + return + } + http.Redirect(w, r, ctx.AccountURL(), http.StatusFound) } diff --git a/pkg/importer/twitter/twitter_test.go b/pkg/importer/twitter/twitter_test.go new file mode 100644 index 000000000..3b49ef73d --- /dev/null +++ b/pkg/importer/twitter/twitter_test.go @@ -0,0 +1,86 @@ +/* +Copyright 2014 The Camlistore Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package twitter + +import ( + "fmt" + "net/http" + "os" + "path/filepath" + "testing" + + "camlistore.org/pkg/context" + "camlistore.org/pkg/types" + + "camlistore.org/third_party/github.com/garyburd/go-oauth/oauth" +) + +func TestGetUserID(t *testing.T) { + im := &imp{ + &oauth.Credentials{ + Token: "foo", + Secret: "bar", + }, + } + ctx := context.New() + ctx.SetHTTPClient(&http.Client{ + Transport: newFakeTransport(map[string]func() *http.Response{ + apiURL + userInfoAPIPath: fileResponder(filepath.FromSlash("testdata/verify_credentials-res.json")), + }), + }) + inf, err := im.getUserInfo(ctx) + if err != nil { + t.Fatal(err) + } + want := userInfo{ + ID: "2325935334", + ScreenName: "lejatorn", + Name: "Mathieu Lonjaret", + } + if inf != want { + t.Errorf("user info = %+v; want %+v", inf, want) + } +} + +// TODO(mpl): remove and use common code once https://camlistore-review.googlesource.com/2547 is in. + +func newFakeTransport(urls map[string]func() *http.Response) http.RoundTripper { + return fakeTransport{urls} +} + +type fakeTransport struct { + m map[string]func() *http.Response +} + +func (ft fakeTransport) RoundTrip(req *http.Request) (res *http.Response, err error) { + urls := req.URL.String() + fn, ok := ft.m[urls] + if !ok { + return nil, fmt.Errorf("Unexpected fakeTransport URL requested: %s", urls) + } + return fn(), nil +} + +func fileResponder(filename string) func() *http.Response { + return func() *http.Response { + f, err := os.Open(filename) + if err != nil { + return &http.Response{StatusCode: 404, Status: "404 Not Found", Body: types.EmptyBody} + } + return &http.Response{StatusCode: 200, Status: "200 OK", Body: f} + } +} diff --git a/server/camlistored/camlistored.go b/server/camlistored/camlistored.go index ee9417928..1681c13f6 100644 --- a/server/camlistored/camlistored.go +++ b/server/camlistored/camlistored.go @@ -75,7 +75,7 @@ import ( //_ "camlistore.org/pkg/importer/flickr" _ "camlistore.org/pkg/importer/foursquare" //_ "camlistore.org/pkg/importer/picasa" - //_ "camlistore.org/pkg/importer/twitter" + _ "camlistore.org/pkg/importer/twitter" ) var (