Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge "twitter importer: fixed to new API"

This commit is contained in:
Brad Fitzpatrick 2014-04-23 15:48:24 +00:00 committed by Gerrit Code Review
parent 5ace3bea0f ef5acd75e3
commit cea6b15013
6 changed files with 268 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pkg/importer/importer.go
View File

@ -88,7 +88,6 @@ var importers = make(map[string]Importer)
func init() {
Register("flickr", TODOImporter)
Register("picasa", TODOImporter)
Register("twitter", TODOImporter)
}
// Register registers a site-specific importer. It should only be called from init,
@ -759,12 +758,14 @@ func (ia *importerAcct) serveHTTPPost(w http.ResponseWriter, r *http.Request) {
}
func (ia *importerAcct) setup(w http.ResponseWriter, r *http.Request) {
ia.im.impl.ServeSetup(w, r, &SetupContext{
if err := ia.im.impl.ServeSetup(w, r, &SetupContext{
Context: context.TODO(),
Host: ia.im.host,
AccountNode: ia.acct,
ia: ia,
})
}); err != nil {
log.Printf("%v", err)
}
}
func (ia *importerAcct) start() {

14
pkg/importer/twitter/README
View File

@ -3,16 +3,4 @@ Twitter Importer
This is a Camlistore importer for Twitter.
To use:
1) Visit https://apps.twitter.com/ and "Create a new app" to get an API key and
secret. Note that you *must* specify a callback URL in the application
settings UI. It doesn't matter what this URL is because Camlistore
overrides it at runtime. It just has to be some valid non-localhost URL.
2) Start the devcam server with twitterapikey flag:
$ devcam server -twitterapikey=<apikey>:<secret>
3) Navigate to http://<server>/importer-twitter/login
4) Watch import progress on the command line
Go to http[s]://your_camlistore_server/importer/twitter

9
pkg/importer/twitter/testdata/verify_credentials-res.json vendored Executable file
View File

@ -0,0 +1,9 @@
{
"id":2325935334,
"id_str":"2325935334",
"name":"Mathieu Lonjaret",
"screen_name":"lejatorn",
"location":"",
"description":"potato, clever label, trendy word",
"url":"https:\/\/t.co\/TF5K7idMNj"
}

293
pkg/importer/twitter/twitter.go
View File

@ -30,81 +30,107 @@ import (
"camlistore.org/pkg/context"
"camlistore.org/pkg/httputil"
"camlistore.org/pkg/importer"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/schema"
"camlistore.org/third_party/github.com/garyburd/go-oauth/oauth"
)
const (
apiURL = "https://api.twitter.com/1.1/"
apiURL = "https://api.twitter.com/1.1/"
temporaryCredentialRequestURL = "https://api.twitter.com/oauth/request_token"
resourceOwnerAuthorizationURL = "https://api.twitter.com/oauth/authorize"
tokenRequestURL = "https://api.twitter.com/oauth/access_token"
userInfoAPIPath = "account/verify_credentials.json"
// Permanode attributes on account node:
acctAttrUserID = "twitterUserID"
acctAttrScreenName = "twitterScreenName"
acctAttrUserFirst = "twitterFirstName"
acctAttrUserLast = "twitterLastName"
acctAttrAccessToken = "oauthAccessToken"
tweetRequestLimit = 200 // max number of tweets we can get in a user_timeline request
)
func init() {
importer.Register("twitter", &imp{})
}
var _ importer.ImporterSetupHTMLer = (*imp)(nil)
var (
oauthClient = &oauth.Client{
TemporaryCredentialRequestURI: "https://api.twitter.com/oauth/request_token",
ResourceOwnerAuthorizationURI: "https://api.twitter.com/oauth/authorize",
TokenRequestURI: "https://api.twitter.com/oauth/access_token",
TemporaryCredentialRequestURI: temporaryCredentialRequestURL,
ResourceOwnerAuthorizationURI: resourceOwnerAuthorizationURL,
TokenRequestURI: tokenRequestURL,
}
)
func init() {
importer.Register("twitter", newFromConfig)
}
type imp struct {
host *importer.Host
userid string // empty if the user isn't authenticated
cred *oauth.Credentials
// cred are the various credentials passed around during OAUTH. First the temporary
// ones, then the access token and secret.
cred *oauth.Credentials
}
func newFromConfig(cfg jsonconfig.Obj, host *importer.Host) (importer.Importer, error) {
apiKey := cfg.RequiredString("apiKey")
if err := cfg.Validate(); err != nil {
return nil, err
}
parts := strings.Split(apiKey, ":")
if len(parts) != 2 {
return nil, fmt.Errorf("Twitter importer: Invalid apiKey configuration: %q", apiKey)
}
func (im *imp) NeedsAPIKey() bool { return true }
oauthClient.Credentials = oauth.Credentials{
Token: parts[0],
Secret: parts[1],
func (im *imp) IsAccountReady(acctNode *importer.Object) (ok bool, err error) {
if acctNode.Attr(acctAttrUserID) != "" && acctNode.Attr(acctAttrAccessToken) != "" {
return true, nil
}
return &imp{
host: host,
cred: &oauthClient.Credentials,
}, nil
return false, nil
}
func (im *imp) CanHandleURL(url string) bool { return false }
func (im *imp) ImportURL(url string) error { panic("unused") }
func (im *imp) Prefix() string {
// This should only get called when we're importing, so it's OK to
// assume we're authenticated.
return fmt.Sprintf("twitter:%s", im.userid)
}
func (im *imp) String() string {
// We use this in logging when we're not authenticated, so it should do
// something reasonable in that case.
userId := "<unauthenticated>"
if im.userid != "" {
userId = im.userid
func (im *imp) SummarizeAccount(acct *importer.Object) string {
ok, err := im.IsAccountReady(acct)
if err != nil {
return "Not configured; error = " + err.Error()
}
return fmt.Sprintf("twitter:%s", userId)
if !ok {
return "Not configured"
}
if acct.Attr(acctAttrUserFirst) == "" && acct.Attr(acctAttrUserLast) == "" {
return fmt.Sprintf("@%s", acct.Attr(acctAttrScreenName))
}
return fmt.Sprintf("@%s (%s %s)", acct.Attr(acctAttrScreenName),
acct.Attr(acctAttrUserFirst), acct.Attr(acctAttrUserLast))
}
func (im *imp) Run(ctx *context.Context) error {
log.Print("Twitter running...")
func (im *imp) AccountSetupHTML(host *importer.Host) string {
base := host.ImporterBaseURL() + "twitter"
return fmt.Sprintf(`
<h1>Configuring Twitter</h1>
<p>Visit <a href='https://apps.twitter.com/'>https://apps.twitter.com/</a> and click "Create New App".</p>
<p>Use the following settings:</p>
<ul>
<li>Name: Does not matter. (camlistore-importer).</li>
<li>Description: Does not matter. (imports twitter data into camlistore).</li>
<li>Website: <b>%s</b></li>
<li>Callback URL: <b>%s</b></li>
</ul>
<p>Click "Create your Twitter application".You should be redirected to the Application Management page of your newly created application.
</br>Go to the API Keys tab. Copy the "API key" and "API secret" into the "Client ID" and "Client Secret" boxes above.</p>
`, base, base+"/callback")
}
if err := im.importTweets(ctx); err != nil {
// A run is our state for a given run of the importer.
type run struct {
*importer.RunContext
im *imp
}
func (im *imp) Run(ctx *importer.RunContext) error {
r := &run{
RunContext: ctx,
im: im,
}
userID := ctx.AccountNode().Attr(acctAttrUserID)
if userID == "" {
return errors.New("UserID hasn't been set by account setup.")
}
if err := r.importTweets(userID); err != nil {
return err
}
return nil
}
@ -114,22 +140,25 @@ type tweetItem struct {
CreatedAt string `json:"created_at"`
}
func (im *imp) importTweets(ctx *context.Context) error {
func (r *run) importTweets(userID string) error {
maxId := ""
continueRequests := true
for continueRequests {
if ctx.IsCanceled() {
if r.Context.IsCanceled() {
log.Printf("Twitter importer: interrupted")
return context.ErrCanceled
}
var resp []*tweetItem
if err := im.doAPI(&resp, "statuses/user_timeline.json", "count", strconv.Itoa(tweetRequestLimit), "max_id", maxId); err != nil {
if err := r.im.doAPI(r.Context, &resp, "statuses/user_timeline.json",
"user_id", userID,
"count", strconv.Itoa(tweetRequestLimit),
"max_id", maxId); err != nil {
return err
}
tweetsNode, err := im.getTopLevelNode("tweets", "Tweets")
tweetsNode, err := r.getTopLevelNode("tweets", "Tweets")
if err != nil {
return err
}
@ -144,11 +173,11 @@ func (im *imp) importTweets(ctx *context.Context) error {
}
for _, tweet := range resp {
if ctx.IsCanceled() {
if r.Context.IsCanceled() {
log.Printf("Twitter importer: interrupted")
return context.ErrCanceled
}
err = im.importTweet(tweetsNode, tweet)
err = r.importTweet(tweetsNode, tweet)
if err != nil {
log.Printf("Twitter importer: error importing tweet %s %v", tweet.Id, err)
continue
@ -159,7 +188,7 @@ func (im *imp) importTweets(ctx *context.Context) error {
return nil
}
func (im *imp) importTweet(parent *importer.Object, tweet *tweetItem) error {
func (r *run) importTweet(parent *importer.Object, tweet *tweetItem) error {
tweetNode, err := parent.ChildPathObject(tweet.Id)
if err != nil {
return err
@ -169,8 +198,7 @@ func (im *imp) importTweet(parent *importer.Object, tweet *tweetItem) error {
createdTime, err := time.Parse(time.RubyDate, tweet.CreatedAt)
if err != nil {
log.Printf("Twitter importer: error parsing time %s %v", tweet.Id, err)
return err
return fmt.Errorf("could not parse time %q: %v", tweet.CreatedAt, err)
}
// TODO: import photos referenced in tweets
@ -182,136 +210,149 @@ func (im *imp) importTweet(parent *importer.Object, tweet *tweetItem) error {
"title", title)
}
// utility
func (im *imp) getTopLevelNode(path string, title string) (*importer.Object, error) {
root, err := im.getRootNode()
func (r *run) getTopLevelNode(path string, title string) (*importer.Object, error) {
tweets, err := r.RootNode().ChildPathObject(path)
if err != nil {
return nil, err
}
photos, err := root.ChildPathObject(path)
if err != nil {
if err := tweets.SetAttr("title", title); err != nil {
return nil, err
}
if err := photos.SetAttr("title", title); err != nil {
return nil, err
}
return photos, nil
return tweets, nil
}
func (im *imp) getRootNode() (*importer.Object, error) {
root, err := im.host.RootObject()
if err != nil {
return nil, err
}
// TODO(mpl): move to an api.go when we it gets bigger.
title := fmt.Sprintf("Twitter (%s)", im.userid)
if err := root.SetAttr("title", title); err != nil {
return nil, err
}
return root, nil
type userInfo struct {
ID string `json:"id_str"`
ScreenName string `json:"screen_name"`
Name string `json:"name,omitempty"`
}
// twitter api builders
func (im *imp) getUserInfo(ctx *context.Context) (userInfo, error) {
var ui userInfo
if err := im.doAPI(ctx, &ui, userInfoAPIPath); err != nil {
return ui, err
}
if ui.ID == "" {
return ui, fmt.Errorf("No userid returned")
}
return ui, nil
}
func (im *imp) doAPI(result interface{}, apiPath string, keyval ...string) error {
func (im *imp) doAPI(ctx *context.Context, result interface{}, apiPath string, keyval ...string) error {
if len(keyval)%2 == 1 {
panic("Incorrect number of keyval arguments")
panic("Incorrect number of keyval arguments. must be even.")
}
if im.cred == nil {
return fmt.Errorf("No authentication creds")
}
if im.userid == "" {
return fmt.Errorf("No user id")
}
form := url.Values{}
form.Set("user_id", im.userid)
for i := 0; i < len(keyval); i += 2 {
if keyval[i+1] != "" {
form.Set(keyval[i], keyval[i+1])
}
}
res, err := im.doGet(apiURL+apiPath, form)
fullURL := apiURL + apiPath
res, err := im.doGet(ctx, fullURL, form)
if err != nil {
return err
}
err = httputil.DecodeJSON(res, result)
if err != nil {
log.Printf("Error parsing response for %s: %s", apiURL, err)
return fmt.Errorf("could not parse response for %s: %v", fullURL, err)
}
return err
return nil
}
func (im *imp) doGet(url string, form url.Values) (*http.Response, error) {
func (im *imp) doGet(ctx *context.Context, url string, form url.Values) (*http.Response, error) {
if im.cred == nil {
return nil, errors.New("Not logged in. Go to /importer-twitter/login.")
return nil, errors.New("No OAUTH credentials. Not logged in?")
}
res, err := oauthClient.Get(im.host.HTTPClient(), im.cred, url, form)
res, err := oauthClient.Get(ctx.HTTPClient(), im.cred, url, form)
if err != nil {
return nil, err
return nil, fmt.Errorf("Error fetching %s: %v", url, err)
}
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Auth request failed with: %s", res.Status)
return nil, fmt.Errorf("Get request on %s failed with: %s", url, res.Status)
}
return res, nil
}
// auth endpoints
func (im *imp) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/login") {
im.serveLogin(w, r)
} else if strings.HasSuffix(r.URL.Path, "/callback") {
im.serveCallback(w, r)
} else {
httputil.BadRequestError(w, "Unknown path: %s", r.URL.Path)
func auth(ctx *importer.SetupContext) (*oauth.Credentials, error) {
clientId, secret, err := ctx.Credentials()
if err != nil {
return nil, err
}
return &oauth.Credentials{
Token: clientId,
Secret: secret,
}, nil
}
func (im *imp) serveLogin(w http.ResponseWriter, r *http.Request) {
callback := im.host.BaseURL + "callback"
tempCred, err := oauthClient.RequestTemporaryCredentials(im.host.HTTPClient(), callback, nil)
func (im *imp) ServeSetup(w http.ResponseWriter, r *http.Request, ctx *importer.SetupContext) error {
cred, err := auth(ctx)
if err != nil {
httputil.ServeError(w, r, fmt.Errorf("Twitter importer: Error getting temp cred: %s", err))
return
err = fmt.Errorf("Error getting API credentials: %v", err)
httputil.ServeError(w, r, err)
return err
}
oauthClient.Credentials = *cred
tempCred, err := oauthClient.RequestTemporaryCredentials(ctx.HTTPClient(), ctx.CallbackURL(), nil)
if err != nil {
err = fmt.Errorf("Error getting temp cred: %v", err)
httputil.ServeError(w, r, err)
}
im.cred = tempCred
authURL := oauthClient.AuthorizationURL(tempCred, nil)
http.Redirect(w, r, authURL, 302)
return nil
}
func (im *imp) serveCallback(w http.ResponseWriter, r *http.Request) {
func (im *imp) ServeCallback(w http.ResponseWriter, r *http.Request, ctx *importer.SetupContext) {
if im.cred.Token != r.FormValue("oauth_token") {
httputil.BadRequestError(w, "Twitter importer: unexpected oauth_token")
log.Printf("unexpected oauth_token: got %v, want %v", r.FormValue("oauth_token"), im.cred.Token)
httputil.BadRequestError(w, "unexpected oauth_token")
return
}
tokenCred, vals, err := oauthClient.RequestToken(im.host.HTTPClient(), im.cred, r.FormValue("oauth_verifier"))
tokenCred, vals, err := oauthClient.RequestToken(ctx.Context.HTTPClient(), im.cred, r.FormValue("oauth_verifier"))
if err != nil {
httputil.ServeError(w, r, fmt.Errorf("Twitter importer: error getting request token: %s ", err))
httputil.ServeError(w, r, fmt.Errorf("Error getting request token: %v ", err))
return
}
userid := vals.Get("user_id")
if userid == "" {
httputil.ServeError(w, r, fmt.Errorf("Couldn't get user id: %v", err))
return
}
im.cred = tokenCred
userid := vals.Get("user_id")
if userid == "" {
log.Printf("Couldn't get user id: %v", err)
http.Error(w, "can't get user id", 500)
u, err := im.getUserInfo(ctx.Context)
if err != nil {
httputil.ServeError(w, r, fmt.Errorf("Couldn't get user info: %v", err))
return
}
im.userid = userid
http.Redirect(w, r, im.host.BaseURL+"?mode=start", 302)
firstName, lastName := "", ""
if u.Name != "" {
if pieces := strings.Fields(u.Name); len(pieces) == 2 {
firstName = pieces[0]
lastName = pieces[1]
}
}
if err := ctx.AccountNode.SetAttrs(
acctAttrUserID, u.ID,
acctAttrUserFirst, firstName,
acctAttrUserLast, lastName,
acctAttrScreenName, u.ScreenName,
acctAttrAccessToken, tokenCred.Token,
); err != nil {
httputil.ServeError(w, r, fmt.Errorf("Error setting attribute: %v", err))
return
}
http.Redirect(w, r, ctx.AccountURL(), http.StatusFound)
}

86
pkg/importer/twitter/twitter_test.go Normal file
View File

@ -0,0 +1,86 @@
/*
Copyright 2014 The Camlistore Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package twitter
import (
"fmt"
"net/http"
"os"
"path/filepath"
"testing"
"camlistore.org/pkg/context"
"camlistore.org/pkg/types"
"camlistore.org/third_party/github.com/garyburd/go-oauth/oauth"
)
func TestGetUserID(t *testing.T) {
im := &imp{
&oauth.Credentials{
Token: "foo",
Secret: "bar",
},
}
ctx := context.New()
ctx.SetHTTPClient(&http.Client{
Transport: newFakeTransport(map[string]func() *http.Response{
apiURL + userInfoAPIPath: fileResponder(filepath.FromSlash("testdata/verify_credentials-res.json")),
}),
})
inf, err := im.getUserInfo(ctx)
if err != nil {
t.Fatal(err)
}
want := userInfo{
ID: "2325935334",
ScreenName: "lejatorn",
Name: "Mathieu Lonjaret",
}
if inf != want {
t.Errorf("user info = %+v; want %+v", inf, want)
}
}
// TODO(mpl): remove and use common code once https://camlistore-review.googlesource.com/2547 is in.
func newFakeTransport(urls map[string]func() *http.Response) http.RoundTripper {
return fakeTransport{urls}
}
type fakeTransport struct {
m map[string]func() *http.Response
}
func (ft fakeTransport) RoundTrip(req *http.Request) (res *http.Response, err error) {
urls := req.URL.String()
fn, ok := ft.m[urls]
if !ok {
return nil, fmt.Errorf("Unexpected fakeTransport URL requested: %s", urls)
}
return fn(), nil
}
func fileResponder(filename string) func() *http.Response {
return func() *http.Response {
f, err := os.Open(filename)
if err != nil {
return &http.Response{StatusCode: 404, Status: "404 Not Found", Body: types.EmptyBody}
}
return &http.Response{StatusCode: 200, Status: "200 OK", Body: f}
}
}

2
server/camlistored/camlistored.go
View File

@ -75,7 +75,7 @@ import (
//_ "camlistore.org/pkg/importer/flickr"
_ "camlistore.org/pkg/importer/foursquare"
//_ "camlistore.org/pkg/importer/picasa"
//_ "camlistore.org/pkg/importer/twitter"
_ "camlistore.org/pkg/importer/twitter"
)
var (