Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Importer overhaul. Only Foursquare is currently working. 

The rest are currently stubbed out and will need updating
to the new APIs.

Change-Id: I9d70302b3ac1026192413bf9dcd3c8f1eb420349
This commit is contained in:
Brad Fitzpatrick 2014-03-30 15:44:26 -07:00
parent bf4426e35e
commit bf2a7b60a3
14 changed files with 2209 additions and 448 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
config/dev-server-config.json
View File

@ -271,47 +271,28 @@
}
},
"/importer-dummy/": {
"handler": "importer-dummy",
"handlerArgs": {
"url": "http://localhost:8080/foo.json",
"username": "alice",
"authToken": "xyz"
}
"/importer/": {
"handler": "importer",
"handlerArgs": {
"dummy": {
"clientID": "dummyID",
"clientSecret": "foobar"
},
"flickr": {
"clientSecret": ["_env", "${CAMLI_FLICKR_API_KEY}", ""]
},
"foursquare": {
"clientSecret": ["_env", "${CAMLI_FOURSQUARE_API_KEY}", ""]
},
"picasa": {
"clientSecret": ["_env", "${CAMLI_PICASA_API_KEY}", ""]
},
"twitter": {
"clientSecret": ["_env", "${CAMLI_TWITTER_API_KEY}", ""]
}
}
},
"/importer-flickr/": {
"handler": "importer-flickr",
"enabled": ["_env", "${CAMLI_FLICKR_ENABLED}", false],
"handlerArgs": {
"apiKey": ["_env", "${CAMLI_FLICKR_API_KEY}", ""]
}
},
"/importer-foursquare/": {
"handler": "importer-foursquare",
"enabled": ["_env", "${CAMLI_FOURSQUARE_ENABLED}", false],
"handlerArgs": {
"apiKey": ["_env", "${CAMLI_FOURSQUARE_API_KEY}", ""]
}
},
"/importer-picasa/": {
"handler": "importer-picasa",
"enabled": ["_env", "${CAMLI_PICASA_ENABLED}", false],
"handlerArgs": {
"apiKey": ["_env", "${CAMLI_PICASA_API_KEY}", ""]
}
},
"/importer-twitter/": {
"handler": "importer-twitter",
"enabled": ["_env", "${CAMLI_TWITTER_ENABLED}", false],
"handlerArgs": {
"apiKey": ["_env", "${CAMLI_TWITTER_API_KEY}", ""]
}
},
"/share/": {
"handler": "share",
"handlerArgs": {

6
pkg/context/context.go
View File

@ -51,8 +51,10 @@ func New() *Context {
// HTTPClient returns the HTTP Client to use for this context.
func (c *Context) HTTPClient() *http.Client {
if cl := c.httpClient; cl != nil {
return cl
if c != nil {
if cl := c.httpClient; cl != nil {
return cl
}
}
return http.DefaultClient
}

10
pkg/importer/README Normal file
View File

@ -0,0 +1,10 @@
Permanode type:
camliNodeType: "importer"
importerType: "twitter"
authClientID: "xxx" // e.g. api token
authClientSecret: "sdkojfsldfjlsdkf"
camliNodeType: "importerAccount"
importerType: "twitter"
twitterAccount: "bradfitz"

10
pkg/importer/dummy/dummy.go
View File

@ -31,10 +31,13 @@ import (
)
func init() {
importer.Register("dummy", newFromConfig)
importer.Register("dummy", importer.TODOImporter)
importer.Register("flickr", importer.TODOImporter)
importer.Register("picasa", importer.TODOImporter)
importer.Register("twitter", importer.TODOImporter)
}
func newFromConfig(cfg jsonconfig.Obj, host *importer.Host) (importer.Importer, error) {
func newFromConfig(cfg jsonconfig.Obj, host *importer.Host) (*imp, error) {
im := &imp{
url: cfg.RequiredString("url"),
username: cfg.RequiredString("username"),
@ -54,9 +57,6 @@ type imp struct {
host *importer.Host
}
func (im *imp) CanHandleURL(url string) bool { return false }
func (im *imp) ImportURL(url string) error { panic("unused") }
func (im *imp) Prefix() string {
return fmt.Sprintf("dummy:%s", im.username)
}

90
pkg/importer/foursquare/api.go Normal file
View File

@ -0,0 +1,90 @@
/*
Copyright 2014 The Camlistore Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Types for Foursquare's JSON API.
package foursquare
type user struct {
Id string
FirstName string
LastName string
}
type userInfo struct {
Response struct {
User user
}
}
type checkinsList struct {
Response struct {
Checkins struct {
Items []*checkinItem
}
}
}
type checkinItem struct {
Id string
CreatedAt int64 // unix time in seconds from 4sq
Venue venueItem
}
type venueItem struct {
Id string // eg 42474900f964a52087201fe3 from 4sq
Name string
Location *venueLocationItem
Categories []*venueCategory
}
func (vi *venueItem) primaryCategory() *venueCategory {
for _, c := range vi.Categories {
if c.Primary {
return c
}
}
return nil
}
func (vi *venueItem) icon() string {
c := vi.primaryCategory()
if c == nil || c.Icon == nil || c.Icon.Prefix == "" {
return ""
}
return c.Icon.Prefix + "bg_88" + c.Icon.Suffix
}
type venueLocationItem struct {
Address string
City string
PostalCode string
State string
Country string // 4sq provides "US"
Lat float64
Lng float64
}
type venueCategory struct {
Primary bool
Name string
Icon *categoryIcon
}
type categoryIcon struct {
Prefix string
Suffix string
}

382
pkg/importer/foursquare/foursquare.go
View File

@ -24,7 +24,6 @@ import (
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"time"
@ -32,213 +31,157 @@ import (
"camlistore.org/pkg/context"
"camlistore.org/pkg/httputil"
"camlistore.org/pkg/importer"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/schema"
"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
)
const (
apiURL = "https://api.foursquare.com/v2/"
apiURL = "https://api.foursquare.com/v2/"
authURL = "https://foursquare.com/oauth2/authenticate"
tokenURL = "https://foursquare.com/oauth2/access_token"
// Permanode attributes on account node:
acctAttrUserId = "foursquareUserId"
acctAttrUserFirst = "foursquareFirstName"
acctAttrUserLast = "foursquareLastName"
acctAttrAccessToken = "oauthAccessToken"
)
func init() {
importer.Register("foursquare", newFromConfig)
importer.Register("foursquare", &imp{
imageFileRef: make(map[string]blob.Ref),
})
}
var _ importer.ImporterSetupHTMLer = (*imp)(nil)
type imp struct {
host *importer.Host
tokenCache oauth.Cache
oauthConfig *oauth.Config
tokenCache oauth.Cache
// no locking, serial access only
mu sync.Mutex // guards following
imageFileRef map[string]blob.Ref // url to file schema blob
mu sync.Mutex
user string
}
func newFromConfig(cfg jsonconfig.Obj, host *importer.Host) (importer.Importer, error) {
apiKey := cfg.RequiredString("apiKey")
if err := cfg.Validate(); err != nil {
return nil, err
func (im *imp) NeedsAPIKey() bool { return true }
func (im *imp) IsAccountReady(acctNode *importer.Object) (ok bool, err error) {
if acctNode.Attr(acctAttrUserId) != "" && acctNode.Attr(acctAttrAccessToken) != "" {
return true, nil
}
parts := strings.Split(apiKey, ":")
if len(parts) != 2 {
return nil, fmt.Errorf("Foursquare importer: Invalid apiKey configuration: %q", apiKey)
return false, nil
}
func (im *imp) SummarizeAccount(acct *importer.Object) string {
ok, err := im.IsAccountReady(acct)
if err != nil {
return "Not configured; error = " + err.Error()
}
clientID, clientSecret := parts[0], parts[1]
im := &imp{
host: host,
tokenCache: &tokenCache{},
imageFileRef: make(map[string]blob.Ref),
oauthConfig: &oauth.Config{
ClientId: clientID,
ClientSecret: clientSecret,
AuthURL: "https://foursquare.com/oauth2/authenticate",
TokenURL: "https://foursquare.com/oauth2/access_token",
RedirectURL: host.BaseURL + "callback",
},
if !ok {
return "Not configured"
}
// TODO: schedule work?
return im, nil
}
type tokenCache struct {
mu sync.Mutex
token *oauth.Token
}
func (tc *tokenCache) Token() (*oauth.Token, error) {
tc.mu.Lock()
defer tc.mu.Unlock()
if tc.token == nil {
return nil, errors.New("no token")
if acct.Attr(acctAttrUserFirst) == "" && acct.Attr(acctAttrUserLast) == "" {
return fmt.Sprintf("userid %s", acct.Attr(acctAttrUserId))
}
return tc.token, nil
return fmt.Sprintf("userid %s (%s %s)", acct.Attr(acctAttrUserId),
acct.Attr(acctAttrUserFirst), acct.Attr(acctAttrUserLast))
}
func (tc *tokenCache) PutToken(t *oauth.Token) error {
tc.mu.Lock()
defer tc.mu.Unlock()
tc.token = t
return nil
func (im *imp) AccountSetupHTML(host *importer.Host) string {
base := host.ImporterBaseURL() + "foursquare"
return fmt.Sprintf(`
<h1>Configuring Foursquare</h1>
<p>Visit <a href='https://foursquare.com/developers/apps'>https://foursquare.com/developers/apps</a> and click "Create a new app".</p>
<p>Use the following settings:</p>
<ul>
<li>Download / welcome page url: <b>%s</b></li>
<li>Your privacy policy url: <b>%s</b></li>
<li>Redirect URI(s): <b>%s</b></li>
</ul>
<p>Click "SAVE CHANGES". Copy the "Client ID" and "Client Secret" into the boxes above.</p>
`, base, base+"/privacy", base+"/callback")
}
func (im *imp) CanHandleURL(url string) bool { return false }
func (im *imp) ImportURL(url string) error { panic("unused") }
func (im *imp) Prefix() string {
im.mu.Lock()
defer im.mu.Unlock()
if im.user == "" {
// This should only get called when we're importing, but check anyway.
panic("Prefix called before authenticated")
// A run is our state for a given run of the importer.
type run struct {
*importer.RunContext
im *imp
oauthConfig *oauth.Config
}
func (r *run) token() string {
return r.RunContext.AccountNode().Attr(acctAttrAccessToken)
}
func (r *run) initRoot() error {
root := r.RootNode()
user := r.AccountNode().Attr("foursquareUser")
if user == "" {
return errors.New("The 'foursquareUser' attribute on the account node is empty.")
}
return fmt.Sprintf("foursquare:%s", im.user)
title := fmt.Sprintf("Foursquare (%s)", user)
return root.SetAttr("title", title)
}
func (im *imp) String() string {
im.mu.Lock()
defer im.mu.Unlock()
userId := "<unauthenticated>"
if im.user != "" {
userId = im.user
}
return fmt.Sprintf("foursquare:%s", userId)
}
func (im *imp) Run(ctx *context.Context) error {
// TODO: plumb context and monitor it for cancelation.
if err := im.importCheckins(); err != nil {
func (im *imp) Run(ctx *importer.RunContext) error {
clientId, secret, err := ctx.Credentials()
if err != nil {
return err
}
return nil
}
// structures from json
type userInfo struct {
Response struct {
User struct {
Id string
}
r := &run{
RunContext: ctx,
im: im,
oauthConfig: &oauth.Config{
ClientId: clientId,
ClientSecret: secret,
AuthURL: authURL,
TokenURL: tokenURL,
},
}
}
type checkinsList struct {
Response struct {
Checkins struct {
Items []*checkinItem
}
}
}
type checkinItem struct {
Id string
CreatedAt int64 // unix time in seconds from 4sq
Venue venueItem
}
type venueItem struct {
Id string // eg 42474900f964a52087201fe3 from 4sq
Name string
Location *venueLocationItem
Categories []*venueCategory
}
func (vi *venueItem) primaryCategory() *venueCategory {
for _, c := range vi.Categories {
if c.Primary {
return c
}
if err := r.importCheckins(); err != nil {
return err
}
return nil
}
func (vi *venueItem) icon() string {
c := vi.primaryCategory()
if c == nil || c.Icon == nil || c.Icon.Prefix == "" {
return ""
}
return c.Icon.Prefix + "bg_88" + c.Icon.Suffix
}
type venueLocationItem struct {
Address string
City string
PostalCode string
State string
Country string // 4sq provides "US"
Lat float64
Lng float64
}
type venueCategory struct {
Primary bool
Name string
Icon *categoryIcon
}
type categoryIcon struct {
Prefix string
Suffix string
}
// data import methods
// urlFileRef slurps urlstr from the net, writes to a file and returns its
// fileref or "" on error
func (im *imp) urlFileRef(urlstr string) string {
func (r *run) urlFileRef(urlstr string) string {
im := r.im
im.mu.Lock()
if br, ok := im.imageFileRef[urlstr]; ok {
im.mu.Unlock()
return br.String()
}
res, err := im.host.HTTPClient().Get(urlstr)
im.mu.Unlock()
res, err := r.Host.HTTPClient().Get(urlstr)
if err != nil {
log.Printf("couldn't get image: %v", err)
return ""
}
defer res.Body.Close()
fileRef, err := schema.WriteFileFromReader(im.host.Target(), "category.png", res.Body)
fileRef, err := schema.WriteFileFromReader(r.Host.Target(), "category.png", res.Body)
if err != nil {
log.Printf("couldn't write file: %v", err)
return ""
}
im.mu.Lock()
defer im.mu.Unlock()
im.imageFileRef[urlstr] = fileRef
return fileRef.String()
}
func (im *imp) importCheckins() error {
func (r *run) importCheckins() error {
limit := 100
offset := 0
continueRequests := true
for continueRequests {
resp := checkinsList{}
if err := im.doAPI(&resp, "users/self/checkins", "limit", strconv.Itoa(limit), "offset", strconv.Itoa(offset)); err != nil {
if err := r.im.doAPI(r.Context, r.token(), &resp, "users/self/checkins", "limit", strconv.Itoa(limit), "offset", strconv.Itoa(offset)); err != nil {
return err
}
@ -250,24 +193,24 @@ func (im *imp) importCheckins() error {
offset += itemcount
}
checkinsNode, err := im.getTopLevelNode("checkins", "Checkins")
checkinsNode, err := r.getTopLevelNode("checkins", "Checkins")
if err != nil {
return err
}
placesNode, err := im.getTopLevelNode("places", "Places")
placesNode, err := r.getTopLevelNode("places", "Places")
if err != nil {
return err
}
for _, checkin := range resp.Response.Checkins.Items {
placeRef, err := im.importPlace(placesNode, &checkin.Venue)
placeRef, err := r.importPlace(placesNode, &checkin.Venue)
if err != nil {
log.Printf("Foursquare importer: error importing place %s %v", checkin.Venue.Id, err)
continue
}
err = im.importCheckin(checkinsNode, checkin, placeRef)
err = r.importCheckin(checkinsNode, checkin, placeRef)
if err != nil {
log.Printf("Foursquare importer: error importing checkin %s %v", checkin.Id, err)
continue
@ -278,7 +221,7 @@ func (im *imp) importCheckins() error {
return nil
}
func (im *imp) importCheckin(parent *importer.Object, checkin *checkinItem, placeRef blob.Ref) error {
func (r *run) importCheckin(parent *importer.Object, checkin *checkinItem, placeRef blob.Ref) error {
checkinNode, err := parent.ChildPathObject(checkin.Id)
if err != nil {
return err
@ -290,7 +233,7 @@ func (im *imp) importCheckin(parent *importer.Object, checkin *checkinItem, plac
"foursquareId", checkin.Id,
"foursquareVenuePermanode", placeRef.String(),
"camliNodeType", "foursquare.com:checkin",
"camliContentImage", im.urlFileRef(checkin.Venue.icon()),
"camliContentImage", r.urlFileRef(checkin.Venue.icon()),
"startDate", schema.RFC3339FromTime(time.Unix(checkin.CreatedAt, 0)),
"title", title); err != nil {
return err
@ -299,7 +242,7 @@ func (im *imp) importCheckin(parent *importer.Object, checkin *checkinItem, plac
return nil
}
func (im *imp) importPlace(parent *importer.Object, place *venueItem) (placeRef blob.Ref, err error) {
func (r *run) importPlace(parent *importer.Object, place *venueItem) (placeRef blob.Ref, err error) {
placeNode, err := parent.ChildPathObject(place.Id)
if err != nil {
return placeRef, err
@ -313,7 +256,7 @@ func (im *imp) importPlace(parent *importer.Object, place *venueItem) (placeRef
if err := placeNode.SetAttrs(
"foursquareId", place.Id,
"camliNodeType", "foursquare.com:venue",
"camliContentImage", im.urlFileRef(place.icon()),
"camliContentImage", r.urlFileRef(place.icon()),
"foursquareCategoryName", catName,
"title", place.Name,
"streetAddress", place.Location.Address,
@ -329,13 +272,8 @@ func (im *imp) importPlace(parent *importer.Object, place *venueItem) (placeRef
return placeNode.PermanodeRef(), nil
}
func (im *imp) getTopLevelNode(path string, title string) (*importer.Object, error) {
root, err := im.getRootNode()
if err != nil {
return nil, err
}
childObject, err := root.ChildPathObject(path)
func (r *run) getTopLevelNode(path string, title string) (*importer.Object, error) {
childObject, err := r.RootNode().ChildPathObject(path)
if err != nil {
return nil, err
}
@ -346,60 +284,31 @@ func (im *imp) getTopLevelNode(path string, title string) (*importer.Object, err
return childObject, nil
}
func (im *imp) getRootNode() (*importer.Object, error) {
root, err := im.host.RootObject()
if err != nil {
return nil, err
func (im *imp) getUserInfo(ctx *context.Context, accessToken string) (user, error) {
var ui userInfo
if err := im.doAPI(ctx, accessToken, &ui, "users/self"); err != nil {
return user{}, err
}
if root.Attr("title") == "" {
im.mu.Lock()
user := im.user
im.mu.Unlock()
title := fmt.Sprintf("Foursquare (%s)", user)
if err := root.SetAttr("title", title); err != nil {
return nil, err
}
if ui.Response.User.Id == "" {
return user{}, fmt.Errorf("No userid returned")
}
return root, nil
return ui.Response.User, nil
}
func (im *imp) getUserId() (string, error) {
user := userInfo{}
if err := im.doAPI(&user, "users/self"); err != nil {
return "", err
}
if user.Response.User.Id == "" {
return "", fmt.Errorf("No username specified")
}
return user.Response.User.Id, nil
}
// foursquare api builders
func (im *imp) doAPI(result interface{}, apiPath string, keyval ...string) error {
func (im *imp) doAPI(ctx *context.Context, accessToken string, result interface{}, apiPath string, keyval ...string) error {
if len(keyval)%2 == 1 {
panic("Incorrect number of keyval arguments")
}
token, err := im.tokenCache.Token()
if err != nil {
return fmt.Errorf("Token error: %v", err)
}
form := url.Values{}
form.Set("v", "20140225") // 4sq requires this to version their API
form.Set("oauth_token", token.AccessToken)
form.Set("oauth_token", accessToken)
for i := 0; i < len(keyval); i += 2 {
form.Set(keyval[i], keyval[i+1])
}
fullURL := apiURL + apiPath
res, err := im.doGet(fullURL, form)
res, err := doGet(ctx, fullURL, form)
if err != nil {
return err
}
@ -410,36 +319,56 @@ func (im *imp) doAPI(result interface{}, apiPath string, keyval ...string) error
return err
}
func (im *imp) doGet(url string, form url.Values) (*http.Response, error) {
func doGet(ctx *context.Context, url string, form url.Values) (*http.Response, error) {
requestURL := url + "?" + form.Encode()
req, err := http.NewRequest("GET", requestURL, nil)
if err != nil {
return nil, err
}
res, err := im.host.HTTPClient().Do(req)
res, err := ctx.HTTPClient().Do(req)
if err != nil {
log.Printf("Error fetching %s: %v", url, err)
return nil, err
}
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Get request on %s failed with: %s", requestURL, res.Status)
}
return res, nil
}
func auth(ctx *importer.SetupContext) (*oauth.Config, error) {
clientId, secret, err := ctx.Credentials()
if err != nil {
return nil, err
}
return &oauth.Config{
ClientId: clientId,
ClientSecret: secret,
AuthURL: authURL,
TokenURL: tokenURL,
RedirectURL: ctx.CallbackURL(),
}, nil
}
// possibly common methods for accessing oauth2 sites
func (im *imp) serveLogin(w http.ResponseWriter, r *http.Request) {
state := "no_clue_what_this_is" // TODO: ask adg to document this. or send him a CL.
authURL := im.oauthConfig.AuthCodeURL(state)
http.Redirect(w, r, authURL, 302)
func (im *imp) ServeSetup(w http.ResponseWriter, r *http.Request, ctx *importer.SetupContext) error {
oauthConfig, err := auth(ctx)
if err == nil {
state := "no_clue_what_this_is" // TODO: ask adg to document this. or send him a CL.
http.Redirect(w, r, oauthConfig.AuthCodeURL(state), 302)
}
return err
}
func (im *imp) serveCallback(w http.ResponseWriter, r *http.Request) {
func (im *imp) ServeCallback(w http.ResponseWriter, r *http.Request, ctx *importer.SetupContext) {
oauthConfig, err := auth(ctx)
if err != nil {
httputil.ServeError(w, r, fmt.Errorf("Error getting oauth config: %v", err))
return
}
if r.Method != "GET" {
http.Error(w, "Expected a GET", 400)
return
@ -449,7 +378,7 @@ func (im *imp) serveCallback(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Expected a code", 400)
return
}
transport := &oauth.Transport{Config: im.oauthConfig}
transport := &oauth.Transport{Config: oauthConfig}
token, err := transport.Exchange(code)
log.Printf("Token = %#v, error %v", token, err)
if err != nil {
@ -457,25 +386,22 @@ func (im *imp) serveCallback(w http.ResponseWriter, r *http.Request) {
http.Error(w, "token exchange error", 500)
return
}
im.tokenCache.PutToken(token)
userid, err := im.getUserId()
u, err := im.getUserInfo(ctx.Context, token.AccessToken)
if err != nil {
log.Printf("Couldn't get username: %v", err)
http.Error(w, "can't get username", 500)
return
}
im.user = userid
http.Redirect(w, r, im.host.BaseURL+"?mode=start", 302)
}
func (im *imp) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/login") {
im.serveLogin(w, r)
} else if strings.HasSuffix(r.URL.Path, "/callback") {
im.serveCallback(w, r)
} else {
httputil.BadRequestError(w, "Unknown path: %s", r.URL.Path)
if err := ctx.AccountNode.SetAttrs(
acctAttrUserId, u.Id,
acctAttrUserFirst, u.FirstName,
acctAttrUserLast, u.LastName,
acctAttrAccessToken, token.AccessToken,
); err != nil {
httputil.ServeError(w, r, fmt.Errorf("Error setting attribute: %v", err))
return
}
http.Redirect(w, r, ctx.AccountURL(), http.StatusFound)
}

76
pkg/importer/foursquare/foursquare_test.go Normal file
View File

@ -0,0 +1,76 @@
/*
Copyright 2014 The Camlistore Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package foursquare
import (
"fmt"
"net/http"
"os"
"testing"
"camlistore.org/pkg/context"
"camlistore.org/pkg/types"
)
func TestGetUserId(t *testing.T) {
im := &imp{}
ctx := context.New()
ctx.SetHTTPClient(&http.Client{
Transport: newFakeTransport(map[string]func() *http.Response{
"https://api.foursquare.com/v2/users/self?oauth_token=footoken&v=20140225": fileResponder("testdata/users-me-res.json"),
}),
})
inf, err := im.getUserInfo(ctx, "footoken")
if err != nil {
t.Fatal(err)
}
want := user{
Id: "13674",
FirstName: "Brad",
LastName: "Fitzpatrick",
}
if inf != want {
t.Errorf("user info = %+v; want %+v", inf, want)
}
}
func newFakeTransport(urls map[string]func() *http.Response) http.RoundTripper {
return fakeTransport{urls}
}
type fakeTransport struct {
m map[string]func() *http.Response
}
func (ft fakeTransport) RoundTrip(req *http.Request) (res *http.Response, err error) {
urls := req.URL.String()
fn, ok := ft.m[urls]
if !ok {
return nil, fmt.Errorf("Unexpected fakeTransport URL requested: %s", urls)
}
return fn(), nil
}
func fileResponder(filename string) func() *http.Response {
return func() *http.Response {
f, err := os.Open(filename)
if err != nil {
return &http.Response{StatusCode: 404, Status: "404 Not Found", Body: types.EmptyBody}
}
return &http.Response{StatusCode: 200, Status: "200 OK", Body: f}
}
}

791
pkg/importer/foursquare/testdata/users-me-res.json vendored Normal file
View File

@ -0,0 +1,791 @@
{
"meta": {
"code": 200
},
"notifications": [
{
"type": "notificationTray",
"item": {
"unreadCount": 4
}
}
],
"response": {
"user": {
"id": "13674",
"firstName": "Brad",
"lastName": "Fitzpatrick",
"gender": "male",
"relationship": "self",
"photo": {
"prefix": "https:\/\/irs0.4sqi.net\/img\/user\/",
"suffix": "\/CKG5FOF2WMCMPD3E.jpg"
},
"friends": {
"count": 174,
"groups": [
{
"type": "friends",
"name": "Mutual friends",
"count": 0,
"items": []
},
{
"type": "others",
"name": "Other friends",
"count": 174,
"items": [
{
"id": "83878",
"firstName": "Randal",
"lastName": "Schwartz",
"gender": "male",
"relationship": "friend",
"photo": {
"prefix": "https:\/\/irs2.4sqi.net\/img\/user\/",
"suffix": "\/41NCM3VIMA30PNZ3.jpg"
},
"tips": {
"count": 28
},
"lists": {
"groups": [
{
"type": "created",
"count": 4,
"items": []
}
]
},
"homeCity": "Beaverton, OR",
"bio": "Yeah, \u2022that\u2022 Randal Schwartz. I'm also a low-carb high-fat (ketogenic) consumer, so if you have location advice around that, let me know!",
"contact": {
"phone": "",
"email": "merlyn.foursquare@stonehenge.com",
"twitter": "merlyn",
"facebook": "504874371"
}
},
{
"id": "38677952",
"firstName": "Nori",
"lastName": "Heikkinen",
"gender": "female",
"relationship": "friend",
"photo": {
"prefix": "https:\/\/irs0.4sqi.net\/img\/user\/",
"suffix": "\/VDOVTY0YUYUJ10QK.jpg"
},
"tips": {
"count": 4
},
"lists": {
"groups": [
{
"type": "created",
"count": 5,
"items": []
}
]
},
"homeCity": "Baltimore, MD",
"bio": "",
"contact": {
"email": "nori.heikkinen@gmail.com",
"twitter": "n0r1",
"facebook": "677985398"
}
},
{
"id": "64376555",
"firstName": "Miguel",
"lastName": "de Icaza",
"gender": "male",
"relationship": "friend",
"photo": {
"prefix": "https:\/\/irs0.4sqi.net\/img\/user\/",
"suffix": "\/LP510B3INEKRTNI2.jpg"
},
"tips": {
"count": 2
},
"lists": {
"groups": [
{
"type": "created",
"count": 1,
"items": []
}
]
},
"homeCity": "Boston, MA",
"bio": "",
"contact": {
"email": "miguel.de.icaza@gmail.com",
"facebook": "532065026"
}
},
{
"id": "50291",
"firstName": "Nat",
"lastName": "Friedman",
"gender": "male",
"relationship": "friend",
"photo": {
"prefix": "https:\/\/irs3.4sqi.net\/img\/user\/",
"suffix": "\/50291_1259165803611.jpg"
},
"tips": {
"count": 12
},
"lists": {
"groups": [
{
"type": "created",
"count": 2,
"items": []
}
]
},
"homeCity": "San Francisco",
"bio": "",
"contact": {
"phone": "",
"email": "nat@nat.org",
"twitter": "natfriedman",
"facebook": "547946582"
}
},
{
"id": "618",
"firstName": "Chris",
"lastName": "Messina",
"gender": "male",
"relationship": "friend",
"photo": {
"prefix": "https:\/\/irs0.4sqi.net\/img\/user\/",
"suffix": "\/-31MXPPWS4MBOET0B.jpg"
},
"tips": {
"count": 389
},
"lists": {
"groups": [
{
"type": "created",
"count": 17,
"items": []
}
]
},
"homeCity": "San Francisco, CA",
"bio": "Bachelor of Arts.\r\n#godfather (http:\/\/nyti.ms\/lJ6Kdj)\r\nI am not the actor.",
"contact": {
"phone": "",
"email": "chris.messina@gmail.com",
"twitter": "chrismessina",
"facebook": "502411873"
},
"superuser": 2
},
{
"id": "21279572",
"firstName": "Barry",
"lastName": "Abrahamson",
"gender": "male",
"relationship": "friend",
"photo": {
"prefix": "https:\/\/irs2.4sqi.net\/img\/user\/",
"suffix": "\/MAV25J2AQ3IORLLM.jpg"
},
"tips": {
"count": 2
},
"lists": {
"groups": [
{
"type": "created",
"count": 2,
"items": []
}
]
},
"homeCity": "Houston, Texas",
"bio": "",
"contact": {
"email": "barry@yourang.org",
"twitter": "bazza",
"facebook": "732341470"
}
},
{
"id": "76974963",
"firstName": "Tiffany",
"lastName": "Precissi",
"gender": "female",
"relationship": "friend",
"photo": {
"prefix": "https:\/\/irs2.4sqi.net\/img\/user\/",
"suffix": "\/blank_girl.png"
},
"tips": {
"count": 0
},
"lists": {
"groups": [
{
"type": "created",
"count": 1,
"items": []
}
]
},
"homeCity": "Stockton, CA",
"bio": "",
"contact": {
"email": "tifflynn@gmail.com",
"twitter": "xo_tiff4ny",
"facebook": "665546095"
}
},
{
"id": "4478390",
"firstName": "Julie",
"lastName": "Parent",
"gender": "female",
"relationship": "friend",
"photo": {
"prefix": "https:\/\/irs2.4sqi.net\/img\/user\/",
"suffix": "\/blank_girl.png"
},
"tips": {
"count": 0
},
"lists": {
"groups": [
{
"type": "created",
"count": 1,
"items": []
}
]
},
"homeCity": "San Francisco, CA",
"bio": "",
"contact": {
"email": "jparent@gmail.com",
"twitter": "jewree",
"facebook": "1513396"
}
},
{
"id": "68959715",
"firstName": "Ekaterina",
"lastName": "Ustinova",
"gender": "female",
"relationship": "friend",
"photo": {
"prefix": "https:\/\/irs2.4sqi.net\/img\/user\/",
"suffix": "\/blank_girl.png"
},
"tips": {
"count": 0
},
"lists": {
"groups": [
{
"type": "created",
"count": 1,
"items": []
}
]
},
"homeCity": "New York, NY",
"bio": "",
"contact": {
"email": "ustinoid@gmail.com",
"facebook": "100000011060984"
}
},
{
"id": "67841916",
"firstName": "Katherine",
"lastName": "Deyo",
"gender": "female",
"relationship": "friend",
"photo": {
"prefix": "https:\/\/irs2.4sqi.net\/img\/user\/",
"suffix": "\/blank_girl.png"
},
"tips": {
"count": 0
},
"lists": {
"groups": [
{
"type": "created",
"count": 1,
"items": []
}
]
},
"homeCity": "San Francisco, CA",
"bio": "I wanna be myself",
"contact": {
"email": "katherine_w_deyo@gmail.com"
}
}
]
}
]
},
"tips": {
"count": 18
},
"homeCity": "San Francisco, CA",
"bio": "",
"contact": {
"phone": "5035551212",
"email": "brad@danga.com",
"twitter": "bradfitz",
"facebook": "500033387"
},
"superuser": 1,
"checkinPings": "off",
"pings": false,
"type": "user",
"badges": {
"count": 65,
"items": [
{
"id": "518577cd498ebaa83dc8f7e0",
"badgeId": "4ebb078f7bebd6a83f1176bd",
"name": "Hot Tamale",
"unlockMessage": "You unlocked the Hot Tamale badge!",
"description": "Rice, beans, cheese, cilantro \u2013 why eat anything else when you can get all the important food groups wrapped into one delicious pound of foil? Now pass those nachos, will ya? It\u2019s time to guac and roll.\n\nThat's 45 different Mexican restaurants! Your taste buds must be scorched. Don't worry, some tequila shots should probably fix that. Congrats on Level 10 Hot Tamale status!",
"level": 10,
"badgeText": "Rice, beans, cheese, cilantro \u2013 why eat anything else when you can get all the important food groups wrapped into one delicious pound of foil? Now pass those nachos, will ya? It\u2019s time to guac and roll.",
"levelText": "That's 45 different Mexican restaurants! Your taste buds must be scorched. Don't worry, some tequila shots should probably fix that. Congrats on Level 10 Hot Tamale status!",
"categorySummary": "Mexican Restaurants",
"image": {
"prefix": "https:\/\/playfoursquare.s3.amazonaws.com\/badge\/",
"sizes": [
57,
114,
200,
300,
400
],
"name": "\/L2RRMCA2PGOBSFRN_10.png"
},
"unlocks": [
{
"checkins": [
{
"id": "518577cc498ebaa83dc8f148",
"createdAt": 1367701452,
"type": "checkin",
"shout": "Van exchange point for The Relay. Not actually going to church.",
"timeZoneOffset": -420,
"venue": {
"id": "4bdc6861c79cc9285e6586e9",
"name": "Crosswalk Community Church",
"contact": {},
"location": {
"lat": 38.30073598016023,
"lng": -122.30450377008302,
"postalCode": "94558",
"cc": "US",
"country": "United States"
},
"categories": [
{
"id": "4bf58dd8d48988d1c1941735",
"name": "Mexican Restaurant",
"pluralName": "Mexican Restaurants",
"shortName": "Mexican",
"icon": {
"prefix": "https:\/\/ss1.4sqi.net\/img\/categories_v2\/food\/mexican_",
"suffix": ".png"
},
"primary": true
}
],
"verified": false,
"stats": {
"checkinsCount": 248,
"usersCount": 90,
"tipCount": 4
}
},
"photos": {
"count": 0,
"items": []
},
"posts": {
"count": 0,
"textCount": 0
},
"comments": {
"count": 1
},
"source": {
"name": "foursquare for Android",
"url": "https:\/\/foursquare.com\/download\/#\/android"
}
}
]
}
]
}
]
},
"mayorships": {
"count": 4,
"items": []
},
"checkins": {
"count": 3272,
"items": [
{
"id": "53396b10498e2c3aed309903",
"createdAt": 1396271888,
"type": "checkin",
"shout": "SFO-PDX",
"timeZoneOffset": -420,
"venue": {
"id": "4a7601b6f964a520efe11fe3",
"name": "Alaska Airlines Board Room",
"contact": {
"twitter": "alaskaair"
},
"location": {
"address": "Terminal 1",
"crossStreet": "at SFO Airport",
"lat": 37.61343253150299,
"lng": -122.3850667476654,
"postalCode": "94128",
"cc": "US",
"city": "San Francisco",
"state": "CA",
"country": "United States"
},
"categories": [
{
"id": "4eb1bc533b7b2c5b1d4306cb",
"name": "Airport Lounge",
"pluralName": "Airport Lounges",
"shortName": "Lounge",
"icon": {
"prefix": "https:\/\/ss1.4sqi.net\/img\/categories_v2\/travel\/airport_lounge_",
"suffix": ".png"
},
"primary": true
}
],
"verified": false,
"stats": {
"checkinsCount": 1318,
"usersCount": 822,
"tipCount": 22
},
"url": "http:\/\/alaskaair.com",
"likes": {
"count": 6,
"groups": [
{
"type": "others",
"count": 6,
"items": [
{
"id": "6446336",
"firstName": "Aaron",
"lastName": "C.",
"gender": "male",
"photo": {
"prefix": "https:\/\/irs3.4sqi.net\/img\/user\/",
"suffix": "\/BVDOLAQG4BYFXHV3.jpg"
}
},
{
"id": "15728179",
"firstName": "Christopher",
"lastName": "P.",
"gender": "male",
"photo": {
"prefix": "https:\/\/irs3.4sqi.net\/img\/user\/",
"suffix": "\/Q5AUNW2UZDUDG10E.jpg"
}
},
{
"id": "7709153",
"firstName": "Farhad",
"lastName": "M.",
"gender": "male",
"photo": {
"prefix": "https:\/\/irs1.4sqi.net\/img\/user\/",
"suffix": "\/VZBNN1XYBEVWEAIO.jpg"
}
},
{
"id": "181603",
"firstName": "Jeffrey-Ryan",
"lastName": "B.",
"gender": "male",
"photo": {
"prefix": "https:\/\/irs2.4sqi.net\/img\/user\/",
"suffix": "\/FP5Q5W1FHULIZOCV.jpg"
}
}
]
}
],
"summary": "Aaron Chaffee, Christopher Potter, Farhad M & 3 others"
},
"like": false,
"beenHere": {
"count": 1,
"marked": true
}
},
"likes": {
"count": 1,
"groups": [
{
"type": "friends",
"count": 1,
"items": [
{
"id": "431392",
"firstName": "Owen",
"lastName": "Thomas",
"gender": "male",
"relationship": "friend",
"photo": {
"prefix": "https:\/\/irs0.4sqi.net\/img\/user\/",
"suffix": "\/GEDBNXFSYUXRUFLD.gif"
}
}
]
}
],
"summary": "Owen Thomas"
},
"like": false,
"photos": {
"count": 0,
"items": []
},
"posts": {
"count": 0,
"textCount": 0
},
"comments": {
"count": 0
},
"source": {
"name": "foursquare for Android",
"url": "https:\/\/foursquare.com\/download\/#\/android"
}
}
]
},
"following": {
"count": 1,
"groups": [
{
"type": "following",
"name": "Mutual following",
"count": 0,
"items": []
},
{
"type": "others",
"name": "Other following",
"count": 1,
"items": [
{
"id": "13276",
"firstName": "Loic",
"lastName": "L.",
"gender": "male",
"relationship": "followingThem",
"photo": {
"prefix": "https:\/\/irs0.4sqi.net\/img\/user\/",
"suffix": "\/MHWAEBRGHQOMJE22.jpg"
},
"type": "celebrity",
"followers": {
"count": 8132,
"groups": []
},
"tips": {
"count": 16
},
"lists": {
"groups": [
{
"type": "created",
"count": 1,
"items": []
}
]
},
"homeCity": "San Francisco",
"bio": "LeWeb and Seesmic founder, love creating things",
"contact": {
"twitter": "loic",
"facebook": "1417669498"
}
}
]
}
]
},
"requests": {
"count": 280
},
"lists": {
"count": 1,
"groups": [
{
"type": "created",
"count": 1,
"items": [
{
"id": "13674\/todos",
"name": "My to-do list",
"description": "",
"user": {
"id": "13674",
"firstName": "Brad",
"lastName": "Fitzpatrick",
"gender": "male",
"relationship": "self",
"photo": {
"prefix": "https:\/\/irs0.4sqi.net\/img\/user\/",
"suffix": "\/CKG5FOF2WMCMPD3E.jpg"
}
},
"editable": false,
"public": false,
"collaborative": false,
"url": "\/bradfitz\/list\/todos",
"canonicalUrl": "https:\/\/foursquare.com\/bradfitz\/list\/todos",
"followers": {
"count": 0
},
"listItems": {
"count": 3
}
}
]
},
{
"type": "followed",
"count": 0,
"items": []
}
]
},
"photos": {
"count": 11,
"items": [
{
"id": "530278c311d26be8ab5da961",
"createdAt": 1392670915,
"source": {
"name": "foursquare for Android",
"url": "https:\/\/foursquare.com\/download\/#\/android"
},
"prefix": "https:\/\/irs1.4sqi.net\/img\/general\/",
"suffix": "\/13674_4TxZ1OeQuFwOlprqcI1lWGZN4Or2f4Oal1rGup6ZPS4.jpg",
"width": 960,
"height": 720,
"visibility": "public",
"venue": {
"id": "40870b00f964a520aaf21ee3",
"name": "The Liberties",
"contact": {
"phone": "4152826789",
"formattedPhone": "(415) 282-6789",
"twitter": "thelibertiesbar"
},
"location": {
"address": "998 Guerrero St",
"crossStreet": "at 22nd St",
"lat": 37.75523648445705,
"lng": -122.4232582553582,
"postalCode": "94110",
"cc": "US",
"city": "San Francisco",
"state": "CA",
"country": "United States"
},
"categories": [
{
"id": "4bf58dd8d48988d116941735",
"name": "Bar",
"pluralName": "Bars",
"shortName": "Bar",
"icon": {
"prefix": "https:\/\/ss1.4sqi.net\/img\/categories_v2\/nightlife\/bar_",
"suffix": ".png"
},
"primary": true
}
],
"verified": true,
"stats": {
"checkinsCount": 4526,
"usersCount": 2147,
"tipCount": 35
},
"url": "http:\/\/www.theliberties.com",
"likes": {
"count": 21,
"groups": [
{
"type": "others",
"count": 20,
"items": []
}
],
"summary": "You and 20 others"
},
"like": true,
"menu": {
"type": "Menu",
"label": "Menu",
"anchor": "View Menu",
"url": "https:\/\/foursquare.com\/v\/the-liberties\/40870b00f964a520aaf21ee3\/menu",
"mobileUrl": "https:\/\/foursquare.com\/v\/40870b00f964a520aaf21ee3\/device_menu"
},
"beenHere": {
"count": 24,
"marked": true
},
"venuePage": {
"id": "46953806"
},
"storeId": ""
},
"checkin": {
"id": "5302789511d2c9d07c49130e",
"createdAt": 1392670869,
"type": "checkin",
"timeZoneOffset": -480
}
}
]
},
"scores": {
"recent": 116,
"max": 272,
"checkinsCount": 27
},
"createdAt": 1242876758,
"referralId": "u-13674"
}
}
}

198
pkg/importer/html.go Normal file
View File

@ -0,0 +1,198 @@
/*
Copyright 2013 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package importer
import (
"bytes"
"fmt"
"html/template"
"net/http"
"strings"
"time"
)
func execTemplate(w http.ResponseWriter, r *http.Request, data interface{}) {
tmplName := strings.TrimPrefix(fmt.Sprintf("%T", data), "importer.")
var buf bytes.Buffer
err := tmpl.ExecuteTemplate(&buf, tmplName, data)
if err != nil {
http.Error(w, fmt.Sprintf("Error executing template %q: %v", tmplName, err), 500)
return
}
w.Write(buf.Bytes())
}
type importersRootPage struct {
Title string
Body importersRootBody
}
type importersRootBody struct {
BasePath string
Importers []*importer
}
type importerPage struct {
Title string
Body importerBody
}
type importerBody struct {
BasePath string
Importer *importer
SetupHelp template.HTML
}
type acctPage struct {
Title string
Body acctBody
}
type acctBody struct {
Acct *importerAcct
AcctType string
Running bool
LastStatus string
StartedAgo time.Duration // or zero if !Running
LastAgo time.Duration // non-zero if previous run && !Running
LastError string
}
var tmpl = template.Must(template.New("root").Parse(`
{{define "pageTop"}}
<html>
<head>
<title>{{.Title}}</title>
</head>
<body>
<h1>{{.Title}}</h1>
{{end}}
{{define "pageBottom"}}
</body>
</html>
{{end}}
{{define "importersRootPage"}}
{{template "pageTop" .}}
{{template "importersRootBody" .Body}}
{{template "pageBottom"}}
{{end}}
{{define "importersRootBody"}}
<ul>
{{$base := .BasePath}}
{{range .Importers}}
<li><a href="{{$base}}{{.Name}}">{{.Name}}</a></li>
{{end}}
</ul>
{{end}}
{{define "importerPage"}}
{{template "pageTop" .}}
{{template "importerBody" .Body}}
{{template "pageBottom"}}
{{end}}
{{define "importerBody"}}
<p>[<a href="./">&lt;&lt; Back</a>]</p>
<ul>
<li>Importer configuration permanode: {{.Importer.Node.PermanodeRef}}</li>
<li>Status: {{.Importer.Status}}</li>
</ul>
{{if .Importer.ShowClientAuthEditForm}}
<h1>Client ID &amp; Client Secret</h1>
<form method='post'>
<input type='hidden' name="mode" value="saveclientidsecret">
<table border=0 cellpadding=3>
<tr><td align=right>Client ID</td><td><input name="clientID" size=50 value="{{.Importer.ClientID}}"></td></tr>
<tr><td align=right>Client Secret</td><td><input name="clientSecret" size=50 value="{{.Importer.ClientSecret}}"></td></tr>
<tr><td align=right></td><td><input type='submit' value="Save"></td></tr>
</table>
</form>
{{end}}
{{.SetupHelp}}
<h1>Accounts</h1>
<ul>
{{range .Importer.Accounts}}
<li><a href="{{.AccountURL}}">{{.AccountLinkText}}</a> {{.AccountLinkSummary}}</li>
{{end}}
</ul>
{{if .Importer.CanAddNewAccount}}
<form method='post'>
<input type='hidden' name="mode" value="newacct">
<input type='submit' value="Add new account">
</form>
{{end}}
{{end}}
{{define "acctPage"}}
{{template "pageTop" .}}
{{template "acctBody" .Body}}
{{template "pageBottom"}}
{{end}}
{{define "acctBody"}}
<p>[<a href="./">&lt;&lt; Back</a>]</p>
<ul>
<li>Account type: {{.AcctType}}</li>
<li>Account metadata permanode: {{.Acct.AccountObject.PermanodeRef}}</li>
<li>Import root permanode: {{if .Acct.RootObject}}{{.Acct.RootObject.PermanodeRef}}{{else}}(none){{end}}</li>
<li>Configured: {{.Acct.IsAccountReady}}</li>
<li>Running: {{.Running}}</li>
{{if .Running}}
<li>Started: {{.StartedAgo}} ago</li>
<li>Last status: {{.LastStatus}}</li>
{{else}}
{{if .LastAgo}}
<li>Previous run: {{.LastAgo}} ago{{if .LastError}}: {{.LastError}}{{else}} (success){{end}}</li>
{{end}}
{{end}}
</ul>
{{if .Acct.IsAccountReady}}
<form method='post' style='display: inline'>
{{if .Running}}
<input type='hidden' name='mode' value='stop'>
<input type='submit' value='Pause Import'>
{{else}}
<input type='hidden' name='mode' value='start'>
<input type='submit' value='Start Import'>
{{end}}
</form>
{{end}}
<form method='post' style='display: inline'>
<input type='hidden' name='mode' value='login'>
<input type='submit' value='Re-login'>
</form>
<form method='post' style='display: inline'>
<input type='hidden' name='mode' value='delete'>
<input type='submit' value='Delete Account' onclick='return confirm("Delete account?")'>
</form>
{{end}}
`))

888
pkg/importer/importer.go
View File

@ -15,16 +15,18 @@ limitations under the License.
*/
// Package importer imports content from third-party websites.
//
// TODO(bradfitz): Finish this. Barely started.
package importer
import (
"errors"
"fmt"
"html/template"
"log"
"net/http"
"sort"
"strings"
"sync"
"time"
"camlistore.org/pkg/blob"
"camlistore.org/pkg/blobserver"
@ -38,29 +40,358 @@ import (
"camlistore.org/pkg/syncutil"
)
// A Host is the environment hosting an importer.
type Host struct {
BaseURL string
const (
attrNodeType = "camliNodeType"
nodeTypeImporter = "importer"
nodeTypeImporterAccount = "importerAccount"
imp Importer
target blobserver.StatReceiver
search *search.Handler
signer *schema.Signer
attrImporterType = "importerType" // => "twitter", "foursquare", etc
attrClientID = "authClientID"
attrClientSecret = "authClientSecret"
attrImportRoot = "importRoot"
)
// An Importer imports from a third-party site.
type Importer interface {
// Run runs a full or incremental import.
//
// The importer should continually or periodically monitor the
// context's Done channel to exit early if requested. The
// return value should be context.ErrCanceled if the importer
// exits for that reason.
Run(*RunContext) error
// NeedsAPIKey reports whether this importer requires an API key
// (OAuth2 client_id & client_secret, or equivalent).
// If the API only requires a username & password, or a flow to get
// an auth token per-account without an overall API key, importers
// can return false here.
NeedsAPIKey() bool
// IsAccountReady reports whether the provided account node
// is configured.
IsAccountReady(acctNode *Object) (ok bool, err error)
SummarizeAccount(acctNode *Object) string
ServeSetup(w http.ResponseWriter, r *http.Request, ctx *SetupContext) error
ServeCallback(w http.ResponseWriter, r *http.Request, ctx *SetupContext)
}
type ImporterSetupHTMLer interface {
AccountSetupHTML(*Host) string
}
var importers = make(map[string]Importer)
// Register registers a site-specific importer. It should only be called from init,
// and not from concurrent goroutines.
func Register(name string, im Importer) {
if _, dup := importers[name]; dup {
panic("Dup registration of importer " + name)
}
importers[name] = im
}
func init() {
// Register the meta "importer" handler, which handles all other handlers.
blobserver.RegisterHandlerConstructor("importer", newFromConfig)
}
func newFromConfig(ld blobserver.Loader, cfg jsonconfig.Obj) (http.Handler, error) {
h := &Host{
baseURL: ld.BaseURL(),
importerBase: ld.BaseURL() + ld.MyPrefix(),
imp: make(map[string]*importer),
}
for k, impl := range importers {
h.importers = append(h.importers, k)
var clientID, clientSecret string
if impConf := cfg.OptionalObject(k); impConf != nil {
clientID = impConf.OptionalString("clientID", "")
clientSecret = impConf.OptionalString("clientSecret", "")
// Special case: allow clientSecret to be of form "clientID:clientSecret"
// if the clientID is empty.
if clientID == "" && strings.Contains(clientSecret, ":") {
if f := strings.SplitN(clientSecret, ":", 2); len(f) == 2 {
clientID, clientSecret = f[0], f[1]
}
}
if err := impConf.Validate(); err != nil {
return nil, fmt.Errorf("Invalid static configuration for importer %q: %v", k, err)
}
}
if clientSecret != "" && clientID == "" {
return nil, fmt.Errorf("Invalid static configuration for importer %q: clientSecret specified without clientID", k)
}
imp := &importer{
host: h,
name: k,
impl: impl,
clientID: clientID,
clientSecret: clientSecret,
}
h.imp[k] = imp
}
if err := cfg.Validate(); err != nil {
return nil, err
}
sort.Strings(h.importers)
return h, nil
}
var _ blobserver.HandlerIniter = (*Host)(nil)
type SetupContext struct {
*context.Context
Host *Host
AccountNode *Object
ia *importerAcct
}
func (sc *SetupContext) Credentials() (clientID, clientSecret string, err error) {
return sc.ia.im.credentials()
}
func (sc *SetupContext) CallbackURL() string {
return sc.Host.ImporterBaseURL() + sc.ia.im.name + "/callback?acct=" + sc.AccountNode.PermanodeRef().String()
}
// AccountURL returns the URL to an account of an importer
// (http://host/importer/TYPE/sha1-sd8fsd7f8sdf7).
func (sc *SetupContext) AccountURL() string {
return sc.Host.ImporterBaseURL() + sc.ia.im.name + "/" + sc.AccountNode.PermanodeRef().String()
}
// RunContext is the context provided for a given Run of an importer, importing
// a certain account on a certain importer.
type RunContext struct {
*context.Context
Host *Host
ia *importerAcct
mu sync.Mutex // guards following
lastProgress *ProgressMessage
}
// Credentials returns the credentials for the importer. This is
// typically the OAuth1, OAuth2, or equivalent client ID (api token)
// and client secret (api secret).
func (rc *RunContext) Credentials() (clientID, clientSecret string, err error) {
return rc.ia.im.credentials()
}
// AccountNode returns the permanode storing account information for this permanode.
// It will contain the attributes:
// * camliNodeType = "importerAccount"
// * importerType = "registered-type"
//
// You must not change the camliNodeType or importerType.
//
// You should use this permanode to store state about where your
// importer left off, if it can efficiently resume later (without
// missing anything).
func (rc *RunContext) AccountNode() *Object { return rc.ia.acct }
// AccountNode returns the initially-empty permanode storing the root
// of this account's data. You can change anything at will. This will
// typically be modeled as a dynamic directory (with camliPath:xxxx
// attributes), where each path element is either a file, object, or
// another dynamic directory.
func (rc *RunContext) RootNode() *Object { return rc.ia.root }
// Host is the HTTP handler and state for managing all the importers
// linked into the binary, even if they're not configured.
type Host struct {
importers []string // sorted; e.g. dummy flickr foursquare picasa twitter
imp map[string]*importer
baseURL string
importerBase string
target blobserver.StatReceiver
search *search.Handler
signer *schema.Signer
// client optionally specifies how to fetch external network
// resources. If nil, http.DefaultClient is used.
client *http.Client
transport http.RoundTripper
mu sync.Mutex
ctx *context.Context
running bool
lastProgress *ProgressMessage
lastRunErr error
}
func (h *Host) String() string {
return fmt.Sprintf("%T(%s)", h, h.imp)
func (h *Host) InitHandler(hl blobserver.FindHandlerByTyper) error {
_, handler, err := hl.FindHandlerByType("root")
if err != nil || handler == nil {
return errors.New("importer requires a 'root' handler")
}
rh := handler.(*server.RootHandler)
searchHandler, ok := rh.SearchHandler()
if !ok {
return errors.New("importer requires a 'root' handler with 'searchRoot' defined.")
}
h.search = searchHandler
if rh.Storage == nil {
return errors.New("importer requires a 'root' handler with 'blobRoot' defined.")
}
h.target = rh.Storage
_, handler, _ = hl.FindHandlerByType("jsonsign")
if sigh, ok := handler.(*signhandler.Handler); ok {
h.signer = sigh.Signer()
}
if h.signer == nil {
return errors.New("importer requires a 'jsonsign' handler")
}
return nil
}
// ServeHTTP serves:
// http://host/importer/
// http://host/importer/twitter/
// http://host/importer/twitter/callback
// http://host/importer/twitter/sha1-abcabcabcabcabc (single account)
func (h *Host) ServeHTTP(w http.ResponseWriter, r *http.Request) {
suffix := httputil.PathSuffix(r)
seg := strings.Split(suffix, "/")
if suffix == "" || len(seg) == 0 {
h.serveImportersRoot(w, r)
return
}
impName := seg[0]
imp, ok := h.imp[impName]
if !ok {
http.NotFound(w, r)
return
}
if len(seg) == 1 || seg[1] == "" {
h.serveImporter(w, r, imp)
return
}
if seg[1] == "callback" {
h.serveImporterAcctCallback(w, r, imp)
return
}
acctRef, ok := blob.Parse(seg[1])
if !ok {
http.NotFound(w, r)
return
}
h.serveImporterAccount(w, r, imp, acctRef)
}
// Serves list of importers at http://host/importer/
func (h *Host) serveImportersRoot(w http.ResponseWriter, r *http.Request) {
body := importersRootBody{
BasePath: httputil.PathBase(r),
Importers: make([]*importer, 0, len(h.imp)),
}
for _, v := range h.importers {
body.Importers = append(body.Importers, h.imp[v])
}
execTemplate(w, r, importersRootPage{
Title: "Importers",
Body: body,
})
}
// Serves list of accounts at http://host/importer/twitter
func (h *Host) serveImporter(w http.ResponseWriter, r *http.Request, imp *importer) {
if r.Method == "POST" {
h.serveImporterPost(w, r, imp)
return
}
var setup string
node, _ := imp.Node()
if setuper, ok := imp.impl.(ImporterSetupHTMLer); ok && node != nil {
setup = setuper.AccountSetupHTML(h)
}
execTemplate(w, r, importerPage{
Title: "Importer - " + imp.Name(),
Body: importerBody{
Importer: imp,
SetupHelp: template.HTML(setup),
},
})
}
// Serves oauth callback at http://host/importer/TYPE/callback
func (h *Host) serveImporterAcctCallback(w http.ResponseWriter, r *http.Request, imp *importer) {
if r.Method != "GET" {
http.Error(w, "invalid method", 400)
return
}
acctRef, ok := blob.Parse(r.FormValue("acct"))
if !ok {
http.Error(w, "missing 'acct' blobref param", 400)
return
}
ia, err := imp.account(acctRef)
if err != nil {
http.Error(w, "invalid 'acct' param: "+err.Error(), 400)
return
}
imp.impl.ServeCallback(w, r, &SetupContext{
Context: context.TODO(),
Host: h,
AccountNode: ia.acct,
ia: ia,
})
}
func (h *Host) serveImporterPost(w http.ResponseWriter, r *http.Request, imp *importer) {
switch r.FormValue("mode") {
default:
http.Error(w, "Unknown mode.", 400)
case "newacct":
ia, err := imp.newAccount()
if err != nil {
http.Error(w, err.Error(), 500)
return
}
ia.setup(w, r)
return
case "saveclientidsecret":
n, err := imp.Node()
if err != nil {
http.Error(w, "Error getting node: "+err.Error(), 500)
return
}
if err := n.SetAttrs(
attrClientID, r.FormValue("clientID"),
attrClientSecret, r.FormValue("clientSecret"),
); err != nil {
http.Error(w, "Error saving node: "+err.Error(), 500)
return
}
http.Redirect(w, r, h.ImporterBaseURL()+imp.name, http.StatusFound)
}
}
// Serves details of accounts at http://host/importer/twitter/sha1-23098429382934
func (h *Host) serveImporterAccount(w http.ResponseWriter, r *http.Request, imp *importer, acctRef blob.Ref) {
ia, err := imp.account(acctRef)
if err != nil {
http.Error(w, "Unknown or invalid importer account "+acctRef.String()+": "+err.Error(), 400)
return
}
ia.ServeHTTP(w, r)
}
// BaseURL returns the root of the whole server, without trailing
// slash.
func (h *Host) BaseURL() string {
return h.baseURL
}
// ImporterBaseURL returns the URL base of the importer handler,
// including trailing slash.
func (h *Host) ImporterBaseURL() string {
return h.importerBase
}
func (h *Host) Target() blobserver.StatReceiver {
@ -71,59 +402,400 @@ func (h *Host) Search() *search.Handler {
return h.search
}
func (h *Host) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if httputil.PathSuffix(r) == "" {
switch r.FormValue("mode") {
case "":
case "start":
h.start()
case "stop":
h.stop()
default:
fmt.Fprintf(w, "Unknown mode")
}
h.mu.Lock()
defer h.mu.Unlock()
fmt.Fprintf(w, "I am an importer of type %T; running=%v; last progress=%#v",
h.imp, h.running, h.lastProgress)
} else {
// TODO(aa): Remove this temporary hack once the UI has a way to configure importers.
h.imp.ServeHTTP(w, r)
}
// importer is an importer for a certain site, but not a specific account on that site.
type importer struct {
host *Host
name string // importer name e.g. "twitter"
impl Importer
// If statically configured in config file, else
// they come from the importer node's attributes.
clientID string
clientSecret string
nodemu sync.Mutex // guards nodeCache
nodeCache *Object // or nil if unset
acctmu sync.Mutex
acct map[blob.Ref]*importerAcct // key: account permanode
}
func (h *Host) start() {
h.mu.Lock()
defer h.mu.Unlock()
if h.running {
func (im *importer) Name() string { return im.name }
func (im *importer) StaticConfig() bool { return im.clientSecret != "" }
// URL returns the importer's URL without trailing slash.
func (im *importer) URL() string { return im.host.ImporterBaseURL() + im.name }
func (im *importer) ShowClientAuthEditForm() bool {
if im.StaticConfig() {
// Don't expose the server's statically-configured client secret
// to the user. (e.g. a hosted multi-user configuation)
return false
}
return im.impl.NeedsAPIKey()
}
func (im *importer) CanAddNewAccount() bool {
if !im.impl.NeedsAPIKey() {
return true
}
id, sec, err := im.credentials()
return id != "" && sec != "" && err == nil
}
func (im *importer) ClientID() (v string, err error) {
v, _, err = im.credentials()
return
}
func (im *importer) ClientSecret() (v string, err error) {
_, v, err = im.credentials()
return
}
func (im *importer) Status() (status string, err error) {
if !im.impl.NeedsAPIKey() {
return "no configuration required", nil
}
if im.StaticConfig() {
return "API key configured on server", nil
}
n, err := im.Node()
if err != nil {
return
}
if n.Attr(attrClientID) != "" && n.Attr(attrClientSecret) != "" {
return "API key configured on node", nil
}
return "API key (client ID & Secret) not configured", nil
}
func (im *importer) credentials() (clientID, clientSecret string, err error) {
if im.StaticConfig() {
return im.clientID, im.clientSecret, nil
}
n, err := im.Node()
if err != nil {
return
}
return n.Attr(attrClientID), n.Attr(attrClientSecret), nil
}
func (im *importer) deleteAccount(acctRef blob.Ref) {
im.acctmu.Lock()
delete(im.acct, acctRef)
im.acctmu.Unlock()
}
func (im *importer) account(nodeRef blob.Ref) (*importerAcct, error) {
im.acctmu.Lock()
ia, ok := im.acct[nodeRef]
im.acctmu.Unlock()
if ok {
return ia, nil
}
acct, err := im.host.ObjectFromRef(nodeRef)
if err != nil {
return nil, err
}
if acct.Attr(attrNodeType) != nodeTypeImporterAccount {
return nil, errors.New("account has wrong node type")
}
if acct.Attr(attrImporterType) != im.name {
return nil, errors.New("account has wrong importer type")
}
var root *Object
if v := acct.Attr(attrImportRoot); v != "" {
rootRef, ok := blob.Parse(v)
if !ok {
return nil, errors.New("invalid import root attribute")
}
root, err = im.host.ObjectFromRef(rootRef)
if err != nil {
return nil, err
}
} else {
root, err = im.host.NewObject()
if err != nil {
return nil, err
}
if err := acct.SetAttr(attrImportRoot, root.PermanodeRef().String()); err != nil {
return nil, err
}
}
ia = &importerAcct{
im: im,
acct: acct,
root: root,
}
im.acctmu.Lock()
defer im.acctmu.Unlock()
im.addAccountLocked(ia)
return ia, nil
}
func (im *importer) newAccount() (*importerAcct, error) {
acct, err := im.host.NewObject()
if err != nil {
return nil, err
}
root, err := im.host.NewObject()
if err != nil {
return nil, err
}
if err := acct.SetAttrs(
attrNodeType, nodeTypeImporterAccount,
attrImporterType, im.name,
attrImportRoot, root.PermanodeRef().String(),
); err != nil {
return nil, err
}
ia := &importerAcct{
im: im,
acct: acct,
root: root,
}
im.acctmu.Lock()
defer im.acctmu.Unlock()
im.addAccountLocked(ia)
return ia, nil
}
func (im *importer) addAccountLocked(ia *importerAcct) {
if im.acct == nil {
im.acct = make(map[blob.Ref]*importerAcct)
}
im.acct[ia.acct.PermanodeRef()] = ia
}
func (im *importer) Accounts() ([]*importerAcct, error) {
var accts []*importerAcct
res, err := im.host.search.Query(&search.SearchQuery{
Expression: fmt.Sprintf("attr:%s:%s attr:%s:%s",
attrNodeType, nodeTypeImporterAccount,
attrImporterType, im.name,
),
})
if err != nil {
return nil, err
}
for _, res := range res.Blobs {
ia, err := im.account(res.Blob)
if err != nil {
return nil, err
}
accts = append(accts, ia)
}
return accts, nil
}
// node returns the importer node. (not specific to a certain account
// on that importer site)
//
// It is a permanode with:
// camliNodeType: "importer"
// importerType: "twitter"
// And optionally:
// authClientID: "xxx" // e.g. api token
// authClientSecret: "sdkojfsldfjlsdkf"
func (im *importer) Node() (*Object, error) {
im.nodemu.Lock()
defer im.nodemu.Unlock()
if im.nodeCache != nil {
return im.nodeCache, nil
}
expr := fmt.Sprintf("attr:%s:%s attr:%s:%s",
attrNodeType, nodeTypeImporter,
attrImporterType, im.name,
)
res, err := im.host.search.Query(&search.SearchQuery{
Limit: 10, // only expect 1
Expression: expr,
})
if err != nil {
return nil, err
}
if len(res.Blobs) > 1 {
return nil, fmt.Errorf("Ambiguous; too many permanodes matched query %q: %v", expr, res.Blobs)
}
if len(res.Blobs) == 1 {
return im.host.ObjectFromRef(res.Blobs[0].Blob)
}
o, err := im.host.NewObject()
if err != nil {
return nil, err
}
if err := o.SetAttrs(
attrNodeType, nodeTypeImporter,
attrImporterType, im.name,
); err != nil {
return nil, err
}
im.nodeCache = o
return o, nil
}
// importerAcct is a long-lived type representing account
type importerAcct struct {
im *importer
acct *Object
root *Object
mu sync.Mutex
current *RunContext // or nil if not running
stopped bool // stop requested (context canceled)
lastRunErr error
lastRunStart time.Time
lastRunDone time.Time
}
func (ia *importerAcct) delete() error {
if err := ia.acct.SetAttrs(
attrNodeType, nodeTypeImporter+"-deleted",
); err != nil {
return err
}
ia.im.deleteAccount(ia.acct.PermanodeRef())
return nil
}
func (ia *importerAcct) IsAccountReady() (bool, error) {
return ia.im.impl.IsAccountReady(ia.acct)
}
func (ia *importerAcct) AccountObject() *Object { return ia.acct }
func (ia *importerAcct) RootObject() *Object { return ia.root }
func (ia *importerAcct) AccountURL() string {
return ia.im.URL() + "/" + ia.acct.PermanodeRef().String()
}
func (ia *importerAcct) AccountLinkText() string {
return ia.acct.PermanodeRef().String()
}
func (ia *importerAcct) AccountLinkSummary() string {
return ia.im.impl.SummarizeAccount(ia.acct)
}
func (ia *importerAcct) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method == "POST" {
ia.serveHTTPPost(w, r)
return
}
ia.mu.Lock()
defer ia.mu.Unlock()
body := acctBody{
Acct: ia,
AcctType: fmt.Sprintf("%T", ia.im.impl),
}
if run := ia.current; run != nil {
body.Running = true
body.StartedAgo = time.Since(ia.lastRunStart)
run.mu.Lock()
body.LastStatus = fmt.Sprintf("%+v", run.lastProgress)
run.mu.Unlock()
} else if !ia.lastRunDone.IsZero() {
body.LastAgo = time.Since(ia.lastRunDone)
if ia.lastRunErr != nil {
body.LastError = ia.lastRunErr.Error()
}
}
title := fmt.Sprintf("%s account: ", ia.im.name)
if summary := ia.im.impl.SummarizeAccount(ia.acct); summary != "" {
title += summary
} else {
title += ia.acct.PermanodeRef().String()
}
execTemplate(w, r, acctPage{
Title: title,
Body: body,
})
}
func (ia *importerAcct) serveHTTPPost(w http.ResponseWriter, r *http.Request) {
// TODO: XSRF token
switch r.FormValue("mode") {
case "":
// Nothing.
case "start":
ia.start()
case "stop":
ia.stop()
case "login":
ia.setup(w, r)
return
case "delete":
ia.stop() // can't hurt
if err := ia.delete(); err != nil {
http.Error(w, err.Error(), 500)
return
}
http.Redirect(w, r, ia.im.URL(), http.StatusFound)
return
default:
http.Error(w, "Unknown mode", 400)
return
}
http.Redirect(w, r, ia.AccountURL(), http.StatusFound)
}
func (ia *importerAcct) setup(w http.ResponseWriter, r *http.Request) {
ia.im.impl.ServeSetup(w, r, &SetupContext{
Context: context.TODO(),
Host: ia.im.host,
AccountNode: ia.acct,
ia: ia,
})
}
func (ia *importerAcct) start() {
ia.mu.Lock()
defer ia.mu.Unlock()
if ia.current != nil {
return
}
ctx := context.New()
h.ctx = ctx
h.running = true
rc := &RunContext{
Context: ctx,
Host: ia.im.host,
ia: ia,
}
ia.current = rc
ia.stopped = false
ia.lastRunStart = time.Now()
go func() {
log.Printf("Starting importer %s", h)
err := h.imp.Run(ctx)
log.Printf("Starting importer %s", ia)
err := ia.im.impl.Run(rc)
if err != nil {
log.Printf("Importer %s error: %v", h, err)
log.Printf("Importer %s error: %v", ia.im.name, err)
} else {
log.Printf("Importer %s finished.", h)
log.Printf("Importer %s finished.", ia.im.name)
}
h.mu.Lock()
defer h.mu.Unlock()
h.running = false
h.lastRunErr = err
ia.mu.Lock()
defer ia.mu.Unlock()
ia.current = nil
ia.stopped = false
ia.lastRunDone = time.Now()
ia.lastRunErr = err
}()
}
func (h *Host) stop() {
h.mu.Lock()
defer h.mu.Unlock()
if !h.running {
func (ia *importerAcct) stop() {
ia.mu.Lock()
defer ia.mu.Unlock()
if ia.current == nil || ia.stopped {
return
}
h.running = false
h.ctx.Cancel()
ia.current.Context.Cancel()
ia.stopped = true
}
// HTTPClient returns the HTTP client to use.
@ -272,7 +944,7 @@ func (h *Host) RootObject() (*Object, error) {
res, err := h.search.GetPermanodesWithAttr(&search.WithAttrRequest{
N: 2, // only expect 1
Attr: "camliImportRoot",
Value: h.imp.Prefix(),
Value: "TODO", // h.imp.Prefix(),
})
if err != nil {
log.Printf("RootObject searching GetPermanodesWithAttr: %v", err)
@ -284,13 +956,15 @@ func (h *Host) RootObject() (*Object, error) {
return nil, err
}
log.Printf("No root object found. Created %v", obj.pn)
if err := obj.SetAttr("camliImportRoot", h.imp.Prefix()); err != nil {
return nil, err
}
/*
if err := obj.SetAttr("camliImportRoot", h.imp.Prefix()); err != nil {
return nil, err
}
*/
return obj, nil
}
if len(res.WithAttr) > 1 {
return nil, fmt.Errorf("Found %d import roots for %q; want 1", len(res.WithAttr), h.imp.Prefix())
return nil, fmt.Errorf("Found %d import roots for %q; want 1", len(res.WithAttr), "xxx" /* h.imp.Prefix() */)
}
pn := res.WithAttr[0].Permanode
return h.ObjectFromRef(pn)
@ -318,95 +992,3 @@ func (h *Host) ObjectFromRef(permanodeRef blob.Ref) (*Object, error) {
attr: map[string][]string(db.Permanode.Attr),
}, nil
}
// An Importer imports from a third-party site.
type Importer interface {
// Run runs a full or increment import.
//
// The importer should continually or periodically monitor the
// context's Done channel to exit early if requested. The
// return value should be context.ErrCanceled if the importer
// exits for that reason.
Run(*context.Context) error
// Prefix returns the unique prefix for this importer.
// It should be of the form "serviceType:username".
// Further colons are added to form the names of planned
// permanodes.
Prefix() string
// CanHandleURL returns whether a URL (such as one a user is
// viewing in their browser and dragged onto Camlistore) is a
// form recognized by this importer. If so, its full metadata
// and full data (e.g. unscaled image) can be fetched, rather
// than just fetching the HTML of the URL.
//
// TODO: implement and use this. For now importers can return
// stub these and return false/errors. They're unused.
CanHandleURL(url string) bool
ImportURL(url string) error
ServeHTTP(w http.ResponseWriter, r *http.Request)
}
// Constructor is the function type that importers must register at init time.
type Constructor func(jsonconfig.Obj, *Host) (Importer, error)
var (
mu sync.Mutex
ctors = make(map[string]Constructor)
)
func Register(name string, fn Constructor) {
mu.Lock()
defer mu.Unlock()
if _, dup := ctors[name]; dup {
panic("Dup registration of importer " + name)
}
ctors[name] = fn
}
func Create(name string, hl blobserver.Loader, baseURL string, cfg jsonconfig.Obj) (*Host, error) {
mu.Lock()
defer mu.Unlock()
fn := ctors[name]
if fn == nil {
return nil, fmt.Errorf("Unknown importer type %q", name)
}
h := &Host{
BaseURL: baseURL,
}
imp, err := fn(cfg, h)
if err != nil {
return nil, err
}
h.imp = imp
return h, nil
}
func (h *Host) InitHandler(hl blobserver.FindHandlerByTyper) error {
_, handler, err := hl.FindHandlerByType("root")
if err != nil || handler == nil {
return errors.New("importer requires a 'root' handler")
}
rh := handler.(*server.RootHandler)
searchHandler, ok := rh.SearchHandler()
if !ok {
return errors.New("importer requires a 'root' handler with 'searchRoot' defined.")
}
h.search = searchHandler
if rh.Storage == nil {
return errors.New("importer requires a 'root' handler with 'blobRoot' defined.")
}
h.target = rh.Storage
_, handler, _ = hl.FindHandlerByType("jsonsign")
if sigh, ok := handler.(*signhandler.Handler); ok {
h.signer = sigh.Signer()
}
if h.signer == nil {
return errors.New("importer requires a 'jsonsign' handler")
}
return nil
}

66
pkg/importer/importer_test.go Normal file
View File

@ -0,0 +1,66 @@
/*
Copyright 2014 The Camlistore Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package importer
import (
"testing"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/test"
)
func init() {
Register("dummy1", TODOImporter)
Register("dummy2", TODOImporter)
}
func TestStaticConfig(t *testing.T) {
ld := test.NewLoader()
h, err := newFromConfig(ld, jsonconfig.Obj{
"dummy1": map[string]interface{}{
"clientID": "id1",
"clientSecret": "secret1",
},
"dummy2": map[string]interface{}{
"clientSecret": "id2:secret2",
},
})
if err != nil {
t.Fatal(err)
}
host := h.(*Host)
if g, w := host.imp["dummy1"].clientID, "id1"; g != w {
t.Errorf("dummy1 id = %q; want %q", g, w)
}
if g, w := host.imp["dummy1"].clientSecret, "secret1"; g != w {
t.Errorf("dummy1 secret = %q; want %q", g, w)
}
if g, w := host.imp["dummy2"].clientID, "id2"; g != w {
t.Errorf("dummy2 id = %q; want %q", g, w)
}
if g, w := host.imp["dummy2"].clientSecret, "secret2"; g != w {
t.Errorf("dummy2 secret = %q; want %q", g, w)
}
if _, err := newFromConfig(ld, jsonconfig.Obj{"dummy1": map[string]interface{}{"bogus": ""}}); err == nil {
t.Errorf("expected error from unknown key")
}
if _, err := newFromConfig(ld, jsonconfig.Obj{"dummy1": map[string]interface{}{"clientSecret": "x"}}); err == nil {
t.Errorf("expected error from secret without id")
}
}

53
pkg/importer/noop.go Normal file
View File

@ -0,0 +1,53 @@
/*
Copyright 2014 The Camlistore Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package importer
import (
"errors"
"fmt"
"net/http"
)
var TODOImporter Importer = todoImp{}
type todoImp struct{}
func (todoImp) NeedsAPIKey() bool { return false }
func (todoImp) Run(*RunContext) error {
return errors.New("fake error from todo importer")
}
func (todoImp) IsAccountReady(acctNode *Object) (ok bool, err error) {
return
}
func (todoImp) SummarizeAccount(acctNode *Object) string { return "" }
func (todoImp) ServeSetup(w http.ResponseWriter, r *http.Request, ctx *SetupContext) error {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
fmt.Fprintf(w, "The Setup page for the TODO importer.\nnode = %v\ncallback = %s\naccount URL = %s\n",
ctx.AccountNode,
ctx.CallbackURL(),
"ctx.AccountURL()")
return nil
}
func (todoImp) ServeCallback(w http.ResponseWriter, r *http.Request, ctx *SetupContext) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
fmt.Fprintf(w, "The callback page for the TODO importer.\n")
}

22
pkg/serverinit/serverinit.go
View File

@ -40,7 +40,6 @@ import (
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/blobserver/handlers"
"camlistore.org/pkg/httputil"
"camlistore.org/pkg/importer"
"camlistore.org/pkg/index"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/types/serverconfig"
@ -334,23 +333,10 @@ func (hl *handlerLoader) setupHandler(prefix string) {
return
}
var hh http.Handler
if strings.HasPrefix(h.htype, "importer-") {
itype := strings.TrimPrefix(h.htype, "importer-")
imp, err := importer.Create(itype, hl, hl.baseURL+h.prefix, h.conf)
if err != nil {
exitFailure("error instantiating importer for prefix %q, type %q: %v",
h.prefix, itype, err)
}
hh = imp
} else {
var err error
hh, err = blobserver.CreateHandler(h.htype, hl, h.conf)
if err != nil {
exitFailure("error instantiating handler for prefix %q, type %q: %v",
h.prefix, h.htype, err)
}
hh, err := blobserver.CreateHandler(h.htype, hl, h.conf)
if err != nil {
exitFailure("error instantiating handler for prefix %q, type %q: %v",
h.prefix, h.htype, err)
}
hl.handler[prefix] = hh

6
server/camlistored/camlistored.go
View File

@ -81,10 +81,10 @@ import (
// Importers:
_ "camlistore.org/pkg/importer/dummy"
_ "camlistore.org/pkg/importer/flickr"
//_ "camlistore.org/pkg/importer/flickr"
_ "camlistore.org/pkg/importer/foursquare"
_ "camlistore.org/pkg/importer/picasa"
_ "camlistore.org/pkg/importer/twitter"
//_ "camlistore.org/pkg/importer/picasa"
//_ "camlistore.org/pkg/importer/twitter"
)
var (