From e3d71c34d19f062efdb1ca5b61a6d76c0f1803b4 Mon Sep 17 00:00:00 2001 From: Aaron Boodman Date: Sun, 17 Nov 2013 21:50:13 -0800 Subject: [PATCH] Actually import the first 100 photos from Flickr. Currently this just imports the title, description, and image data. Change-Id: I95de17cddb0ac18eec05f8ee288764b04cd7f406 --- pkg/importer/flickr/flickr.go | 121 +++++++++++++++++++++++++++++----- pkg/importer/importer.go | 26 +++++--- 2 files changed, 122 insertions(+), 25 deletions(-) diff --git a/pkg/importer/flickr/flickr.go b/pkg/importer/flickr/flickr.go index c80c33436..b7755cd62 100644 --- a/pkg/importer/flickr/flickr.go +++ b/pkg/importer/flickr/flickr.go @@ -21,11 +21,13 @@ import ( "encoding/json" "errors" "fmt" + "log" "net/http" "net/url" "camlistore.org/pkg/importer" "camlistore.org/pkg/jsonconfig" + "camlistore.org/pkg/schema" "camlistore.org/third_party/github.com/garyburd/go-oauth/oauth" ) @@ -104,7 +106,7 @@ type searchPhotosResult struct { Pages int Perpage int Total int `json:",string"` - Photo []photoMeta + Photo []*photoMeta } Stat string @@ -112,7 +114,7 @@ type searchPhotosResult struct { func (im *imp) Run(intr importer.Interrupt) error { resp := searchPhotosResult{} - if err := im.flickrRequest(url.Values{ + if err := im.flickrAPIRequest(url.Values{ "method": {"flickr.photos.search"}, "user_id": {"me"}, "extras": {"description, date_upload, date_taken, original_format, last_update, geo, tags, machine_tags, views, media, url_o"}}, @@ -120,31 +122,118 @@ func (im *imp) Run(intr importer.Interrupt) error { return err } - for _, item := range resp.Photos.Photo { - camliIdFramgment := fmt.Sprintf("photo-%s", item.Id) - photoContentHint := item.Lastupdate - fmt.Println(camliIdFramgment, photoContentHint) - // TODO(aa): Stuff + photos, err := im.getPhotosNode() + if err != nil { + return err } + log.Printf("Importing %d photos into permanode %s", + len(resp.Photos.Photo), photos.PermanodeRef().String()) + + for _, item := range resp.Photos.Photo { + if err := im.importPhoto(photos, item); err != nil { + log.Printf("Flickr importer: error importing %s: %s", item.Id, err) + continue + } + } + return nil } -func (im *imp) flickrRequest(form url.Values, result interface{}) error { - if im.user == nil { - return errors.New("Not logged in. Go to /importer-flickr/login.") - } +// TODO(aa): +// * Parallelize: http://golang.org/doc/effective_go.html#concurrency +// * Record lastmodified and don't reimport photos that haven't changed +// * Do more than one "page" worth of results +// * Report progress and errors back through host interface +// * All the rest of the metadata (see photoMeta) +// * What happens when changes at Flickr conflict with changes made through the Camlistore UI? +// * Test! +func (im *imp) importPhoto(parent *importer.Object, photo *photoMeta) error { + filename := fmt.Sprintf("%s.%s", photo.Id, photo.Originalformat) - form.Set("format", "json") - form.Set("nojsoncallback", "1") - res, err := oauthClient.Get(im.host.HTTPClient(), im.user.Cred, apiURL, form) + res, err := im.flickrRequest(photo.URL, url.Values{}) + if err != nil { + log.Printf("Flickr importer: Could not fetch %s: %s", photo.URL, err) + return err + } + defer res.Body.Close() + + fileRef, err := schema.WriteFileFromReader(im.host.Target(), filename, res.Body) if err != nil { return err } - if res.StatusCode != http.StatusOK { - return fmt.Errorf("Auth request failed with: %s", res.Status) + photoNode, err := parent.ChildPathObject(filename) + if err != nil { + return err } + if err := photoNode.SetAttr("camliContent", fileRef.String()); err != nil { + return err + } + + if photo.Title != "" { + photoNode.SetAttr("title", photo.Title) + } + if photo.Description.Content != "" { + photoNode.SetAttr("description", photo.Description.Content) + } + + return nil +} + +func (im *imp) getPhotosNode() (*importer.Object, error) { + root, err := im.getRootNode() + if err != nil { + return nil, err + } + + photos, err := root.ChildPathObject("photos") + if err != nil { + return nil, err + } + + if err := photos.SetAttr("title", "Photos"); err != nil { + return nil, err + } + return photos, nil +} + +func (im *imp) getRootNode() (*importer.Object, error) { + root, err := im.host.RootObject() + if err != nil { + return nil, err + } + + if err := root.SetAttr("title", "Flickr Import Root"); err != nil { + return nil, err + } + return root, nil +} + +func (im *imp) flickrAPIRequest(form url.Values, result interface{}) error { + form.Set("format", "json") + form.Set("nojsoncallback", "1") + res, err := im.flickrRequest(apiURL, form) + if err != nil { + return err + } defer res.Body.Close() return json.NewDecoder(res.Body).Decode(result) } + +func (im *imp) flickrRequest(url string, form url.Values) (*http.Response, error) { + if im.user == nil { + return nil, errors.New("Not logged in. Go to /importer-flickr/login.") + } + + res, err := oauthClient.Get(im.host.HTTPClient(), im.user.Cred, url, form) + if err != nil { + return nil, err + } + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Auth request failed with: %s", res.Status) + } + + return res, nil +} diff --git a/pkg/importer/importer.go b/pkg/importer/importer.go index eae6c8378..5f82359da 100644 --- a/pkg/importer/importer.go +++ b/pkg/importer/importer.go @@ -194,6 +194,9 @@ func (o *Object) Attrs(attr string) []string { } func (o *Object) SetAttr(key, value string) error { + if o.Attr("key") == value { + return nil + } _, err := o.h.upload(schema.NewSetAttributeClaim(o.pn, key, value)) if err != nil { return err @@ -211,21 +214,27 @@ func (o *Object) SetAttr(key, value string) error { // from the permanode o, given by the "camliPath:xxxx" attribute, // where xxx is the provided path. func (o *Object) ChildPathObject(path string) (*Object, error) { - if v := o.Attr("camliPath:" + path); v != "" { + attrName := "camliPath:" + path + if v := o.Attr(attrName); v != "" { br, ok := blob.Parse(v) if ok { return o.h.ObjectFromRef(br) } } - // TODO: else, create a new permanode w/ the schema - // package. sign + upload it. See how SetAttr does it above. - // Then once you have the permanode's blobref, call SetAttr - // on this node's "camliPath:foo" to that blobref, and call - // the load path as seen earlier in this function. + childBlobRef, err := o.h.upload(schema.NewUnsignedPermanode()) + if err != nil { + return nil, err + } - log.Printf("TODO: ChildPathObject not implemented") - return nil, errors.New("TODO: ChildPathObject not implemented") + if err := o.SetAttr(attrName, childBlobRef.String()); err != nil { + return nil, err + } + + return &Object{ + h: o.h, + pn: childBlobRef, + }, nil } // RootObject returns the root permanode for this importer account. @@ -254,7 +263,6 @@ func (h *Host) RootObject() (*Object, error) { return nil, fmt.Errorf("Found %d import roots for %q; want 1", len(res.WithAttr), h.imp.Prefix()) } pn := res.WithAttr[0].Permanode - log.Printf("Root permanode of %s is %v", h, pn) return h.ObjectFromRef(pn) }