From 29c63cc2ab6d37270b37ef333ff4663d198b5065 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 29 Jul 2014 11:44:44 -0700 Subject: [PATCH] picasa, picago: support video, change struct definitions, add start of more tests --- pkg/importer/picasa/picasa.go | 5 +- pkg/importer/picasa/testdata.go | 8 +- .../github.com/tgulacsi/picago/atom.go | 20 +- .../github.com/tgulacsi/picago/atom_test.go | 37 ++++ third_party/github.com/tgulacsi/picago/get.go | 153 ++++++++----- .../picago/testdata/gallery-with-a-video.xml | 206 ++++++++++++++++++ 6 files changed, 361 insertions(+), 68 deletions(-) create mode 100644 third_party/github.com/tgulacsi/picago/testdata/gallery-with-a-video.xml diff --git a/pkg/importer/picasa/picasa.go b/pkg/importer/picasa/picasa.go index d2328b4b8..27a77bd1a 100644 --- a/pkg/importer/picasa/picasa.go +++ b/pkg/importer/picasa/picasa.go @@ -17,9 +17,6 @@ limitations under the License. // Package picasa implements an importer for picasa.com accounts. package picasa -// TODO: videos don't import correctly. it currently imports a still -// preview frame from the video, instead of the video bytes. - import ( "errors" "fmt" @@ -55,7 +52,7 @@ const ( // complete run. Otherwise, if the importer runs to // completion, this version number is recorded on the account // permanode and subsequent importers can stop early. - runCompleteVersion = "1" + runCompleteVersion = "2" ) func init() { diff --git a/pkg/importer/picasa/testdata.go b/pkg/importer/picasa/testdata.go index 3d1d1c252..7839ce5d5 100644 --- a/pkg/importer/picasa/testdata.go +++ b/pkg/importer/picasa/testdata.go @@ -124,7 +124,7 @@ func fakeAlbum(counter int) picago.Entry { author := picago.Author{ Name: "fakeAuthorName", } - media := picago.Media{ + media := &picago.Media{ Description: "fakeAlbumDescription", Keywords: "fakeKeyword1,fakeKeyword2", } @@ -186,11 +186,11 @@ func fakePhotoEntry(photoNbr int, albumNbr int) picago.Entry { URL: "https://camlistore.org/pic/pudgy2.png", Type: "image/png", } - media := picago.Media{ + media := &picago.Media{ Title: "fakePhotoTitle", Description: "fakePhotoDescription", Keywords: "fakeKeyword1,fakeKeyword2", - Content: mediaContent, + Content: []picago.MediaContent{mediaContent}, } // to be consistent, all the pics times should be anterior to their respective albums times. whatever. day := time.Hour * 24 @@ -199,7 +199,7 @@ func fakePhotoEntry(photoNbr int, albumNbr int) picago.Entry { published := created.Add(day) updated := published.Add(day) - exif := picago.Exif{ + exif := &picago.Exif{ FStop: 7.7, Make: "whatisthis?", // not obvious to me, needs doc in picago Model: "potato", diff --git a/third_party/github.com/tgulacsi/picago/atom.go b/third_party/github.com/tgulacsi/picago/atom.go index 887a7f1bb..fe7dafe6b 100644 --- a/third_party/github.com/tgulacsi/picago/atom.go +++ b/third_party/github.com/tgulacsi/picago/atom.go @@ -40,8 +40,8 @@ type Entry struct { Location string `xml:"http://schemas.google.com/photos/2007 location"` NumPhotos int `xml:"numphotos"` Content EntryContent `xml:"content"` - Media Media `xml:"group"` - Exif Exif `xml:"tags"` + Media *Media `xml:"group"` + Exif *Exif `xml:"tags"` Point string `xml:"where>Point>pos"` } @@ -64,15 +64,19 @@ type Link struct { } type Media struct { - Title string `xml:"http://search.yahoo.com/mrss title"` - Description string `xml:"description"` - Keywords string `xml:"keywords"` - Content MediaContent `xml:"content"` + Title string `xml:"http://search.yahoo.com/mrss title"` + Description string `xml:"description"` + Keywords string `xml:"keywords"` + Content []MediaContent `xml:"content"` + Thumbnail []MediaContent `xml:"thumbnail"` } type MediaContent struct { - URL string `xml:"url,attr"` - Type string `xml:"type,attr"` + URL string `xml:"url,attr"` + Type string `xml:"type,attr"` + Width int `xml:"width,attr"` + Height int `xml:"height,attr"` + Medium string `xml:"medium,attr"` // "image" or "video" for Picasa at least } type EntryContent struct { diff --git a/third_party/github.com/tgulacsi/picago/atom_test.go b/third_party/github.com/tgulacsi/picago/atom_test.go index 88d052329..9add57cc8 100644 --- a/third_party/github.com/tgulacsi/picago/atom_test.go +++ b/third_party/github.com/tgulacsi/picago/atom_test.go @@ -6,6 +6,7 @@ package picago import ( "encoding/xml" + "os" "testing" ) @@ -250,3 +251,39 @@ func TestAtom(t *testing.T) { t.Logf("result: %#v", result) } } + +func mustParseAtom(t *testing.T, file string) *Atom { + f, err := os.Open(file) + if err != nil { + t.Fatal(err) + } + defer f.Close() + a := new(Atom) + if err := xml.NewDecoder(f).Decode(a); err != nil { + t.Fatal(err) + } + return a +} + +func TestVideoInGallery(t *testing.T) { + atom := mustParseAtom(t, "testdata/gallery-with-a-video.xml") + if len(atom.Entries) != 3 { + t.Fatalf("num entries = %d; want 3", len(atom.Entries)) + } + p, err := atom.Entries[2].photo() + if err != nil { + t.Fatal(err) + } + if p.Type != "video/mpeg4" { + t.Errorf("type = %q; want video/mpeg4", p.Type) + } + if got, want := p.URL, "https://foo.googlevideo.com/bar.mp4"; got != want { + t.Errorf("URL = %q; want %q", got, want) + } + + for i, ent := range atom.Entries { + t.Logf("%d. Media = %#v", i, ent.Media) + p, _ := ent.photo() + t.Logf("%d. %#v", i, p) + } +} diff --git a/third_party/github.com/tgulacsi/picago/get.go b/third_party/github.com/tgulacsi/picago/get.go index 75715c5ed..fceedc13b 100644 --- a/third_party/github.com/tgulacsi/picago/get.go +++ b/third_party/github.com/tgulacsi/picago/get.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "io/ioutil" - "log" "net/http" neturl "net/url" "os" @@ -26,7 +25,7 @@ const ( userURL = "https://picasaweb.google.com/data/feed/api/user/{userID}/contacts?kind=user" ) -var DebugDir string +var DebugDir = os.Getenv("PICAGO_DEBUG_DIR") type User struct { ID, URI, Name, Thumbnail string @@ -40,13 +39,27 @@ type Album struct { URL string } +// A Photo is a photo (or video) in a Picasaweb (or G+) gallery. type Photo struct { - ID, Title, Summary, Description, Location string - Keywords []string - Published, Updated time.Time - Latitude, Longitude float64 - URL, Type string - Exif Exif + ID, Title, Summary, Description string + Keywords []string + Published, Updated time.Time + + // Latitude and Longitude optionally contain the GPS coordinates + // of the photo. + Latitude, Longitude float64 + + // Location is free-form text describing the location of the + // photo. + Location string + + // URL is the URL of the photo or video. + URL string + + // Type is the Content-Type. + Type string + + Exif *Exif } // Filename returns the filename of the photo (from title or ID + type). @@ -111,16 +124,22 @@ func getAlbums(albums []Album, client *http.Client, url string, startIndex int) break } } + var des string + var kw []string + if entry.Media != nil { + des = entry.Media.Description + kw = strings.Split(entry.Media.Keywords, ",") + } albums = append(albums, Album{ ID: entry.ID, Name: entry.Name, Summary: entry.Summary, Title: entry.Title, - Description: entry.Media.Description, + Description: des, Location: entry.Location, AuthorName: entry.Author.Name, AuthorURI: entry.Author.URI, - Keywords: strings.Split(entry.Media.Keywords, ","), + Keywords: kw, Published: entry.Published, Updated: entry.Updated, URL: albumURL, @@ -162,55 +181,85 @@ func getPhotos(photos []Photo, client *http.Client, url string, startIndex int) if len(feed.Entries) == 0 { return nil, false, nil } - if cap(photos)-len(photos) < len(feed.Entries) { - photos = append(photos, make([]Photo, 0, len(feed.Entries))...) - } for _, entry := range feed.Entries { - var lat, long float64 - i := strings.Index(entry.Point, " ") - if i >= 1 { - lat, err = strconv.ParseFloat(entry.Point[:i], 64) - if err != nil { - log.Printf("cannot parse %q as latitude: %v", entry.Point[:i], err) - } - long, err = strconv.ParseFloat(entry.Point[i+1:], 64) - if err != nil { - log.Printf("cannot parse %q as longitude: %v", entry.Point[i+1:], err) - } + p, err := entry.photo() + if err != nil { + return nil, false, err } - if entry.Point != "" && lat == 0 && long == 0 { - log.Fatalf("point=%q but couldn't parse it as lat/long", entry.Point) - } - url, typ := entry.Content.URL, entry.Content.Type - if url == "" { - url, typ = entry.Media.Content.URL, entry.Media.Content.Type - } - title := entry.Title - if title == "" { - title = entry.Media.Title - } - photos = append(photos, Photo{ - ID: entry.ID, - Exif: entry.Exif, - Summary: entry.Summary, - Title: title, - Description: entry.Media.Description, - Location: entry.Location, - //AuthorName: entry.Author.Name, - //AuthorURI: entry.Author.URI, - Keywords: strings.Split(entry.Media.Keywords, ","), - Published: entry.Published, - Updated: entry.Updated, - URL: url, - Type: typ, - Latitude: lat, - Longitude: long, - }) + photos = append(photos, p) } // startIndex starts with 1, we need to compensate for it. return photos, startIndex+len(feed.Entries) <= feed.NumPhotos, nil } +func (e *Entry) photo() (p Photo, err error) { + var lat, long float64 + i := strings.Index(e.Point, " ") + if i >= 1 { + lat, err = strconv.ParseFloat(e.Point[:i], 64) + if err != nil { + return p, fmt.Errorf("cannot parse %q as latitude: %v", e.Point[:i], err) + } + long, err = strconv.ParseFloat(e.Point[i+1:], 64) + if err != nil { + return p, fmt.Errorf("cannot parse %q as longitude: %v", e.Point[i+1:], err) + } + } + if e.Point != "" && lat == 0 && long == 0 { + return p, fmt.Errorf("point=%q but couldn't parse it as lat/long", e.Point) + } + p = Photo{ + ID: e.ID, + Exif: e.Exif, + Summary: e.Summary, + Title: e.Title, + Location: e.Location, + Published: e.Published, + Updated: e.Updated, + Latitude: lat, + Longitude: long, + } + if e.Media != nil { + p.Keywords = strings.Split(e.Media.Keywords, ",") + p.Description = e.Media.Description + if mc, ok := e.Media.bestContent(); ok { + p.URL, p.Type = mc.URL, mc.Type + } + if p.Title == "" { + p.Title = e.Media.Title + } + } + return p, nil +} + +func (m *Media) bestContent() (ret MediaContent, ok bool) { + // Find largest non-Flash video. + var bestPixels int64 + for _, mc := range m.Content { + thisPixels := int64(mc.Width) * int64(mc.Height) + if mc.Medium == "video" && mc.Type != "application/x-shockwave-flash" && thisPixels > bestPixels { + ret = mc + ok = true + bestPixels = thisPixels + } + } + if ok { + return + } + + // Else, just find largest anything. + bestPixels = 0 + for _, mc := range m.Content { + thisPixels := int64(mc.Width) * int64(mc.Height) + if thisPixels > bestPixels { + ret = mc + ok = true + bestPixels = thisPixels + } + } + return +} + func downloadAndParse(client *http.Client, url string) (*Atom, error) { resp, err := client.Get(url) if err != nil { diff --git a/third_party/github.com/tgulacsi/picago/testdata/gallery-with-a-video.xml b/third_party/github.com/tgulacsi/picago/testdata/gallery-with-a-video.xml new file mode 100644 index 000000000..9fd910600 --- /dev/null +++ b/third_party/github.com/tgulacsi/picago/testdata/gallery-with-a-video.xml @@ -0,0 +1,206 @@ + + + https://picasaweb.google.com/data/feed/api/user/default/albumid/6040139514831220113 + 2014-07-28T23:07:15.698Z + + Biking with Blake + Description is biking up San Bruno mountain. + +And a newline. + protected + https://lh4.googleusercontent.com/-VSf28XLm47g/U9Li4v9QrZE/AAAAAAAAAD0/Zkcd4B_xKl8/s160-c/BikingWithBlake.jpg + + + + + + + + + + Gast Erson + https://picasaweb.google.com/114403741484702971746 + + Picasaweb + 3 + 1 + 1000 + 6040139514831220113 + BikingWithBlake + San Bruno Mt, CA + protected + 1406012400000 + 3 + 1997 + 4037418 + 114403741484702971746 + Gast Erson + true + true + + https://picasaweb.google.com/data/entry/api/user/114403741484702971746/albumid/6040139514831220113/photoid/6040139511962430354 + 2014-07-25T23:06:10.000Z + 2014-07-25T23:06:14.015Z + + fail.png + + + + + + + + + + + 6040139511962430354 + 3 + -1.0 + 6040139514831220113 + only_you + 922 + 392 + 58730 + es-upload-highlights + + 1406329570000 + 19 + true + 0 + shared_group_6040139511962430354 + ALL_RIGHTS_RESERVED + + + 600da24a1bc3ddb90000000000000000 + + + + Gast Erson + + + + + + fail.png + + + + https://picasaweb.google.com/data/entry/api/user/114403741484702971746/albumid/6040139514831220113/photoid/6040140028386335042 + 2014-07-25T23:08:10.000Z + 2014-07-28T23:07:15.698Z + + IMG_2034.JPG + This is a caption + + + + + + + + + + 6040140028386335042 + 16 + 0.0 + 6040139514831220113 + only_you + 1183 + 872 + 689067 + es-pc-add-photos + + 1406256485000 + 61 + true + 1 + shared_group_6040140028386335042 + ALL_RIGHTS_RESERVED + + + 2.2 + Apple + iPhone 5s + 0.001242236 + false + 4.12 + 32 + 1406231285000 + 6e37fb5bf62bde9e0000000000000000 + + + + Gast Erson + This is a caption + + + + + IMG_2034.JPG + + + + 37.6955972 -122.4339361 + + + + + https://picasaweb.google.com/data/entry/api/user/114403741484702971746/albumid/6040139514831220113/photoid/6041225428268790466 + 2014-07-28T21:20:04.000Z + 2014-07-28T22:20:42.270Z + + VID_20140728_141919.mp4 + + + + + + + + + + + 6041225428268790466 + 9 + 1.0 + 6040139514831220113 + only_you + final + + 854 + 480 + 3289621 + pwa + + 1406607562000 + 45 + true + 0 + shared_group_6041225428268790466 + ALL_RIGHTS_RESERVED + + + 78d317eb33e596180000000000000000 + + + + + + + + + Gast Erson + + keyboard, stuff + + + + VID_20140728_141919.mp4 + + + + 37.7447 -122.434 + + + +