mirror of https://github.com/perkeep/perkeep.git
geocode: add support for openstreetmap geocoding api
This commit is contained in:
parent
ddbeb109b1
commit
5af672029e
|
@ -3,12 +3,12 @@
|
||||||
Geocoding is the process of converting a location name (like `nyc` or
|
Geocoding is the process of converting a location name (like `nyc` or
|
||||||
`Argentina`) into GPS coordinates and bounding box(es).
|
`Argentina`) into GPS coordinates and bounding box(es).
|
||||||
|
|
||||||
Perkeep's location search currently requires Google's Geocoding API,
|
Perkeep's location search will use Google's Geocoding API if a key is provided,
|
||||||
which now requires an API key. We do not currently provide a default,
|
otherwise it falls back to using OpenStreetMap's API.
|
||||||
shared API key to use by default. (We might in the future.)
|
|
||||||
|
|
||||||
For now, you need to manually get your own Geocoding API key from Google and place it
|
To use Google's Geocoding API, you need to manually get your own API key from
|
||||||
in your Perkeep configuration directory (run `pk env configdir` to
|
Google and place it in your Perkeep configuration directory (run `pk env
|
||||||
find your configuration directory) in a file named `google-geocode.key`.
|
configdir` to find your configuration directory) in a file named
|
||||||
|
`google-geocode.key`.
|
||||||
|
|
||||||
To get the key, see https://developers.google.com/maps/documentation/geocoding/start#get-a-key
|
To get the Google API key, see https://developers.google.com/maps/documentation/geocoding/get-api-key
|
||||||
|
|
|
@ -24,15 +24,19 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"perkeep.org/internal/osutil"
|
"perkeep.org/internal/osutil"
|
||||||
|
"perkeep.org/pkg/buildinfo"
|
||||||
|
|
||||||
"go4.org/ctxutil"
|
"go4.org/ctxutil"
|
||||||
|
"go4.org/legal"
|
||||||
"go4.org/syncutil/singleflight"
|
"go4.org/syncutil/singleflight"
|
||||||
"go4.org/wkfs"
|
"go4.org/wkfs"
|
||||||
"golang.org/x/net/context/ctxhttp"
|
"golang.org/x/net/context/ctxhttp"
|
||||||
|
@ -113,7 +117,7 @@ func GetAPIKey() (string, error) {
|
||||||
return key, nil
|
return key, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var ErrNoGoogleKey = errors.New("geocode: geocoding is not configured; see https://perkeep.org/doc/geocoding")
|
var ErrNoGoogleKey = errors.New("geocode: Google API key not configured, using OpenStreetMap; see https://perkeep.org/doc/geocoding")
|
||||||
|
|
||||||
// Lookup returns rectangles for the given address. Currently the only
|
// Lookup returns rectangles for the given address. Currently the only
|
||||||
// implementation is the Google geocoding service.
|
// implementation is the Google geocoding service.
|
||||||
|
@ -130,36 +134,44 @@ func Lookup(ctx context.Context, address string) ([]Rect, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
key, err := GetAPIKey()
|
key, err := GetAPIKey()
|
||||||
if err != nil {
|
if err != nil && err != ErrNoGoogleKey {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
rectsi, err := sf.Do(address, func() (interface{}, error) {
|
rectsi, err := sf.Do(address, func() (interface{}, error) {
|
||||||
// TODO: static data files from OpenStreetMap, Wikipedia, etc?
|
if key != "" {
|
||||||
urlStr := "https://maps.googleapis.com/maps/api/geocode/json?address=" + url.QueryEscape(address) + "&sensor=false&key=" + url.QueryEscape(key)
|
return lookupGoogle(ctx, address, key)
|
||||||
res, err := ctxhttp.Get(ctx, ctxutil.Client(ctx), urlStr)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("geocode: HTTP error doing Google lookup: %v", err)
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer res.Body.Close()
|
|
||||||
rects, err := decodeGoogleResponse(res.Body)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("geocode: error decoding Google geocode response for %q: %v", address, err)
|
|
||||||
} else {
|
} else {
|
||||||
log.Printf("geocode: Google lookup (%q) = %#v", address, rects)
|
return lookupOpenStreetMap(ctx, address)
|
||||||
}
|
}
|
||||||
if err == nil {
|
|
||||||
mu.Lock()
|
|
||||||
cache[address] = rects
|
|
||||||
mu.Unlock()
|
|
||||||
}
|
|
||||||
return rects, err
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return rectsi.([]Rect), nil
|
rects = rectsi.([]Rect)
|
||||||
|
|
||||||
|
mu.Lock()
|
||||||
|
cache[address] = rects
|
||||||
|
mu.Unlock()
|
||||||
|
return rects, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func lookupGoogle(ctx context.Context, address string, key string) ([]Rect, error) {
|
||||||
|
// TODO: static data files from OpenStreetMap, Wikipedia, etc?
|
||||||
|
urlStr := "https://maps.googleapis.com/maps/api/geocode/json?address=" + url.QueryEscape(address) + "&sensor=false&key=" + url.QueryEscape(key)
|
||||||
|
res, err := ctxhttp.Get(ctx, ctxutil.Client(ctx), urlStr)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("geocode: HTTP error doing Google lookup: %v", err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer res.Body.Close()
|
||||||
|
rects, err := decodeGoogleResponse(res.Body)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("geocode: error decoding Google geocode response for %q: %v", address, err)
|
||||||
|
} else {
|
||||||
|
log.Printf("geocode: Google lookup (%q) = %#v", address, rects)
|
||||||
|
}
|
||||||
|
return rects, err
|
||||||
}
|
}
|
||||||
|
|
||||||
type googleResTop struct {
|
type googleResTop struct {
|
||||||
|
@ -194,3 +206,67 @@ func decodeGoogleResponse(r io.Reader) (rects []Rect, err error) {
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var openstreetmapUserAgent = fmt.Sprintf("perkeep/%v", buildinfo.Summary())
|
||||||
|
|
||||||
|
func lookupOpenStreetMap(ctx context.Context, address string) ([]Rect, error) {
|
||||||
|
// TODO: static data files from OpenStreetMap, Wikipedia, etc?
|
||||||
|
urlStr := "https://nominatim.openstreetmap.org/search?format=json&limit=1&q=" + url.QueryEscape(address)
|
||||||
|
req, err := http.NewRequest("GET", urlStr, nil)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("geocode: HTTP error doing OpenStreetMap lookup: %v", err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Nominatim Usage Policy requires a user agent (https://operations.osmfoundation.org/policies/nominatim/)
|
||||||
|
req.Header.Set("User-Agent", openstreetmapUserAgent)
|
||||||
|
res, err := ctxhttp.Do(ctx, ctxutil.Client(ctx), req)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("geocode: HTTP error doing OpenStreetMap lookup: %v", err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer res.Body.Close()
|
||||||
|
rects, err := decodeOpenStreetMapResponse(res.Body)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("geocode: error decoding OpenStreetMap geocode response for %q: %v", address, err)
|
||||||
|
} else {
|
||||||
|
log.Printf("geocode: OpenStreetMap lookup (%q) = %#v", address, rects)
|
||||||
|
}
|
||||||
|
return rects, err
|
||||||
|
}
|
||||||
|
|
||||||
|
type openstreetmapResult struct {
|
||||||
|
// BoundingBox is encoded as four floats (encoded as strings) in order: SW Lat, NE Lat, SW Long, NE Long
|
||||||
|
BoundingBox []string `json:"boundingbox"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeOpenStreetMapResponse(r io.Reader) (rects []Rect, err error) {
|
||||||
|
var osmResults []*openstreetmapResult
|
||||||
|
if err := json.NewDecoder(r).Decode(&osmResults); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, res := range osmResults {
|
||||||
|
if len(res.BoundingBox) == 4 {
|
||||||
|
var coords []float64
|
||||||
|
for _, b := range res.BoundingBox {
|
||||||
|
f, err := strconv.ParseFloat(b, 64)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
coords = append(coords, f)
|
||||||
|
}
|
||||||
|
rect := Rect{
|
||||||
|
NorthEast: LatLong{Lat: coords[1], Long: coords[3]},
|
||||||
|
SouthWest: LatLong{Lat: coords[0], Long: coords[2]},
|
||||||
|
}
|
||||||
|
rects = append(rects, rect)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
legal.RegisterLicense(`
|
||||||
|
Mapping data and services copyright OpenStreetMap contributors, ODbL 1.0.
|
||||||
|
https://osm.org/copyright.`)
|
||||||
|
}
|
||||||
|
|
|
@ -236,3 +236,57 @@ var googleUSA = `
|
||||||
"status" : "OK"
|
"status" : "OK"
|
||||||
}
|
}
|
||||||
`
|
`
|
||||||
|
|
||||||
|
func TestDecodeOpenStreetMapResponse(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
res string
|
||||||
|
want []Rect
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "moscow",
|
||||||
|
res: openstreetmapMoscow,
|
||||||
|
want: []Rect{
|
||||||
|
{
|
||||||
|
NorthEast: LatLong{pf("55.9577717"), pf("37.9674277")},
|
||||||
|
SouthWest: LatLong{pf("55.4913076"), pf("37.290502")},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
rects, err := decodeOpenStreetMapResponse(strings.NewReader(tt.res))
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Decoding %s: %v", tt.name, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(rects, tt.want) {
|
||||||
|
t.Errorf("Test %s: wrong rects\n Got %#v\nWant %#v", tt.name, rects, tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// https://nominatim.openstreetmap.org/search?format=json&limit=1&q=moscow
|
||||||
|
var openstreetmapMoscow = `
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"place_id": 282700412,
|
||||||
|
"licence": "Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright",
|
||||||
|
"osm_type": "relation",
|
||||||
|
"osm_id": 2555133,
|
||||||
|
"boundingbox": [
|
||||||
|
"55.4913076",
|
||||||
|
"55.9577717",
|
||||||
|
"37.290502",
|
||||||
|
"37.9674277"
|
||||||
|
],
|
||||||
|
"lat": "55.7504461",
|
||||||
|
"lon": "37.6174943",
|
||||||
|
"display_name": "Москва, Центральный федеральный округ, Россия",
|
||||||
|
"class": "place",
|
||||||
|
"type": "city",
|
||||||
|
"importance": 0.7908193282833463,
|
||||||
|
"icon": "https://nominatim.openstreetmap.org/ui/mapicons//poi_place_city.p.20.png"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
`
|
||||||
|
|
|
@ -57,6 +57,10 @@ const helpHTML string = `<html>
|
||||||
|
|
||||||
<h3>Anything Else?</h3>
|
<h3>Anything Else?</h3>
|
||||||
<p>See the Perkeep <a href='https://perkeep.org/doc/'>online documentation</a> and <a href='https://perkeep.org/community'>community contacts</a>.</p>
|
<p>See the Perkeep <a href='https://perkeep.org/doc/'>online documentation</a> and <a href='https://perkeep.org/community'>community contacts</a>.</p>
|
||||||
|
|
||||||
|
<h3>Attribution</h3>
|
||||||
|
<p>Various mapping data and services <a href="https://osm.org/copyright">copyright OpenStreetMap contributors</a>, ODbL 1.0.</p>
|
||||||
|
|
||||||
</body>
|
</body>
|
||||||
</html>`
|
</html>`
|
||||||
|
|
||||||
|
|
|
@ -381,9 +381,9 @@ func checkGeoKey() error {
|
||||||
}
|
}
|
||||||
if env.OnGCE() {
|
if env.OnGCE() {
|
||||||
keyPath = strings.TrimPrefix(keyPath, "/gcs/")
|
keyPath = strings.TrimPrefix(keyPath, "/gcs/")
|
||||||
return fmt.Errorf("for location related requests to properly work, you need to create a Google Geocoding API Key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ), and save it in your VM's configuration bucket as: %v", keyPath)
|
return fmt.Errorf("using OpenStreetMap for location related requests. To use the Google Geocoding API, create a key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ) and save it in your VM's configuration bucket as: %v", keyPath)
|
||||||
}
|
}
|
||||||
return fmt.Errorf("for location related requests to properly work, you need to create a Google Geocoding API Key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ), and save it in Perkeep's configuration directory as: %v", keyPath)
|
return fmt.Errorf("using OpenStreetMap for location related requests. To use the Google Geocoding API, create a key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ) and save it in Perkeep's configuration directory as: %v", keyPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
// main wraps Main so tests (which generate their own func main) can still run Main.
|
// main wraps Main so tests (which generate their own func main) can still run Main.
|
||||||
|
|
Loading…
Reference in New Issue