geocode: add support for openstreetmap geocoding api

This commit is contained in:
Will Norris 2022-05-03 16:29:24 -07:00 committed by Brad Fitzpatrick
parent ddbeb109b1
commit 5af672029e
5 changed files with 164 additions and 30 deletions

View File

@ -3,12 +3,12 @@
Geocoding is the process of converting a location name (like `nyc` or Geocoding is the process of converting a location name (like `nyc` or
`Argentina`) into GPS coordinates and bounding box(es). `Argentina`) into GPS coordinates and bounding box(es).
Perkeep's location search currently requires Google's Geocoding API, Perkeep's location search will use Google's Geocoding API if a key is provided,
which now requires an API key. We do not currently provide a default, otherwise it falls back to using OpenStreetMap's API.
shared API key to use by default. (We might in the future.)
For now, you need to manually get your own Geocoding API key from Google and place it To use Google's Geocoding API, you need to manually get your own API key from
in your Perkeep configuration directory (run `pk env configdir` to Google and place it in your Perkeep configuration directory (run `pk env
find your configuration directory) in a file named `google-geocode.key`. configdir` to find your configuration directory) in a file named
`google-geocode.key`.
To get the key, see https://developers.google.com/maps/documentation/geocoding/start#get-a-key To get the Google API key, see https://developers.google.com/maps/documentation/geocoding/get-api-key

View File

@ -24,15 +24,19 @@ import (
"fmt" "fmt"
"io" "io"
"log" "log"
"net/http"
"net/url" "net/url"
"os" "os"
"path/filepath" "path/filepath"
"strconv"
"strings" "strings"
"sync" "sync"
"perkeep.org/internal/osutil" "perkeep.org/internal/osutil"
"perkeep.org/pkg/buildinfo"
"go4.org/ctxutil" "go4.org/ctxutil"
"go4.org/legal"
"go4.org/syncutil/singleflight" "go4.org/syncutil/singleflight"
"go4.org/wkfs" "go4.org/wkfs"
"golang.org/x/net/context/ctxhttp" "golang.org/x/net/context/ctxhttp"
@ -113,7 +117,7 @@ func GetAPIKey() (string, error) {
return key, nil return key, nil
} }
var ErrNoGoogleKey = errors.New("geocode: geocoding is not configured; see https://perkeep.org/doc/geocoding") var ErrNoGoogleKey = errors.New("geocode: Google API key not configured, using OpenStreetMap; see https://perkeep.org/doc/geocoding")
// Lookup returns rectangles for the given address. Currently the only // Lookup returns rectangles for the given address. Currently the only
// implementation is the Google geocoding service. // implementation is the Google geocoding service.
@ -130,36 +134,44 @@ func Lookup(ctx context.Context, address string) ([]Rect, error) {
} }
key, err := GetAPIKey() key, err := GetAPIKey()
if err != nil { if err != nil && err != ErrNoGoogleKey {
return nil, err return nil, err
} }
rectsi, err := sf.Do(address, func() (interface{}, error) { rectsi, err := sf.Do(address, func() (interface{}, error) {
// TODO: static data files from OpenStreetMap, Wikipedia, etc? if key != "" {
urlStr := "https://maps.googleapis.com/maps/api/geocode/json?address=" + url.QueryEscape(address) + "&sensor=false&key=" + url.QueryEscape(key) return lookupGoogle(ctx, address, key)
res, err := ctxhttp.Get(ctx, ctxutil.Client(ctx), urlStr)
if err != nil {
log.Printf("geocode: HTTP error doing Google lookup: %v", err)
return nil, err
}
defer res.Body.Close()
rects, err := decodeGoogleResponse(res.Body)
if err != nil {
log.Printf("geocode: error decoding Google geocode response for %q: %v", address, err)
} else { } else {
log.Printf("geocode: Google lookup (%q) = %#v", address, rects) return lookupOpenStreetMap(ctx, address)
} }
if err == nil {
mu.Lock()
cache[address] = rects
mu.Unlock()
}
return rects, err
}) })
if err != nil { if err != nil {
return nil, err return nil, err
} }
return rectsi.([]Rect), nil rects = rectsi.([]Rect)
mu.Lock()
cache[address] = rects
mu.Unlock()
return rects, nil
}
func lookupGoogle(ctx context.Context, address string, key string) ([]Rect, error) {
// TODO: static data files from OpenStreetMap, Wikipedia, etc?
urlStr := "https://maps.googleapis.com/maps/api/geocode/json?address=" + url.QueryEscape(address) + "&sensor=false&key=" + url.QueryEscape(key)
res, err := ctxhttp.Get(ctx, ctxutil.Client(ctx), urlStr)
if err != nil {
log.Printf("geocode: HTTP error doing Google lookup: %v", err)
return nil, err
}
defer res.Body.Close()
rects, err := decodeGoogleResponse(res.Body)
if err != nil {
log.Printf("geocode: error decoding Google geocode response for %q: %v", address, err)
} else {
log.Printf("geocode: Google lookup (%q) = %#v", address, rects)
}
return rects, err
} }
type googleResTop struct { type googleResTop struct {
@ -194,3 +206,67 @@ func decodeGoogleResponse(r io.Reader) (rects []Rect, err error) {
} }
return return
} }
var openstreetmapUserAgent = fmt.Sprintf("perkeep/%v", buildinfo.Summary())
func lookupOpenStreetMap(ctx context.Context, address string) ([]Rect, error) {
// TODO: static data files from OpenStreetMap, Wikipedia, etc?
urlStr := "https://nominatim.openstreetmap.org/search?format=json&limit=1&q=" + url.QueryEscape(address)
req, err := http.NewRequest("GET", urlStr, nil)
if err != nil {
log.Printf("geocode: HTTP error doing OpenStreetMap lookup: %v", err)
return nil, err
}
// Nominatim Usage Policy requires a user agent (https://operations.osmfoundation.org/policies/nominatim/)
req.Header.Set("User-Agent", openstreetmapUserAgent)
res, err := ctxhttp.Do(ctx, ctxutil.Client(ctx), req)
if err != nil {
log.Printf("geocode: HTTP error doing OpenStreetMap lookup: %v", err)
return nil, err
}
defer res.Body.Close()
rects, err := decodeOpenStreetMapResponse(res.Body)
if err != nil {
log.Printf("geocode: error decoding OpenStreetMap geocode response for %q: %v", address, err)
} else {
log.Printf("geocode: OpenStreetMap lookup (%q) = %#v", address, rects)
}
return rects, err
}
type openstreetmapResult struct {
// BoundingBox is encoded as four floats (encoded as strings) in order: SW Lat, NE Lat, SW Long, NE Long
BoundingBox []string `json:"boundingbox"`
}
func decodeOpenStreetMapResponse(r io.Reader) (rects []Rect, err error) {
var osmResults []*openstreetmapResult
if err := json.NewDecoder(r).Decode(&osmResults); err != nil {
return nil, err
}
for _, res := range osmResults {
if len(res.BoundingBox) == 4 {
var coords []float64
for _, b := range res.BoundingBox {
f, err := strconv.ParseFloat(b, 64)
if err != nil {
return nil, err
}
coords = append(coords, f)
}
rect := Rect{
NorthEast: LatLong{Lat: coords[1], Long: coords[3]},
SouthWest: LatLong{Lat: coords[0], Long: coords[2]},
}
rects = append(rects, rect)
}
}
return
}
func init() {
legal.RegisterLicense(`
Mapping data and services copyright OpenStreetMap contributors, ODbL 1.0.
https://osm.org/copyright.`)
}

View File

@ -236,3 +236,57 @@ var googleUSA = `
"status" : "OK" "status" : "OK"
} }
` `
func TestDecodeOpenStreetMapResponse(t *testing.T) {
tests := []struct {
name string
res string
want []Rect
}{
{
name: "moscow",
res: openstreetmapMoscow,
want: []Rect{
{
NorthEast: LatLong{pf("55.9577717"), pf("37.9674277")},
SouthWest: LatLong{pf("55.4913076"), pf("37.290502")},
},
},
},
}
for _, tt := range tests {
rects, err := decodeOpenStreetMapResponse(strings.NewReader(tt.res))
if err != nil {
t.Errorf("Decoding %s: %v", tt.name, err)
continue
}
if !reflect.DeepEqual(rects, tt.want) {
t.Errorf("Test %s: wrong rects\n Got %#v\nWant %#v", tt.name, rects, tt.want)
}
}
}
// https://nominatim.openstreetmap.org/search?format=json&limit=1&q=moscow
var openstreetmapMoscow = `
[
{
"place_id": 282700412,
"licence": "Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright",
"osm_type": "relation",
"osm_id": 2555133,
"boundingbox": [
"55.4913076",
"55.9577717",
"37.290502",
"37.9674277"
],
"lat": "55.7504461",
"lon": "37.6174943",
"display_name": "Москва, Центральный федеральный округ, Россия",
"class": "place",
"type": "city",
"importance": 0.7908193282833463,
"icon": "https://nominatim.openstreetmap.org/ui/mapicons//poi_place_city.p.20.png"
}
]
`

View File

@ -57,6 +57,10 @@ const helpHTML string = `<html>
<h3>Anything Else?</h3> <h3>Anything Else?</h3>
<p>See the Perkeep <a href='https://perkeep.org/doc/'>online documentation</a> and <a href='https://perkeep.org/community'>community contacts</a>.</p> <p>See the Perkeep <a href='https://perkeep.org/doc/'>online documentation</a> and <a href='https://perkeep.org/community'>community contacts</a>.</p>
<h3>Attribution</h3>
<p>Various mapping data and services <a href="https://osm.org/copyright">copyright OpenStreetMap contributors</a>, ODbL 1.0.</p>
</body> </body>
</html>` </html>`

View File

@ -381,9 +381,9 @@ func checkGeoKey() error {
} }
if env.OnGCE() { if env.OnGCE() {
keyPath = strings.TrimPrefix(keyPath, "/gcs/") keyPath = strings.TrimPrefix(keyPath, "/gcs/")
return fmt.Errorf("for location related requests to properly work, you need to create a Google Geocoding API Key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ), and save it in your VM's configuration bucket as: %v", keyPath) return fmt.Errorf("using OpenStreetMap for location related requests. To use the Google Geocoding API, create a key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ) and save it in your VM's configuration bucket as: %v", keyPath)
} }
return fmt.Errorf("for location related requests to properly work, you need to create a Google Geocoding API Key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ), and save it in Perkeep's configuration directory as: %v", keyPath) return fmt.Errorf("using OpenStreetMap for location related requests. To use the Google Geocoding API, create a key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ) and save it in Perkeep's configuration directory as: %v", keyPath)
} }
// main wraps Main so tests (which generate their own func main) can still run Main. // main wraps Main so tests (which generate their own func main) can still run Main.