Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

geocode: add support for openstreetmap geocoding api

This commit is contained in:
Will Norris 2022-05-03 16:29:24 -07:00 committed by Brad Fitzpatrick
parent ddbeb109b1
commit 5af672029e
5 changed files with 164 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
doc/geocoding.md
View File

@ -3,12 +3,12 @@
Geocoding is the process of converting a location name (like `nyc` or
`Argentina`) into GPS coordinates and bounding box(es).
Perkeep's location search currently requires Google's Geocoding API,
which now requires an API key. We do not currently provide a default,
shared API key to use by default. (We might in the future.)
Perkeep's location search will use Google's Geocoding API if a key is provided,
otherwise it falls back to using OpenStreetMap's API.
For now, you need to manually get your own Geocoding API key from Google and place it
in your Perkeep configuration directory (run `pk env configdir` to
find your configuration directory) in a file named `google-geocode.key`.
To use Google's Geocoding API, you need to manually get your own API key from
Google and place it in your Perkeep configuration directory (run `pk env
configdir` to find your configuration directory) in a file named
`google-geocode.key`.
To get the key, see https://developers.google.com/maps/documentation/geocoding/start#get-a-key
To get the Google API key, see https://developers.google.com/maps/documentation/geocoding/get-api-key

118
internal/geocode/geocode.go
View File

@ -24,15 +24,19 @@ import (
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"perkeep.org/internal/osutil"
"perkeep.org/pkg/buildinfo"
"go4.org/ctxutil"
"go4.org/legal"
"go4.org/syncutil/singleflight"
"go4.org/wkfs"
"golang.org/x/net/context/ctxhttp"
@ -113,7 +117,7 @@ func GetAPIKey() (string, error) {
return key, nil
}
var ErrNoGoogleKey = errors.New("geocode: geocoding is not configured; see https://perkeep.org/doc/geocoding")
var ErrNoGoogleKey = errors.New("geocode: Google API key not configured, using OpenStreetMap; see https://perkeep.org/doc/geocoding")
// Lookup returns rectangles for the given address. Currently the only
// implementation is the Google geocoding service.
@ -130,36 +134,44 @@ func Lookup(ctx context.Context, address string) ([]Rect, error) {
}
key, err := GetAPIKey()
if err != nil {
if err != nil && err != ErrNoGoogleKey {
return nil, err
}
rectsi, err := sf.Do(address, func() (interface{}, error) {
// TODO: static data files from OpenStreetMap, Wikipedia, etc?
urlStr := "https://maps.googleapis.com/maps/api/geocode/json?address=" + url.QueryEscape(address) + "&sensor=false&key=" + url.QueryEscape(key)
res, err := ctxhttp.Get(ctx, ctxutil.Client(ctx), urlStr)
if err != nil {
log.Printf("geocode: HTTP error doing Google lookup: %v", err)
return nil, err
}
defer res.Body.Close()
rects, err := decodeGoogleResponse(res.Body)
if err != nil {
log.Printf("geocode: error decoding Google geocode response for %q: %v", address, err)
if key != "" {
return lookupGoogle(ctx, address, key)
} else {
log.Printf("geocode: Google lookup (%q) = %#v", address, rects)
return lookupOpenStreetMap(ctx, address)
}
if err == nil {
mu.Lock()
cache[address] = rects
mu.Unlock()
}
return rects, err
})
if err != nil {
return nil, err
}
return rectsi.([]Rect), nil
rects = rectsi.([]Rect)
mu.Lock()
cache[address] = rects
mu.Unlock()
return rects, nil
}
func lookupGoogle(ctx context.Context, address string, key string) ([]Rect, error) {
// TODO: static data files from OpenStreetMap, Wikipedia, etc?
urlStr := "https://maps.googleapis.com/maps/api/geocode/json?address=" + url.QueryEscape(address) + "&sensor=false&key=" + url.QueryEscape(key)
res, err := ctxhttp.Get(ctx, ctxutil.Client(ctx), urlStr)
if err != nil {
log.Printf("geocode: HTTP error doing Google lookup: %v", err)
return nil, err
}
defer res.Body.Close()
rects, err := decodeGoogleResponse(res.Body)
if err != nil {
log.Printf("geocode: error decoding Google geocode response for %q: %v", address, err)
} else {
log.Printf("geocode: Google lookup (%q) = %#v", address, rects)
}
return rects, err
}
type googleResTop struct {
@ -194,3 +206,67 @@ func decodeGoogleResponse(r io.Reader) (rects []Rect, err error) {
}
return
}
var openstreetmapUserAgent = fmt.Sprintf("perkeep/%v", buildinfo.Summary())
func lookupOpenStreetMap(ctx context.Context, address string) ([]Rect, error) {
// TODO: static data files from OpenStreetMap, Wikipedia, etc?
urlStr := "https://nominatim.openstreetmap.org/search?format=json&limit=1&q=" + url.QueryEscape(address)
req, err := http.NewRequest("GET", urlStr, nil)
if err != nil {
log.Printf("geocode: HTTP error doing OpenStreetMap lookup: %v", err)
return nil, err
}
// Nominatim Usage Policy requires a user agent (https://operations.osmfoundation.org/policies/nominatim/)
req.Header.Set("User-Agent", openstreetmapUserAgent)
res, err := ctxhttp.Do(ctx, ctxutil.Client(ctx), req)
if err != nil {
log.Printf("geocode: HTTP error doing OpenStreetMap lookup: %v", err)
return nil, err
}
defer res.Body.Close()
rects, err := decodeOpenStreetMapResponse(res.Body)
if err != nil {
log.Printf("geocode: error decoding OpenStreetMap geocode response for %q: %v", address, err)
} else {
log.Printf("geocode: OpenStreetMap lookup (%q) = %#v", address, rects)
}
return rects, err
}
type openstreetmapResult struct {
// BoundingBox is encoded as four floats (encoded as strings) in order: SW Lat, NE Lat, SW Long, NE Long
BoundingBox []string `json:"boundingbox"`
}
func decodeOpenStreetMapResponse(r io.Reader) (rects []Rect, err error) {
var osmResults []*openstreetmapResult
if err := json.NewDecoder(r).Decode(&osmResults); err != nil {
return nil, err
}
for _, res := range osmResults {
if len(res.BoundingBox) == 4 {
var coords []float64
for _, b := range res.BoundingBox {
f, err := strconv.ParseFloat(b, 64)
if err != nil {
return nil, err
}
coords = append(coords, f)
}
rect := Rect{
NorthEast: LatLong{Lat: coords[1], Long: coords[3]},
SouthWest: LatLong{Lat: coords[0], Long: coords[2]},
}
rects = append(rects, rect)
}
}
return
}
func init() {
legal.RegisterLicense(`
Mapping data and services copyright OpenStreetMap contributors, ODbL 1.0.
https://osm.org/copyright.`)
}

54
internal/geocode/geocode_test.go
View File

@ -236,3 +236,57 @@ var googleUSA = `
"status" : "OK"
}
`
func TestDecodeOpenStreetMapResponse(t *testing.T) {
tests := []struct {
name string
res string
want []Rect
}{
{
name: "moscow",
res: openstreetmapMoscow,
want: []Rect{
{
NorthEast: LatLong{pf("55.9577717"), pf("37.9674277")},
SouthWest: LatLong{pf("55.4913076"), pf("37.290502")},
},
},
},
}
for _, tt := range tests {
rects, err := decodeOpenStreetMapResponse(strings.NewReader(tt.res))
if err != nil {
t.Errorf("Decoding %s: %v", tt.name, err)
continue
}
if !reflect.DeepEqual(rects, tt.want) {
t.Errorf("Test %s: wrong rects\n Got %#v\nWant %#v", tt.name, rects, tt.want)
}
}
}
// https://nominatim.openstreetmap.org/search?format=json&limit=1&q=moscow
var openstreetmapMoscow = `
[
{
"place_id": 282700412,
"licence": "Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright",
"osm_type": "relation",
"osm_id": 2555133,
"boundingbox": [
"55.4913076",
"55.9577717",
"37.290502",
"37.9674277"
],
"lat": "55.7504461",
"lon": "37.6174943",
"display_name": "Москва, Центральный федеральный округ, Россия",
"class": "place",
"type": "city",
"importance": 0.7908193282833463,
"icon": "https://nominatim.openstreetmap.org/ui/mapicons//poi_place_city.p.20.png"
}
]
`

4
pkg/server/help.go
View File

@ -57,6 +57,10 @@ const helpHTML string = `<html>
<h3>Anything Else?</h3>
<p>See the Perkeep <a href='https://perkeep.org/doc/'>online documentation</a> and <a href='https://perkeep.org/community'>community contacts</a>.</p>
<h3>Attribution</h3>
<p>Various mapping data and services <a href="https://osm.org/copyright">copyright OpenStreetMap contributors</a>, ODbL 1.0.</p>
</body>
</html>`

4
server/perkeepd/perkeepd.go
View File

@ -381,9 +381,9 @@ func checkGeoKey() error {
}
if env.OnGCE() {
keyPath = strings.TrimPrefix(keyPath, "/gcs/")
return fmt.Errorf("for location related requests to properly work, you need to create a Google Geocoding API Key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ), and save it in your VM's configuration bucket as: %v", keyPath)
return fmt.Errorf("using OpenStreetMap for location related requests. To use the Google Geocoding API, create a key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ) and save it in your VM's configuration bucket as: %v", keyPath)
}
return fmt.Errorf("for location related requests to properly work, you need to create a Google Geocoding API Key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ), and save it in Perkeep's configuration directory as: %v", keyPath)
return fmt.Errorf("using OpenStreetMap for location related requests. To use the Google Geocoding API, create a key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ) and save it in Perkeep's configuration directory as: %v", keyPath)
}
// main wraps Main so tests (which generate their own func main) can still run Main.