geocode: add support for openstreetmap geocoding api

doc/geocoding.md

@@ -3,12 +3,12 @@
 Geocoding is the process of converting a location name (like `nyc` or
 `Argentina`) into GPS coordinates and bounding box(es).
 
-Perkeep's location search currently requires Google's Geocoding API,
-which now requires an API key. We do not currently provide a default,
-shared API key to use by default. (We might in the future.)
+Perkeep's location search will use Google's Geocoding API if a key is provided,
+otherwise it falls back to using OpenStreetMap's API.
 
-For now, you need to manually get your own Geocoding API key from Google and place it
-in your Perkeep configuration directory (run `pk env configdir` to
-find your configuration directory) in a file named `google-geocode.key`.
+To use Google's Geocoding API, you need to manually get your own API key from
+Google and place it in your Perkeep configuration directory (run `pk env
+configdir` to find your configuration directory) in a file named
+`google-geocode.key`.
 
-To get the key, see https://developers.google.com/maps/documentation/geocoding/start#get-a-key
+To get the Google API key, see https://developers.google.com/maps/documentation/geocoding/get-api-key

internal/geocode/geocode.go

@@ -24,15 +24,19 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 
 	"perkeep.org/internal/osutil"
+	"perkeep.org/pkg/buildinfo"
 
 	"go4.org/ctxutil"
+	"go4.org/legal"
 	"go4.org/syncutil/singleflight"
 	"go4.org/wkfs"
 	"golang.org/x/net/context/ctxhttp"
@@ -113,7 +117,7 @@ func GetAPIKey() (string, error) {
 	return key, nil
 }
 
-var ErrNoGoogleKey = errors.New("geocode: geocoding is not configured; see https://perkeep.org/doc/geocoding")
+var ErrNoGoogleKey = errors.New("geocode: Google API key not configured, using OpenStreetMap; see https://perkeep.org/doc/geocoding")
 
 // Lookup returns rectangles for the given address. Currently the only
 // implementation is the Google geocoding service.
@@ -130,11 +134,29 @@ func Lookup(ctx context.Context, address string) ([]Rect, error) {
 	}
 
 	key, err := GetAPIKey()
-	if err != nil {
+	if err != nil && err != ErrNoGoogleKey {
 		return nil, err
 	}
 
 	rectsi, err := sf.Do(address, func() (interface{}, error) {
+		if key != "" {
+			return lookupGoogle(ctx, address, key)
+		} else {
+			return lookupOpenStreetMap(ctx, address)
+		}
+	})
+	if err != nil {
+		return nil, err
+	}
+	rects = rectsi.([]Rect)
+
+	mu.Lock()
+	cache[address] = rects
+	mu.Unlock()
+	return rects, nil
+}
+
+func lookupGoogle(ctx context.Context, address string, key string) ([]Rect, error) {
 	// TODO: static data files from OpenStreetMap, Wikipedia, etc?
 	urlStr := "https://maps.googleapis.com/maps/api/geocode/json?address=" + url.QueryEscape(address) + "&sensor=false&key=" + url.QueryEscape(key)
 	res, err := ctxhttp.Get(ctx, ctxutil.Client(ctx), urlStr)
@@ -149,17 +171,7 @@ func Lookup(ctx context.Context, address string) ([]Rect, error) {
 	} else {
 		log.Printf("geocode: Google lookup (%q) = %#v", address, rects)
 	}
-		if err == nil {
-			mu.Lock()
-			cache[address] = rects
-			mu.Unlock()
-		}
 	return rects, err
-	})
-	if err != nil {
-		return nil, err
-	}
-	return rectsi.([]Rect), nil
 }
 
 type googleResTop struct {
@@ -194,3 +206,67 @@ func decodeGoogleResponse(r io.Reader) (rects []Rect, err error) {
 	}
 	return
 }
+
+var openstreetmapUserAgent = fmt.Sprintf("perkeep/%v", buildinfo.Summary())
+
+func lookupOpenStreetMap(ctx context.Context, address string) ([]Rect, error) {
+	// TODO: static data files from OpenStreetMap, Wikipedia, etc?
+	urlStr := "https://nominatim.openstreetmap.org/search?format=json&limit=1&q=" + url.QueryEscape(address)
+	req, err := http.NewRequest("GET", urlStr, nil)
+	if err != nil {
+		log.Printf("geocode: HTTP error doing OpenStreetMap lookup: %v", err)
+		return nil, err
+	}
+	// Nominatim Usage Policy requires a user agent (https://operations.osmfoundation.org/policies/nominatim/)
+	req.Header.Set("User-Agent", openstreetmapUserAgent)
+	res, err := ctxhttp.Do(ctx, ctxutil.Client(ctx), req)
+	if err != nil {
+		log.Printf("geocode: HTTP error doing OpenStreetMap lookup: %v", err)
+		return nil, err
+	}
+	defer res.Body.Close()
+	rects, err := decodeOpenStreetMapResponse(res.Body)
+	if err != nil {
+		log.Printf("geocode: error decoding OpenStreetMap geocode response for %q: %v", address, err)
+	} else {
+		log.Printf("geocode: OpenStreetMap lookup (%q) = %#v", address, rects)
+	}
+	return rects, err
+}
+
+type openstreetmapResult struct {
+	// BoundingBox is encoded as four floats (encoded as strings) in order: SW Lat, NE Lat, SW Long, NE Long
+	BoundingBox []string `json:"boundingbox"`
+}
+
+func decodeOpenStreetMapResponse(r io.Reader) (rects []Rect, err error) {
+	var osmResults []*openstreetmapResult
+	if err := json.NewDecoder(r).Decode(&osmResults); err != nil {
+		return nil, err
+	}
+	for _, res := range osmResults {
+		if len(res.BoundingBox) == 4 {
+			var coords []float64
+			for _, b := range res.BoundingBox {
+				f, err := strconv.ParseFloat(b, 64)
+				if err != nil {
+					return nil, err
+				}
+				coords = append(coords, f)
+			}
+			rect := Rect{
+				NorthEast: LatLong{Lat: coords[1], Long: coords[3]},
+				SouthWest: LatLong{Lat: coords[0], Long: coords[2]},
+			}
+			rects = append(rects, rect)
+		}
+	}
+
+	return
+}
+
+func init() {
+	legal.RegisterLicense(`
+Mapping data and services copyright OpenStreetMap contributors, ODbL 1.0. 
+https://osm.org/copyright.`)
+}

internal/geocode/geocode_test.go

@@ -236,3 +236,57 @@ var googleUSA = `
    "status" : "OK"
 }
 `
+
+func TestDecodeOpenStreetMapResponse(t *testing.T) {
+	tests := []struct {
+		name string
+		res  string
+		want []Rect
+	}{
+		{
+			name: "moscow",
+			res:  openstreetmapMoscow,
+			want: []Rect{
+				{
+					NorthEast: LatLong{pf("55.9577717"), pf("37.9674277")},
+					SouthWest: LatLong{pf("55.4913076"), pf("37.290502")},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		rects, err := decodeOpenStreetMapResponse(strings.NewReader(tt.res))
+		if err != nil {
+			t.Errorf("Decoding %s: %v", tt.name, err)
+			continue
+		}
+		if !reflect.DeepEqual(rects, tt.want) {
+			t.Errorf("Test %s: wrong rects\n Got %#v\nWant %#v", tt.name, rects, tt.want)
+		}
+	}
+}
+
+// https://nominatim.openstreetmap.org/search?format=json&limit=1&q=moscow
+var openstreetmapMoscow = `
+[
+   {
+     "place_id": 282700412,
+     "licence": "Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright",
+     "osm_type": "relation",
+     "osm_id": 2555133,
+     "boundingbox": [
+       "55.4913076",
+       "55.9577717",
+       "37.290502",
+       "37.9674277"
+     ],
+     "lat": "55.7504461",
+     "lon": "37.6174943",
+     "display_name": "Москва, Центральный федеральный округ, Россия",
+     "class": "place",
+     "type": "city",
+     "importance": 0.7908193282833463,
+     "icon": "https://nominatim.openstreetmap.org/ui/mapicons//poi_place_city.p.20.png"
+   }
+ ]
+ `

pkg/server/help.go

@@ -57,6 +57,10 @@ const helpHTML string = `<html>
 
 			<h3>Anything Else?</h3>
 			<p>See the Perkeep <a href='https://perkeep.org/doc/'>online documentation</a> and <a href='https://perkeep.org/community'>community contacts</a>.</p>
+
+			<h3>Attribution</h3>
+			<p>Various mapping data and services <a href="https://osm.org/copyright">copyright OpenStreetMap contributors</a>, ODbL 1.0.</p>
+
 		</body>
 	</html>`
 

server/perkeepd/perkeepd.go

@@ -381,9 +381,9 @@ func checkGeoKey() error {
 	}
 	if env.OnGCE() {
 		keyPath = strings.TrimPrefix(keyPath, "/gcs/")
-		return fmt.Errorf("for location related requests to properly work, you need to create a Google Geocoding API Key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ), and save it in your VM's configuration bucket as: %v", keyPath)
+		return fmt.Errorf("using OpenStreetMap for location related requests. To use the Google Geocoding API, create a key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ) and save it in your VM's configuration bucket as: %v", keyPath)
 	}
-	return fmt.Errorf("for location related requests to properly work, you need to create a Google Geocoding API Key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ), and save it in Perkeep's configuration directory as: %v", keyPath)
+	return fmt.Errorf("using OpenStreetMap for location related requests. To use the Google Geocoding API, create a key (see https://developers.google.com/maps/documentation/geocoding/get-api-key ) and save it in Perkeep's configuration directory as: %v", keyPath)
 }
 
 // main wraps Main so tests (which generate their own func main) can still run Main.