Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

importer: linkify blobrefs, and start of automatic recurring importing 

Change-Id: Ia172fe3e1cf762d328ad8a1886f545b4885337b0
This commit is contained in:
Brad Fitzpatrick 2014-05-09 20:28:53 -07:00
parent bf24666080
commit bfede766b3
2 changed files with 49 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
pkg/importer/html.go
View File

@ -23,12 +23,14 @@ import (
"net/http" "net/http"
"strings" "strings"
"time" "time"
"camlistore.org/pkg/blob"
) )
func execTemplate(w http.ResponseWriter, r *http.Request, data interface{}) { func (h *Host) execTemplate(w http.ResponseWriter, r *http.Request, data interface{}) {
tmplName := strings.TrimPrefix(fmt.Sprintf("%T", data), "importer.") tmplName := strings.TrimPrefix(fmt.Sprintf("%T", data), "importer.")
var buf bytes.Buffer var buf bytes.Buffer
err := tmpl.ExecuteTemplate(&buf, tmplName, data) err := h.tmpl.ExecuteTemplate(&buf, tmplName, data)
if err != nil { if err != nil {
http.Error(w, fmt.Sprintf("Error executing template %q: %v", tmplName, err), 500) http.Error(w, fmt.Sprintf("Error executing template %q: %v", tmplName, err), 500)
return return
@ -72,7 +74,11 @@ type acctBody struct {
LastError string LastError string
} }
var tmpl = template.Must(template.New("root").Parse(` var tmpl = template.Must(template.New("root").Funcs(map[string]interface{}{
"bloblink": func(br blob.Ref) string {
panic("should be overridden; this one won't be called")
},
}).Parse(`
{{define "pageTop"}} {{define "pageTop"}}
<html> <html>
<head> <head>
@ -113,7 +119,7 @@ var tmpl = template.Must(template.New("root").Parse(`
{{define "importerBody"}} {{define "importerBody"}}
<p>[<a href="{{.Host.ImporterBaseURL}}">&lt;&lt; Back</a>]</p> <p>[<a href="{{.Host.ImporterBaseURL}}">&lt;&lt; Back</a>]</p>
<ul> <ul>
<li>Importer configuration permanode: {{.Importer.Node.PermanodeRef}}</li> <li>Importer configuration permanode: {{.Importer.Node.PermanodeRef | bloblink}}</li>
<li>Status: {{.Importer.Status}}</li> <li>Status: {{.Importer.Status}}</li>
</ul> </ul>
@ -157,8 +163,8 @@ var tmpl = template.Must(template.New("root").Parse(`
<p>[<a href="./">&lt;&lt; Back</a>]</p> <p>[<a href="./">&lt;&lt; Back</a>]</p>
<ul> <ul>
<li>Account type: {{.AcctType}}</li> <li>Account type: {{.AcctType}}</li>
<li>Account metadata permanode: {{.Acct.AccountObject.PermanodeRef}}</li> <li>Account metadata permanode: {{.Acct.AccountObject.PermanodeRef | bloblink}}</li>
<li>Import root permanode: {{if .Acct.RootObject}}{{.Acct.RootObject.PermanodeRef}}{{else}}(none){{end}}</li> <li>Import root permanode: {{if .Acct.RootObject}}{{.Acct.RootObject.PermanodeRef | bloblink}}{{else}}(none){{end}}</li>
<li>Configured: {{.Acct.IsAccountReady}}</li> <li>Configured: {{.Acct.IsAccountReady}}</li>
<li>Summary: {{.Acct.AccountLinkSummary}}</li> <li>Summary: {{.Acct.AccountLinkSummary}}</li>
<li>Import interval: {{if .Acct.RefreshInterval}}{{.Acct.RefreshInterval}}{{else}}(manual){{end}}</li> <li>Import interval: {{if .Acct.RefreshInterval}}{{.Acct.RefreshInterval}}{{else}}(manual){{end}}</li>

40
pkg/importer/importer.go
View File

@ -111,6 +111,19 @@ func newFromConfig(ld blobserver.Loader, cfg jsonconfig.Obj) (http.Handler, erro
importerBase: ld.BaseURL() + ld.MyPrefix(), importerBase: ld.BaseURL() + ld.MyPrefix(),
imp: make(map[string]*importer), imp: make(map[string]*importer),
} }
var err error
h.tmpl, err = tmpl.Clone()
if err != nil {
return nil, err
}
h.tmpl = h.tmpl.Funcs(map[string]interface{}{
"bloblink": func(br blob.Ref) template.HTML {
if h.uiPrefix == "" {
return template.HTML(br.String())
}
return template.HTML(fmt.Sprintf("<a href=\"%s?b=%s\">%s</a>", h.uiPrefix, br, br))
},
})
for k, impl := range importers { for k, impl := range importers {
h.importers = append(h.importers, k) h.importers = append(h.importers, k)
var clientID, clientSecret string var clientID, clientSecret string
@ -214,6 +227,7 @@ func (rc *RunContext) RootNode() *Object { return rc.ia.root }
// Host is the HTTP handler and state for managing all the importers // Host is the HTTP handler and state for managing all the importers
// linked into the binary, even if they're not configured. // linked into the binary, even if they're not configured.
type Host struct { type Host struct {
tmpl *template.Template
importers []string // sorted; e.g. dummy flickr foursquare picasa twitter importers []string // sorted; e.g. dummy flickr foursquare picasa twitter
imp map[string]*importer imp map[string]*importer
baseURL string baseURL string
@ -221,6 +235,7 @@ type Host struct {
target blobserver.StatReceiver target blobserver.StatReceiver
search *search.Handler search *search.Handler
signer *schema.Signer signer *schema.Signer
uiPrefix string // or empty if no UI handler
// client optionally specifies how to fetch external network // client optionally specifies how to fetch external network
// resources. If nil, http.DefaultClient is used. // resources. If nil, http.DefaultClient is used.
@ -229,6 +244,10 @@ type Host struct {
} }
func (h *Host) InitHandler(hl blobserver.FindHandlerByTyper) error { func (h *Host) InitHandler(hl blobserver.FindHandlerByTyper) error {
if prefix, _, err := hl.FindHandlerByType("ui"); err == nil {
h.uiPrefix = prefix
}
_, handler, err := hl.FindHandlerByType("root") _, handler, err := hl.FindHandlerByType("root")
if err != nil || handler == nil { if err != nil || handler == nil {
return errors.New("importer requires a 'root' handler") return errors.New("importer requires a 'root' handler")
@ -251,6 +270,7 @@ func (h *Host) InitHandler(hl blobserver.FindHandlerByTyper) error {
if h.signer == nil { if h.signer == nil {
return errors.New("importer requires a 'jsonsign' handler") return errors.New("importer requires a 'jsonsign' handler")
} }
go h.startPeriodicImporters()
return nil return nil
} }
@ -299,7 +319,7 @@ func (h *Host) serveImportersRoot(w http.ResponseWriter, r *http.Request) {
for _, v := range h.importers { for _, v := range h.importers {
body.Importers = append(body.Importers, h.imp[v]) body.Importers = append(body.Importers, h.imp[v])
} }
execTemplate(w, r, importersRootPage{ h.execTemplate(w, r, importersRootPage{
Title: "Importers", Title: "Importers",
Body: body, Body: body,
}) })
@ -318,7 +338,7 @@ func (h *Host) serveImporter(w http.ResponseWriter, r *http.Request, imp *import
setup = setuper.AccountSetupHTML(h) setup = setuper.AccountSetupHTML(h)
} }
execTemplate(w, r, importerPage{ h.execTemplate(w, r, importerPage{
Title: "Importer - " + imp.Name(), Title: "Importer - " + imp.Name(),
Body: importerBody{ Body: importerBody{
Host: h, Host: h,
@ -391,6 +411,20 @@ func (h *Host) serveImporterAccount(w http.ResponseWriter, r *http.Request, imp
ia.ServeHTTP(w, r) ia.ServeHTTP(w, r)
} }
func (h *Host) startPeriodicImporters() {
res, err := h.search.Query(&search.SearchQuery{
Expression: "attr:camliNodeType:importerAccount",
Describe: &search.DescribeRequest{
Depth: 1,
},
})
if err != nil {
log.Printf("periodic importer search fail: %v", err)
return
}
log.Printf("TODO: periodic importer search: %#v", res)
}
// BaseURL returns the root of the whole server, without trailing // BaseURL returns the root of the whole server, without trailing
// slash. // slash.
func (h *Host) BaseURL() string { func (h *Host) BaseURL() string {
@ -742,7 +776,7 @@ func (ia *importerAcct) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} else { } else {
title += ia.acct.PermanodeRef().String() title += ia.acct.PermanodeRef().String()
} }
execTemplate(w, r, acctPage{ ia.im.host.execTemplate(w, r, acctPage{
Title: title, Title: title,
Body: body, Body: body,
}) })