Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

More status handler HTML+JSON, more sync status. 

Change-Id: I0381853191d5b871af649d102b976e592def791f
This commit is contained in:
Brad Fitzpatrick 2014-03-16 20:13:47 -07:00
parent bfb0d1e8de
commit bf28dd4488
6 changed files with 183 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/blobserver/registry.go
View File

@ -42,6 +42,10 @@ type FindHandlerByTyper interface {
// construction of all handlers), then prefix and handler will
// both be non-nil when err is nil.
FindHandlerByType(handlerType string) (prefix string, handler interface{}, err error)
// AllHandlers returns a map from prefix to handler type, and
// a map from prefix to handler.
AllHandlers() (map[string]string, map[string]interface{})
}
type Loader interface {

26
pkg/server/root.go
View File

@ -28,6 +28,7 @@ import (
"camlistore.org/pkg/auth"
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/buildinfo"
"camlistore.org/pkg/images"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/osutil"
@ -48,6 +49,7 @@ type RootHandler struct {
BlobRoot string
SearchRoot string
statusRoot string
Prefix string // root handler's prefix
Storage blobserver.Storage // of BlobRoot, or nil
@ -78,6 +80,7 @@ func newRootFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handle
SearchRoot: conf.OptionalString("searchRoot", ""),
OwnerName: conf.OptionalString("ownerName", username),
Username: osutil.Username(),
Prefix: ld.MyPrefix(),
}
root.Stealth = conf.OptionalBool("stealth", false)
root.statusRoot = conf.OptionalString("statusRoot", "")
@ -143,16 +146,21 @@ func (rh *RootHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
serveStaticFile(rw, req, Files, "favicon.ico")
return
}
configLink := ""
if auth.IsLocalhost(req) && !isDevServer() {
configLink = "<p>If you're coming from localhost, configure your Camlistore server at <a href='/setup'>/setup</a>.</p>"
f := func(p string, a ...interface{}) {
fmt.Fprintf(rw, p, a...)
}
fmt.Fprintf(rw, "<html><body>This is camlistored, a "+
"<a href='http://camlistore.org'>Camlistore</a> server."+
"%s"+
"<p>To manage your content, access the <a href='/ui/'>/ui/</a>.</p></body></html>\n",
configLink)
f("<html><body><p>This is camlistored (%s), a "+
"<a href='http://camlistore.org'>Camlistore</a> server.</p>", buildinfo.Version())
if auth.IsLocalhost(req) && !isDevServer() {
f("<p>If you're coming from localhost, configure your Camlistore server at <a href='/setup'>/setup</a>.</p>")
}
if rh.ui != nil {
f("<p>To manage your content, access the <a href='%s'>%s</a>.</p>", rh.ui.prefix, rh.ui.prefix)
}
if rh.statusRoot != "" {
f("<p>To view status, see <a href='%s'>%s</a>", rh.statusRoot, rh.statusRoot)
}
fmt.Fprintf(rw, "</body></html>")
}
func isDevServer() bool {

98
pkg/server/status.go
View File

@ -17,16 +17,24 @@ limitations under the License.
package server
import (
"encoding/json"
"fmt"
"html"
"log"
"net/http"
"strings"
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/buildinfo"
"camlistore.org/pkg/httputil"
"camlistore.org/pkg/index"
"camlistore.org/pkg/jsonconfig"
)
// StatusHandler publishes server status information.
type StatusHandler struct {
prefix string
handlerFinder blobserver.FindHandlerByTyper
}
func init() {
@ -37,30 +45,96 @@ func newStatusFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Hand
if err := conf.Validate(); err != nil {
return nil, err
}
return &StatusHandler{}, nil
return &StatusHandler{
prefix: ld.MyPrefix(),
handlerFinder: ld,
}, nil
}
func (sh *StatusHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
suffix := httputil.PathSuffix(req)
if req.Method != "GET" {
http.Error(rw, "Illegal URL.", http.StatusMethodNotAllowed)
if !httputil.IsGet(req) {
http.Error(rw, "Illegal status method.", http.StatusMethodNotAllowed)
return
}
if suffix == "status.json" {
sh.serveStatus(rw, req)
return
switch suffix {
case "status.json":
sh.serveStatusJSON(rw, req)
case "":
sh.serveStatusHTML(rw, req)
default:
http.Error(rw, "Illegal status path.", 404)
}
http.Error(rw, "Illegal URL.", 404)
}
type statusResponse struct {
Version string `json:"version"`
type status struct {
Version string `json:"version"`
Error string `json:"error,omitempty"`
SyncStatus []syncStatus `json:"sync"`
Storage map[string]storageStatus `json:"storage"`
rootPrefix string
}
func (sh *StatusHandler) serveStatus(rw http.ResponseWriter, req *http.Request) {
res := &statusResponse{
type storageStatus struct {
Primary bool `json:"primary,omitempty"`
IsIndex bool `json:"isIndex,omitempty"`
Type string `json:"type"`
ApproxBlobs int `json:"approximateBlobs"`
ApproxBytes int `json:"approximateBytes"`
ImplStatus interface{} `json:"implStatus,omitempty"`
}
func (sh *StatusHandler) currentStatus() *status {
res := &status{
Version: buildinfo.Version(),
Storage: make(map[string]storageStatus),
}
_, hi, err := sh.handlerFinder.FindHandlerByType("root")
if err != nil {
res.Error = fmt.Sprintf("Error finding root handler: %v", err)
return res
}
rh := hi.(*RootHandler)
res.rootPrefix = rh.Prefix
for _, sh := range rh.sync {
res.SyncStatus = append(res.SyncStatus, sh.currentStatus())
}
httputil.ReturnJSON(rw, res)
types, handlers := sh.handlerFinder.AllHandlers()
// Storage
for pfx, typ := range types {
if !strings.HasPrefix(typ, "storage-") {
continue
}
h := handlers[pfx]
_, isIndex := h.(*index.Index)
res.Storage[pfx] = storageStatus{
Type: strings.TrimPrefix(typ, "storage-"),
Primary: pfx == rh.BlobRoot,
IsIndex: isIndex,
}
}
return res
}
func (sh *StatusHandler) serveStatusJSON(rw http.ResponseWriter, req *http.Request) {
httputil.ReturnJSON(rw, sh.currentStatus())
}
func (sh *StatusHandler) serveStatusHTML(rw http.ResponseWriter, req *http.Request) {
st := sh.currentStatus()
f := func(p string, a ...interface{}) {
fmt.Fprintf(rw, p, a...)
}
f("<html><head><title>Status</title></head>")
f("<body><h2>Status</h2>")
f("<p>As JSON: <a href='status.json'>status.json</a>; and the <a href='%s?camli.mode=config'>discovery JSON</a>.</p>", st.rootPrefix)
f("<p>Not yet pretty HTML UI:</p>")
js, err := json.MarshalIndent(st, "", " ")
if err != nil {
log.Printf("JSON marshal error: %v", err)
}
f("<pre>%s</pre>", html.EscapeString(string(js)))
}

80
pkg/server/sync.go
View File

@ -86,9 +86,12 @@ type SyncHandler struct {
totalErrors int64
vshards []string // validation shards. if 0, validation not running
vshardDone int // shards validated
vmissing int64 // missing blobs found during validat
vdestCount int // number of blobs seen on dest during validate
vdestBytes int64 // number of blob bytes seen on dest during validate
vshardErrs []string
vmissing int64 // missing blobs found during validat
vdestCount int // number of blobs seen on dest during validate
vdestBytes int64 // number of blob bytes seen on dest during validate
vsrcCount int // number of blobs seen on src during validate
vsrcBytes int64 // number of blob bytes seen on src during validate
}
var (
@ -234,7 +237,6 @@ func newIdleSyncHandler(fromName, toName string) *SyncHandler {
}
func (sh *SyncHandler) discoveryMap() map[string]interface{} {
// TODO(mpl): more status info
return map[string]interface{}{
"from": sh.fromName,
"to": sh.toName,
@ -242,6 +244,41 @@ func (sh *SyncHandler) discoveryMap() map[string]interface{} {
}
}
// syncStatus is a snapshot of the current status, for display by the
// status handler (status.go) in both JSON and HTML forms.
type syncStatus struct {
sh *SyncHandler
From string `json:"from"`
FromDesc string `json:"fromDesc"`
To string `json:"to"`
ToDesc string `json:"toDesc"`
DestIsIndex bool `json:"destIsIndex,omitempty"`
BlobsToCopy int `json:"blobsToCopy"`
BytesToCopy int64 `json:"bytesToCopy"`
LastCopySecAgo int `json:"lastCopySecondsAgo,omitempty"`
}
func (sh *SyncHandler) currentStatus() syncStatus {
sh.mu.Lock()
defer sh.mu.Unlock()
ago := 0
if !sh.recentCopyTime.IsZero() {
ago = int(time.Now().Sub(sh.recentCopyTime).Seconds())
}
return syncStatus{
sh: sh,
From: sh.fromName,
FromDesc: storageDesc(sh.from),
To: sh.toName,
ToDesc: storageDesc(sh.to),
DestIsIndex: sh.toIndex,
BlobsToCopy: len(sh.needCopy),
BytesToCopy: sh.bytesRemain,
LastCopySecAgo: ago,
}
}
// readQueueToMemory slurps in the pending queue from disk (or
// wherever) to memory. Even with millions of blobs, it's not much
// memory. The point of the persistent queue is to survive restarts if
@ -280,6 +317,8 @@ func (sh *SyncHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
return
}
// TODO: remove this lock and instead just call currentStatus,
// and transition to using that here.
sh.mu.Lock()
defer sh.mu.Unlock()
f := func(p string, a ...interface{}) {
@ -311,7 +350,7 @@ func (sh *SyncHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
f("<h2>Validation</h2>")
if len(sh.vshards) == 0 {
f("Disabled")
f("Validation disabled")
token := xsrftoken.Generate(auth.ProcessRandom(), "user", "runFullValidate")
f("<form method='POST'><input type='hidden' name='mode' value='validate'><input type='hidden' name='token' value='%s'><input type='submit' value='Start validation'></form>", token)
} else {
@ -321,9 +360,14 @@ func (sh *SyncHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
sh.vshardDone,
len(sh.vshards),
100*float64(sh.vshardDone)/float64(len(sh.vshards)))
f("<li>Blobs found missing + fixed: %d</li>", sh.vmissing)
f("<li>Source blobs seen: %d</li>", sh.vsrcCount)
f("<li>Source bytes seen: %d</li>", sh.vsrcBytes)
f("<li>Dest blobs seen: %d</li>", sh.vdestCount)
f("<li>Dest bytes seen: %d</li>", sh.vdestBytes)
f("<li>Blobs found missing + fixed: %d</li>", sh.vmissing)
if len(sh.vshardErrs) > 0 {
f("<li>Validation errors: %s</li>", sh.vshardErrs)
}
f("</ul>")
}
@ -510,11 +554,6 @@ func (sh *SyncHandler) copyBlob(sb blob.SizedRef) (err error) {
sh.copying[br] = cs
sh.mu.Unlock()
if strings.Contains(storageDesc(sh.to), "bradfitz-camlistore-pt") {
//sh.logf("LIES NOT ACTUALLY COPYING")
//return nil
}
if sb.Size > constants.MaxBlobSize {
return fmt.Errorf("blob size %d too large; max blob size is %d", sb.Size, constants.MaxBlobSize)
}
@ -651,12 +690,14 @@ func (sh *SyncHandler) runFullValidation() {
func (sh *SyncHandler) validateShardPrefix(pfx string) (err error) {
defer func() {
if err != nil {
sh.logf("Failed to validate prefix %s: %v", pfx, err)
return
}
sh.mu.Lock()
sh.vshardDone++
if err != nil {
errs := fmt.Sprintf("Failed to validate prefix %s: %v", pfx, err)
sh.logf("%s", errs)
sh.vshardErrs = append(sh.vshardErrs, errs)
} else {
sh.vshardDone++
}
sh.mu.Unlock()
}()
ctx := context.New()
@ -705,12 +746,15 @@ func (sh *SyncHandler) startValidatePrefix(ctx *context.Context, pfx string, doD
if !strings.HasPrefix(sb.Ref.String(), pfx) {
return errNotPrefix
}
sh.mu.Lock()
if doDest {
sh.mu.Lock()
sh.vdestCount++
sh.vdestBytes += int64(sb.Size)
sh.mu.Unlock()
} else {
sh.vsrcCount++
sh.vsrcBytes += int64(sb.Size)
}
sh.mu.Unlock()
return nil
case <-ctx.Done():
return context.ErrCanceled

10
pkg/serverinit/serverinit.go
View File

@ -208,6 +208,16 @@ func (hl *handlerLoader) FindHandlerByType(htype string) (prefix string, handler
return
}
func (hl *handlerLoader) AllHandlers() (types map[string]string, handlers map[string]interface{}) {
types = make(map[string]string)
handlers = make(map[string]interface{})
for pfx, config := range hl.config {
types[pfx] = config.htype
handlers[pfx] = hl.handler[pfx]
}
return
}
func (hl *handlerLoader) setupAll() {
for prefix := range hl.config {
hl.setupHandler(prefix)

4
pkg/test/loader.go
View File

@ -41,6 +41,10 @@ func (ld *Loader) FindHandlerByType(handlerType string) (prefix string, handler
panic("NOIMPL")
}
func (ld *Loader) AllHandlers() (map[string]string, map[string]interface{}) {
panic("NOIMPL")
}
func (ld *Loader) MyPrefix() string {
return "/lies/"
}