From e55ec0897b8fdcbc4df6278fa4b02efa6a2f8bfb Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sat, 12 Jun 2010 14:45:58 -0700 Subject: [PATCH] start of storage server --- camlistored/.gitignore | 3 + camlistored/Makefile | 6 ++ camlistored/README | 1 + camlistored/camlistored.go | 172 +++++++++++++++++++++++++++++++++++++ camlistored/test-put.pl | 20 +++++ 5 files changed, 202 insertions(+) create mode 100644 camlistored/.gitignore create mode 100644 camlistored/Makefile create mode 100644 camlistored/README create mode 100644 camlistored/camlistored.go create mode 100755 camlistored/test-put.pl diff --git a/camlistored/.gitignore b/camlistored/.gitignore new file mode 100644 index 000000000..366e65435 --- /dev/null +++ b/camlistored/.gitignore @@ -0,0 +1,3 @@ +camlistored +camlistored.8 +*~ diff --git a/camlistored/Makefile b/camlistored/Makefile new file mode 100644 index 000000000..870002902 --- /dev/null +++ b/camlistored/Makefile @@ -0,0 +1,6 @@ +camlistored: camlistored.go + 8g camlistored.go + 8l -o camlistored camlistored.8 + +clean: + rm -f camlistored camlistored.8 diff --git a/camlistored/README b/camlistored/README new file mode 100644 index 000000000..f83d497c6 --- /dev/null +++ b/camlistored/README @@ -0,0 +1 @@ +The Camli storage node daemon ("layer 1"). diff --git a/camlistored/camlistored.go b/camlistored/camlistored.go new file mode 100644 index 000000000..71b75d8ea --- /dev/null +++ b/camlistored/camlistored.go @@ -0,0 +1,172 @@ +// Copyright 2010 Brad Fitzpatrick +// +// See LICENSE. + +package main + +import "crypto/sha1" +import "flag" +import "fmt" +import "hash" +import "http" +import "io" +import "io/ioutil" +import "os" +import "regexp" + +var listen *string = flag.String("listen", "0.0.0.0:3179", "host:port to listen on") +var storageRoot *string = flag.String("root", "/tmp/camliroot", "Root directory to store files") + +var sharedSecret string + +var kPutPattern *regexp.Regexp = regexp.MustCompile(`^/camli/(sha1)-([a-f0-9]+)$`) + +func badRequestError(conn *http.Conn, errorMessage string) { + conn.WriteHeader(http.StatusBadRequest) + fmt.Fprintf(conn, "%s\n", errorMessage) +} + +func serverError(conn *http.Conn, err os.Error) { + conn.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(conn, "Server error: %s\n", err) +} + +func handleCamli(conn *http.Conn, req *http.Request) { + if (req.Method == "PUT") { + handlePut(conn, req) + return + } + + badRequestError(conn, "Unsupported method.") +} + +func handlePut(conn *http.Conn, req *http.Request) { + groups := kPutPattern.MatchStrings(req.URL.Path) + if (len(groups) != 3) { + badRequestError(conn, "Malformed PUT URL.") + fmt.Println("PUT URL: ", req.URL.Path) + return + } + + hashFunc := groups[1] + digest := groups[2] + + if (hashFunc == "sha1" && len(digest) != 40) { + badRequestError(conn, "invalid length for sha1 hash") + return + } + + // TODO(bradfitz): authn/authz checks here. + + hashedDirectory := fmt.Sprintf("%s/%s/%s", + *storageRoot, + digest[0:3], + digest[3:6]) + + err := os.MkdirAll(hashedDirectory, 0700) + if err != nil { + serverError(conn, err) + return + } + + fileBaseName := fmt.Sprintf("%s-%s.dat", hashFunc, digest) + + tempFile, err := ioutil.TempFile(hashedDirectory, fileBaseName + ".tmp") + if err != nil { + serverError(conn, err) + return + } + + success := false // set true later + defer func() { + if !success { + fmt.Println("Removing temp file: ", tempFile.Name()) + os.Remove(tempFile.Name()) + } + }(); + + written, err := io.Copy(tempFile, req.Body) + if err != nil { + serverError(conn, err); return + } + if _, err = tempFile.Seek(0, 0); err != nil { + serverError(conn, err); return + } + + var hasher hash.Hash; + switch (hashFunc) { + case "sha1": + hasher = sha1.New(); + break; + } + if (hasher == nil) { + badRequestError(conn, "unsupported hash function.") + return; + } + io.Copy(hasher, tempFile) + if fmt.Sprintf("%x", hasher.Sum()) != digest { + badRequestError(conn, "digest didn't match as declared.") + return; + } + if err = tempFile.Close(); err != nil { + serverError(conn, err); return + } + + fileName := fmt.Sprintf("%s/%s", hashedDirectory, fileBaseName) + if err = os.Rename(tempFile.Name(), fileName); err != nil { + serverError(conn, err); return + } + + stat, err := os.Lstat(fileName) + if err != nil { + serverError(conn, err); return; + } + if !stat.IsRegular() || stat.Size != written { + serverError(conn, os.NewError("Written size didn't match.")) + // Unlink it? Bogus? Naah, better to not lose data. + // We can clean it up later in a GC phase. + return + } + + success = true + fmt.Fprint(conn, "OK") +} + +func HandleRoot(conn *http.Conn, req *http.Request) { + fmt.Fprintf(conn, ` +This is camlistored, a Camlistore storage daemon. +`); +} + +func main() { + flag.Parse() + + sharedSecret = os.Getenv("CAMLI_PASSWORD") + if len(sharedSecret) == 0 { + fmt.Fprintf(os.Stderr, + "No CAMLI_PASSWORD environment variable set.\n") + os.Exit(1) + } + + { + fi, err := os.Stat(*storageRoot) + if err != nil || !fi.IsDirectory() { + fmt.Fprintf(os.Stderr, + "Storage root '%s' doesn't exist or is not a directory.\n", + *storageRoot) + os.Exit(1) + } + } + + mux := http.NewServeMux() + mux.HandleFunc("/", HandleRoot) + mux.HandleFunc("/camli/", handleCamli) + + fmt.Printf("Starting to listen on http://%v/\n", *listen) + err := http.ListenAndServe(*listen, mux) + if err != nil { + fmt.Fprintf(os.Stderr, + "Error in http server: %v\n", err) + os.Exit(1) + } +} diff --git a/camlistored/test-put.pl b/camlistored/test-put.pl new file mode 100755 index 000000000..1c1dd57b6 --- /dev/null +++ b/camlistored/test-put.pl @@ -0,0 +1,20 @@ +#!/usr/bin/perl + +use strict; +my $file = shift; +die "Usage: test-put [base_url]" unless -f $file; +my $sha1 = `sha1sum $file`; +$sha1 =~ s!\s.*!!s; + +my $url = shift; +$url ||= "http://127.0.0.1:3179"; +$url =~ s!/$!!; + +# Bogus sha1: +#$sha1 = "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15"; + +$url .= "/camli/sha1-$sha1"; + +system("curl", "-T", $file, $url) and die "Curl failed."; + +