camput: ignore files like .DS_Store with -filenodes

Also made initTrustedCertsOnce a field of the Client
object, as it is a similar change.

http://camlistore.org/issue/104

Change-Id: Iabbd7f06e06d31265f390a23c4bdaac956f856f7

cmd/camput/files.go

@@ -223,6 +223,9 @@ func (c *fileCmd) RunCommand(args []string) error {
 			t.Start()
 			lastPut, err = t.Wait()
 		} else {
+			if up.fileOpts.wantFilePermanode() && up.Client.IsIgnoredFile(filename) {
+				continue
+			}
 			lastPut, err = up.UploadFile(filename)
 		}
 		if handleResult("file", lastPut, err) != nil {
@@ -874,6 +877,9 @@ func (t *TreeUpload) statPath(fullPath string, fi os.FileInfo) (nod *node, err e
 			t.stattedc <- nod
 		}
 	}()
+	if t.up.fileOpts.wantFilePermanode() && t.up.Client.IsIgnoredFile(fullPath) {
+		return nil, nil
+	}
 	if fi == nil {
 		fi, err = t.up.lstat(fullPath)
 		if err != nil {
@@ -905,7 +911,9 @@ func (t *TreeUpload) statPath(fullPath string, fi os.FileInfo) (nod *node, err e
 		if err != nil {
 			return nil, err
 		}
-		n.children = append(n.children, depn)
+		if depn != nil {
+			n.children = append(n.children, depn)
+		}
 	}
 	return n, nil
 }

cmd/camput/init.go

@@ -186,6 +186,7 @@ func (c *initCmd) RunCommand(args []string) error {
 		m["server"] = "http://localhost:3179/"
 		m["selfPubKeyDir"] = blobDir
 		m["auth"] = "localhost"
+		m["ignoredFiles"] = []string{".DS_Store"}
 
 		jsonBytes, err := json.MarshalIndent(m, "", "  ")
 		if err != nil {

config/dev-client-dir/client-config.json

@@ -1,6 +1,6 @@
 {
   "server": ["_env", "${CAMLI_SERVER}", "http://localhost:3179/"],
-
+  "ignoredFiles": [".DS_Store"],
   "auth": ["_env", "${CAMLI_AUTH}" ],
   "selfPubKeyDir": ["_env", "${CAMLI_DEV_KEYBLOBS}" ],
   "secretRing": ["_env", "${CAMLI_SECRET_RING}"],

pkg/client/client.go

@@ -30,6 +30,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"sync"
@@ -74,6 +75,7 @@ type Client struct {
 	httpClient *http.Client
 	haveCache  HaveCache
 
+	initTrustedCertsOnce sync.Once
 	// We define a certificate fingerprint as the 10 digits lowercase prefix
 	// of the SHA1 of the complete certificate (in ASN.1 DER encoding).
 	// It is the same as what 'openssl x509 -fingerprint' shows and what
@@ -89,6 +91,12 @@ type Client struct {
 	// if set, we also skip the check against trustedCerts
 	InsecureTLS bool
 
+	initIgnoredFilesOnce sync.Once
+	// list of files that camput should ignore when using -filenodes.
+	// Defaults to empty, but camput init creates a config with a non
+	// empty list.
+	ignoredFiles []string
+
 	pendStatMu sync.Mutex           // guards pendStat
 	pendStat   map[string][]statReq // blobref -> reqs; for next batch(es)
 
@@ -833,3 +841,16 @@ func (c *Client) UploadPlannedPermanode(key string, sigTime time.Time) (*PutResu
 	}
 	return c.uploadString(signed)
 }
+
+// IsIgnoredFile returns whether the file name in fullpath
+// is in the list of file names that should be ignored when
+// uploading with camput -filenodes.
+func (c *Client) IsIgnoredFile(fullpath string) bool {
+	filename := filepath.Base(fullpath)
+	for _, v := range c.getIgnoredFiles() {
+		if filename == v {
+			return true
+		}
+	}
+	return false
+}

pkg/client/config.go

@@ -271,8 +271,6 @@ func (c *Client) GetBlobFetcher() blob.SeekFetcher {
 // See Client.trustedCerts in client.go
 const trustedCerts = "trustedCerts"
 
-var initTrustedCertsOnce sync.Once
-
 func (c *Client) initTrustedCerts() {
 	if e := os.Getenv("CAMLI_TRUSTED_CERT"); e != "" {
 		c.trustedCerts = []string{e}
@@ -295,6 +293,37 @@ func (c *Client) initTrustedCerts() {
 }
 
 func (c *Client) GetTrustedCerts() []string {
-	initTrustedCertsOnce.Do(func() { c.initTrustedCerts() })
+	c.initTrustedCertsOnce.Do(func() { c.initTrustedCerts() })
 	return c.trustedCerts
 }
+
+// config[ignoredFiles] is the list of files that camput should ignore
+// and not try to upload when using -filenodes.
+// See Client.ignoredFiles in client.go
+const ignoredFiles = "ignoredFiles"
+
+func (c *Client) initIgnoredFiles() {
+	if e := os.Getenv("CAMLI_IGNORED_FILES"); e != "" {
+		c.ignoredFiles = []string{e}
+		return
+	}
+	c.ignoredFiles = []string{}
+	configOnce.Do(parseConfig)
+	val, ok := config[ignoredFiles].([]interface{})
+	if !ok {
+		return
+	}
+	for _, v := range val {
+		ignoredFile, ok := v.(string)
+		if !ok {
+			log.Printf("ignoredFile: was expecting a string, got %T", v)
+			return
+		}
+		c.ignoredFiles = append(c.ignoredFiles, ignoredFile)
+	}
+}
+
+func (c *Client) getIgnoredFiles() []string {
+	c.initIgnoredFilesOnce.Do(func() { c.initIgnoredFiles() })
+	return c.ignoredFiles
+}