optimization

dav_multiuser_cmd/main.go

@@ -16,6 +16,7 @@ import (
 var KV *diskv.Diskv
 
 func init() {
+	log.SetFlags(log.LstdFlags | log.Lmicroseconds)
 	KV = diskv.New(diskv.Options{
 		BasePath: ".diskv",
 		Transform: func(s string) []string {

ipfs_api/api.go

@@ -4,6 +4,7 @@ import (
 	"log"
 
 	"github.com/ipfs/go-ipfs-api"
+	"github.com/streamrail/concurrent-map"
 	"net/http"
 )
 
@@ -15,6 +16,7 @@ type MyShell struct {
 	shell.Shell
 	Url    string
 	Client *http.Client
+	cache  cmap.ConcurrentMap
 }
 
 func reset() {
@@ -22,6 +24,7 @@ func reset() {
 		sh = &MyShell{
 			Url:    Addr,
 			Client: http.DefaultClient,
+			cache:  cmap.New(),
 		}
 		sh.Shell = *shell.NewShellWithClient(sh.Url, sh.Client)
 		if id, err := sh.ID(); err == nil {

ipfs_api/cached_read.go

@@ -0,0 +1,40 @@
+package ipfs_api
+
+import (
+	"github.com/ipfs/go-ipfs-api"
+	. "github.com/kpmy/ypk/tc"
+	"github.com/mattetti/filebuffer"
+	"io"
+)
+
+func (sh *MyShell) FileList(hash string) (ret *shell.UnixLsObject, err error) {
+	if x, ok := sh.cache.Get(hash); !ok {
+		if ret, err = sh.Shell.FileList(hash); err == nil {
+			sh.cache.Set(hash, ret)
+		}
+	} else {
+		ret = x.(*shell.UnixLsObject)
+	}
+	return
+}
+
+const BufferLimit = 2048
+
+func (sh *MyShell) CacheCat(hash string) (ret io.ReadCloser, err error) {
+	if x, ok := sh.cache.Get(hash); !ok {
+		var old io.ReadCloser
+		if old, err = sh.Shell.Cat(hash); err == nil {
+			buf := filebuffer.New(nil)
+			io.CopyN(buf, old, BufferLimit+1)
+			Assert(buf.Buff.Len() <= BufferLimit, 40, "buffer too large")
+			buf.Seek(0, io.SeekStart)
+			sh.cache.Set(hash, buf)
+			ret = buf
+		}
+	} else {
+		buf := x.(*filebuffer.Buffer)
+		buf.Seek(0, io.SeekStart)
+		ret = buf
+	}
+	return
+}

wdfs/chain.go

@@ -41,13 +41,23 @@ func newChain(root *chain, filepath string) (ret *chain) {
 					}
 				}
 			} else {
-				Halt(100, root.name, err)
+				Halt(100, root.name, " ", err)
 			}
 		}
 	}
 	return
 }
 
+func (root *chain) clone() (ret *chain) {
+	Assert(root.down == nil, 21)
+	ret = &chain{}
+	ret.up = root.up
+	ret.upd = root.upd
+	ret.link = root.link
+	ret.UnixLsObject = root.UnixLsObject
+	return
+}
+
 func (root *chain) tail() (ret *chain) {
 	for ret = root; ret.down != nil; ret = ret.down {
 	}
@@ -91,6 +101,7 @@ func (c *chain) Mode() os.FileMode {
 	} else if c.Type == "File" {
 		return 0
 	}
+	Halt(100, c.Type)
 	panic(100)
 }
 

wdfs/files.go

@@ -70,16 +70,6 @@ func (f *file) Sys() interface{} {
 	return nil
 }
 
-func (f *file) Close() error {
-	if f.wr != nil {
-		close(f.wr)
-		f.wg.Wait()
-	} else if !f.ch.exists() {
-		f.update(nil)
-	}
-	return nil
-}
-
 func (f *file) Read(p []byte) (n int, err error) {
 	if f.links == nil {
 		f.links, _ = ipfs_api.Shell().List(f.ch.Hash)
@@ -145,6 +135,16 @@ func (f *file) Stat() (os.FileInfo, error) {
 
 const emptyFileHash = "QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH"
 
+func (f *file) Close() error {
+	if f.wr != nil {
+		close(f.wr)
+		f.wg.Wait()
+	} else if !f.ch.exists() {
+		f.update(nil)
+	}
+	return nil
+}
+
 func (f *file) update(data io.ReadCloser) {
 	if !fn.IsNil(data) {
 		f.ch.Hash, _ = ipfs_api.Shell().Add(data)
@@ -165,17 +165,34 @@ func (f *file) update(data io.ReadCloser) {
 	head.link.update(head.Hash)
 }
 
+const BufferLimit = 1024 * 128
+
+type ioFile interface {
+	io.Seeker
+	io.ReadCloser
+	io.Writer
+}
+
 func (f *file) Write(p []byte) (n int, err error) {
 	if f.wr == nil {
 		f.wr = make(chan *block, 16)
 		f.wg = new(sync.WaitGroup)
 		f.wg.Add(1)
 		go func(f *file) {
-			tmp, _ := ioutil.TempFile(os.TempDir(), "mipfs")
+			var tmp ioFile
+			buf := filebuffer.New(nil)
+			tmp = buf
 			for w := range f.wr {
 				tmp.Seek(w.pos, io.SeekStart)
 				w.data.Seek(0, io.SeekStart)
 				io.Copy(tmp, w.data)
+				if !fn.IsNil(buf) && buf.Buff.Len() > BufferLimit {
+					tf, _ := ioutil.TempFile(os.TempDir(), "mipfs")
+					buf.Seek(0, io.SeekStart)
+					io.Copy(tf, buf)
+					tmp = tf
+					buf = nil
+				}
 			}
 			tmp.Seek(0, io.SeekStart)
 			f.update(tmp)
@@ -195,7 +212,7 @@ func (f *file) readPropsModel() {
 		ls, _ := ipfs_api.Shell().FileList(f.ch.up.Hash)
 		pm := propLinksMap(ls)
 		if p, ok := pm[f.ch.name]; ok {
-			rd, _ := ipfs_api.Shell().Cat(p.Hash)
+			rd, _ := ipfs_api.Shell().CacheCat(p.Hash)
 			if el, err := dom.Decode(rd); err == nil {
 				f.props = el.Model()
 			} else {

wdfs/locations.go

@@ -68,6 +68,8 @@ func (l *loc) Seek(offset int64, whence int) (int64, error) {
 	return 0, webdav.ErrForbidden
 }
 
+const DetailedPropsLimit = 64
+
 func (l *loc) Readdir(count int) (ret []os.FileInfo, err error) {
 	ls, _ := ipfs_api.Shell().FileList(l.ch.Hash)
 	for _, lo := range ls.Links {
@@ -77,15 +79,19 @@ func (l *loc) Readdir(count int) (ret []os.FileInfo, err error) {
 		case "Directory":
 			if !strings.HasPrefix(lo.Name, "*") {
 				filepath := l.ch.Hash + "/" + lo.Name
-				tail := newChain(l.ch, filepath).tail()
+				tail := newChain(l.ch.clone(), filepath).tail()
 				var fi os.FileInfo
 				if tail.IsDir() {
 					_l := &loc{ch: tail}
-					_l.readPropsModel()
+					if len(ls.Links) <= DetailedPropsLimit {
+						_l.readPropsModel()
+					}
 					fi = _l
 				} else {
 					_f := &file{ch: tail}
-					_f.readPropsModel()
+					if len(ls.Links) <= DetailedPropsLimit {
+						_f.readPropsModel()
+					}
 					fi = _f
 				}
 				ret = append(ret, fi)
@@ -109,7 +115,7 @@ func (l *loc) readPropsModel() {
 	ls, _ := ipfs_api.Shell().FileList(l.ch.Hash)
 	pm := propLinksMap(ls)
 	if p, ok := pm["*"]; ok {
-		rd, _ := ipfs_api.Shell().Cat(p.Hash)
+		rd, _ := ipfs_api.Shell().CacheCat(p.Hash)
 		if el, err := dom.Decode(rd); err == nil {
 			l.props = el.Model()
 		} else {