Check maximum manifest size + error flow

event/common.go

@@ -252,6 +252,7 @@ const (
 	// File Handling Events
 	ShareManifest              = Type("ShareManifest")
 	ManifestSizeReceived       = Type("ManifestSizeReceived")
+	ManifestError      = Type("ManifestError")
 	ManifestReceived           = Type("ManifestReceived")
 	ManifestSaved              = Type("ManifestSaved")
 	FileDownloadProgressUpdate = Type("FileDownloadProgressUpdate")

peer/cwtch_peer.go

@@ -842,7 +842,15 @@ func (cp *cwtchPeer) eventHandler() {
 				if scope == attr.PublicScope {
 					if strings.HasSuffix(path, ".manifest.size") {
 						fileKey := strings.Replace(path, ".manifest.size", "", 1)
-						cp.eventBus.Publish(event.NewEvent(event.ManifestSizeReceived, map[event.Field]string{event.FileKey: fileKey, event.ManifestSize: val, event.Handle: onion}))
+						size,err := strconv.Atoi(val)
+						// if size is valid and below the maximum size for a manifest
+						// this is to prevent malicious sharers from using large amounts of memory when distributing
+						// a manifest as we reconstruct this in-memory
+						if err == nil && size < files.MaxManifestSize {
+							cp.eventBus.Publish(event.NewEvent(event.ManifestSizeReceived, map[event.Field]string{event.FileKey: fileKey, event.ManifestSize: val, event.Handle: onion}))
+						} else {
+							cp.eventBus.Publish(event.NewEvent(event.ManifestError, map[event.Field]string{event.FileKey: fileKey, event.Handle: onion}))
+						}
 					} else {
 						cp.SetContactAttribute(onion, attr.GetPeerScope(path), val)
 					}

protocol/files/filesharing_subsystem.go

@@ -25,6 +25,7 @@ type FileSharingSubSystem struct {
 }
 
 
+
 // ShareFile given a file key and a serialized manifest, allow the serialized manifest to be downloaded
 // by Cwtch profiles in possession of the fileKey
 func (fsss *FileSharingSubSystem) ShareFile(fileKey string, serializedManifest string) {
@@ -81,10 +82,12 @@ func (fsss *FileSharingSubSystem) RequestManifestParts(fileKey string) []model.P
 		for i := 0; i < len(serializedManifest); i += DefaultChunkSize {
 			offset := i * DefaultChunkSize
 			end := (i + 1) * DefaultChunkSize
+			// truncate end
 			if end > len(serializedManifest) {
 				end = len(serializedManifest)
 			}
 			chunk := serializedManifest[offset:end]
+			// request this manifest part
 			messages = append(messages, model.PeerMessage{
 				Context: event.ContextSendManifest,
 				ID:      fmt.Sprintf("%s.%d", fileKey, uint64(i)),
@@ -98,7 +101,7 @@ func (fsss *FileSharingSubSystem) RequestManifestParts(fileKey string) []model.P
 // ReceiveManifestPart given a manifestKey reconstruct part the manifest from the provided part
 func (fsss *FileSharingSubSystem) ReceiveManifestPart(manifestKey string, part []byte) (fileKey string, serializedManifest string) {
 	fileKeyParts := strings.Split(manifestKey, ".")
-	if len(fileKeyParts) == 3 {
+	if len(fileKeyParts) == 3 { // rootHash.nonce.manifestPart
 		fileKey = fmt.Sprintf("%s.%s", fileKeyParts[0], fileKeyParts[1])
 		log.Debugf("manifest filekey: %s", fileKey)
 		manifestPart, err := strconv.Atoi(fileKeyParts[2])
@@ -136,6 +139,7 @@ func (fsss *FileSharingSubSystem) ReceiveManifestPart(manifestKey string, part [
 // ProcessChunkRequest given a fileKey, and a chunk request, compile a set of responses for each requested Chunk
 func (fsss *FileSharingSubSystem) ProcessChunkRequest(fileKey string, serializedChunkRequest []byte) []model.PeerMessage {
 	log.Debugf("chunk request: %v", fileKey)
+	// fileKey is rootHash.nonce
 	manifestI, exists := fsss.activeShares.Load(fileKey)
 	var messages []model.PeerMessage
 	if exists {
@@ -169,7 +173,8 @@ func (fsss *FileSharingSubSystem) ProcessChunk(chunkKey string, chunk []byte) (f
 	fileKeyParts := strings.Split(chunkKey, ".")
 	downloaded = false
 	log.Debugf("got chunk for %s", fileKeyParts)
-	if len(fileKeyParts) == 3 {
+	if len(fileKeyParts) == 3 { // fileKey is rootHash.nonce.chunk
+		// recalculate file key
 		fileKey = fmt.Sprintf("%s.%s", fileKeyParts[0], fileKeyParts[1])
 		derivedChunkID, err := strconv.Atoi(fileKeyParts[2])
 		if err == nil {
@@ -178,10 +183,15 @@ func (fsss *FileSharingSubSystem) ProcessChunk(chunkKey string, chunk []byte) (f
 			manifestI, exists := fsss.activeDownloads.Load(fileKey)
 			if exists {
 				manifest := manifestI.(*Manifest)
-				log.Debugf("found active manifest %v", manifest)
-				progress, err = manifest.StoreChunk(uint64(chunkID), chunk)
 				totalChunks = uint64(len(manifest.Chunks))
+				log.Debugf("found active manifest %v", manifest)
+				progress, err = manifest.StoreChunk(chunkID, chunk)
 				log.Debugf("attempts to store chunk %v %v", progress, err)
+				// malicious contacts who share conversations can share random chunks
+				// these will not match the chunk hash and as such will fail.
+				// at this point we can't differentiate between a malicious chunk and failure to store a
+				// legitimate chunk, so if there is an error we silently drop it and expect the higher level callers (e.g. the ui)
+				//to detect and respond to missing chunks if it detects them..
 				if err == nil {
 					if progress == totalChunks {
 						if manifest.VerifyFile() == nil {
@@ -191,8 +201,6 @@ func (fsss *FileSharingSubSystem) ProcessChunk(chunkKey string, chunk []byte) (f
 							downloaded = true
 						}
 					}
-				} else {
-					// TODO if a chunk fails to save (possibly because its data hash didn't match the manifest), re-request it
 				}
 			}
 		}

protocol/files/manifest.go

@@ -2,6 +2,7 @@ package files
 
 import (
 	"bufio"
+	"crypto/sha256"
 	"crypto/sha512"
 	"crypto/subtle"
 	"encoding/json"
@@ -19,6 +20,13 @@ type Chunk []byte
 // DefaultChunkSize is the default value of a manifest chunk
 const DefaultChunkSize = 4096
 
+// MaxManifestSize is the maximum size of a manifest (in DefaultChunkSize)
+// Because we reconstruct the manifest in memory we have to practically limit this size.
+// 2622000 * 4096 ~= 10GB using 4096 byte chunks
+// This makes the actual manifest size ~125Mb which seems reasonable for a 10Gb file.
+// most file transfers are expected to have manifest that are much smaller.
+const MaxManifestSize = 2622000
+
 // Manifest is a collection of hashes and other metadata needed to reconstruct a file and verify contents given a root hash
 type Manifest struct {
 	Chunks           []Chunk
@@ -62,7 +70,7 @@ func CreateManifest(path string) (*Manifest, error) {
 			}
 			break
 		}
-		hash := sha512.New()
+		hash := sha256.New()
 		hash.Write(buf[0:n])
 		rootHash.Write(buf[0:n])
 		chunkHash := hash.Sum(nil)
@@ -172,7 +180,7 @@ func (m *Manifest) StoreChunk(id uint64, contents []byte) (uint64, error) {
 	}
 
 	// Validate the chunk hash
-	hash := sha512.New()
+	hash := sha256.New()
 	hash.Write(contents)
 	chunkHash := hash.Sum(nil)
 

protocol/files/testdata/cwtch.png.manifest

@@ -1 +1 @@
-{"Chunks":["2614xLRT0mPJYSBejVhB+xfFMSe5sM72qSTDBn3KqmmcK89oIwVmLr+2X5K3Wpl2f1oo3byyU86zdJp//dwTVg==","H2GcT8AZcy65Bbov34aVFGini5c4awpAT9Pcj4qamN56RkHQiLdtAtY4Lo7YlD2XQvfGF7bbfTt15EUcY8G2+Q==","YmvYk+5Ds1JB6K5YYZsNCfaMn7Qciql4iWpTxgHMz2a3uTpibGTQPW/ja2B6x7hplbhyCzkqlPS5XZAM4bVxxA==","1WrybcOYtRmU3QaGv/VEjlRSj9rNvFFrB7pw4E2NvN5rDt2zjMfHTIAT69pVHUPm7oRjYPGX16w0rg9CKv7Eow==","lz5h++lfGOodW+/xwpsfQz3HdTqTZYczGqnpkpoCH/6hfUbGRlnOfWeGpovKR9c6azzJf1ciMjHCsE3T7ZiNKA==","gLf58jjGMM/SXpj/1DDk9qae/EBHuXdZ9lPZXybu0/ZNwsPHjf7hqgcVlKQnx8nfRIuy/LR5WTdGZPAklWRtbg==","ujddGeMM9b7uTIqopSW63aDZYsrvHbXluhcpBUECqr2BBxrxvOK+rHFrk8HKk1JjYWyma9UnTmWGUWj5QHGsoQ==","dYzRAfWCgrxgB82+JcqnXLiwa4tKNDb4PzBXuW5uPKhYIuydL/FOJi1SWGrBP1Es+9UmmPnFaFyy8IXVUOLeMg==","Saubl7U9KePb/4VCm2X3yFrNFoO0au93JFu0By1klmHR+3Vz7ayWX3aCdnSwLY3IBdjcsvZv9xQ0IafdSy6JSw==","enA+R1ufZNlsZrwgJb+HQQtuBLlJM0sCzpJ2lhGdTs2yYMBFpVwNo8/Z/1o4OOqUqVnlF4u/A3L7R1lgDn7VYA==","ywykeEMBsuTgLwTXu/4kQc79JDmv0LVZPo+spV0PhrNBX24PNlm7Fo3n7H83N5UMtSpTW7PtvXOyBFd+m1hxQQ==","RFahJ53i1/LWZuAFoa1ey7/qPjEeVyG6L4UQgXfKdfxUj/XoMqDXKAg8SOfwj3GgSsdTmZJ2xEMRN3BqF9Hu8g==","Fdu2XF7GIWeGejgqiMdKZEYG2lZCVBplX6Y2P8VPziAZVnc9PXvl7scLZ7/n0TkaBqLKOgruKPQyQM3ZrfmSVg=="],"FileName":"testdata/cwtch.png","RootHash":"jw7XO7sw20W2p0CxJRyuApRfSOT5kUZNXzYHaFxF3NE2oyXasuX2QpzitxXmArILWxa/dDj7YjX+/pEq3O21/Q==","FileSizeInBytes":51791,"ChunkSizeInBytes":4096}
+{"Chunks":["BXbFagOrWyDwcsnW+f1O6fddCqJywEISjUrzI31FAE0=","1SZcGk0NSduL093Hh0hZ4WVcx2o6VKgL3kUy2WqmdLY=","R4wwVcR4andJJ0fkXlp/td1ZSjH7xHi3Egh8aloWONA=","TAuI06kog7TYVDSO8AgWprAGY8LSlGBwqZvpgMymhZE=","XQLxqLjiM0qIAeOmGIrZJkyuCEfJ4t+ikgbV1ohudiY=","aXInp/WF58A5/TGkwAwniNvIU2ZlRjVtrpClw0sBcVM=","oSCjcrenQ4+Pix4jtgNCRt40K0kQ41eCumSJO0Gqo/0=","FebZSfHuyVdRWkS8/IaWA6UooEURkf9vPxnqZXKII8g=","tITbm77ca1YmExGzbX4WBP5fAOh4bUzDtceN1VBYcBI=","VJd8rWuMtrZzqobdKam0n6t4Vgo72GcsNRNzMk46PsI=","7ywzxLV44HVk9wz+QQHvvVQJAFkTU6/pHyVFjE0uF40=","PoHUwEoQOSXv8ZpJ9bGeCZqiwY34bXcFcBki2OPxd8o=","eogaSYPKrl0MFEqVP1mwUMczMCcnjjwUmUz/0DsAF48="],"FileName":"testdata/cwtch.png","RootHash":"jw7XO7sw20W2p0CxJRyuApRfSOT5kUZNXzYHaFxF3NE2oyXasuX2QpzitxXmArILWxa/dDj7YjX+/pEq3O21/Q==","FileSizeInBytes":51791,"ChunkSizeInBytes":4096}