diff --git a/event/common.go b/event/common.go index e2deea6..f586685 100644 --- a/event/common.go +++ b/event/common.go @@ -252,6 +252,7 @@ const ( // File Handling Events ShareManifest = Type("ShareManifest") ManifestSizeReceived = Type("ManifestSizeReceived") + ManifestError = Type("ManifestError") ManifestReceived = Type("ManifestReceived") ManifestSaved = Type("ManifestSaved") FileDownloadProgressUpdate = Type("FileDownloadProgressUpdate") diff --git a/peer/cwtch_peer.go b/peer/cwtch_peer.go index 7dd385f..c510ad3 100644 --- a/peer/cwtch_peer.go +++ b/peer/cwtch_peer.go @@ -842,7 +842,15 @@ func (cp *cwtchPeer) eventHandler() { if scope == attr.PublicScope { if strings.HasSuffix(path, ".manifest.size") { fileKey := strings.Replace(path, ".manifest.size", "", 1) - cp.eventBus.Publish(event.NewEvent(event.ManifestSizeReceived, map[event.Field]string{event.FileKey: fileKey, event.ManifestSize: val, event.Handle: onion})) + size,err := strconv.Atoi(val) + // if size is valid and below the maximum size for a manifest + // this is to prevent malicious sharers from using large amounts of memory when distributing + // a manifest as we reconstruct this in-memory + if err == nil && size < files.MaxManifestSize { + cp.eventBus.Publish(event.NewEvent(event.ManifestSizeReceived, map[event.Field]string{event.FileKey: fileKey, event.ManifestSize: val, event.Handle: onion})) + } else { + cp.eventBus.Publish(event.NewEvent(event.ManifestError, map[event.Field]string{event.FileKey: fileKey, event.Handle: onion})) + } } else { cp.SetContactAttribute(onion, attr.GetPeerScope(path), val) } diff --git a/protocol/files/filesharing_subsystem.go b/protocol/files/filesharing_subsystem.go index e900fdd..3380ef7 100644 --- a/protocol/files/filesharing_subsystem.go +++ b/protocol/files/filesharing_subsystem.go @@ -25,6 +25,7 @@ type FileSharingSubSystem struct { } + // ShareFile given a file key and a serialized manifest, allow the serialized manifest to be downloaded // by Cwtch profiles in possession of the fileKey func (fsss *FileSharingSubSystem) ShareFile(fileKey string, serializedManifest string) { @@ -81,10 +82,12 @@ func (fsss *FileSharingSubSystem) RequestManifestParts(fileKey string) []model.P for i := 0; i < len(serializedManifest); i += DefaultChunkSize { offset := i * DefaultChunkSize end := (i + 1) * DefaultChunkSize + // truncate end if end > len(serializedManifest) { end = len(serializedManifest) } chunk := serializedManifest[offset:end] + // request this manifest part messages = append(messages, model.PeerMessage{ Context: event.ContextSendManifest, ID: fmt.Sprintf("%s.%d", fileKey, uint64(i)), @@ -98,7 +101,7 @@ func (fsss *FileSharingSubSystem) RequestManifestParts(fileKey string) []model.P // ReceiveManifestPart given a manifestKey reconstruct part the manifest from the provided part func (fsss *FileSharingSubSystem) ReceiveManifestPart(manifestKey string, part []byte) (fileKey string, serializedManifest string) { fileKeyParts := strings.Split(manifestKey, ".") - if len(fileKeyParts) == 3 { + if len(fileKeyParts) == 3 { // rootHash.nonce.manifestPart fileKey = fmt.Sprintf("%s.%s", fileKeyParts[0], fileKeyParts[1]) log.Debugf("manifest filekey: %s", fileKey) manifestPart, err := strconv.Atoi(fileKeyParts[2]) @@ -136,6 +139,7 @@ func (fsss *FileSharingSubSystem) ReceiveManifestPart(manifestKey string, part [ // ProcessChunkRequest given a fileKey, and a chunk request, compile a set of responses for each requested Chunk func (fsss *FileSharingSubSystem) ProcessChunkRequest(fileKey string, serializedChunkRequest []byte) []model.PeerMessage { log.Debugf("chunk request: %v", fileKey) + // fileKey is rootHash.nonce manifestI, exists := fsss.activeShares.Load(fileKey) var messages []model.PeerMessage if exists { @@ -169,7 +173,8 @@ func (fsss *FileSharingSubSystem) ProcessChunk(chunkKey string, chunk []byte) (f fileKeyParts := strings.Split(chunkKey, ".") downloaded = false log.Debugf("got chunk for %s", fileKeyParts) - if len(fileKeyParts) == 3 { + if len(fileKeyParts) == 3 { // fileKey is rootHash.nonce.chunk + // recalculate file key fileKey = fmt.Sprintf("%s.%s", fileKeyParts[0], fileKeyParts[1]) derivedChunkID, err := strconv.Atoi(fileKeyParts[2]) if err == nil { @@ -178,10 +183,15 @@ func (fsss *FileSharingSubSystem) ProcessChunk(chunkKey string, chunk []byte) (f manifestI, exists := fsss.activeDownloads.Load(fileKey) if exists { manifest := manifestI.(*Manifest) - log.Debugf("found active manifest %v", manifest) - progress, err = manifest.StoreChunk(uint64(chunkID), chunk) totalChunks = uint64(len(manifest.Chunks)) + log.Debugf("found active manifest %v", manifest) + progress, err = manifest.StoreChunk(chunkID, chunk) log.Debugf("attempts to store chunk %v %v", progress, err) + // malicious contacts who share conversations can share random chunks + // these will not match the chunk hash and as such will fail. + // at this point we can't differentiate between a malicious chunk and failure to store a + // legitimate chunk, so if there is an error we silently drop it and expect the higher level callers (e.g. the ui) + //to detect and respond to missing chunks if it detects them.. if err == nil { if progress == totalChunks { if manifest.VerifyFile() == nil { @@ -191,8 +201,6 @@ func (fsss *FileSharingSubSystem) ProcessChunk(chunkKey string, chunk []byte) (f downloaded = true } } - } else { - // TODO if a chunk fails to save (possibly because its data hash didn't match the manifest), re-request it } } } diff --git a/protocol/files/manifest.go b/protocol/files/manifest.go index 67d5fc6..1668a7f 100644 --- a/protocol/files/manifest.go +++ b/protocol/files/manifest.go @@ -2,6 +2,7 @@ package files import ( "bufio" + "crypto/sha256" "crypto/sha512" "crypto/subtle" "encoding/json" @@ -19,6 +20,13 @@ type Chunk []byte // DefaultChunkSize is the default value of a manifest chunk const DefaultChunkSize = 4096 +// MaxManifestSize is the maximum size of a manifest (in DefaultChunkSize) +// Because we reconstruct the manifest in memory we have to practically limit this size. +// 2622000 * 4096 ~= 10GB using 4096 byte chunks +// This makes the actual manifest size ~125Mb which seems reasonable for a 10Gb file. +// most file transfers are expected to have manifest that are much smaller. +const MaxManifestSize = 2622000 + // Manifest is a collection of hashes and other metadata needed to reconstruct a file and verify contents given a root hash type Manifest struct { Chunks []Chunk @@ -62,7 +70,7 @@ func CreateManifest(path string) (*Manifest, error) { } break } - hash := sha512.New() + hash := sha256.New() hash.Write(buf[0:n]) rootHash.Write(buf[0:n]) chunkHash := hash.Sum(nil) @@ -172,7 +180,7 @@ func (m *Manifest) StoreChunk(id uint64, contents []byte) (uint64, error) { } // Validate the chunk hash - hash := sha512.New() + hash := sha256.New() hash.Write(contents) chunkHash := hash.Sum(nil) diff --git a/protocol/files/testdata/cwtch.png.manifest b/protocol/files/testdata/cwtch.png.manifest index 826a093..f8099fc 100644 --- a/protocol/files/testdata/cwtch.png.manifest +++ b/protocol/files/testdata/cwtch.png.manifest @@ -1 +1 @@ -{"Chunks":["2614xLRT0mPJYSBejVhB+xfFMSe5sM72qSTDBn3KqmmcK89oIwVmLr+2X5K3Wpl2f1oo3byyU86zdJp//dwTVg==","H2GcT8AZcy65Bbov34aVFGini5c4awpAT9Pcj4qamN56RkHQiLdtAtY4Lo7YlD2XQvfGF7bbfTt15EUcY8G2+Q==","YmvYk+5Ds1JB6K5YYZsNCfaMn7Qciql4iWpTxgHMz2a3uTpibGTQPW/ja2B6x7hplbhyCzkqlPS5XZAM4bVxxA==","1WrybcOYtRmU3QaGv/VEjlRSj9rNvFFrB7pw4E2NvN5rDt2zjMfHTIAT69pVHUPm7oRjYPGX16w0rg9CKv7Eow==","lz5h++lfGOodW+/xwpsfQz3HdTqTZYczGqnpkpoCH/6hfUbGRlnOfWeGpovKR9c6azzJf1ciMjHCsE3T7ZiNKA==","gLf58jjGMM/SXpj/1DDk9qae/EBHuXdZ9lPZXybu0/ZNwsPHjf7hqgcVlKQnx8nfRIuy/LR5WTdGZPAklWRtbg==","ujddGeMM9b7uTIqopSW63aDZYsrvHbXluhcpBUECqr2BBxrxvOK+rHFrk8HKk1JjYWyma9UnTmWGUWj5QHGsoQ==","dYzRAfWCgrxgB82+JcqnXLiwa4tKNDb4PzBXuW5uPKhYIuydL/FOJi1SWGrBP1Es+9UmmPnFaFyy8IXVUOLeMg==","Saubl7U9KePb/4VCm2X3yFrNFoO0au93JFu0By1klmHR+3Vz7ayWX3aCdnSwLY3IBdjcsvZv9xQ0IafdSy6JSw==","enA+R1ufZNlsZrwgJb+HQQtuBLlJM0sCzpJ2lhGdTs2yYMBFpVwNo8/Z/1o4OOqUqVnlF4u/A3L7R1lgDn7VYA==","ywykeEMBsuTgLwTXu/4kQc79JDmv0LVZPo+spV0PhrNBX24PNlm7Fo3n7H83N5UMtSpTW7PtvXOyBFd+m1hxQQ==","RFahJ53i1/LWZuAFoa1ey7/qPjEeVyG6L4UQgXfKdfxUj/XoMqDXKAg8SOfwj3GgSsdTmZJ2xEMRN3BqF9Hu8g==","Fdu2XF7GIWeGejgqiMdKZEYG2lZCVBplX6Y2P8VPziAZVnc9PXvl7scLZ7/n0TkaBqLKOgruKPQyQM3ZrfmSVg=="],"FileName":"testdata/cwtch.png","RootHash":"jw7XO7sw20W2p0CxJRyuApRfSOT5kUZNXzYHaFxF3NE2oyXasuX2QpzitxXmArILWxa/dDj7YjX+/pEq3O21/Q==","FileSizeInBytes":51791,"ChunkSizeInBytes":4096} \ No newline at end of file +{"Chunks":["BXbFagOrWyDwcsnW+f1O6fddCqJywEISjUrzI31FAE0=","1SZcGk0NSduL093Hh0hZ4WVcx2o6VKgL3kUy2WqmdLY=","R4wwVcR4andJJ0fkXlp/td1ZSjH7xHi3Egh8aloWONA=","TAuI06kog7TYVDSO8AgWprAGY8LSlGBwqZvpgMymhZE=","XQLxqLjiM0qIAeOmGIrZJkyuCEfJ4t+ikgbV1ohudiY=","aXInp/WF58A5/TGkwAwniNvIU2ZlRjVtrpClw0sBcVM=","oSCjcrenQ4+Pix4jtgNCRt40K0kQ41eCumSJO0Gqo/0=","FebZSfHuyVdRWkS8/IaWA6UooEURkf9vPxnqZXKII8g=","tITbm77ca1YmExGzbX4WBP5fAOh4bUzDtceN1VBYcBI=","VJd8rWuMtrZzqobdKam0n6t4Vgo72GcsNRNzMk46PsI=","7ywzxLV44HVk9wz+QQHvvVQJAFkTU6/pHyVFjE0uF40=","PoHUwEoQOSXv8ZpJ9bGeCZqiwY34bXcFcBki2OPxd8o=","eogaSYPKrl0MFEqVP1mwUMczMCcnjjwUmUz/0DsAF48="],"FileName":"testdata/cwtch.png","RootHash":"jw7XO7sw20W2p0CxJRyuApRfSOT5kUZNXzYHaFxF3NE2oyXasuX2QpzitxXmArILWxa/dDj7YjX+/pEq3O21/Q==","FileSizeInBytes":51791,"ChunkSizeInBytes":4096} \ No newline at end of file