Check maximum manifest size + error flow
This commit is contained in:
parent
c3376e9d3a
commit
ade5f944b3
|
@ -252,6 +252,7 @@ const (
|
||||||
// File Handling Events
|
// File Handling Events
|
||||||
ShareManifest = Type("ShareManifest")
|
ShareManifest = Type("ShareManifest")
|
||||||
ManifestSizeReceived = Type("ManifestSizeReceived")
|
ManifestSizeReceived = Type("ManifestSizeReceived")
|
||||||
|
ManifestError = Type("ManifestError")
|
||||||
ManifestReceived = Type("ManifestReceived")
|
ManifestReceived = Type("ManifestReceived")
|
||||||
ManifestSaved = Type("ManifestSaved")
|
ManifestSaved = Type("ManifestSaved")
|
||||||
FileDownloadProgressUpdate = Type("FileDownloadProgressUpdate")
|
FileDownloadProgressUpdate = Type("FileDownloadProgressUpdate")
|
||||||
|
|
|
@ -842,7 +842,15 @@ func (cp *cwtchPeer) eventHandler() {
|
||||||
if scope == attr.PublicScope {
|
if scope == attr.PublicScope {
|
||||||
if strings.HasSuffix(path, ".manifest.size") {
|
if strings.HasSuffix(path, ".manifest.size") {
|
||||||
fileKey := strings.Replace(path, ".manifest.size", "", 1)
|
fileKey := strings.Replace(path, ".manifest.size", "", 1)
|
||||||
cp.eventBus.Publish(event.NewEvent(event.ManifestSizeReceived, map[event.Field]string{event.FileKey: fileKey, event.ManifestSize: val, event.Handle: onion}))
|
size,err := strconv.Atoi(val)
|
||||||
|
// if size is valid and below the maximum size for a manifest
|
||||||
|
// this is to prevent malicious sharers from using large amounts of memory when distributing
|
||||||
|
// a manifest as we reconstruct this in-memory
|
||||||
|
if err == nil && size < files.MaxManifestSize {
|
||||||
|
cp.eventBus.Publish(event.NewEvent(event.ManifestSizeReceived, map[event.Field]string{event.FileKey: fileKey, event.ManifestSize: val, event.Handle: onion}))
|
||||||
|
} else {
|
||||||
|
cp.eventBus.Publish(event.NewEvent(event.ManifestError, map[event.Field]string{event.FileKey: fileKey, event.Handle: onion}))
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
cp.SetContactAttribute(onion, attr.GetPeerScope(path), val)
|
cp.SetContactAttribute(onion, attr.GetPeerScope(path), val)
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,6 +25,7 @@ type FileSharingSubSystem struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// ShareFile given a file key and a serialized manifest, allow the serialized manifest to be downloaded
|
// ShareFile given a file key and a serialized manifest, allow the serialized manifest to be downloaded
|
||||||
// by Cwtch profiles in possession of the fileKey
|
// by Cwtch profiles in possession of the fileKey
|
||||||
func (fsss *FileSharingSubSystem) ShareFile(fileKey string, serializedManifest string) {
|
func (fsss *FileSharingSubSystem) ShareFile(fileKey string, serializedManifest string) {
|
||||||
|
@ -81,10 +82,12 @@ func (fsss *FileSharingSubSystem) RequestManifestParts(fileKey string) []model.P
|
||||||
for i := 0; i < len(serializedManifest); i += DefaultChunkSize {
|
for i := 0; i < len(serializedManifest); i += DefaultChunkSize {
|
||||||
offset := i * DefaultChunkSize
|
offset := i * DefaultChunkSize
|
||||||
end := (i + 1) * DefaultChunkSize
|
end := (i + 1) * DefaultChunkSize
|
||||||
|
// truncate end
|
||||||
if end > len(serializedManifest) {
|
if end > len(serializedManifest) {
|
||||||
end = len(serializedManifest)
|
end = len(serializedManifest)
|
||||||
}
|
}
|
||||||
chunk := serializedManifest[offset:end]
|
chunk := serializedManifest[offset:end]
|
||||||
|
// request this manifest part
|
||||||
messages = append(messages, model.PeerMessage{
|
messages = append(messages, model.PeerMessage{
|
||||||
Context: event.ContextSendManifest,
|
Context: event.ContextSendManifest,
|
||||||
ID: fmt.Sprintf("%s.%d", fileKey, uint64(i)),
|
ID: fmt.Sprintf("%s.%d", fileKey, uint64(i)),
|
||||||
|
@ -98,7 +101,7 @@ func (fsss *FileSharingSubSystem) RequestManifestParts(fileKey string) []model.P
|
||||||
// ReceiveManifestPart given a manifestKey reconstruct part the manifest from the provided part
|
// ReceiveManifestPart given a manifestKey reconstruct part the manifest from the provided part
|
||||||
func (fsss *FileSharingSubSystem) ReceiveManifestPart(manifestKey string, part []byte) (fileKey string, serializedManifest string) {
|
func (fsss *FileSharingSubSystem) ReceiveManifestPart(manifestKey string, part []byte) (fileKey string, serializedManifest string) {
|
||||||
fileKeyParts := strings.Split(manifestKey, ".")
|
fileKeyParts := strings.Split(manifestKey, ".")
|
||||||
if len(fileKeyParts) == 3 {
|
if len(fileKeyParts) == 3 { // rootHash.nonce.manifestPart
|
||||||
fileKey = fmt.Sprintf("%s.%s", fileKeyParts[0], fileKeyParts[1])
|
fileKey = fmt.Sprintf("%s.%s", fileKeyParts[0], fileKeyParts[1])
|
||||||
log.Debugf("manifest filekey: %s", fileKey)
|
log.Debugf("manifest filekey: %s", fileKey)
|
||||||
manifestPart, err := strconv.Atoi(fileKeyParts[2])
|
manifestPart, err := strconv.Atoi(fileKeyParts[2])
|
||||||
|
@ -136,6 +139,7 @@ func (fsss *FileSharingSubSystem) ReceiveManifestPart(manifestKey string, part [
|
||||||
// ProcessChunkRequest given a fileKey, and a chunk request, compile a set of responses for each requested Chunk
|
// ProcessChunkRequest given a fileKey, and a chunk request, compile a set of responses for each requested Chunk
|
||||||
func (fsss *FileSharingSubSystem) ProcessChunkRequest(fileKey string, serializedChunkRequest []byte) []model.PeerMessage {
|
func (fsss *FileSharingSubSystem) ProcessChunkRequest(fileKey string, serializedChunkRequest []byte) []model.PeerMessage {
|
||||||
log.Debugf("chunk request: %v", fileKey)
|
log.Debugf("chunk request: %v", fileKey)
|
||||||
|
// fileKey is rootHash.nonce
|
||||||
manifestI, exists := fsss.activeShares.Load(fileKey)
|
manifestI, exists := fsss.activeShares.Load(fileKey)
|
||||||
var messages []model.PeerMessage
|
var messages []model.PeerMessage
|
||||||
if exists {
|
if exists {
|
||||||
|
@ -169,7 +173,8 @@ func (fsss *FileSharingSubSystem) ProcessChunk(chunkKey string, chunk []byte) (f
|
||||||
fileKeyParts := strings.Split(chunkKey, ".")
|
fileKeyParts := strings.Split(chunkKey, ".")
|
||||||
downloaded = false
|
downloaded = false
|
||||||
log.Debugf("got chunk for %s", fileKeyParts)
|
log.Debugf("got chunk for %s", fileKeyParts)
|
||||||
if len(fileKeyParts) == 3 {
|
if len(fileKeyParts) == 3 { // fileKey is rootHash.nonce.chunk
|
||||||
|
// recalculate file key
|
||||||
fileKey = fmt.Sprintf("%s.%s", fileKeyParts[0], fileKeyParts[1])
|
fileKey = fmt.Sprintf("%s.%s", fileKeyParts[0], fileKeyParts[1])
|
||||||
derivedChunkID, err := strconv.Atoi(fileKeyParts[2])
|
derivedChunkID, err := strconv.Atoi(fileKeyParts[2])
|
||||||
if err == nil {
|
if err == nil {
|
||||||
|
@ -178,10 +183,15 @@ func (fsss *FileSharingSubSystem) ProcessChunk(chunkKey string, chunk []byte) (f
|
||||||
manifestI, exists := fsss.activeDownloads.Load(fileKey)
|
manifestI, exists := fsss.activeDownloads.Load(fileKey)
|
||||||
if exists {
|
if exists {
|
||||||
manifest := manifestI.(*Manifest)
|
manifest := manifestI.(*Manifest)
|
||||||
log.Debugf("found active manifest %v", manifest)
|
|
||||||
progress, err = manifest.StoreChunk(uint64(chunkID), chunk)
|
|
||||||
totalChunks = uint64(len(manifest.Chunks))
|
totalChunks = uint64(len(manifest.Chunks))
|
||||||
|
log.Debugf("found active manifest %v", manifest)
|
||||||
|
progress, err = manifest.StoreChunk(chunkID, chunk)
|
||||||
log.Debugf("attempts to store chunk %v %v", progress, err)
|
log.Debugf("attempts to store chunk %v %v", progress, err)
|
||||||
|
// malicious contacts who share conversations can share random chunks
|
||||||
|
// these will not match the chunk hash and as such will fail.
|
||||||
|
// at this point we can't differentiate between a malicious chunk and failure to store a
|
||||||
|
// legitimate chunk, so if there is an error we silently drop it and expect the higher level callers (e.g. the ui)
|
||||||
|
//to detect and respond to missing chunks if it detects them..
|
||||||
if err == nil {
|
if err == nil {
|
||||||
if progress == totalChunks {
|
if progress == totalChunks {
|
||||||
if manifest.VerifyFile() == nil {
|
if manifest.VerifyFile() == nil {
|
||||||
|
@ -191,8 +201,6 @@ func (fsss *FileSharingSubSystem) ProcessChunk(chunkKey string, chunk []byte) (f
|
||||||
downloaded = true
|
downloaded = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
// TODO if a chunk fails to save (possibly because its data hash didn't match the manifest), re-request it
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,6 +2,7 @@ package files
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
|
"crypto/sha256"
|
||||||
"crypto/sha512"
|
"crypto/sha512"
|
||||||
"crypto/subtle"
|
"crypto/subtle"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
@ -19,6 +20,13 @@ type Chunk []byte
|
||||||
// DefaultChunkSize is the default value of a manifest chunk
|
// DefaultChunkSize is the default value of a manifest chunk
|
||||||
const DefaultChunkSize = 4096
|
const DefaultChunkSize = 4096
|
||||||
|
|
||||||
|
// MaxManifestSize is the maximum size of a manifest (in DefaultChunkSize)
|
||||||
|
// Because we reconstruct the manifest in memory we have to practically limit this size.
|
||||||
|
// 2622000 * 4096 ~= 10GB using 4096 byte chunks
|
||||||
|
// This makes the actual manifest size ~125Mb which seems reasonable for a 10Gb file.
|
||||||
|
// most file transfers are expected to have manifest that are much smaller.
|
||||||
|
const MaxManifestSize = 2622000
|
||||||
|
|
||||||
// Manifest is a collection of hashes and other metadata needed to reconstruct a file and verify contents given a root hash
|
// Manifest is a collection of hashes and other metadata needed to reconstruct a file and verify contents given a root hash
|
||||||
type Manifest struct {
|
type Manifest struct {
|
||||||
Chunks []Chunk
|
Chunks []Chunk
|
||||||
|
@ -62,7 +70,7 @@ func CreateManifest(path string) (*Manifest, error) {
|
||||||
}
|
}
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
hash := sha512.New()
|
hash := sha256.New()
|
||||||
hash.Write(buf[0:n])
|
hash.Write(buf[0:n])
|
||||||
rootHash.Write(buf[0:n])
|
rootHash.Write(buf[0:n])
|
||||||
chunkHash := hash.Sum(nil)
|
chunkHash := hash.Sum(nil)
|
||||||
|
@ -172,7 +180,7 @@ func (m *Manifest) StoreChunk(id uint64, contents []byte) (uint64, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate the chunk hash
|
// Validate the chunk hash
|
||||||
hash := sha512.New()
|
hash := sha256.New()
|
||||||
hash.Write(contents)
|
hash.Write(contents)
|
||||||
chunkHash := hash.Sum(nil)
|
chunkHash := hash.Sum(nil)
|
||||||
|
|
||||||
|
|
|
@ -1 +1 @@
|
||||||
{"Chunks":["2614xLRT0mPJYSBejVhB+xfFMSe5sM72qSTDBn3KqmmcK89oIwVmLr+2X5K3Wpl2f1oo3byyU86zdJp//dwTVg==","H2GcT8AZcy65Bbov34aVFGini5c4awpAT9Pcj4qamN56RkHQiLdtAtY4Lo7YlD2XQvfGF7bbfTt15EUcY8G2+Q==","YmvYk+5Ds1JB6K5YYZsNCfaMn7Qciql4iWpTxgHMz2a3uTpibGTQPW/ja2B6x7hplbhyCzkqlPS5XZAM4bVxxA==","1WrybcOYtRmU3QaGv/VEjlRSj9rNvFFrB7pw4E2NvN5rDt2zjMfHTIAT69pVHUPm7oRjYPGX16w0rg9CKv7Eow==","lz5h++lfGOodW+/xwpsfQz3HdTqTZYczGqnpkpoCH/6hfUbGRlnOfWeGpovKR9c6azzJf1ciMjHCsE3T7ZiNKA==","gLf58jjGMM/SXpj/1DDk9qae/EBHuXdZ9lPZXybu0/ZNwsPHjf7hqgcVlKQnx8nfRIuy/LR5WTdGZPAklWRtbg==","ujddGeMM9b7uTIqopSW63aDZYsrvHbXluhcpBUECqr2BBxrxvOK+rHFrk8HKk1JjYWyma9UnTmWGUWj5QHGsoQ==","dYzRAfWCgrxgB82+JcqnXLiwa4tKNDb4PzBXuW5uPKhYIuydL/FOJi1SWGrBP1Es+9UmmPnFaFyy8IXVUOLeMg==","Saubl7U9KePb/4VCm2X3yFrNFoO0au93JFu0By1klmHR+3Vz7ayWX3aCdnSwLY3IBdjcsvZv9xQ0IafdSy6JSw==","enA+R1ufZNlsZrwgJb+HQQtuBLlJM0sCzpJ2lhGdTs2yYMBFpVwNo8/Z/1o4OOqUqVnlF4u/A3L7R1lgDn7VYA==","ywykeEMBsuTgLwTXu/4kQc79JDmv0LVZPo+spV0PhrNBX24PNlm7Fo3n7H83N5UMtSpTW7PtvXOyBFd+m1hxQQ==","RFahJ53i1/LWZuAFoa1ey7/qPjEeVyG6L4UQgXfKdfxUj/XoMqDXKAg8SOfwj3GgSsdTmZJ2xEMRN3BqF9Hu8g==","Fdu2XF7GIWeGejgqiMdKZEYG2lZCVBplX6Y2P8VPziAZVnc9PXvl7scLZ7/n0TkaBqLKOgruKPQyQM3ZrfmSVg=="],"FileName":"testdata/cwtch.png","RootHash":"jw7XO7sw20W2p0CxJRyuApRfSOT5kUZNXzYHaFxF3NE2oyXasuX2QpzitxXmArILWxa/dDj7YjX+/pEq3O21/Q==","FileSizeInBytes":51791,"ChunkSizeInBytes":4096}
|
{"Chunks":["BXbFagOrWyDwcsnW+f1O6fddCqJywEISjUrzI31FAE0=","1SZcGk0NSduL093Hh0hZ4WVcx2o6VKgL3kUy2WqmdLY=","R4wwVcR4andJJ0fkXlp/td1ZSjH7xHi3Egh8aloWONA=","TAuI06kog7TYVDSO8AgWprAGY8LSlGBwqZvpgMymhZE=","XQLxqLjiM0qIAeOmGIrZJkyuCEfJ4t+ikgbV1ohudiY=","aXInp/WF58A5/TGkwAwniNvIU2ZlRjVtrpClw0sBcVM=","oSCjcrenQ4+Pix4jtgNCRt40K0kQ41eCumSJO0Gqo/0=","FebZSfHuyVdRWkS8/IaWA6UooEURkf9vPxnqZXKII8g=","tITbm77ca1YmExGzbX4WBP5fAOh4bUzDtceN1VBYcBI=","VJd8rWuMtrZzqobdKam0n6t4Vgo72GcsNRNzMk46PsI=","7ywzxLV44HVk9wz+QQHvvVQJAFkTU6/pHyVFjE0uF40=","PoHUwEoQOSXv8ZpJ9bGeCZqiwY34bXcFcBki2OPxd8o=","eogaSYPKrl0MFEqVP1mwUMczMCcnjjwUmUz/0DsAF48="],"FileName":"testdata/cwtch.png","RootHash":"jw7XO7sw20W2p0CxJRyuApRfSOT5kUZNXzYHaFxF3NE2oyXasuX2QpzitxXmArILWxa/dDj7YjX+/pEq3O21/Q==","FileSizeInBytes":51791,"ChunkSizeInBytes":4096}
|
Loading…
Reference in New Issue