Refactor storage interface to return io.Reader and Writer instances

Doing so will enable encrypting/decrypting files in a 'pipeline' without
having to write the intermediate results to a file or store them in
their entirety in memory.

This commit also updates the existing storage classes (local and FTP) to
meet the new interface definition.
This commit is contained in:
2013-09-09 22:59:18 -04:00
parent 1254a7fb45
commit e2ae508382
5 changed files with 104 additions and 69 deletions

View File

@ -11,6 +11,7 @@ import (
"errors"
"flag"
"fmt"
"io"
"io/ioutil"
"os"
"os/user"
@ -204,7 +205,19 @@ func ProcessLocalEvent(globals AsinkGlobals, event *asink.Event) {
//upload file to remote storage
StatStartUpload()
err = globals.storage.Put(cachedFilename, event.Hash)
uploadWriteCloser, err := globals.storage.Put(event.Hash)
if err != nil {
panic(err)
}
defer uploadWriteCloser.Close()
uploadFile, err := os.Open(cachedFilename)
if err != nil {
panic(err)
}
defer uploadFile.Close()
_, err = io.Copy(uploadWriteCloser, uploadFile)
StatStopUpload()
if err != nil {
panic(err)
@ -270,9 +283,15 @@ func ProcessRemoteEvent(globals AsinkGlobals, event *asink.Event) {
panic(err) //TODO handle sensibly
}
tmpfilename := outfile.Name()
outfile.Close()
StatStartDownload()
err = globals.storage.Get(tmpfilename, event.Hash)
downloadReadCloser, err := globals.storage.Get(event.Hash)
if err != nil {
panic(err)
}
defer downloadReadCloser.Close()
_, err = io.Copy(outfile, downloadReadCloser)
outfile.Close()
StatStopDownload()
if err != nil {
panic(err) //TODO handle sensibly