mirror of
https://github.com/aclindsa/ofxgo.git
synced 2024-10-30 05:20:05 -04:00
0d93a42626
Some financial institutions (*cough* Wells Fargo *cough*) export OFX files as a single line, which is technically valid according to the v1 spec. In order to parse them correctly, `readSGMLHeaders` now uses a regular expression that allows for all whitespace/line breaks to be optionally excluded. A new sample response (wellsfargo.qfx) has been added to document this behaviour.
506 lines
14 KiB
Go
506 lines
14 KiB
Go
package ofxgo
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"reflect"
|
|
"regexp"
|
|
"strings"
|
|
|
|
"github.com/aclindsa/xml"
|
|
)
|
|
|
|
// Response is the top-level object returned from a parsed OFX response file.
|
|
// It can be inspected by using type assertions or switches on the message set
|
|
// you're interested in.
|
|
type Response struct {
|
|
Version ofxVersion // OFX header version
|
|
Signon SignonResponse //<SIGNONMSGSETV1>
|
|
Signup []Message //<SIGNUPMSGSETV1>
|
|
Bank []Message //<BANKMSGSETV1>
|
|
CreditCard []Message //<CREDITCARDMSGSETV1>
|
|
Loan []Message //<LOANMSGSETV1>
|
|
InvStmt []Message //<INVSTMTMSGSETV1>
|
|
InterXfer []Message //<INTERXFERMSGSETV1>
|
|
WireXfer []Message //<WIREXFERMSGSETV1>
|
|
Billpay []Message //<BILLPAYMSGSETV1>
|
|
Email []Message //<EMAILMSGSETV1>
|
|
SecList []Message //<SECLISTMSGSETV1>
|
|
PresDir []Message //<PRESDIRMSGSETV1>
|
|
PresDlv []Message //<PRESDLVMSGSETV1>
|
|
Prof []Message //<PROFMSGSETV1>
|
|
Image []Message //<IMAGEMSGSETV1>
|
|
}
|
|
|
|
func (or *Response) readSGMLHeaders(r *bufio.Reader) error {
|
|
b, err := r.ReadSlice('<')
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
s := string(b)
|
|
err = r.UnreadByte()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// According to the latest OFX SGML spec (1.6), headers should be CRLF-separated
|
|
// and written as KEY:VALUE. However, some banks include a whitespace after the
|
|
// colon (KEY: VALUE), while others include no line breaks at all. The spec doesn't
|
|
// require a line break after the OFX headers, but it is allowed, and will be
|
|
// optionally captured & discarded by the trailing `\s*`. Valid SGML headers must
|
|
// always be present in exactly this order, so a regular expression is acceptable.
|
|
headerExp := regexp.MustCompile(
|
|
`OFXHEADER:\s*(?P<OFXHEADER>\d+)\s*` +
|
|
`DATA:\s*(?P<DATA>[A-Z]+)\s*` +
|
|
`VERSION:\s*(?P<VERSION>\d+)\s*` +
|
|
`SECURITY:\s*(?P<SECURITY>[\w]+)\s*` +
|
|
`ENCODING:\s*(?P<ENCODING>[A-Z0-9-]+)\s*` +
|
|
`CHARSET:\s*(?P<CHARSET>[\w-]+)\s*` +
|
|
`COMPRESSION:\s*(?P<COMPRESSION>[A-Z]+)\s*` +
|
|
`OLDFILEUID:\s*(?P<OLDFILEUID>[\w-]+)\s*` +
|
|
`NEWFILEUID:\s*(?P<NEWFILEUID>[\w-]+)\s*`)
|
|
|
|
matches := headerExp.FindStringSubmatch(s)
|
|
if len(matches) == 0 {
|
|
return errors.New("OFX headers malformed")
|
|
}
|
|
|
|
for i, name := range headerExp.SubexpNames() {
|
|
if i == 0 {
|
|
continue
|
|
}
|
|
|
|
headerValue := matches[i]
|
|
switch name {
|
|
case "OFXHEADER":
|
|
if headerValue != "100" {
|
|
return errors.New("OFXHEADER is not 100")
|
|
}
|
|
case "DATA":
|
|
if headerValue != "OFXSGML" {
|
|
return errors.New("OFX DATA header does not contain OFXSGML")
|
|
}
|
|
case "VERSION":
|
|
err := or.Version.FromString(headerValue)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if or.Version > OfxVersion160 {
|
|
return errors.New("OFX VERSION > 160 in SGML header")
|
|
}
|
|
case "SECURITY":
|
|
if headerValue != "NONE" {
|
|
return errors.New("OFX SECURITY header not NONE")
|
|
}
|
|
case "COMPRESSION":
|
|
if headerValue != "NONE" {
|
|
return errors.New("OFX COMPRESSION header not NONE")
|
|
}
|
|
case "ENCODING", "CHARSET", "OLDFILEUID", "NEWFILEUID":
|
|
// TODO: check/handle these headers?
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (or *Response) readXMLHeaders(decoder *xml.Decoder) error {
|
|
var tok xml.Token
|
|
tok, err := nextNonWhitespaceToken(decoder)
|
|
if err != nil {
|
|
return err
|
|
} else if xmlElem, ok := tok.(xml.ProcInst); !ok || xmlElem.Target != "xml" {
|
|
return errors.New("Missing xml processing instruction")
|
|
}
|
|
|
|
// parse the OFX header
|
|
tok, err = nextNonWhitespaceToken(decoder)
|
|
if err != nil {
|
|
return err
|
|
} else if ofxElem, ok := tok.(xml.ProcInst); ok && ofxElem.Target == "OFX" {
|
|
var seenHeader, seenVersion bool = false, false
|
|
|
|
headers := bytes.TrimSpace(ofxElem.Inst)
|
|
for len(headers) > 0 {
|
|
tmp := bytes.SplitN(headers, []byte("=\""), 2)
|
|
if len(tmp) != 2 {
|
|
return errors.New("Malformed OFX header")
|
|
}
|
|
header := string(tmp[0])
|
|
headers = tmp[1]
|
|
tmp = bytes.SplitN(headers, []byte("\""), 2)
|
|
if len(tmp) != 2 {
|
|
return errors.New("Malformed OFX header")
|
|
}
|
|
value := string(tmp[0])
|
|
headers = bytes.TrimSpace(tmp[1])
|
|
|
|
switch header {
|
|
case "OFXHEADER":
|
|
if value != "200" {
|
|
return errors.New("OFXHEADER is not 200")
|
|
}
|
|
seenHeader = true
|
|
case "VERSION":
|
|
err := or.Version.FromString(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
seenVersion = true
|
|
|
|
if or.Version < OfxVersion200 {
|
|
return errors.New("OFX VERSION < 200 in XML header")
|
|
}
|
|
case "SECURITY":
|
|
if value != "NONE" {
|
|
return errors.New("OFX SECURITY header not NONE")
|
|
}
|
|
case "OLDFILEUID", "NEWFILEUID":
|
|
// TODO check/handle these headers?
|
|
default:
|
|
return errors.New("Invalid OFX header: " + header)
|
|
}
|
|
}
|
|
|
|
if !seenHeader {
|
|
return errors.New("OFXHEADER version missing")
|
|
}
|
|
if !seenVersion {
|
|
return errors.New("OFX VERSION header missing")
|
|
}
|
|
|
|
} else {
|
|
return errors.New("Missing xml 'OFX' processing instruction")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Number of bytes of response to read when attempting to figure out whether
|
|
// we're using OFX or SGML
|
|
const guessVersionCheckBytes = 1024
|
|
|
|
// Defaults to XML if it can't determine the version or if there is any
|
|
// ambiguity
|
|
// Returns false for SGML, true (for XML) otherwise.
|
|
func guessVersion(r *bufio.Reader) (bool, error) {
|
|
b, _ := r.Peek(guessVersionCheckBytes)
|
|
if b == nil {
|
|
return false, errors.New("Failed to read OFX header")
|
|
}
|
|
sgmlIndex := bytes.Index(b, []byte("OFXHEADER:"))
|
|
xmlIndex := bytes.Index(b, []byte("OFXHEADER="))
|
|
if sgmlIndex < 0 {
|
|
return true, nil
|
|
} else if xmlIndex < 0 {
|
|
return false, nil
|
|
} else {
|
|
return xmlIndex <= sgmlIndex, nil
|
|
}
|
|
}
|
|
|
|
// A map of message set tags to a map of transaction wrapper tags to the
|
|
// reflect.Type of the struct for that transaction type. Used when decoding
|
|
// Responses. Newly-implemented response transaction types *must* be added to
|
|
// this map in order to be unmarshalled.
|
|
var responseTypes = map[string]map[string]reflect.Type{
|
|
SignupRs.String(): {
|
|
(&AcctInfoResponse{}).Name(): reflect.TypeOf(AcctInfoResponse{})},
|
|
BankRs.String(): {
|
|
(&StatementResponse{}).Name(): reflect.TypeOf(StatementResponse{})},
|
|
CreditCardRs.String(): {
|
|
(&CCStatementResponse{}).Name(): reflect.TypeOf(CCStatementResponse{})},
|
|
LoanRs.String(): {},
|
|
InvStmtRs.String(): {
|
|
(&InvStatementResponse{}).Name(): reflect.TypeOf(InvStatementResponse{})},
|
|
InterXferRs.String(): {},
|
|
WireXferRs.String(): {},
|
|
BillpayRs.String(): {},
|
|
EmailRs.String(): {},
|
|
SecListRs.String(): {
|
|
(&SecListResponse{}).Name(): reflect.TypeOf(SecListResponse{}),
|
|
(&SecurityList{}).Name(): reflect.TypeOf(SecurityList{})},
|
|
PresDirRs.String(): {},
|
|
PresDlvRs.String(): {},
|
|
ProfRs.String(): {
|
|
(&ProfileResponse{}).Name(): reflect.TypeOf(ProfileResponse{})},
|
|
ImageRs.String(): {},
|
|
}
|
|
|
|
func decodeMessageSet(d *xml.Decoder, start xml.StartElement, msgs *[]Message, version ofxVersion) error {
|
|
setTypes, ok := responseTypes[start.Name.Local]
|
|
if !ok {
|
|
return errors.New("Invalid message set: " + start.Name.Local)
|
|
}
|
|
for {
|
|
tok, err := nextNonWhitespaceToken(d)
|
|
if err != nil {
|
|
return err
|
|
} else if end, ok := tok.(xml.EndElement); ok && end.Name.Local == start.Name.Local {
|
|
// If we found the end of our starting element, we're done parsing
|
|
return nil
|
|
} else if startElement, ok := tok.(xml.StartElement); ok {
|
|
responseType, ok := setTypes[startElement.Name.Local]
|
|
if !ok {
|
|
// If you are a developer and received this message after you
|
|
// thought you added a new transaction type, make sure you
|
|
// added it to the responseTypes map above
|
|
return errors.New("Unsupported response transaction for " +
|
|
start.Name.Local + ": " + startElement.Name.Local)
|
|
}
|
|
response := reflect.New(responseType).Interface()
|
|
responseMessage := response.(Message)
|
|
if err := d.DecodeElement(responseMessage, &startElement); err != nil {
|
|
return err
|
|
}
|
|
*msgs = append(*msgs, responseMessage)
|
|
} else {
|
|
return errors.New("Didn't find an opening element")
|
|
}
|
|
}
|
|
}
|
|
|
|
// ParseResponse parses and validates an OFX response in SGML or XML into a
|
|
// Response object from the given io.Reader
|
|
//
|
|
// It is commonly used as part of Client.Request(), but may be used on its own
|
|
// to parse already-downloaded OFX files (such as those from 'Web Connect'). It
|
|
// performs version autodetection if it can and attempts to be as forgiving as
|
|
// possible about the input format.
|
|
func ParseResponse(reader io.Reader) (*Response, error) {
|
|
resp, err := DecodeResponse(reader)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
_, err = resp.Valid()
|
|
return resp, err
|
|
}
|
|
|
|
// DecodeResponse parses an OFX response in SGML or XML into a Response object
|
|
// from the given io.Reader
|
|
func DecodeResponse(reader io.Reader) (*Response, error) {
|
|
var or Response
|
|
|
|
r := bufio.NewReaderSize(reader, guessVersionCheckBytes)
|
|
xmlVersion, err := guessVersion(r)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// parse SGML headers before creating XML decoder
|
|
if !xmlVersion {
|
|
if err := or.readSGMLHeaders(r); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
decoder := xml.NewDecoder(r)
|
|
if !xmlVersion {
|
|
decoder.Strict = false
|
|
decoder.AutoCloseAfterCharData = ofxLeafElements
|
|
}
|
|
decoder.CharsetReader = func(charset string, input io.Reader) (io.Reader, error) {
|
|
return input, nil
|
|
}
|
|
|
|
if xmlVersion {
|
|
// parse the xml header
|
|
if err := or.readXMLHeaders(decoder); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
tok, err := nextNonWhitespaceToken(decoder)
|
|
if err != nil {
|
|
return nil, err
|
|
} else if ofxStart, ok := tok.(xml.StartElement); !ok || ofxStart.Name.Local != "OFX" {
|
|
return nil, errors.New("Missing opening OFX xml element")
|
|
}
|
|
|
|
// Unmarshal the signon message
|
|
tok, err = nextNonWhitespaceToken(decoder)
|
|
if err != nil {
|
|
return nil, err
|
|
} else if signonStart, ok := tok.(xml.StartElement); ok && signonStart.Name.Local == SignonRs.String() {
|
|
if err := decoder.Decode(&or.Signon); err != nil {
|
|
return nil, err
|
|
}
|
|
} else {
|
|
return nil, errors.New("Missing opening SIGNONMSGSRSV1 xml element")
|
|
}
|
|
|
|
tok, err = nextNonWhitespaceToken(decoder)
|
|
if err != nil {
|
|
return nil, err
|
|
} else if signonEnd, ok := tok.(xml.EndElement); !ok || signonEnd.Name.Local != SignonRs.String() {
|
|
return nil, errors.New("Missing closing SIGNONMSGSRSV1 xml element")
|
|
}
|
|
|
|
var messageSlices = map[string]*[]Message{
|
|
SignupRs.String(): &or.Signup,
|
|
BankRs.String(): &or.Bank,
|
|
CreditCardRs.String(): &or.CreditCard,
|
|
LoanRs.String(): &or.Loan,
|
|
InvStmtRs.String(): &or.InvStmt,
|
|
InterXferRs.String(): &or.InterXfer,
|
|
WireXferRs.String(): &or.WireXfer,
|
|
BillpayRs.String(): &or.Billpay,
|
|
EmailRs.String(): &or.Email,
|
|
SecListRs.String(): &or.SecList,
|
|
PresDirRs.String(): &or.PresDir,
|
|
PresDlvRs.String(): &or.PresDlv,
|
|
ProfRs.String(): &or.Prof,
|
|
ImageRs.String(): &or.Image,
|
|
}
|
|
|
|
for {
|
|
tok, err = nextNonWhitespaceToken(decoder)
|
|
if err != nil {
|
|
return nil, err
|
|
} else if ofxEnd, ok := tok.(xml.EndElement); ok && ofxEnd.Name.Local == "OFX" {
|
|
return &or, nil // found closing XML element, so we're done
|
|
} else if start, ok := tok.(xml.StartElement); ok {
|
|
slice, ok := messageSlices[start.Name.Local]
|
|
if !ok {
|
|
return nil, errors.New("Invalid message set: " + start.Name.Local)
|
|
}
|
|
if err := decodeMessageSet(decoder, start, slice, or.Version); err != nil {
|
|
return nil, err
|
|
}
|
|
} else {
|
|
return nil, errors.New("Found unexpected token")
|
|
}
|
|
}
|
|
}
|
|
|
|
// Valid returns whether the Response is valid according to the OFX spec
|
|
func (or *Response) Valid() (bool, error) {
|
|
var errs errInvalid
|
|
if ok, err := or.Signon.Valid(or.Version); !ok {
|
|
errs.AddErr(err)
|
|
}
|
|
for _, messageSet := range [][]Message{
|
|
or.Signup,
|
|
or.Bank,
|
|
or.CreditCard,
|
|
or.Loan,
|
|
or.InvStmt,
|
|
or.InterXfer,
|
|
or.WireXfer,
|
|
or.Billpay,
|
|
or.Email,
|
|
or.SecList,
|
|
or.PresDir,
|
|
or.PresDlv,
|
|
or.Prof,
|
|
or.Image,
|
|
} {
|
|
for _, message := range messageSet {
|
|
if ok, err := message.Valid(or.Version); !ok {
|
|
errs.AddErr(err)
|
|
}
|
|
}
|
|
}
|
|
err := errs.ErrOrNil()
|
|
return err == nil, err
|
|
}
|
|
|
|
// Marshal this Response into its SGML/XML representation held in a bytes.Buffer
|
|
//
|
|
// If error is non-nil, this bytes.Buffer is ready to be sent to an OFX client
|
|
func (or *Response) Marshal() (*bytes.Buffer, error) {
|
|
var b bytes.Buffer
|
|
|
|
// Write the header appropriate to our version
|
|
writeHeader(&b, or.Version, false)
|
|
|
|
encoder := xml.NewEncoder(&b)
|
|
encoder.Indent("", " ")
|
|
|
|
ofxElement := xml.StartElement{Name: xml.Name{Local: "OFX"}}
|
|
|
|
if err := encoder.EncodeToken(ofxElement); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if ok, err := or.Signon.Valid(or.Version); !ok {
|
|
return nil, err
|
|
}
|
|
signonMsgSet := xml.StartElement{Name: xml.Name{Local: SignonRs.String()}}
|
|
if err := encoder.EncodeToken(signonMsgSet); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := encoder.Encode(&or.Signon); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := encoder.EncodeToken(signonMsgSet.End()); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
messageSets := []struct {
|
|
Messages []Message
|
|
Type messageType
|
|
}{
|
|
{or.Signup, SignupRs},
|
|
{or.Bank, BankRs},
|
|
{or.CreditCard, CreditCardRs},
|
|
{or.Loan, LoanRs},
|
|
{or.InvStmt, InvStmtRs},
|
|
{or.InterXfer, InterXferRs},
|
|
{or.WireXfer, WireXferRs},
|
|
{or.Billpay, BillpayRs},
|
|
{or.Email, EmailRs},
|
|
{or.SecList, SecListRs},
|
|
{or.PresDir, PresDirRs},
|
|
{or.PresDlv, PresDlvRs},
|
|
{or.Prof, ProfRs},
|
|
{or.Image, ImageRs},
|
|
}
|
|
for _, set := range messageSets {
|
|
if err := encodeMessageSet(encoder, set.Messages, set.Type, or.Version); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if err := encoder.EncodeToken(ofxElement.End()); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := encoder.Flush(); err != nil {
|
|
return nil, err
|
|
}
|
|
return &b, nil
|
|
}
|
|
|
|
// errInvalid represents validation failures while parsing an OFX response
|
|
// If an institution returns slightly malformed data, ParseResponse will return a best-effort parsed response and a validation error.
|
|
type errInvalid []error
|
|
|
|
func (e errInvalid) Error() string {
|
|
var errStrings []string
|
|
for _, err := range e {
|
|
errStrings = append(errStrings, err.Error())
|
|
}
|
|
return fmt.Sprintf("Validation failed: %s", strings.Join(errStrings, "; "))
|
|
}
|
|
|
|
func (e *errInvalid) AddErr(err error) {
|
|
if err != nil {
|
|
if errs, ok := err.(errInvalid); ok {
|
|
*e = append(*e, errs...)
|
|
} else {
|
|
*e = append(*e, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (e errInvalid) ErrOrNil() error {
|
|
if len(e) > 0 {
|
|
return e
|
|
}
|
|
return nil
|
|
}
|