Merge pull request #187 from moov-io/download-fixes

fix: download fed files in-memory instead of flat files

Author: adamdecaf

CHANGELOG.md

@@ -1,3 +1,9 @@
+## v0.9.0 (Released 2022-08-03)
+
+IMPROVEMENTS
+
+- Remove `DOWNLOAD_DIRECTORY` and store downloaded files in memory.
+
 ## v0.8.1 (Released 2022-08-02)
 
 IMPROVEMENTS

README.md

@@ -187,7 +187,6 @@ PONG
 | `FEDWIRE_DATA_PATH` | Filepath to Fedwire data file | `./data/fpddir.txt` |
 | `FRB_ROUTING_NUMBER` | Federal Reserve Board eServices (ABA) routing number used to download FedACH and FedWire files | Empty |
 | `FRB_DOWNLOAD_CODE` | Federal Reserve Board eServices (ABA) download code used to download FedACH and FedWire files | Empty |
-| `DOWNLOAD_DIRECTORY` | Directory for saving downloaded eServices files into | OS Temp Dir |
 | `LOG_FORMAT` | Format for logging lines to be written as. | Options: `json`, `plain` - Default: `plain` |
 | `HTTP_BIND_ADDRESS` | Address for Fed to bind its HTTP server on. This overrides the command-line flag `-http.addr`. | Default: `:8086` |
 | `HTTP_ADMIN_BIND_ADDRESS` | Address for Fed to bind its admin HTTP server on. This overrides the command-line flag `-admin.addr`. | Default: `:9096` |

cmd/server/main.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"os/signal"
@@ -111,7 +112,11 @@ func main() {
 
 	// Start our searcher
 	searcher := &searcher{logger: logger}
-	if err := setupSearcher(logger, searcher, fedACHDataFile(logger), fedWireDataFile(logger)); err != nil {
+
+	fedACHData := fedACHDataFile(logger)
+	fedWireData := fedWireDataFile(logger)
+
+	if err := setupSearcher(logger, searcher, fedACHData, fedWireData); err != nil {
 		logger.Logf("read: %v", err)
 		os.Exit(1)
 	}
@@ -154,23 +159,20 @@ func addPingRoute(r *mux.Router) {
 	})
 }
 
-func setupSearcher(logger log.Logger, s *searcher, achFile, wireFile *os.File) error {
+func setupSearcher(logger log.Logger, s *searcher, achFile, wireFile io.Reader) error {
 	if achFile == nil {
 		return errors.New("missing fedach data file")
 	}
 	if wireFile == nil {
 		return errors.New("missing fedwire data file")
 	}
 
-	logger.Logf("search: loading %s for ACH data", achFile.Name())
 	if err := s.readFEDACHData(achFile); err != nil {
-		return fmt.Errorf("error reading ACH file at %s: %v", achFile.Name(), err)
+		return fmt.Errorf("error reading ACH data: %v", err)
 	}
 
-	logger.Logf("search: loading %s for Wire data", wireFile.Name())
 	if err := s.readFEDWIREData(wireFile); err != nil {
-		return fmt.Errorf("error reading wire file at %s: %v", wireFile.Name(), err)
-
+		return fmt.Errorf("error reading wire data: %v", err)
 	}
 	return nil
 }

cmd/server/reader.go

@@ -6,44 +6,49 @@ package main
 
 import (
 	"fmt"
+	"io"
 	"os"
 
 	"github.com/moov-io/base/log"
 	"github.com/moov-io/fed"
 	"github.com/moov-io/fed/pkg/download"
 )
 
-func fedACHDataFile(logger log.Logger) *os.File {
+func fedACHDataFile(logger log.Logger) io.Reader {
 	if file, err := attemptFileDownload(logger, "fedach"); file != nil {
 		return file
 	} else if err != nil {
 		panic(fmt.Sprintf("problem downloading fedach: %v", err))
 	}
 
 	path := readDataFilepath("FEDACH_DATA_PATH", "./data/FedACHdir.txt")
+	logger.Logf("search: loading %s for ACH data", path)
+
 	file, err := os.Open(path)
 	if err != nil {
 		panic(fmt.Sprintf("problem opening %s: %v", path, err))
 	}
 	return file
 }
 
-func fedWireDataFile(logger log.Logger) *os.File {
+func fedWireDataFile(logger log.Logger) io.Reader {
 	if file, err := attemptFileDownload(logger, "fedwire"); file != nil {
 		return file
 	} else if err != nil {
 		panic(fmt.Sprintf("problem downloading fedwire: %v", err))
 	}
 
 	path := readDataFilepath("FEDWIRE_DATA_PATH", "./data/fpddir.txt")
+	logger.Logf("search: loading %s for Wire data", path)
+
 	file, err := os.Open(path)
 	if err != nil {
 		panic(fmt.Sprintf("problem opening %s: %v", path, err))
 	}
 	return file
 }
 
-func attemptFileDownload(logger log.Logger, listName string) (*os.File, error) {
+func attemptFileDownload(logger log.Logger, listName string) (io.Reader, error) {
 	routingNumber := os.Getenv("FRB_ROUTING_NUMBER")
 	downloadCode := os.Getenv("FRB_DOWNLOAD_CODE")
 
@@ -71,14 +76,17 @@ func readDataFilepath(env, fallback string) string {
 
 // readFEDACHData opens and reads FedACHdir.txt then runs ACHDictionary.Read() to
 // parse and define ACHDictionary properties
-func (s *searcher) readFEDACHData(file *os.File) error {
+func (s *searcher) readFEDACHData(reader io.Reader) error {
 	if s.logger != nil {
 		s.logger.Logf("Read of FED data")
 	}
-	defer file.Close()
+
+	if closer, ok := reader.(io.Closer); ok {
+		defer closer.Close()
+	}
 
 	s.ACHDictionary = fed.NewACHDictionary()
-	if err := s.ACHDictionary.Read(file); err != nil {
+	if err := s.ACHDictionary.Read(reader); err != nil {
 		return fmt.Errorf("ERROR: reading FedACHdir.txt %v", err)
 	}
 
@@ -91,14 +99,17 @@ func (s *searcher) readFEDACHData(file *os.File) error {
 
 // readFEDWIREData opens and reads fpddir.txt then runs WIREDictionary.Read() to
 // parse and define WIREDictionary properties
-func (s *searcher) readFEDWIREData(file *os.File) error {
+func (s *searcher) readFEDWIREData(reader io.Reader) error {
 	if s.logger != nil {
 		s.logger.Logf("Read of FED data")
 	}
-	defer file.Close()
+
+	if closer, ok := reader.(io.Closer); ok {
+		defer closer.Close()
+	}
 
 	s.WIREDictionary = fed.NewWIREDictionary()
-	if err := s.WIREDictionary.Read(file); err != nil {
+	if err := s.WIREDictionary.Read(reader); err != nil {
 		return fmt.Errorf("ERROR: reading fpddir.txt %v", err)
 	}
 

pkg/download/download.go

@@ -5,16 +5,13 @@
 package download
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
-	"os"
 	"time"
-
-	"github.com/moov-io/base/strx"
 )
 
 type Client struct {
@@ -55,19 +52,9 @@ func NewClient(opts *ClientOpts) (*Client, error) {
 	}, nil
 }
 
-var (
-	downloadDirectory = strx.Or(os.Getenv("DOWNLOAD_DIRECTORY"), os.TempDir())
-)
-
-func init() {
-	if _, err := os.Stat(downloadDirectory); os.IsNotExist(err) {
-		os.MkdirAll(downloadDirectory, 0777)
-	}
-}
-
-// GetList downloads an FRB list and saves it into a temporary file.
+// GetList downloads an FRB list and saves it into an io.Reader.
 // Example listName values: fedach, fedwire
-func (c *Client) GetList(listName string) (*os.File, error) {
+func (c *Client) GetList(listName string) (io.Reader, error) {
 	where, err := url.Parse(fmt.Sprintf("https://frbservices.org/EPaymentsDirectory/directories/%s?format=json", listName))
 	if err != nil {
 		return nil, fmt.Errorf("url: %v", err)
@@ -85,20 +72,21 @@ func (c *Client) GetList(listName string) (*os.File, error) {
 	if err != nil {
 		return nil, fmt.Errorf("http get: %v", err)
 	}
-	defer resp.Body.Close()
+	if resp != nil && resp.Body != nil {
+		defer resp.Body.Close()
+	}
 
-	out, err := ioutil.TempFile(downloadDirectory, fmt.Sprintf("%s-*", listName))
-	if err != nil {
-		return nil, fmt.Errorf("temp file: %v", err)
+	// Quit if we fail to download
+	if resp.StatusCode >= 299 {
+		return nil, fmt.Errorf("unexpected http status: %d", resp.StatusCode)
 	}
-	if n, err := io.Copy(out, resp.Body); n == 0 || err != nil {
+
+	var out bytes.Buffer
+	if n, err := io.Copy(&out, resp.Body); n == 0 || err != nil {
 		return nil, fmt.Errorf("copying n=%d: %v", n, err)
 	}
-	if err := out.Sync(); err != nil {
-		return nil, fmt.Errorf("sync: %v", err)
-	}
-	if _, err := out.Seek(0, io.SeekStart); err != nil {
-		return nil, fmt.Errorf("seek: %v", err)
+	if out.Len() > 0 {
+		return &out, nil
 	}
-	return out, nil
+	return nil, nil
 }

pkg/download/download_test.go

@@ -10,6 +10,8 @@ import (
 	"io/ioutil"
 	"os"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 func TestClient__fedach(t *testing.T) {
@@ -20,11 +22,10 @@ func TestClient__fedach(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	st, err := fedach.Stat()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if n := st.Size(); n < 1024 {
+	buf, ok := fedach.(*bytes.Buffer)
+	require.True(t, ok)
+
+	if n := buf.Len(); n < 1024 {
 		t.Errorf("unexpected size of %d bytes", n)
 	}
 
@@ -42,11 +43,10 @@ func TestClient__fedwire(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	st, err := fedwire.Stat()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if n := st.Size(); n < 1024 {
+	buf, ok := fedwire.(*bytes.Buffer)
+	require.True(t, ok)
+
+	if n := buf.Len(); n < 1024 {
 		t.Errorf("unexpected size of %d bytes", n)
 	}
 

version.go

@@ -5,4 +5,4 @@
 package fed
 
 // Version is the current version
-const Version = "v0.8.1"
+const Version = "v0.9.0"