From 2fb2dfda7891386c9c97d510b1b5c81d2d287269 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Thu, 9 Dec 2021 11:55:22 +0100 Subject: [PATCH] Moved commonly use file operations to separate package. --- cmd/csaf_provider/actions.go | 3 +- cmd/csaf_provider/create.go | 5 +- cmd/csaf_provider/files.go | 106 ------------------------ cmd/csaf_provider/transaction.go | 7 +- csaf/models.go | 17 +--- csaf/rolie.go | 6 +- util/file.go | 134 +++++++++++++++++++++++++++++++ 7 files changed, 151 insertions(+), 127 deletions(-) create mode 100644 util/file.go diff --git a/cmd/csaf_provider/actions.go b/cmd/csaf_provider/actions.go index 8c5c786..b15b660 100644 --- a/cmd/csaf_provider/actions.go +++ b/cmd/csaf_provider/actions.go @@ -16,6 +16,7 @@ import ( "github.com/ProtonMail/gopenpgp/v2/crypto" "github.com/csaf-poc/csaf_distribution/csaf" + "github.com/csaf-poc/csaf_distribution/util" ) const dateFormat = time.RFC3339 @@ -256,7 +257,7 @@ func (c *controller) upload(r *http.Request) (interface{}, error) { rolie.SortEntriesByUpdated() // Store the feed - if err := saveToFile(feed, rolie); err != nil { + if err := util.WriteToFile(feed, rolie); err != nil { return err } diff --git a/cmd/csaf_provider/create.go b/cmd/csaf_provider/create.go index 29a361d..03f0784 100644 --- a/cmd/csaf_provider/create.go +++ b/cmd/csaf_provider/create.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/csaf-poc/csaf_distribution/csaf" + "github.com/csaf-poc/csaf_distribution/util" ) func ensureFolders(c *config) error { @@ -52,7 +53,7 @@ func createFeedFolders(c *config, wellknown string) error { if _, err := filepath.EvalSymlinks(tlpLink); err != nil { if os.IsNotExist(err) { tlpFolder := filepath.Join(c.Folder, string(t)) - if tlpFolder, err = mkUniqDir(tlpFolder); err != nil { + if tlpFolder, err = util.MakeUniqDir(tlpFolder); err != nil { return err } if err = os.Symlink(tlpFolder, tlpLink); err != nil { @@ -104,5 +105,5 @@ func createProviderMetadata(c *config, wellknownCSAF string) error { keyID, fingerprint := key.GetHexKeyID(), key.GetFingerprint() pm.SetPGP(fingerprint, c.GetOpenPGPURL(keyID)) - return saveToFile(path, pm) + return util.WriteToFile(path, pm) } diff --git a/cmd/csaf_provider/files.go b/cmd/csaf_provider/files.go index 50ba98f..4af9ee9 100644 --- a/cmd/csaf_provider/files.go +++ b/cmd/csaf_provider/files.go @@ -8,102 +8,9 @@ import ( "hash" "io" "io/ioutil" - "math/rand" "os" - "path/filepath" - "strconv" - "time" ) -func deepCopy(dst, src string) error { - - stack := []string{dst, src} - - for len(stack) > 0 { - src = stack[len(stack)-1] - dst = stack[len(stack)-2] - stack = stack[:len(stack)-2] - - if err := func() error { - dir, err := os.Open(src) - if err != nil { - return err - } - defer dir.Close() - - // Use Readdir as we need no sorting. - files, err := dir.Readdir(-1) - if err != nil { - return err - } - - for _, f := range files { - nsrc := filepath.Join(src, f.Name()) - ndst := filepath.Join(dst, f.Name()) - if f.IsDir() { - // Create new sub dir - if err := os.Mkdir(ndst, 0755); err != nil { - return err - } - stack = append(stack, ndst, nsrc) - } else if f.Mode().IsRegular() { - // Create hard link. - if err := os.Link(nsrc, ndst); err != nil { - return err - } - } - } - return nil - }(); err != nil { - return err - } - } - - return nil -} - -func mkUniqFile(prefix string) (string, *os.File, error) { - var file *os.File - name, err := mkUniq(prefix, func(name string) error { - var err error - file, err = os.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) - return err - }) - return name, file, err -} - -func mkUniqDir(prefix string) (string, error) { - return mkUniq(prefix, func(name string) error { return os.Mkdir(name, 0755) }) -} - -func mkUniq(prefix string, create func(string) error) (string, error) { - now := time.Now() - stamp := now.Format("-2006-01-02-150405") - name := prefix + stamp - err := create(name) - if err == nil { - return name, nil - } - if os.IsExist(err) { - rnd := rand.New(rand.NewSource(now.Unix())) - - for i := 0; i < 10000; i++ { - nname := name + "-" + strconv.FormatUint(uint64(rnd.Uint32()&0xff_ffff), 16) - err := create(nname) - if err == nil { - return nname, nil - } - if os.IsExist(err) { - continue - } - return "", err - } - return "", &os.PathError{Op: "mkuniq", Path: name, Err: os.ErrExist} - } - - return "", err -} - func writeHash(fname, name string, h hash.Hash, data []byte) error { if _, err := io.Copy(h, bytes.NewReader(data)); err != nil { @@ -137,16 +44,3 @@ func writeHashedFile(fname, name string, data []byte, armored string) error { } return nil } - -func saveToFile(fname string, wt io.WriterTo) error { - f, err1 := os.Create(fname) - if err1 != nil { - return err1 - } - _, err1 = wt.WriteTo(f) - err2 := f.Close() - if err1 != nil { - return err1 - } - return err2 -} diff --git a/cmd/csaf_provider/transaction.go b/cmd/csaf_provider/transaction.go index 04f4d45..936fd0f 100644 --- a/cmd/csaf_provider/transaction.go +++ b/cmd/csaf_provider/transaction.go @@ -5,6 +5,7 @@ import ( "path/filepath" "github.com/csaf-poc/csaf_distribution/csaf" + "github.com/csaf-poc/csaf_distribution/util" ) func doTransaction( @@ -42,13 +43,13 @@ func doTransaction( folderTLP := filepath.Join(cfg.Folder, string(t)) - newDir, err := mkUniqDir(folderTLP) + newDir, err := util.MakeUniqDir(folderTLP) if err != nil { return err } // Copy old content into new. - if err := deepCopy(newDir, oldDir); err != nil { + if err := util.DeepCopy(newDir, oldDir); err != nil { os.RemoveAll(newDir) return err } @@ -61,7 +62,7 @@ func doTransaction( // Write back provider metadata if its dynamic. if cfg.DynamicProviderMetaData { - newMetaName, newMetaFile, err := mkUniqFile(metadata) + newMetaName, newMetaFile, err := util.MakeUniqFile(metadata) if err != nil { os.RemoveAll(newDir) return err diff --git a/csaf/models.go b/csaf/models.go index b313ab3..ab5bba7 100644 --- a/csaf/models.go +++ b/csaf/models.go @@ -8,6 +8,8 @@ import ( "regexp" "strings" "time" + + "github.com/csaf-poc/csaf_distribution/util" ) // TLPLabel is the traffic light policy of the CSAF. @@ -447,24 +449,13 @@ func NewProviderMetadataDomain(domain string, tlps []TLPLabel) *ProviderMetadata return pm } -type nWriter struct { - io.Writer - n int64 -} - -func (nw *nWriter) Write(p []byte) (int, error) { - n, err := nw.Writer.Write(p) - nw.n += int64(n) - return n, err -} - // WriteTo saves a metadata provider to a writer. func (pmd *ProviderMetadata) WriteTo(w io.Writer) (int64, error) { - nw := nWriter{w, 0} + nw := util.NWriter{w, 0} enc := json.NewEncoder(&nw) enc.SetIndent("", " ") err := enc.Encode(pmd) - return nw.n, err + return nw.N, err } // LoadProviderMetadata loads a metadata provider from a reader. diff --git a/csaf/rolie.go b/csaf/rolie.go index b79c56a..1451111 100644 --- a/csaf/rolie.go +++ b/csaf/rolie.go @@ -5,6 +5,8 @@ import ( "io" "sort" "time" + + "github.com/csaf-poc/csaf_distribution/util" ) // Link for ROLIE. @@ -70,11 +72,11 @@ func LoadROLIEFeed(r io.Reader) (*ROLIEFeed, error) { // WriteTo saves a ROLIE feed to a writer. func (rf *ROLIEFeed) WriteTo(w io.Writer) (int64, error) { - nw := nWriter{w, 0} + nw := util.NWriter{w, 0} enc := json.NewEncoder(&nw) enc.SetIndent("", " ") err := enc.Encode(rf) - return nw.n, err + return nw.N, err } // EntryByID looks up an entry by its ID. diff --git a/util/file.go b/util/file.go new file mode 100644 index 0000000..394983c --- /dev/null +++ b/util/file.go @@ -0,0 +1,134 @@ +package util + +import ( + "io" + "math/rand" + "os" + "path/filepath" + "strconv" + "time" +) + +// NWriter is an io.Writer counting the bytes copied through it. +type NWriter struct { + io.Writer + N int64 +} + +// Write implements the Write method of io.Writer. +func (nw *NWriter) Write(p []byte) (int, error) { + n, err := nw.Writer.Write(p) + nw.N += int64(n) + return n, err +} + +// WriteToFile saves the content of wt into a file names fname. +func WriteToFile(fname string, wt io.WriterTo) error { + f, err1 := os.Create(fname) + if err1 != nil { + return err1 + } + _, err1 = wt.WriteTo(f) + err2 := f.Close() + if err1 != nil { + return err1 + } + return err2 +} + +// DeepCopy copy a directory tree src to tree dst. Files are hard linked. +func DeepCopy(dst, src string) error { + + stack := []string{dst, src} + + for len(stack) > 0 { + src = stack[len(stack)-1] + dst = stack[len(stack)-2] + stack = stack[:len(stack)-2] + + if err := func() error { + dir, err := os.Open(src) + if err != nil { + return err + } + defer dir.Close() + + // Use Readdir as we need no sorting. + files, err := dir.Readdir(-1) + if err != nil { + return err + } + + for _, f := range files { + nsrc := filepath.Join(src, f.Name()) + ndst := filepath.Join(dst, f.Name()) + if f.IsDir() { + // Create new sub dir + if err := os.Mkdir(ndst, 0755); err != nil { + return err + } + stack = append(stack, ndst, nsrc) + } else if f.Mode().IsRegular() { + // Create hard link. + if err := os.Link(nsrc, ndst); err != nil { + return err + } + } + } + return nil + }(); err != nil { + return err + } + } + + return nil +} + +// MakeUniqFile creates a unique named file with the given prefix +// opened in write only mode. +// In case of name collisions the current date plus a random +// number is appended. +func MakeUniqFile(prefix string) (string, *os.File, error) { + var file *os.File + name, err := mkUniq(prefix, func(name string) error { + var err error + file, err = os.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) + return err + }) + return name, file, err +} + +// MakeUniqDir creates a unique named directory with the given prefix. +// In case of name collisions the current date plus a random +// number is appended. +func MakeUniqDir(prefix string) (string, error) { + return mkUniq(prefix, func(name string) error { return os.Mkdir(name, 0755) }) +} + +func mkUniq(prefix string, create func(string) error) (string, error) { + now := time.Now() + stamp := now.Format("-2006-01-02-150405") + name := prefix + stamp + err := create(name) + if err == nil { + return name, nil + } + if os.IsExist(err) { + rnd := rand.New(rand.NewSource(now.Unix())) + + for i := 0; i < 10000; i++ { + nname := name + "-" + strconv.FormatUint(uint64(rnd.Uint32()&0xff_ffff), 16) + err := create(nname) + if err == nil { + return nname, nil + } + if os.IsExist(err) { + continue + } + return "", err + } + return "", &os.PathError{Op: "mkuniq", Path: name, Err: os.ErrExist} + } + + return "", err +}