diff --git a/.gitignore b/.gitignore index f4816cd..88a81d1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ dist/ tmp/ +env/ diff --git a/cmd/ma/main.go b/cmd/ma/main.go index 6646f1e..ce6d8bb 100644 --- a/cmd/ma/main.go +++ b/cmd/ma/main.go @@ -56,12 +56,18 @@ func flags() []cli.Flag { &cli.BoolFlag{ Name: "monochrome", Required: false, - Usage: "disable colored output", + Usage: "disable colored loggingoutput", Value: false, }, &cli.BoolFlag{ Name: "debug", Required: false, + Usage: "enable verbose debugging", + Value: false, + }, + &cli.BoolFlag{ + Name: "trace", + Required: false, Usage: "enable debugging of http requests", Value: false, }, @@ -99,8 +105,8 @@ func mg(c *cli.Context) func() *smugmug.Client { client, err := smugmug.NewClient( smugmug.WithConcurrency(c.Int("concurrency")), smugmug.WithHTTPClient(httpclient), - smugmug.WithPretty(c.Bool("debug")), - smugmug.WithHTTPTracing(c.Bool("debug"))) + smugmug.WithPretty(c.Bool("trace")), + smugmug.WithHTTPTracing(c.Bool("trace"))) if err != nil { panic(err) } @@ -130,7 +136,7 @@ func main() { } grab := &http.Client{} - if c.Bool("debug") { + if c.Bool("trace") { grab.Transport = &httpwares.VerboseTransport{} } @@ -147,7 +153,6 @@ func main() { Metrics: metric, Encoder: json.NewEncoder(writer), Fs: afero.NewOsFs(), - Exif: ma.NewGoExif(), Language: language.English, Start: time.Now(), }, @@ -157,8 +162,6 @@ func main() { }, After: ma.Metrics, Commands: []*cli.Command{ - ma.CommandCopy(), - ma.CommandExif(), ma.CommandExport(), ma.CommandFind(), ma.CommandList(), diff --git a/cp.go b/cp.go deleted file mode 100644 index 68b2218..0000000 --- a/cp.go +++ /dev/null @@ -1,290 +0,0 @@ -package ma - -import ( - "context" - "errors" - "fmt" - "io" - "io/fs" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/hashicorp/go-metrics" - "github.com/rs/zerolog/log" - "github.com/spf13/afero" - "github.com/urfave/cli/v2" - "golang.org/x/sync/errgroup" -) - -const defaultDateFormat = "2006/2006-01/02" - -func split(fullname string) identifier { - dirname, filename := filepath.Split(fullname) - n := strings.LastIndexFunc(filename, func(s rune) bool { - return s == '.' - }) - var basename string - switch n { - case -1: - basename = filename - default: - basename = filename[0:n] - } - return identifier{dirname: filepath.Clean(dirname), basename: basename} -} - -type identifier struct { - dirname string - basename string -} - -type fileset struct { - identifier identifier - files []fs.FileInfo -} - -// dateTime attempts to create a time.Time for for every file in the fileset -func (f *fileset) dateTime(afs afero.Fs, ex Exif) (time.Time, error) { - var times []time.Time - for _, md := range ex.Extract(afs, f.identifier.dirname, f.files...) { - if md.Err != nil { - return time.Time{}, md.Err - } - if !md.DateTime.IsZero() { - times = append(times, md.DateTime) - } - } - switch len(times) { - case 0: - return time.Time{}, nil - case 1: - // no need to sort - default: - // @todo(bzimmer) ensure dates are consistent (within a ~second or so) - sort.SliceStable(times, func(i, j int) bool { - return times[i].Before(times[j]) - }) - } - return times[0], nil -} - -type entangler struct { - fs afero.Fs - exif Exif - metrics *metrics.Metrics - concurrency int - dryrun bool - dateFormat string -} - -func (c *entangler) cp(ctx context.Context, sources []string, destination string) error { - q := make(chan *fileset) - grp, ctx := errgroup.WithContext(ctx) - grp.Go(func() error { - defer close(q) - sets := make(map[identifier][]fs.FileInfo) - for i := range sources { - select { - case <-ctx.Done(): - return ctx.Err() - default: - if err := afero.Walk(c.fs, sources[i], c.filesets(sets)); err != nil { - return err - } - } - } - for id, files := range sets { - select { - case <-ctx.Done(): - return ctx.Err() - case q <- &fileset{identifier: id, files: files}: - c.metrics.IncrCounter([]string{"cp", "filesets"}, 1) - } - } - return nil - }) - for i := 0; i < c.concurrency; i++ { - grp.Go(c.copyFileset(q, destination)) - } - return grp.Wait() -} - -func (c *entangler) copyFileset(q <-chan *fileset, destination string) func() error { //nolint:gocognit - return func() error { - for x := range q { - c.metrics.IncrCounter([]string{"cp", "fileset", "attempt"}, 1) - dt, err := x.dateTime(c.fs, c.exif) - if err != nil { - c.metrics.IncrCounter([]string{"cp", "fileset", "failed", "exif"}, 1) - if errors.Is(err, io.EOF) { - return nil - } - return err - } - if dt.IsZero() { - c.metrics.IncrCounter([]string{"cp", "fileset", "skip", "unsupported"}, 1) - for i := range x.files { - filename := filepath.Join(x.identifier.dirname, x.files[i].Name()) - ext := filepath.Ext(filename) - ext = strings.TrimPrefix(ext, ".") - if ext == "" { - ext = "" - } - log.Warn().Str("filename", filename).Str("reason", "unsupported."+ext).Msg("skip") - c.metrics.IncrCounter([]string{"cp", "skip", "unsupported", ext}, 1) - } - continue - } - df := dt.Format(c.dateFormat) - for i := range x.files { - src := filepath.Join(x.identifier.dirname, x.files[i].Name()) - dst := filepath.Join(destination, df, x.files[i].Name()) - if err = c.copyFile(src, dst); err != nil { - return err - } - } - } - return nil - } -} - -func (c *entangler) copy(src, dst string) error { - dirname, _ := filepath.Split(dst) - if err := c.fs.MkdirAll(dirname, 0755); err != nil { - return err - } - out, err := c.fs.Create(dst) - if err != nil { - return err - } - defer out.Close() - in, err := c.fs.Open(src) - if err != nil { - return err - } - defer in.Close() - _, err = io.Copy(out, in) - if err != nil { - return err - } - err = out.Sync() - if err != nil { - return err - } - info, err := in.Stat() - if err != nil { - return err - } - mtime := info.ModTime() - return c.fs.Chtimes(dst, mtime, mtime) -} - -func (c *entangler) copyFile(source, destination string) error { - defer c.metrics.MeasureSince([]string{"cp", "elapsed", "file"}, time.Now()) - c.metrics.IncrCounter([]string{"cp", "file", "attempt"}, 1) - stat, err := c.fs.Stat(destination) - if err != nil { - if !errors.Is(err, afero.ErrFileNotFound) { - return err - } - } - if stat != nil { - c.metrics.IncrCounter([]string{"cp", "skip", "exists"}, 1) - log.Info().Str("src", source).Str("dst", destination).Str("reason", "exists").Msg("skip") - return nil - } - log.Info().Str("src", source).Str("dst", destination).Msg("cp") - if c.dryrun { - c.metrics.IncrCounter([]string{"cp", "file", "dryrun"}, 1) - return nil - } - if err = c.copy(source, destination); err != nil { - c.metrics.IncrCounter([]string{"cp", "file", "failed"}, 1) - return err - } - c.metrics.IncrCounter([]string{"cp", "file", "success"}, 1) - return nil -} - -// filesets creates filesets from a directory traversal -func (c *entangler) filesets(sets map[identifier][]fs.FileInfo) filepath.WalkFunc { - return func(path string, info fs.FileInfo, err error) error { - if err != nil { - if errors.Is(err, fs.ErrPermission) { - c.metrics.IncrCounter([]string{"cp", "skip", "denied"}, 1) - log.Warn().Str("path", path).Err(err).Msg("skip") - return filepath.SkipDir - } - return err - } - if info.IsDir() { - c.metrics.IncrCounter([]string{"cp", "visited", "directories"}, 1) - return nil - } - if strings.HasPrefix(info.Name(), ".") { - c.metrics.IncrCounter([]string{"cp", "skip", "hidden"}, 1) - return nil - } - c.metrics.IncrCounter([]string{"cp", "visited", "files"}, 1) - - id := split(path) - sets[id] = append(sets[id], info) - - return nil - } -} - -func cp(c *cli.Context) error { - if c.NArg() < 2 { - return fmt.Errorf("expected 2+ arguments, not {%d}", c.NArg()) - } - defer runtime(c).Metrics.MeasureSince([]string{"cp", "elapsed"}, time.Now()) - en := &entangler{ - fs: runtime(c).Fs, - metrics: runtime(c).Metrics, - concurrency: c.Int("concurrency"), - dryrun: c.Bool("dryrun"), - dateFormat: c.String("format"), - exif: runtime(c).Exif, - } - args := c.Args().Slice() - destination, err := filepath.Abs(args[len(args)-1]) - if err != nil { - return err - } - return en.cp(c.Context, args[0:len(args)-1], destination) -} - -func CommandCopy() *cli.Command { - return &cli.Command{ - Name: "cp", - HelpName: "cp", - Usage: "Copy files to a date-formatted directory structure", - Description: "Copy files from a source(s) to a destination using the image date to layout the directory structure", - ArgsUsage: " [, ...] ", - Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "dryrun", - Aliases: []string{"n"}, - Usage: "prepare to copy but don't actually do it", - Value: false, - Required: false, - }, - &cli.StringFlag{ - Name: "format", - Usage: "the date format used for the destination directory", - Value: defaultDateFormat, - Required: false, - }, - &cli.IntFlag{ - Name: "concurrency", - Aliases: []string{"c"}, - Usage: "the number of concurrent copy operations", - Value: 2, - }, - }, - Action: cp, - } -} diff --git a/cp_test.go b/cp_test.go deleted file mode 100644 index 0134380..0000000 --- a/cp_test.go +++ /dev/null @@ -1,371 +0,0 @@ -package ma_test - -import ( - "context" - "io" - "io/fs" - "os" - "strings" - "testing" - "time" - - "github.com/rs/zerolog/log" - "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/urfave/cli/v2" - - "github.com/bzimmer/ma" -) - -func createTestFile(t *testing.T, afs afero.Fs) afero.File { - if err := afs.MkdirAll("/foo/bar", 0755); err != nil { - t.Error(err) - } - fp, err := afs.Create("/foo/bar/Nikon_D70.jpg") - if err != nil { - t.Error(err) - } - defer fp.Close() - if err = copyFile(fp, "testdata/Nikon_D70.jpg"); err != nil { - t.Error(err) - } - return fp -} - -func TestCopy(t *testing.T) { - a := assert.New(t) - tests := []harness{ - { - name: "one argument", - args: []string{"cp", "/foo/bar"}, - err: "expected 2+ arguments", - }, - { - name: "empty directory", - args: []string{"cp", "/foo/bar", "/foo/baz"}, - counters: map[string]int{ - "cp.visited.directories": 1, - }, - before: func(c *cli.Context) error { - a.NoError(runtime(c).Fs.MkdirAll("/foo/bar", 0755)) - return nil - }, - }, - { - name: "hidden files", - args: []string{"cp", "/foo/bar", "/foo/baz"}, - counters: map[string]int{ - "cp.visited.directories": 1, - "cp.skip.hidden": 2, - }, - before: func(c *cli.Context) error { - a.NoError(runtime(c).Fs.MkdirAll("/foo/bar", 0755)) - fp, err := runtime(c).Fs.Create("/foo/bar/.something") - a.NoError(err) - a.NoError(fp.Close()) - fp, err = runtime(c).Fs.Create("/foo/bar/.else") - a.NoError(err) - a.NoError(fp.Close()) - return nil - }, - }, - { - name: "filename with no extension", - args: []string{"cp", "/foo/bar", "/foo/baz"}, - counters: map[string]int{ - "cp.visited.directories": 1, - "cp.skip.unsupported.": 1, - }, - before: func(c *cli.Context) error { - a.NoError(runtime(c).Fs.MkdirAll("/foo/bar", 0755)) - fp, err := runtime(c).Fs.Create("/foo/bar/something") - a.NoError(err) - a.NoError(fp.Close()) - return nil - }, - }, - { - name: "unsupported files", - args: []string{"cp", "/foo/bar", "/foo/baz"}, - counters: map[string]int{ - "cp.visited.directories": 2, - "cp.skip.unsupported.UKN": 1, - "cp.skip.unsupported.txt": 1, - }, - before: func(c *cli.Context) error { - a.NoError(runtime(c).Fs.MkdirAll("/foo/bar/boo", 0755)) - fp, err := runtime(c).Fs.Create("/foo/bar/DSC18920.UKN") - a.NoError(err) - a.NoError(fp.Close()) - fp, err = runtime(c).Fs.Create("/foo/bar/schedule.txt") - a.NoError(err) - a.NoError(fp.Close()) - return nil - }, - }, - { - name: "single image dng", - args: []string{"cp", "/foo/bar", "/foo/baz"}, - counters: map[string]int{ - "cp.visited.directories": 1, - }, - before: func(c *cli.Context) error { - image := createTestFile(t, runtime(c).Fs) - a.NoError(image.Close()) - // a bit of hack to test reading the entire contents of a .dng file - // the exif parser doesn't care about file extensions, it sees only bytes - name := image.Name() - name = strings.Replace(name, ".jpg", ".dng", 1) - a.NoError(runtime(c).Fs.Rename(image.Name(), name)) - return nil - }, - }, - { - name: "single image", - args: []string{"cp", "/foo/bar", "/foo/baz"}, - counters: map[string]int{ - "cp.visited.directories": 1, - }, - before: func(c *cli.Context) error { - image := createTestFile(t, runtime(c).Fs) - a.NoError(image.Close()) - - tm := time.Date(2008, time.March, 15, 11, 22, 0, 0, time.Local) - a.NoError(runtime(c).Fs.Chtimes(image.Name(), tm, tm)) - - dst, err := runtime(c).Fs.Stat(image.Name()) - a.NoError(err) - a.NotNil(dst) - log.Info().Str("src", image.Name()).Time("dst", dst.ModTime()).Msg("set test modification times") - return nil - }, - after: func(c *cli.Context) error { - dst, err := runtime(c).Fs.Stat("/foo/baz/2008/2008-03/15/Nikon_D70.jpg") - a.NoError(err) - a.NotNil(dst) - if false { - // @todo memfs chtimes doesn't seem to work properly -- need to investigate - t := time.Date(2008, time.March, 15, 11, 22, 0, 0, time.Local) - log.Info().Time("src", t).Time("dst", dst.ModTime()).Msg("asserting modification times") - a.Equalf(t, dst.ModTime(), "expected identical modification times") - } - return nil - }, - }, - { - name: "image exists", - args: []string{"cp", "/foo/bar", "/foo/baz"}, - counters: map[string]int{ - "cp.visited.directories": 1, - "cp.skip.exists": 1, - }, - before: func(c *cli.Context) error { - a.NoError(runtime(c).Fs.MkdirAll("/foo/bar", 0755)) - for _, filename := range []string{"/foo/bar/Nikon_D70.xmp", "/foo/baz/2008/2008-03/15/Nikon_D70.jpg"} { - fp, err := runtime(c).Fs.Create(filename) - a.NoError(err) - a.NoError(fp.Close()) - image, err := runtime(c).Fs.Create("/foo/bar/Nikon_D70.jpg") - a.NoError(err) - fp, err = os.Open("testdata/Nikon_D70.jpg") - a.NoError(err) - _, err = io.Copy(image, fp) - a.NoError(err) - a.NoError(image.Sync()) - a.NoError(image.Close()) - a.NoError(fp.Close()) - } - return nil - }, - }, - { - name: "image + xmp", - args: []string{"cp", "/foo/bar", "/foo/baz"}, - counters: map[string]int{ - "cp.visited.directories": 1, - }, - before: func(c *cli.Context) error { - fp := createTestFile(t, runtime(c).Fs) - a.NoError(fp.Close()) - fp, err := runtime(c).Fs.Create("/foo/bar/Nikon_D70.xmp") - a.NoError(err) - a.NoError(fp.Close()) - return nil - }, - after: func(c *cli.Context) error { - stat, err := runtime(c).Fs.Stat("/foo/baz/2008/2008-03/15/Nikon_D70.jpg") - a.NoError(err) - a.NotNil(stat) - stat, err = runtime(c).Fs.Stat("/foo/baz/2008/2008-03/15/Nikon_D70.xmp") - a.NoError(err) - a.NotNil(stat) - return nil - }, - }, - { - name: "two valid files", - args: []string{"cp", "/foo/bar", "/foo/baz"}, - counters: map[string]int{ - "cp.visited.directories": 1, - "cp.visited.files": 2, - }, - before: func(c *cli.Context) error { - fp := createTestFile(t, runtime(c).Fs) - a.NoError(fp.Close()) - fp, err := runtime(c).Fs.Create("/foo/bar/Nikon_D70_0.jpg") - a.NoError(err) - a.NoError(copyFile(fp, "testdata/Nikon_D70.jpg")) - a.NoError(fp.Close()) - return nil - }, - after: func(c *cli.Context) error { - stat, err := runtime(c).Fs.Stat("/foo/baz/2008/2008-03/15/Nikon_D70.jpg") - a.NoError(err) - a.NotNil(stat) - stat, err = runtime(c).Fs.Stat("/foo/baz/2008/2008-03/15/Nikon_D70_0.jpg") - a.NoError(err) - a.NotNil(stat) - return nil - }, - }, - { - name: "image + xmp dry-run", - args: []string{"cp", "-n", "/foo/bar", "/foo/baz"}, - counters: map[string]int{ - "cp.visited.directories": 1, - "cp.file.dryrun": 2, - }, - before: func(c *cli.Context) error { - fp := createTestFile(t, runtime(c).Fs) - a.NoError(fp.Close()) - fp, err := runtime(c).Fs.Create("/foo/bar/Nikon_D70.xmp") - a.NoError(err) - a.NoError(fp.Close()) - return nil - }, - after: func(c *cli.Context) error { - _, err := runtime(c).Fs.Stat("/foo/baz/2008/2008-03/15/Nikon_D70.jpg") - a.Error(err) - a.True(os.IsNotExist(err)) - _, err = runtime(c).Fs.Stat("/foo/baz/2008/2008-03/15/Nikon_D70.xmp") - a.Error(err) - a.True(os.IsNotExist(err)) - return nil - }, - }, - { - name: "image + xmp in different directories", - args: []string{"cp", "/foo/bar", "/foo/baz"}, - counters: map[string]int{ - "cp.visited.files": 2, - "cp.visited.directories": 2, - "cp.fileset.skip.unsupported": 1, - }, - before: func(c *cli.Context) error { - fp := createTestFile(t, runtime(c).Fs) - a.NoError(fp.Close()) - a.NoError(runtime(c).Fs.MkdirAll("/foo/bar/boo", 0755)) - fp, err := runtime(c).Fs.Create("/foo/bar/boo/Nikon_D70.xmp") - a.NoError(err) - a.NoError(fp.Close()) - return nil - }, - after: func(c *cli.Context) error { - stat, err := runtime(c).Fs.Stat("/foo/baz/2008/2008-03/15/Nikon_D70.jpg") - a.NoError(err) - a.NotNil(stat) - _, err = runtime(c).Fs.Stat("/foo/baz/2008/2008-03/15/Nikon_D70.xmp") - a.Error(err) - a.True(os.IsNotExist(err)) - return nil - }, - }, - { - name: "image with garbage exif", - args: []string{"cp", "/foo/bar", "/foo/baz"}, - counters: map[string]int{ - "cp.filesets": 1, - "cp.visited.directories": 1, - "cp.visited.files": 1, - "cp.fileset.failed.exif": 1, - }, - before: func(c *cli.Context) error { - fp, err := runtime(c).Fs.Create("/foo/bar/Nikon_D70.jpg") - a.NoError(err) - a.NoError(copyFile(fp, "testdata/user_cmac.json")) - a.NoError(fp.Close()) - return nil - }, - }, - { - name: "fail on copy", - args: []string{"cp", "/foo/bar", "/foo/baz"}, - err: "operation not permitted", - counters: map[string]int{ - "cp.visited.directories": 1, - }, - before: func(c *cli.Context) error { - fp := createTestFile(t, runtime(c).Fs) - a.NoError(fp.Close()) - runtime(c).Fs = afero.NewReadOnlyFs(runtime(c).Fs) - return nil - }, - after: func(c *cli.Context) error { - stat, err := runtime(c).Fs.Stat("/foo/baz/2008/2008-03/15/Nikon_D70.jpg") - a.True(os.IsNotExist(err)) - a.Nil(stat) - return nil - }, - }, - { - name: "directory without read/execute permissions", - args: []string{"cp", "/foo/bar", "/foo/baz"}, - counters: map[string]int{ - "cp.skip.denied": 1, - "cp.visited.directories": 5, - }, - before: func(c *cli.Context) error { - a.NoError(runtime(c).Fs.MkdirAll("/foo/bar/boo0", 0755)) - a.NoError(runtime(c).Fs.MkdirAll("/foo/bar/boo1", 0755)) - a.NoError(runtime(c).Fs.MkdirAll("/foo/bar/boo2", 0600)) - a.NoError(runtime(c).Fs.MkdirAll("/foo/bar/boo3", 0755)) - runtime(c).Fs = &ErrFs{Fs: runtime(c).Fs, err: fs.ErrPermission, name: "/foo/bar/boo2"} - return nil - }, - }, - { - name: "directory walk error", - args: []string{"cp", "/foo/bar", "/foo/baz"}, - err: "invalid argument", - before: func(c *cli.Context) error { - a.NoError(runtime(c).Fs.MkdirAll("/foo/bar/boo0", 0755)) - a.NoError(runtime(c).Fs.MkdirAll("/foo/bar/boo3", 0755)) - runtime(c).Fs = &ErrFs{Fs: runtime(c).Fs, err: fs.ErrInvalid, name: "/foo/bar/boo3"} - return nil - }, - }, - { - name: "canceled context", - args: []string{"cp", "/foo/bar", "/foo/baz"}, - err: context.Canceled.Error(), - before: func(c *cli.Context) error { - fp := createTestFile(t, runtime(c).Fs) - a.NoError(fp.Close()) - return nil - }, - context: func(ctx context.Context) context.Context { - ctx, cancel := context.WithCancel(ctx) - cancel() - <-ctx.Done() - return ctx - }, - }, - } - - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - run(t, &tt, nil, ma.CommandCopy) - }) - } -} diff --git a/docs/commands.md b/docs/commands.md index b98f72e..961408c 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -10,12 +10,12 @@ All your media archiving needs! |smugmug-access-token||SMUGMUG_ACCESS_TOKEN|smugmug access token| |smugmug-token-secret||SMUGMUG_TOKEN_SECRET|smugmug token secret| |json|j||emit all results as JSON and print to stdout| -|monochrome|||disable colored output| -|debug|||enable debugging of http requests| +|monochrome|||disable colored loggingoutput| +|debug|||enable verbose debugging| +|trace|||enable debugging of http requests| |help|h||show help| ## Commands -* [cp](#cp) * [envvars](#envvars) * [export](#export) * [find](#find) @@ -39,30 +39,6 @@ All your media archiving needs! * [user](#user) * [version](#version) -### *cp* - -**Description** - -Copy files from a source(s) to a destination using the image date to layout the directory structure - - - -**Syntax** - -```sh -$ ma cp [flags] [, ...] -``` - - -**Flags** - -|Name|Aliases|EnvVars|Description| -|-|-|-|-| -|dryrun|n||prepare to copy but don't actually do it| -|format|||the date format used for the destination directory| -|concurrency|c||the number of concurrent copy operations| - - ### *envvars* **Description** diff --git a/exif.go b/exif.go deleted file mode 100644 index 2458b60..0000000 --- a/exif.go +++ /dev/null @@ -1,117 +0,0 @@ -package ma - -import ( - "errors" - "io" - "io/fs" - "path/filepath" - "strings" - "time" - - "github.com/rs/zerolog/log" - "github.com/rwcarlsen/goexif/exif" - "github.com/spf13/afero" - "github.com/urfave/cli/v2" -) - -var _ Exif = (*goExif)(nil) - -const exifHeaderSize = 4 - -// MetaData represents the EXIF data about a file -type MetaData struct { - // Info is the analyzed file - Info fs.FileInfo - - // Err is non-nil if an error occurred processing the file - Err error - // DateTime is the best effort `DateTimeOriginal` of the file - DateTime time.Time -} - -// Exif extracts EXIF metadata from files -type Exif interface { - // Extract returns metadata about a file - Extract(afs afero.Fs, dirname string, infos ...fs.FileInfo) []MetaData -} - -func NewGoExif() Exif { - return new(goExif) -} - -type goExif struct{} - -func (x *goExif) datetime(afs afero.Fs, filename string) (time.Time, error) { - fp, err := afs.Open(filename) - if err != nil { - return time.Time{}, err - } - defer fp.Close() - m, err := exif.Decode(fp) - if err != nil { - return time.Time{}, err - } - return m.DateTime() -} - -func (x *goExif) Extract(afs afero.Fs, dirname string, infos ...fs.FileInfo) []MetaData { - mds := make([]MetaData, len(infos)) - for i := range infos { - mds[i] = MetaData{Info: infos[i]} - ext := strings.ToLower(filepath.Ext(mds[i].Info.Name())) - switch ext { - case "", ".pxm", ".xmp": - case ".orf", ".mov", ".avi", ".mp4": - mds[i].DateTime = mds[i].Info.ModTime() - default: - if mds[i].Info.Size() < exifHeaderSize { - // the exif header is four bytes long so bail rather than EOF - continue - } - mds[i].DateTime, mds[i].Err = x.datetime(afs, filepath.Join(dirname, mds[i].Info.Name())) - } - } - return mds -} - -func xif(c *cli.Context) error { //nolint:gocognit - afs := runtime(c).Fs - exf := runtime(c).Exif - for i := 0; i < c.NArg(); i++ { - if err := afero.Walk(afs, c.Args().Get(i), func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - dirname, _ := filepath.Split(path) - for _, m := range exf.Extract(afs, dirname, info) { - if m.Err != nil { - if errors.Is(m.Err, io.EOF) { - log.Warn().Time("datetime", m.DateTime).Str("filename", path).Msg(c.Command.Name) - return nil - } - log.Error().Err(m.Err).Time("datetime", m.DateTime).Str("filename", path).Msg(c.Command.Name) - return m.Err - } - log.Info().Str("filename", path).Time("datetime", m.DateTime).Msg(c.Command.Name) - } - return nil - }); err != nil { - return err - } - } - return nil -} - -func CommandExif() *cli.Command { - return &cli.Command{ - Name: "exif", - HelpName: "exif", - Hidden: true, - Usage: "debugging tool for exif data", - Description: "Debugging tool for exif data", - Action: xif, - } -} diff --git a/exif_test.go b/exif_test.go deleted file mode 100644 index fd3825b..0000000 --- a/exif_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package ma_test - -import ( - "io/fs" - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/urfave/cli/v2" - - "github.com/bzimmer/ma" -) - -func TestExif(t *testing.T) { - a := assert.New(t) - tests := []harness{ - { - name: "no arguments", - args: []string{"exif"}, - }, - { - name: "supported exif file", - args: []string{"exif", "/foo/bar/Nikon_D70.jpg"}, - before: func(c *cli.Context) error { - fp := createTestFile(t, runtime(c).Fs) - a.NotNil(fp) - defer fp.Close() - return nil - }, - }, - { - name: "error does not exist", - args: []string{"exif", "/foo/bar/Nikon_D70.jpg"}, - err: os.ErrNotExist.Error(), - }, - { - name: "error opening file", - args: []string{"exif", "/foo/bar/"}, - err: os.ErrPermission.Error(), - before: func(c *cli.Context) error { - fp := createTestFile(t, runtime(c).Fs) - fp.Close() - runtime(c).Fs = &ErrFs{Fs: runtime(c).Fs, err: fs.ErrPermission, name: fp.Name()} - return nil - }, - }, - { - name: "unsupported exif file", - args: []string{"exif", "/foo/bar/Olympus.orf"}, - before: func(c *cli.Context) error { - afs := runtime(c).Fs - a.NoError(afs.MkdirAll("/foo/bar", 0755)) - fp, err := afs.Create("/foo/bar/Olympus.orf") - a.NoError(err) - a.NotNil(fp) - defer fp.Close() - return nil - }, - }, - { - name: "file with no exif data", - args: []string{"exif", "/foo/bar/user_cmac.json"}, - before: func(c *cli.Context) error { - afs := runtime(c).Fs - a.NoError(afs.MkdirAll("/foo/bar", 0755)) - fp, err := afs.Create("/foo/bar/user_cmac.json") - a.NoError(err) - a.NotNil(fp) - a.NoError(copyFile(fp, "testdata/user_cmac.json")) - defer fp.Close() - return nil - }, - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - run(t, &tt, nil, ma.CommandExif) - }) - } -} diff --git a/exiftool.go b/exiftool.go deleted file mode 100644 index 3e5baee..0000000 --- a/exiftool.go +++ /dev/null @@ -1,74 +0,0 @@ -//go:build exiftool - -package ma - -/* -This implementation of an `Exif` leverages the external `exiftool`. While `exiftool` -is definitely more capable of a wide range of files it's also far slower and requires an -external dependency. -*/ - -import ( - "errors" - "io/fs" - "path/filepath" - "strings" - "time" - - "github.com/barasher/go-exiftool" - "github.com/spf13/afero" -) - -var _ Exif = (*perl)(nil) - -const exifTimeLayout = "2006:01:02 15:04:05" - -type ExiftoolOption func(*perl) error - -func NewExiftool(options ...ExiftoolOption) (Exif, error) { - x := new(perl) - for _, opt := range options { - if err := opt(x); err != nil { - return nil, err - } - } - if x.tool == nil { - return nil, errors.New("no exiftool.Tool specified") - } - return x, nil -} - -type perl struct { - tool *exiftool.Exiftool -} - -func (x *perl) Extract(_ afero.Fs, dirname string, infos ...fs.FileInfo) []MetaData { - filenames := make([]string, len(infos)) - for i := range infos { - switch ext := strings.ToLower(filepath.Ext(infos[i].Name())); ext { - case "", ".pxm", ".xmp": - default: - filenames[i] = filepath.Join(dirname, infos[i].Name()) - } - } - mds := make([]MetaData, len(infos)) - - for i, m := range x.tool.ExtractMetadata(filenames...) { - if m.Err != nil { - if filenames[i] != "" { - mds[i].Err = m.Err - } - continue - } - if dto, ok := m.Fields["DateTimeOriginal"]; ok { - timeZone := time.Local - tm, err := time.ParseInLocation(exifTimeLayout, dto.(string), timeZone) - if err != nil { - mds[i].Err = err - continue - } - mds[i].DateTime = tm - } - } - return mds -} diff --git a/go.mod b/go.mod index e42c982..671ea65 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,13 @@ module github.com/bzimmer/ma -go 1.22 +go 1.22.3 require ( - github.com/barasher/go-exiftool v1.10.0 github.com/bzimmer/httpwares v0.1.3 github.com/bzimmer/manual v0.1.5 - github.com/bzimmer/smugmug v0.7.2 + github.com/bzimmer/smugmug v0.7.3 github.com/hashicorp/go-metrics v0.5.3 github.com/rs/zerolog v1.33.0 - github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd github.com/spf13/afero v1.11.0 github.com/stretchr/testify v1.8.2 github.com/urfave/cli/v2 v2.27.2 @@ -27,6 +25,7 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mrjones/oauth v0.0.0-20190623134757-126b35219450 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect golang.org/x/sys v0.20.0 // indirect diff --git a/go.sum b/go.sum index 11d1f8f..58cf2b2 100644 --- a/go.sum +++ b/go.sum @@ -3,8 +3,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/barasher/go-exiftool v1.10.0 h1:f5JY5jc42M7tzR6tbL9508S2IXdIcG9QyieEXNMpIhs= -github.com/barasher/go-exiftool v1.10.0/go.mod h1:F9s/a3uHSM8YniVfwF+sbQUtP8Gmh9nyzigNF+8vsWo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -12,8 +10,8 @@ github.com/bzimmer/httpwares v0.1.3 h1:Haw1fGBRW51iv7O2NIkIZyuUt3XLZZG+ePd6NEwbD github.com/bzimmer/httpwares v0.1.3/go.mod h1:8pi184rxXR7Pbn7cNL8uMPeqYA8+DbQSl4oOQE5q4Vk= github.com/bzimmer/manual v0.1.5 h1:K/MQTTCO6r2HUhwReCAQOUAgbudl6ku6id1ZJpN0O9Y= github.com/bzimmer/manual v0.1.5/go.mod h1:OSIzkDN37qwrQ5M2h9MNsC08uOG1BaHmp9tHeNTdt0Q= -github.com/bzimmer/smugmug v0.7.2 h1:AjcoyMD1gaD4EQkPYWIR3Hw5wPmJxLsIrF8gt+8AjLw= -github.com/bzimmer/smugmug v0.7.2/go.mod h1:+WWiiNIwcqqPKeWWv8UwTyHcpDKti+bqZnGNmCLn7gg= +github.com/bzimmer/smugmug v0.7.3 h1:CM4a+YRXNoWGSpGVlJnDxQfxGm6ZTNKl7hbcb6nALfg= +github.com/bzimmer/smugmug v0.7.3/go.mod h1:8iYBv82RcYc71I1sK0zP/TujaxNU3AF863iBFcrMi5w= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= @@ -53,8 +51,9 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -90,13 +89,13 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= diff --git a/iter.go b/iter.go index d801c1c..9296f73 100644 --- a/iter.go +++ b/iter.go @@ -18,7 +18,7 @@ func imageIterFunc(c *cli.Context, album *smugmug.Album, op string) smugmug.Imag if album != nil && image.Album == nil { image.Album = album } - log.Info(). + log.Debug(). Str("type", "Image"). Str("albumKey", albumKey). Str("imageKey", image.ImageKey). @@ -39,7 +39,7 @@ func albumIterFunc(c *cli.Context, op string) smugmug.AlbumIterFunc { imageq := c.Bool("image") return func(album *smugmug.Album) (bool, error) { runtime(c).Metrics.IncrCounter([]string{op, "album"}, 1) - log.Info(). + log.Debug(). Str("type", smugmug.TypeAlbum). Str("name", album.Name). Str("nodeID", album.NodeID). @@ -67,7 +67,7 @@ func nodeIterFunc(c *cli.Context, recurse bool, op string) smugmug.NodeIterFunc imageq := c.Bool("image") return func(node *smugmug.Node) (bool, error) { runtime(c).Metrics.IncrCounter([]string{op, "node"}, 1) - msg := log.Info() + msg := log.Debug() msg = msg.Str("type", node.Type) msg = msg.Str("name", node.Name) msg = msg.Str("nodeID", node.NodeID) diff --git a/ma_test.go b/ma_test.go index 4e7b699..1b5e309 100644 --- a/ma_test.go +++ b/ma_test.go @@ -107,7 +107,6 @@ func NewTestApp(t *testing.T, tt *harness, cmd *cli.Command, url string) *cli.Ap Encoder: json.NewEncoder(writer), Grab: new(http.Client), Fs: afero.NewMemMapFs(), - Exif: ma.NewGoExif(), Language: language.English, Start: time.Now(), } diff --git a/runtime.go b/runtime.go index 84ee251..7b847c3 100644 --- a/runtime.go +++ b/runtime.go @@ -31,8 +31,6 @@ type Runtime struct { Fs afero.Fs // Grab for bulk querying images Grab Grab - // Exif for accessing EXIF metadata - Exif Exif // Language for title case Language language.Tag // Start time of the execution