From c21ded028a19ad8cfd65f3e3a5fcf9113e05acb7 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Tue, 3 Sep 2024 16:33:15 +1000 Subject: [PATCH] Scan video orientation (#5189) * Adjust video dimensions for side data rotation * Warn user when ffprobe version < 5. Only get rotation data on version >= 5 --- cmd/phasher/main.go | 4 +- internal/manager/init.go | 2 +- internal/manager/manager.go | 6 +- pkg/ffmpeg/codec_hardware.go | 12 ++-- pkg/ffmpeg/ffmpeg.go | 15 +++-- pkg/ffmpeg/ffmpeg_test.go | 40 ++++++------ pkg/ffmpeg/ffprobe.go | 120 ++++++++++++++++++++++++++++++++--- pkg/ffmpeg/stream.go | 4 +- pkg/ffmpeg/types.go | 3 + pkg/file/image/scan.go | 2 +- pkg/file/video/scan.go | 4 +- pkg/image/thumbnail.go | 4 +- 12 files changed, 163 insertions(+), 53 deletions(-) diff --git a/cmd/phasher/main.go b/cmd/phasher/main.go index d4bf79590..864195631 100644 --- a/cmd/phasher/main.go +++ b/cmd/phasher/main.go @@ -18,7 +18,7 @@ func customUsage() { flag.PrintDefaults() } -func printPhash(ff *ffmpeg.FFMpeg, ffp ffmpeg.FFProbe, inputfile string, quiet *bool) error { +func printPhash(ff *ffmpeg.FFMpeg, ffp *ffmpeg.FFProbe, inputfile string, quiet *bool) error { ffvideoFile, err := ffp.NewVideoFile(inputfile) if err != nil { return err @@ -80,7 +80,7 @@ func main() { ffmpegPath, ffprobePath := getPaths() encoder := ffmpeg.NewEncoder(ffmpegPath) // don't need to InitHWSupport, phashing doesn't use hw acceleration - ffprobe := ffmpeg.FFProbe(ffprobePath) + ffprobe := ffmpeg.NewFFProbe(ffprobePath) for _, item := range args { if err := printPhash(encoder, ffprobe, item, quiet); err != nil { diff --git a/internal/manager/init.go b/internal/manager/init.go index 020ba944d..dd1640ed3 100644 --- a/internal/manager/init.go +++ b/internal/manager/init.go @@ -311,7 +311,7 @@ func (s *Manager) RefreshFFMpeg(ctx context.Context) { logger.Debugf("using ffprobe: %s", ffprobePath) s.FFMpeg = ffmpeg.NewEncoder(ffmpegPath) - s.FFProbe = ffmpeg.FFProbe(ffprobePath) + s.FFProbe = ffmpeg.NewFFProbe(ffprobePath) s.FFMpeg.InitHWSupport(ctx) } diff --git a/internal/manager/manager.go b/internal/manager/manager.go index ffba184d2..4827a3e3d 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -43,7 +43,7 @@ type Manager struct { Paths *paths.Paths FFMpeg *ffmpeg.FFMpeg - FFProbe ffmpeg.FFProbe + FFProbe *ffmpeg.FFProbe StreamManager *ffmpeg.StreamManager JobManager *job.Manager @@ -300,7 +300,7 @@ func (s *Manager) Setup(ctx context.Context, input SetupInput) error { } func (s *Manager) validateFFmpeg() error { - if s.FFMpeg == nil || s.FFProbe == "" { + if s.FFMpeg == nil || s.FFProbe == nil { return errors.New("missing ffmpeg and/or ffprobe") } return nil @@ -400,7 +400,7 @@ func (s *Manager) GetSystemStatus() *SystemStatus { } ffprobePath := "" - if s.FFProbe != "" { + if s.FFProbe != nil { ffprobePath = s.FFProbe.Path() } diff --git a/pkg/ffmpeg/codec_hardware.go b/pkg/ffmpeg/codec_hardware.go index 73d825706..5151e7efe 100644 --- a/pkg/ffmpeg/codec_hardware.go +++ b/pkg/ffmpeg/codec_hardware.go @@ -277,15 +277,15 @@ func (f *FFMpeg) hwCodecFilter(args VideoFilter, codec VideoCodec, vf *models.Vi func (f *FFMpeg) hwApplyFullHWFilter(args VideoFilter, codec VideoCodec, fullhw bool) VideoFilter { switch codec { case VideoCodecN264, VideoCodecN264H: - if fullhw && f.version.Gteq(FFMpegVersion{major: 5}) { // Added in FFMpeg 5 + if fullhw && f.version.Gteq(Version{major: 5}) { // Added in FFMpeg 5 args = args.Append("scale_cuda=format=yuv420p") } case VideoCodecV264, VideoCodecVVP9: - if fullhw && f.version.Gteq(FFMpegVersion{major: 3, minor: 1}) { // Added in FFMpeg 3.1 + if fullhw && f.version.Gteq(Version{major: 3, minor: 1}) { // Added in FFMpeg 3.1 args = args.Append("scale_vaapi=format=nv12") } case VideoCodecI264, VideoCodecI264C, VideoCodecIVP9: - if fullhw && f.version.Gteq(FFMpegVersion{major: 3, minor: 3}) { // Added in FFMpeg 3.3 + if fullhw && f.version.Gteq(Version{major: 3, minor: 3}) { // Added in FFMpeg 3.3 args = args.Append("scale_qsv=format=nv12") } } @@ -300,17 +300,17 @@ func (f *FFMpeg) hwApplyScaleTemplate(sargs string, codec VideoCodec, match []in switch codec { case VideoCodecN264, VideoCodecN264H: template = "scale_cuda=$value" - if fullhw && f.version.Gteq(FFMpegVersion{major: 5}) { // Added in FFMpeg 5 + if fullhw && f.version.Gteq(Version{major: 5}) { // Added in FFMpeg 5 template += ":format=yuv420p" } case VideoCodecV264, VideoCodecVVP9: template = "scale_vaapi=$value" - if fullhw && f.version.Gteq(FFMpegVersion{major: 3, minor: 1}) { // Added in FFMpeg 3.1 + if fullhw && f.version.Gteq(Version{major: 3, minor: 1}) { // Added in FFMpeg 3.1 template += ":format=nv12" } case VideoCodecI264, VideoCodecI264C, VideoCodecIVP9: template = "scale_qsv=$value" - if fullhw && f.version.Gteq(FFMpegVersion{major: 3, minor: 3}) { // Added in FFMpeg 3.3 + if fullhw && f.version.Gteq(Version{major: 3, minor: 3}) { // Added in FFMpeg 3.3 template += ":format=nv12" } case VideoCodecM264: diff --git a/pkg/ffmpeg/ffmpeg.go b/pkg/ffmpeg/ffmpeg.go index c32a3e2fd..ce1232e5d 100644 --- a/pkg/ffmpeg/ffmpeg.go +++ b/pkg/ffmpeg/ffmpeg.go @@ -145,6 +145,8 @@ func ResolveFFMpeg(path string, fallbackPath string) string { return ret } +var version_re = regexp.MustCompile(`ffmpeg version n?((\d+)\.(\d+)(?:\.(\d+))?)`) + func (f *FFMpeg) getVersion() error { var args Args args = append(args, "-version") @@ -158,7 +160,6 @@ func (f *FFMpeg) getVersion() error { return err } - version_re := regexp.MustCompile(`ffmpeg version n?((\d+)\.(\d+)(?:\.(\d+))?)`) stdoutStr := stdout.String() match := version_re.FindStringSubmatchIndex(stdoutStr) if match == nil { @@ -183,20 +184,20 @@ func (f *FFMpeg) getVersion() error { if i, err := strconv.Atoi(patchS); err == nil { f.version.patch = i } - logger.Debugf("FFMpeg version %d.%d.%d detected", f.version.major, f.version.minor, f.version.patch) + logger.Debugf("FFMpeg version %s detected", f.version.String()) return nil } // FFMpeg version params -type FFMpegVersion struct { +type Version struct { major int minor int patch int } // Gteq returns true if the version is greater than or equal to the other version. -func (v FFMpegVersion) Gteq(other FFMpegVersion) bool { +func (v Version) Gteq(other Version) bool { if v.major > other.major { return true } @@ -209,10 +210,14 @@ func (v FFMpegVersion) Gteq(other FFMpegVersion) bool { return false } +func (v Version) String() string { + return fmt.Sprintf("%d.%d.%d", v.major, v.minor, v.patch) +} + // FFMpeg provides an interface to ffmpeg. type FFMpeg struct { ffmpeg string - version FFMpegVersion + version Version hwCodecSupport []VideoCodec } diff --git a/pkg/ffmpeg/ffmpeg_test.go b/pkg/ffmpeg/ffmpeg_test.go index 3e9151ed9..a56c7e61a 100644 --- a/pkg/ffmpeg/ffmpeg_test.go +++ b/pkg/ffmpeg/ffmpeg_test.go @@ -6,62 +6,62 @@ import "testing" func TestFFMpegVersion_GreaterThan(t *testing.T) { tests := []struct { name string - this FFMpegVersion - other FFMpegVersion + this Version + other Version want bool }{ { "major greater, minor equal, patch equal", - FFMpegVersion{2, 0, 0}, - FFMpegVersion{1, 0, 0}, + Version{2, 0, 0}, + Version{1, 0, 0}, true, }, { "major greater, minor less, patch less", - FFMpegVersion{2, 1, 1}, - FFMpegVersion{1, 0, 0}, + Version{2, 1, 1}, + Version{1, 0, 0}, true, }, { "major equal, minor greater, patch equal", - FFMpegVersion{1, 1, 0}, - FFMpegVersion{1, 0, 0}, + Version{1, 1, 0}, + Version{1, 0, 0}, true, }, { "major equal, minor equal, patch greater", - FFMpegVersion{1, 0, 1}, - FFMpegVersion{1, 0, 0}, + Version{1, 0, 1}, + Version{1, 0, 0}, true, }, { "major equal, minor equal, patch equal", - FFMpegVersion{1, 0, 0}, - FFMpegVersion{1, 0, 0}, + Version{1, 0, 0}, + Version{1, 0, 0}, true, }, { "major less, minor equal, patch equal", - FFMpegVersion{1, 0, 0}, - FFMpegVersion{2, 0, 0}, + Version{1, 0, 0}, + Version{2, 0, 0}, false, }, { "major equal, minor less, patch equal", - FFMpegVersion{1, 0, 0}, - FFMpegVersion{1, 1, 0}, + Version{1, 0, 0}, + Version{1, 1, 0}, false, }, { "major equal, minor equal, patch less", - FFMpegVersion{1, 0, 0}, - FFMpegVersion{1, 0, 1}, + Version{1, 0, 0}, + Version{1, 0, 1}, false, }, { "major less, minor less, patch less", - FFMpegVersion{1, 0, 0}, - FFMpegVersion{2, 1, 1}, + Version{1, 0, 0}, + Version{2, 1, 1}, false, }, } diff --git a/pkg/ffmpeg/ffprobe.go b/pkg/ffmpeg/ffprobe.go index 31b3cbf00..59f8ed218 100644 --- a/pkg/ffmpeg/ffprobe.go +++ b/pkg/ffmpeg/ffprobe.go @@ -1,12 +1,14 @@ package ffmpeg import ( + "bytes" "encoding/json" "errors" "fmt" "math" "os" "os/exec" + "regexp" "strconv" "strings" "time" @@ -16,6 +18,8 @@ import ( "github.com/stashapp/stash/pkg/logger" ) +const minimumFFProbeVersion = 5 + func ValidateFFProbe(ffprobePath string) error { cmd := stashExec.Command(ffprobePath, "-h") bytes, err := cmd.CombinedOutput() @@ -139,16 +143,94 @@ func (v *VideoFile) TranscodeScale(maxSize int) (int, int) { } // FFProbe provides an interface to the ffprobe executable. -type FFProbe string +type FFProbe struct { + path string + version Version +} func (f *FFProbe) Path() string { - return string(*f) + return f.path +} + +var ffprobeVersionRE = regexp.MustCompile(`ffprobe version n?((\d+)\.(\d+)(?:\.(\d+))?)`) + +func (f *FFProbe) getVersion() error { + var args []string + args = append(args, "-version") + cmd := stashExec.Command(f.path, args...) + + var stdout bytes.Buffer + cmd.Stdout = &stdout + + var err error + if err = cmd.Run(); err != nil { + return err + } + + stdoutStr := stdout.String() + match := ffprobeVersionRE.FindStringSubmatchIndex(stdoutStr) + if match == nil { + return errors.New("version string malformed") + } + + majorS := stdoutStr[match[4]:match[5]] + minorS := stdoutStr[match[6]:match[7]] + + // patch is optional + var patchS string + if match[8] != -1 && match[9] != -1 { + patchS = stdoutStr[match[8]:match[9]] + } + + if i, err := strconv.Atoi(majorS); err == nil { + f.version.major = i + } + if i, err := strconv.Atoi(minorS); err == nil { + f.version.minor = i + } + if i, err := strconv.Atoi(patchS); err == nil { + f.version.patch = i + } + logger.Debugf("FFProbe version %s detected", f.version.String()) + + return nil +} + +// Creates a new FFProbe instance. +func NewFFProbe(path string) *FFProbe { + ret := &FFProbe{ + path: path, + } + if err := ret.getVersion(); err != nil { + logger.Warnf("FFProbe version not detected %v", err) + } + + if ret.version.major != 0 && ret.version.major < minimumFFProbeVersion { + logger.Warnf("FFProbe version %d.%d.%d detected, but %d.x or later is required", ret.version.major, ret.version.minor, ret.version.patch, minimumFFProbeVersion) + } + + return ret } // NewVideoFile runs ffprobe on the given path and returns a VideoFile. func (f *FFProbe) NewVideoFile(videoPath string) (*VideoFile, error) { - args := []string{"-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", "-show_error", videoPath} - cmd := stashExec.Command(string(*f), args...) + args := []string{ + "-v", + "quiet", + "-print_format", "json", + "-show_format", + "-show_streams", + "-show_error", + } + + // show_entries stream_side_data=rotation requires 5.x or later ffprobe + if f.version.major >= 5 { + args = append(args, "-show_entries", "stream_side_data=rotation") + } + + args = append(args, videoPath) + + cmd := stashExec.Command(f.path, args...) out, err := cmd.Output() if err != nil { @@ -167,7 +249,7 @@ func (f *FFProbe) NewVideoFile(videoPath string) (*VideoFile, error) { // Used when the frame count is missing or incorrect. func (f *FFProbe) GetReadFrameCount(path string) (int64, error) { args := []string{"-v", "quiet", "-print_format", "json", "-count_frames", "-show_format", "-show_streams", "-show_error", path} - out, err := stashExec.Command(string(*f), args...).Output() + out, err := stashExec.Command(f.path, args...).Output() if err != nil { return 0, fmt.Errorf("FFProbe encountered an error with <%s>.\nError JSON:\n%s\nError: %s", path, string(out), err.Error()) @@ -246,13 +328,14 @@ func parse(filePath string, probeJSON *FFProbeJSON) (*VideoFile, error) { framerate = 0 } result.FrameRate = math.Round(framerate*100) / 100 - if rotate, err := strconv.ParseInt(videoStream.Tags.Rotate, 10, 64); err == nil && rotate != 180 { + result.Width = videoStream.Width + result.Height = videoStream.Height + + if isRotated(videoStream) { result.Width = videoStream.Height result.Height = videoStream.Width - } else { - result.Width = videoStream.Width - result.Height = videoStream.Height } + result.VideoStreamDuration, err = strconv.ParseFloat(videoStream.Duration, 64) if err != nil { // Revert to the historical behaviour, which is still correct in the vast majority of cases. @@ -263,6 +346,25 @@ func parse(filePath string, probeJSON *FFProbeJSON) (*VideoFile, error) { return result, nil } +func isRotated(s *FFProbeStream) bool { + rotate, _ := strconv.ParseInt(s.Tags.Rotate, 10, 64) + if rotate != 180 && rotate != 0 { + return true + } + + for _, sd := range s.SideDataList { + r := sd.Rotation + if r < 0 { + r = -r + } + if r != 0 && r != 180 { + return true + } + } + + return false +} + func (v *VideoFile) getAudioStream() *FFProbeStream { index := v.getStreamIndex("audio", v.JSON) if index != -1 { diff --git a/pkg/ffmpeg/stream.go b/pkg/ffmpeg/stream.go index b94c03b55..cd043dadc 100644 --- a/pkg/ffmpeg/stream.go +++ b/pkg/ffmpeg/stream.go @@ -23,7 +23,7 @@ const ( type StreamManager struct { cacheDir string encoder *FFMpeg - ffprobe FFProbe + ffprobe *FFProbe config StreamManagerConfig lockManager *fsutil.ReadLockManager @@ -42,7 +42,7 @@ type StreamManagerConfig interface { GetTranscodeHardwareAcceleration() bool } -func NewStreamManager(cacheDir string, encoder *FFMpeg, ffprobe FFProbe, config StreamManagerConfig, lockManager *fsutil.ReadLockManager) *StreamManager { +func NewStreamManager(cacheDir string, encoder *FFMpeg, ffprobe *FFProbe, config StreamManagerConfig, lockManager *fsutil.ReadLockManager) *StreamManager { if cacheDir == "" { logger.Warn("cache directory is not set. Live HLS/DASH transcoding will be disabled") } diff --git a/pkg/ffmpeg/types.go b/pkg/ffmpeg/types.go index c9454cb40..c463e7f97 100644 --- a/pkg/ffmpeg/types.go +++ b/pkg/ffmpeg/types.go @@ -94,4 +94,7 @@ type FFProbeStream struct { MaxBitRate string `json:"max_bit_rate,omitempty"` SampleFmt string `json:"sample_fmt,omitempty"` SampleRate string `json:"sample_rate,omitempty"` + SideDataList []struct { + Rotation int `json:"rotation"` + } `json:"side_data_list"` } diff --git a/pkg/file/image/scan.go b/pkg/file/image/scan.go index 258f3e42b..a1d63f649 100644 --- a/pkg/file/image/scan.go +++ b/pkg/file/image/scan.go @@ -19,7 +19,7 @@ import ( // Decorator adds image specific fields to a File. type Decorator struct { - FFProbe ffmpeg.FFProbe + FFProbe *ffmpeg.FFProbe } func (d *Decorator) Decorate(ctx context.Context, fs models.FS, f models.File) (models.File, error) { diff --git a/pkg/file/video/scan.go b/pkg/file/video/scan.go index ca7d0be96..21be0cd11 100644 --- a/pkg/file/video/scan.go +++ b/pkg/file/video/scan.go @@ -12,11 +12,11 @@ import ( // Decorator adds video specific fields to a File. type Decorator struct { - FFProbe ffmpeg.FFProbe + FFProbe *ffmpeg.FFProbe } func (d *Decorator) Decorate(ctx context.Context, fs models.FS, f models.File) (models.File, error) { - if d.FFProbe == "" { + if d.FFProbe == nil { return f, errors.New("ffprobe not configured") } diff --git a/pkg/image/thumbnail.go b/pkg/image/thumbnail.go index 16191fa55..c65cfc77e 100644 --- a/pkg/image/thumbnail.go +++ b/pkg/image/thumbnail.go @@ -31,7 +31,7 @@ var ( type ThumbnailEncoder struct { FFMpeg *ffmpeg.FFMpeg - FFProbe ffmpeg.FFProbe + FFProbe *ffmpeg.FFProbe ClipPreviewOptions ClipPreviewOptions vips *vipsEncoder } @@ -49,7 +49,7 @@ func GetVipsPath() string { return vipsPath } -func NewThumbnailEncoder(ffmpegEncoder *ffmpeg.FFMpeg, ffProbe ffmpeg.FFProbe, clipPreviewOptions ClipPreviewOptions) ThumbnailEncoder { +func NewThumbnailEncoder(ffmpegEncoder *ffmpeg.FFMpeg, ffProbe *ffmpeg.FFProbe, clipPreviewOptions ClipPreviewOptions) ThumbnailEncoder { ret := ThumbnailEncoder{ FFMpeg: ffmpegEncoder, FFProbe: ffProbe,