From 738c092621ebc97035290cd5946af53f0efa7547 Mon Sep 17 00:00:00 2001 From: Daniel Morgan Date: Mon, 10 Feb 2020 12:58:46 +0100 Subject: [PATCH 01/10] smart content detector - initial commit --- .gitignore | 4 + scenedetect/detectors/__init__.py | 1 + .../detectors/smart_content_detector.py | 193 ++++++++++++++++++ scenedetect/scene_detector.py | 10 + scenedetect/scene_manager.py | 11 +- 5 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 scenedetect/detectors/smart_content_detector.py diff --git a/.gitignore b/.gitignore index 8e4267e8..b269d98d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ build/ dist/ *.egg-info/ manual/_build/ +*.mp4 +*.csv +.spyproject/ +test.py diff --git a/scenedetect/detectors/__init__.py b/scenedetect/detectors/__init__.py index f8600bdb..83787309 100644 --- a/scenedetect/detectors/__init__.py +++ b/scenedetect/detectors/__init__.py @@ -39,6 +39,7 @@ # PySceneDetect Detection Algorithm Imports from scenedetect.detectors.content_detector import ContentDetector from scenedetect.detectors.threshold_detector import ThresholdDetector +from scenedetect.detectors.smart_content_detector import SmartContentDetector # Algorithms being ported: #from scenedetect.detectors.motion_detector import MotionDetector diff --git a/scenedetect/detectors/smart_content_detector.py b/scenedetect/detectors/smart_content_detector.py new file mode 100644 index 00000000..1eff80f1 --- /dev/null +++ b/scenedetect/detectors/smart_content_detector.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- +# +# PySceneDetect: Python-Based Video Scene Detector +# --------------------------------------------------------------- +# [ Site: http://www.bcastell.com/projects/PySceneDetect/ ] +# [ Github: https://github.com/Breakthrough/PySceneDetect/ ] +# [ Documentation: http://pyscenedetect.readthedocs.org/ ] +# +# Copyright (C) 2014-2019 Brandon Castellano . +# +# PySceneDetect is licensed under the BSD 3-Clause License; see the included +# LICENSE file, or visit one of the following pages for details: +# - https://github.com/Breakthrough/PySceneDetect/ +# - http://www.bcastell.com/projects/PySceneDetect/ +# +# This software uses Numpy, OpenCV, click, tqdm, simpletable, and pytest. +# See the included LICENSE files or one of the above URLs for more information. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +""" Module: ``scenedetect.detectors.content_detector`` + +This module implements the :py:class:`ContentDetector`, which compares the +difference in content between adjacent frames against a set threshold/score, +which if exceeded, triggers a scene cut. + +This detector is available from the command-line interface by using the +`detect-content` command. +""" + +# Third-Party Library Imports +import numpy +import cv2 + +# PySceneDetect Library Imports +from scenedetect.scene_detector import SceneDetector + + +class SmartContentDetector(SceneDetector): + """Detects fast cuts using changes in colour and intensity between frames. + + Since the difference between frames is used, unlike the ThresholdDetector, + only fast cuts are detected with this method. To detect slow fades between + content scenes still using HSV information, use the DissolveDetector. + + + """ + + def __init__(self, threshold=30.0, min_scene_len=15): + super(SmartContentDetector, self).__init__() + self.threshold = threshold + self.min_scene_len = min_scene_len # minimum length of any given scene, in frames (int) or FrameTimecode + self.last_frame = None + self.last_scene_cut = None + self.last_hsv = None + self._metric_keys = ['content_val', 'delta_hue', 'delta_sat', 'delta_lum'] + self.running_averages = dict() + self._peaks_found = False + self.cli_name = 'smart-detect-content' + + def process_frame(self, frame_num, frame_img): + # type: (int, numpy.ndarray) -> List[int] + """ Similar to ThresholdDetector, but using the HSV colour space DIFFERENCE instead + of single-frame RGB/grayscale intensity (thus cannot detect slow fades with this method). + + Arguments: + frame_num (int): Frame number of frame that is being passed. + + frame_img (Optional[int]): Decoded frame image (numpy.ndarray) to perform scene + detection on. Can be None *only* if the self.is_processing_required() method + (inhereted from the base SceneDetector class) returns True. + + Returns: + List[int]: List of frames where scene cuts have been detected. There may be 0 + or more frames in the list, and not necessarily the same as frame_num. + """ + + cut_list = [] + metric_keys = self._metric_keys + _unused = '' + + # Initialize last scene cut point at the beginning of the frames of interest. + if self.last_scene_cut is None: + self.last_scene_cut = frame_num + + # We can only start detecting once we have a frame to compare with. + if self.last_frame is not None: + # Change in average of HSV (hsv), (h)ue only, (s)aturation only, (l)uminance only. + # These are refered to in a statsfile as their respective self._metric_keys string. + delta_hsv_avg, delta_h, delta_s, delta_v = 0.0, 0.0, 0.0, 0.0 + + if (self.stats_manager is not None and + self.stats_manager.metrics_exist(frame_num, metric_keys)): + delta_hsv_avg, delta_h, delta_s, delta_v = self.stats_manager.get_metrics( + frame_num, metric_keys) + + else: + num_pixels = frame_img.shape[0] * frame_img.shape[1] + curr_hsv = cv2.split(cv2.cvtColor(frame_img, cv2.COLOR_BGR2HSV)) + last_hsv = self.last_hsv + if not last_hsv: + last_hsv = cv2.split(cv2.cvtColor(self.last_frame, cv2.COLOR_BGR2HSV)) + + delta_hsv = [0, 0, 0, 0] + for i in range(3): + num_pixels = curr_hsv[i].shape[0] * curr_hsv[i].shape[1] + curr_hsv[i] = curr_hsv[i].astype(numpy.int32) + last_hsv[i] = last_hsv[i].astype(numpy.int32) + delta_hsv[i] = numpy.sum( + numpy.abs(curr_hsv[i] - last_hsv[i])) / float(num_pixels) + delta_hsv[3] = sum(delta_hsv[0:3]) / 3.0 + delta_h, delta_s, delta_v, delta_hsv_avg = delta_hsv + + if self.stats_manager is not None: + self.stats_manager.set_metrics(frame_num, { + metric_keys[0]: delta_hsv_avg, + metric_keys[1]: delta_h, + metric_keys[2]: delta_s, + metric_keys[3]: delta_v}) + + self.last_hsv = curr_hsv + + # We consider any frame over the threshold a new scene, but only if + # the minimum scene length has been reached (otherwise it is ignored). + if delta_hsv_avg >= self.threshold and ( + (frame_num - self.last_scene_cut) >= self.min_scene_len): + cut_list.append(frame_num) + self.last_scene_cut = frame_num + + if self.last_frame is not None and self.last_frame is not _unused: + del self.last_frame + + # If we have the next frame computed, don't copy the current frame + # into last_frame since we won't use it on the next call anyways. + if (self.stats_manager is not None and + self.stats_manager.metrics_exist(frame_num+1, metric_keys)): + self.last_frame = _unused + else: + self.last_frame = frame_img.copy() + + return cut_list + + def meta_post_process(self, video_manager, stats_manager, metathreshold=3): + """ + After an initial run through the video to detect content change + between each frame, we try to identify fast cuts as short peaks in the + `content_val` value. If a single frame has a high `content-val` while + the frames around it are low, we can be sure it's fast cut. If several + frames in a row have high `content-val`, it probably isn't a cut -- it + could be fast camera movement or a change in lighting that lasts for + more than a single frame. + """ + revised_cut_list = [] + _, start_timecode, end_timecode = video_manager.get_duration() + start_frame = start_timecode.get_frames() + end_frame = end_timecode.get_frames() + if self._peaks_found is False: + print('Calculating running average of content change by frame...') + for frame in range(start_frame + 3, end_frame - 1): + # If the `content-val` of the frame is more than + # `metathreshold` times the mean `content-val` of the + # frames around it, then we mark it as a cut. + denom = (stats_manager.get_metrics(frame - 2, ['content_val'])[0] + + stats_manager.get_metrics(frame - 1, ['content_val'])[0] + + stats_manager.get_metrics(frame + 1, ['content_val'])[0] + + stats_manager.get_metrics(frame + 2, ['content_val'])[0]) / 4 + if denom != 0: + self.running_averages[frame] = stats_manager.get_metrics( + frame, ['content_val'])[0] / denom + elif denom == 0 and stats_manager.get_metrics( + frame, ['content_val'])[0] >= 2: + self.running_averages[frame] = 99 + else: + self.running_averages[frame] = 0 + self._peaks_found = True + print('Revising cut list based on metathreshold...') + for frame in range(start_frame + 3, end_frame - 1): + if self.running_averages[frame] > metathreshold: + revised_cut_list.append(frame) + return revised_cut_list + + #def post_process(self, frame_num): + # """ TODO: Based on the parameters passed to the ContentDetector constructor, + # ensure that the last scene meets the minimum length requirement, + # otherwise it should be merged with the previous scene. + # """ + # return [] diff --git a/scenedetect/scene_detector.py b/scenedetect/scene_detector.py index e9435231..927f986c 100644 --- a/scenedetect/scene_detector.py +++ b/scenedetect/scene_detector.py @@ -108,6 +108,16 @@ def post_process(self, frame_num): List[int]: List of frame numbers of cuts to be added to the cutting list. """ return [] + + def meta_post_process(self, video_manager, stats_manager, metathreshold): + # type: (video) -> List[int] + """Performs additional post processing based on values for the entire video + rather than frame by frame. + + Returns: + List[int]: revised list of cuts + """ + return [] class SparseSceneDetector(SceneDetector): diff --git a/scenedetect/scene_manager.py b/scenedetect/scene_manager.py index 82cce1cb..f8bf29de 100644 --- a/scenedetect/scene_manager.py +++ b/scenedetect/scene_manager.py @@ -124,7 +124,7 @@ def write_scene_list(output_csv_file, scene_list, cut_list=None): in the video that need to be split to generate individual scenes). If not passed, the start times of each scene (besides the 0th scene) is used instead. """ - # type: (File, List[Tuple[FrameTimecode, FrameTimecode]], Optional[List[FrameTimecode]]) -> None + # type: (File, List[Tuple[FrameTimecode, FrmaeTimecode]], Optional[List[FrameTimecode]]) -> None csv_writer = get_csv_writer(output_csv_file) # Output Timecode List csv_writer.writerow( @@ -514,6 +514,13 @@ def _post_process(self, frame_num): for detector in self._detector_list: self._cutting_list += detector.post_process(frame_num) + def _meta_post_process(self, video_manager, stats_manager): + # type(int, VideoManager) -> None + """ Replaces the cut list with a revised one based on metaanalysis of the + entire video. """ + for detector in self._detector_list: + #if isinstance(detector, SmartContentDetector) is True: + self._cutting_list = detector.meta_post_process(video_manager, stats_manager) def detect_scenes(self, frame_source, end_time=None, frame_skip=0, show_progress=True): @@ -619,6 +626,8 @@ def detect_scenes(self, frame_source, end_time=None, frame_skip=0, self._post_process(curr_frame) num_frames = curr_frame - start_frame + + self._meta_post_process(frame_source, self._stats_manager) finally: From 3dbda0e9c574bf33cfe1c7c80b140885d9b49dc0 Mon Sep 17 00:00:00 2001 From: Daniel Morgan Date: Mon, 10 Feb 2020 17:40:16 +0100 Subject: [PATCH 02/10] content val ratio added as metric key --- .../detectors/smart_content_detector.py | 63 +++++++++++-------- scenedetect/scene_detector.py | 4 +- scenedetect/scene_manager.py | 7 ++- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/scenedetect/detectors/smart_content_detector.py b/scenedetect/detectors/smart_content_detector.py index 1eff80f1..9f2effdb 100644 --- a/scenedetect/detectors/smart_content_detector.py +++ b/scenedetect/detectors/smart_content_detector.py @@ -59,9 +59,11 @@ def __init__(self, threshold=30.0, min_scene_len=15): self.last_frame = None self.last_scene_cut = None self.last_hsv = None - self._metric_keys = ['content_val', 'delta_hue', 'delta_sat', 'delta_lum'] - self.running_averages = dict() - self._peaks_found = False + self._metric_keys = ['content_val', + 'delta_hue', + 'delta_sat', + 'delta_lum', + 'con_val_ratio'] self.cli_name = 'smart-detect-content' def process_frame(self, frame_num, frame_img): @@ -98,7 +100,7 @@ def process_frame(self, frame_num, frame_img): if (self.stats_manager is not None and self.stats_manager.metrics_exist(frame_num, metric_keys)): delta_hsv_avg, delta_h, delta_s, delta_v = self.stats_manager.get_metrics( - frame_num, metric_keys) + frame_num, metric_keys)[:4] else: num_pixels = frame_img.shape[0] * frame_img.shape[1] @@ -146,7 +148,13 @@ def process_frame(self, frame_num, frame_img): return cut_list - def meta_post_process(self, video_manager, stats_manager, metathreshold=3): + def get_content_val(self, frame_num): + """ + Returns the average content change for a frame. + """ + return self.stats_manager.get_metrics(frame_num, ['content_val'])[0] + + def meta_post_process(self, video_manager, metathreshold=3): """ After an initial run through the video to detect content change between each frame, we try to identify fast cuts as short peaks in the @@ -160,30 +168,35 @@ def meta_post_process(self, video_manager, stats_manager, metathreshold=3): _, start_timecode, end_timecode = video_manager.get_duration() start_frame = start_timecode.get_frames() end_frame = end_timecode.get_frames() - if self._peaks_found is False: - print('Calculating running average of content change by frame...') - for frame in range(start_frame + 3, end_frame - 1): + metric_keys = self._metric_keys + + if self.stats_manager is not None: + for frame_num in range(start_frame + 3, end_frame - 1): # If the `content-val` of the frame is more than # `metathreshold` times the mean `content-val` of the # frames around it, then we mark it as a cut. - denom = (stats_manager.get_metrics(frame - 2, ['content_val'])[0] - + stats_manager.get_metrics(frame - 1, ['content_val'])[0] - + stats_manager.get_metrics(frame + 1, ['content_val'])[0] - + stats_manager.get_metrics(frame + 2, ['content_val'])[0]) / 4 - if denom != 0: - self.running_averages[frame] = stats_manager.get_metrics( - frame, ['content_val'])[0] / denom - elif denom == 0 and stats_manager.get_metrics( - frame, ['content_val'])[0] >= 2: - self.running_averages[frame] = 99 + denominator = sum([self.get_content_val(frame_num - 2), + self.get_content_val(frame_num - 1), + self.get_content_val(frame_num + 1), + self.get_content_val(frame_num + 2)]) / 4 + if denominator != 0: + self.stats_manager.set_metrics(frame_num, { + metric_keys[4]: self.get_content_val(frame_num) + / denominator}) + elif denominator == 0 and self.get_content_val(frame_num) >= 2: + self.stats_manager.set_metrics(frame_num, { + metric_keys[4]: 99}) else: - self.running_averages[frame] = 0 - self._peaks_found = True - print('Revising cut list based on metathreshold...') - for frame in range(start_frame + 3, end_frame - 1): - if self.running_averages[frame] > metathreshold: - revised_cut_list.append(frame) - return revised_cut_list + self.stats_manager.set_metrics(frame_num, { + metric_keys[4]: 0}) + for frame_num in range(start_frame + 3, end_frame - 1): + if (self.stats_manager.get_metrics( + frame_num, ['con_val_ratio'])[0] > metathreshold + and self.stats_manager.get_metrics( + frame_num, ['content_val'])[0] > 3): + revised_cut_list.append(frame_num) + return revised_cut_list + return None #def post_process(self, frame_num): # """ TODO: Based on the parameters passed to the ContentDetector constructor, diff --git a/scenedetect/scene_detector.py b/scenedetect/scene_detector.py index 927f986c..61add6de 100644 --- a/scenedetect/scene_detector.py +++ b/scenedetect/scene_detector.py @@ -109,7 +109,7 @@ def post_process(self, frame_num): """ return [] - def meta_post_process(self, video_manager, stats_manager, metathreshold): + def meta_post_process(self, video_manager, metathreshold): # type: (video) -> List[int] """Performs additional post processing based on values for the entire video rather than frame by frame. @@ -117,7 +117,7 @@ def meta_post_process(self, video_manager, stats_manager, metathreshold): Returns: List[int]: revised list of cuts """ - return [] + return None class SparseSceneDetector(SceneDetector): diff --git a/scenedetect/scene_manager.py b/scenedetect/scene_manager.py index f8bf29de..c6d6b2e1 100644 --- a/scenedetect/scene_manager.py +++ b/scenedetect/scene_manager.py @@ -517,10 +517,13 @@ def _post_process(self, frame_num): def _meta_post_process(self, video_manager, stats_manager): # type(int, VideoManager) -> None """ Replaces the cut list with a revised one based on metaanalysis of the - entire video. """ + entire video. + """ for detector in self._detector_list: #if isinstance(detector, SmartContentDetector) is True: - self._cutting_list = detector.meta_post_process(video_manager, stats_manager) + returned_cut_list = detector.meta_post_process(video_manager) + if returned_cut_list is not None: + self._cutting_list = returned_cut_list def detect_scenes(self, frame_source, end_time=None, frame_skip=0, show_progress=True): From 576623eea63d889f24c65615515319d406f05fec Mon Sep 17 00:00:00 2001 From: Daniel Morgan Date: Wed, 12 Feb 2020 14:17:59 +0100 Subject: [PATCH 03/10] add smart-detect-content option for cli --- scenedetect/cli/__init__.py | 42 ++++++++++++++ .../detectors/smart_content_detector.py | 57 +++++++++++-------- 2 files changed, 74 insertions(+), 25 deletions(-) diff --git a/scenedetect/cli/__init__.py b/scenedetect/cli/__init__.py index 45e60dd0..b46328e3 100644 --- a/scenedetect/cli/__init__.py +++ b/scenedetect/cli/__init__.py @@ -441,9 +441,51 @@ def detect_content_command(ctx, threshold, min_scene_len): #, intensity_cutoff): # a frame metric key error when registering the detector. ctx.obj.add_detector(scenedetect.detectors.ContentDetector( threshold=threshold, min_scene_len=min_scene_len)) +@click.command('smart-detect-content') +@click.option( + '--threshold', '-t', metavar='VAL', + type=click.FLOAT, default=30.0, show_default=True, help= + 'Threshold value (float) that the content_val frame metric must exceed to trigger a new scene.' + ' Refers to frame metric content_val in stats file.') +# '--intensity-cutoff', '-i', metavar='VAL', +# type=click.FLOAT, default=None, show_default=True, help= +# '[Optional] Intensity cutoff threshold to disable scene cut detection. Useful for avoiding.' +# ' scene changes triggered by flashes. Refers to frame metric delta_lum in stats file.') +@click.option( + '--min-scene-len', '-m', metavar='TIMECODE', + type=click.STRING, default="0.6s", show_default=True, help= + 'Minimum size/length of any scene. TIMECODE can be specified as exact' + ' number of frames, a time in seconds followed by s, or a timecode in the' + ' format HH:MM:SS or HH:MM:SS.nnn') +@click.option( + '--metathreshold', '-M', metavar='VAL', + type=click.FLOAT, default=3.0, show_default=True, help= + 'Metathreshold value (float): the content_val frame metric must be this many times' + ' the content_val for the surrounding frames to trigger a new scene.') +@click.pass_context +def detect_content_command(ctx, threshold, min_scene_len, metathreshold): #, intensity_cutoff): + """ Perform content detection algorithm on input video(s). + + smart-detect-content + + smart-detect-content --metatheshold 3.2 + """ + #if intensity_cutoff is not None: + # raise NotImplementedError() + min_scene_len = parse_timecode(ctx.obj, min_scene_len) + logging.debug('Detecting content, parameters:\n' + ' threshold: %d, min-scene-len: %d', + threshold, min_scene_len) + + # Initialize detector and add to scene manager. + # Need to ensure that a detector is not added twice, or will cause + # a frame metric key error when registering the detector. + ctx.obj.add_detector(scenedetect.detectors.SmartContentDetector( + threshold=threshold, min_scene_len=min_scene_len, + metathreshold=metathreshold)) @click.command('detect-threshold') @click.option( '--threshold', '-t', metavar='VAL', diff --git a/scenedetect/detectors/smart_content_detector.py b/scenedetect/detectors/smart_content_detector.py index 9f2effdb..80816442 100644 --- a/scenedetect/detectors/smart_content_detector.py +++ b/scenedetect/detectors/smart_content_detector.py @@ -52,18 +52,16 @@ class SmartContentDetector(SceneDetector): """ - def __init__(self, threshold=30.0, min_scene_len=15): + def __init__(self, threshold=30.0, min_scene_len=15, metathreshold=3): super(SmartContentDetector, self).__init__() self.threshold = threshold self.min_scene_len = min_scene_len # minimum length of any given scene, in frames (int) or FrameTimecode + self.metathreshold = metathreshold self.last_frame = None self.last_scene_cut = None self.last_hsv = None - self._metric_keys = ['content_val', - 'delta_hue', - 'delta_sat', - 'delta_lum', - 'con_val_ratio'] + self._metric_keys = ['content_val', 'delta_hue', 'delta_sat', + 'delta_lum', 'con_val_ratio'] self.cli_name = 'smart-detect-content' def process_frame(self, frame_num, frame_img): @@ -154,7 +152,7 @@ def get_content_val(self, frame_num): """ return self.stats_manager.get_metrics(frame_num, ['content_val'])[0] - def meta_post_process(self, video_manager, metathreshold=3): + def meta_post_process(self, video_manager): """ After an initial run through the video to detect content change between each frame, we try to identify fast cuts as short peaks in the @@ -169,31 +167,40 @@ def meta_post_process(self, video_manager, metathreshold=3): start_frame = start_timecode.get_frames() end_frame = end_timecode.get_frames() metric_keys = self._metric_keys + metathreshold = self.metathreshold if self.stats_manager is not None: - for frame_num in range(start_frame + 3, end_frame - 1): + for frame_num in range(start_frame + 3, end_frame - 2): # If the `content-val` of the frame is more than # `metathreshold` times the mean `content-val` of the # frames around it, then we mark it as a cut. - denominator = sum([self.get_content_val(frame_num - 2), - self.get_content_val(frame_num - 1), - self.get_content_val(frame_num + 1), - self.get_content_val(frame_num + 2)]) / 4 + denominator = sum([ + self.get_content_val(frame_num - 2), + self.get_content_val(frame_num - 1), + self.get_content_val(frame_num + 1), + self.get_content_val(frame_num + 2) + ]) / 4 if denominator != 0: - self.stats_manager.set_metrics(frame_num, { - metric_keys[4]: self.get_content_val(frame_num) - / denominator}) - elif denominator == 0 and self.get_content_val(frame_num) >= 2: - self.stats_manager.set_metrics(frame_num, { - metric_keys[4]: 99}) + self.stats_manager.set_metrics( + frame_num, { + metric_keys[4]: + self.get_content_val(frame_num) / denominator + }) + elif denominator == 0 and self.get_content_val(frame_num) >= 5: + # avoid dividing by zero, setting con_val_ratio to + # a really high value + self.stats_manager.set_metrics(frame_num, + {metric_keys[4]: 99}) else: - self.stats_manager.set_metrics(frame_num, { - metric_keys[4]: 0}) - for frame_num in range(start_frame + 3, end_frame - 1): - if (self.stats_manager.get_metrics( - frame_num, ['con_val_ratio'])[0] > metathreshold - and self.stats_manager.get_metrics( - frame_num, ['content_val'])[0] > 3): + # avoid dividing by zero, setting con_val_ratio to zero + # if content_val is still very low + self.stats_manager.set_metrics(frame_num, + {metric_keys[4]: 0}) + for frame_num in range(start_frame + 3, end_frame - 2): + if self.stats_manager.get_metrics( + frame_num, ['con_val_ratio'])[0] > metathreshold: + #and self.stats_manager.get_metrics(frame_num, + # ['content_val'])[0] > 3): revised_cut_list.append(frame_num) return revised_cut_list return None From 348de2eb35bd726b276cc37a551aa55d01223f1c Mon Sep 17 00:00:00 2001 From: Daniel Morgan Date: Thu, 13 Feb 2020 11:24:17 +0100 Subject: [PATCH 04/10] set minimum content_val for triggering cut to 6 --- scenedetect/detectors/smart_content_detector.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scenedetect/detectors/smart_content_detector.py b/scenedetect/detectors/smart_content_detector.py index 80816442..b8456521 100644 --- a/scenedetect/detectors/smart_content_detector.py +++ b/scenedetect/detectors/smart_content_detector.py @@ -52,7 +52,7 @@ class SmartContentDetector(SceneDetector): """ - def __init__(self, threshold=30.0, min_scene_len=15, metathreshold=3): + def __init__(self, threshold=30.0, min_scene_len=15, metathreshold=3.0): super(SmartContentDetector, self).__init__() self.threshold = threshold self.min_scene_len = min_scene_len # minimum length of any given scene, in frames (int) or FrameTimecode @@ -186,7 +186,7 @@ def meta_post_process(self, video_manager): metric_keys[4]: self.get_content_val(frame_num) / denominator }) - elif denominator == 0 and self.get_content_val(frame_num) >= 5: + elif denominator == 0 and self.get_content_val(frame_num) >= 6: # avoid dividing by zero, setting con_val_ratio to # a really high value self.stats_manager.set_metrics(frame_num, @@ -197,10 +197,10 @@ def meta_post_process(self, video_manager): self.stats_manager.set_metrics(frame_num, {metric_keys[4]: 0}) for frame_num in range(start_frame + 3, end_frame - 2): - if self.stats_manager.get_metrics( - frame_num, ['con_val_ratio'])[0] > metathreshold: - #and self.stats_manager.get_metrics(frame_num, - # ['content_val'])[0] > 3): + if (self.stats_manager.get_metrics( + frame_num, ['con_val_ratio'])[0] > metathreshold and + self.stats_manager.get_metrics(frame_num, + ['content_val'])[0] > 6): revised_cut_list.append(frame_num) return revised_cut_list return None From effd4471ec5136798a9095fdc3e2238518477fed Mon Sep 17 00:00:00 2001 From: Daniel Morgan Date: Thu, 13 Feb 2020 11:25:51 +0100 Subject: [PATCH 05/10] remove useless parameters --- scenedetect/scene_detector.py | 2 +- scenedetect/scene_manager.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scenedetect/scene_detector.py b/scenedetect/scene_detector.py index 61add6de..e039dcb4 100644 --- a/scenedetect/scene_detector.py +++ b/scenedetect/scene_detector.py @@ -109,7 +109,7 @@ def post_process(self, frame_num): """ return [] - def meta_post_process(self, video_manager, metathreshold): + def meta_post_process(self, video_manager): # type: (video) -> List[int] """Performs additional post processing based on values for the entire video rather than frame by frame. diff --git a/scenedetect/scene_manager.py b/scenedetect/scene_manager.py index c6d6b2e1..f3417414 100644 --- a/scenedetect/scene_manager.py +++ b/scenedetect/scene_manager.py @@ -514,7 +514,7 @@ def _post_process(self, frame_num): for detector in self._detector_list: self._cutting_list += detector.post_process(frame_num) - def _meta_post_process(self, video_manager, stats_manager): + def _meta_post_process(self, video_manager): # type(int, VideoManager) -> None """ Replaces the cut list with a revised one based on metaanalysis of the entire video. @@ -630,7 +630,7 @@ def detect_scenes(self, frame_source, end_time=None, frame_skip=0, num_frames = curr_frame - start_frame - self._meta_post_process(frame_source, self._stats_manager) + self._meta_post_process(frame_source) finally: From 3e9b905c0876bf39dcf2e91bcf6e121d1576712d Mon Sep 17 00:00:00 2001 From: Walter Schwenger Date: Fri, 28 Feb 2020 09:53:48 -0500 Subject: [PATCH 06/10] Fixed typo and aligned with master --- .gitignore | 4 ---- scenedetect/scene_manager.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index b269d98d..8e4267e8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,3 @@ build/ dist/ *.egg-info/ manual/_build/ -*.mp4 -*.csv -.spyproject/ -test.py diff --git a/scenedetect/scene_manager.py b/scenedetect/scene_manager.py index f3417414..b155ce3e 100644 --- a/scenedetect/scene_manager.py +++ b/scenedetect/scene_manager.py @@ -124,7 +124,7 @@ def write_scene_list(output_csv_file, scene_list, cut_list=None): in the video that need to be split to generate individual scenes). If not passed, the start times of each scene (besides the 0th scene) is used instead. """ - # type: (File, List[Tuple[FrameTimecode, FrmaeTimecode]], Optional[List[FrameTimecode]]) -> None + # type: (File, List[Tuple[FrameTimecode, FrameTimecode]], Optional[List[FrameTimecode]]) -> None csv_writer = get_csv_writer(output_csv_file) # Output Timecode List csv_writer.writerow( From bde85b1ce8ac4604b9dfebe4cfad16acef825fde Mon Sep 17 00:00:00 2001 From: Walter Schwenger Date: Sat, 29 Feb 2020 00:48:29 -0500 Subject: [PATCH 07/10] Refactored to adaptive-content-detector. --- scenedetect/cli/__init__.py | 50 ++++++++++--------- scenedetect/detectors/__init__.py | 2 +- ...tector.py => adaptive_content_detector.py} | 45 +++++------------ scenedetect/scene_detector.py | 12 +---- scenedetect/scene_manager.py | 13 ----- 5 files changed, 41 insertions(+), 81 deletions(-) rename scenedetect/detectors/{smart_content_detector.py => adaptive_content_detector.py} (84%) diff --git a/scenedetect/cli/__init__.py b/scenedetect/cli/__init__.py index b46328e3..219d5dee 100644 --- a/scenedetect/cli/__init__.py +++ b/scenedetect/cli/__init__.py @@ -441,51 +441,52 @@ def detect_content_command(ctx, threshold, min_scene_len): #, intensity_cutoff): # a frame metric key error when registering the detector. ctx.obj.add_detector(scenedetect.detectors.ContentDetector( threshold=threshold, min_scene_len=min_scene_len)) -@click.command('smart-detect-content') + + +@click.command('adaptive-detect-content') @click.option( '--threshold', '-t', metavar='VAL', - type=click.FLOAT, default=30.0, show_default=True, help= - 'Threshold value (float) that the content_val frame metric must exceed to trigger a new scene.' - ' Refers to frame metric content_val in stats file.') -# '--intensity-cutoff', '-i', metavar='VAL', -# type=click.FLOAT, default=None, show_default=True, help= -# '[Optional] Intensity cutoff threshold to disable scene cut detection. Useful for avoiding.' -# ' scene changes triggered by flashes. Refers to frame metric delta_lum in stats file.') + type=click.FLOAT, default=3.0, show_default=True, help= + 'Threshold value (float) that the con_val_ratio frame metric must exceed to trigger a new scene.' + ' Refers to frame metric content_val_ratio in stats file.') @click.option( '--min-scene-len', '-m', metavar='TIMECODE', type=click.STRING, default="0.6s", show_default=True, help= 'Minimum size/length of any scene. TIMECODE can be specified as exact' ' number of frames, a time in seconds followed by s, or a timecode in the' ' format HH:MM:SS or HH:MM:SS.nnn') -@click.option( - '--metathreshold', '-M', metavar='VAL', - type=click.FLOAT, default=3.0, show_default=True, help= - 'Metathreshold value (float): the content_val frame metric must be this many times' - ' the content_val for the surrounding frames to trigger a new scene.') @click.pass_context -def detect_content_command(ctx, threshold, min_scene_len, metathreshold): #, intensity_cutoff): - """ Perform content detection algorithm on input video(s). +def adaptive_detect_content_command(ctx, threshold, min_scene_len): + """ Perform adaptive content detection algorithm on input video(s). - smart-detect-content + adaptive-detect-content - smart-detect-content --metatheshold 3.2 + adaptive-detect-content --threshold 3.2 """ - #if intensity_cutoff is not None: - # raise NotImplementedError() - min_scene_len = parse_timecode(ctx.obj, min_scene_len) - logging.debug('Detecting content, parameters:\n' + logging.debug('Adaptively detecting content, parameters:\n' ' threshold: %d, min-scene-len: %d', threshold, min_scene_len) + # Check for a stats manager, necessary to use the adaptive content detector + if not ctx.obj.stats_manager: + error_strs = [ + 'No stats file specified for use with the adaptive content detector.' + ' Either use a different detector or specify a stats file with -s/--stats\n'] + logging.error('\n'.join(error_strs)) + raise click.BadParameter( + '\n Specifying a stats file -s/--stats is necessary to use the adaptive content detector', + param_hint='adaptive detector + stats file') + # Initialize detector and add to scene manager. # Need to ensure that a detector is not added twice, or will cause # a frame metric key error when registering the detector. - ctx.obj.add_detector(scenedetect.detectors.SmartContentDetector( - threshold=threshold, min_scene_len=min_scene_len, - metathreshold=metathreshold)) + ctx.obj.add_detector(scenedetect.detectors.AdaptiveContentDetector( + video_manager=ctx.obj.video_manager, adaptive_threshold=threshold, min_scene_len=min_scene_len)) + + @click.command('detect-threshold') @click.option( '--threshold', '-t', metavar='VAL', @@ -793,6 +794,7 @@ def colors_command(ctx): add_cli_command(scenedetect_cli, time_command) add_cli_command(scenedetect_cli, detect_content_command) add_cli_command(scenedetect_cli, detect_threshold_command) +add_cli_command(scenedetect_cli, adaptive_detect_content_command) add_cli_command(scenedetect_cli, list_scenes_command) add_cli_command(scenedetect_cli, save_images_command) diff --git a/scenedetect/detectors/__init__.py b/scenedetect/detectors/__init__.py index 83787309..3b3c49a5 100644 --- a/scenedetect/detectors/__init__.py +++ b/scenedetect/detectors/__init__.py @@ -39,7 +39,7 @@ # PySceneDetect Detection Algorithm Imports from scenedetect.detectors.content_detector import ContentDetector from scenedetect.detectors.threshold_detector import ThresholdDetector -from scenedetect.detectors.smart_content_detector import SmartContentDetector +from scenedetect.detectors.adaptive_content_detector import AdaptiveContentDetector # Algorithms being ported: #from scenedetect.detectors.motion_detector import MotionDetector diff --git a/scenedetect/detectors/smart_content_detector.py b/scenedetect/detectors/adaptive_content_detector.py similarity index 84% rename from scenedetect/detectors/smart_content_detector.py rename to scenedetect/detectors/adaptive_content_detector.py index b8456521..18665788 100644 --- a/scenedetect/detectors/smart_content_detector.py +++ b/scenedetect/detectors/adaptive_content_detector.py @@ -42,7 +42,7 @@ from scenedetect.scene_detector import SceneDetector -class SmartContentDetector(SceneDetector): +class AdaptiveContentDetector(SceneDetector): """Detects fast cuts using changes in colour and intensity between frames. Since the difference between frames is used, unlike the ThresholdDetector, @@ -52,17 +52,17 @@ class SmartContentDetector(SceneDetector): """ - def __init__(self, threshold=30.0, min_scene_len=15, metathreshold=3.0): - super(SmartContentDetector, self).__init__() - self.threshold = threshold + def __init__(self, video_manager=None, adaptive_threshold=3.0, min_scene_len=15): + super(AdaptiveContentDetector, self).__init__() + self.video_manager = video_manager self.min_scene_len = min_scene_len # minimum length of any given scene, in frames (int) or FrameTimecode - self.metathreshold = metathreshold + self.adaptive_threshold = adaptive_threshold self.last_frame = None self.last_scene_cut = None self.last_hsv = None self._metric_keys = ['content_val', 'delta_hue', 'delta_sat', 'delta_lum', 'con_val_ratio'] - self.cli_name = 'smart-detect-content' + self.cli_name = 'adaptive-detect-content' def process_frame(self, frame_num, frame_img): # type: (int, numpy.ndarray) -> List[int] @@ -81,14 +81,9 @@ def process_frame(self, frame_num, frame_img): or more frames in the list, and not necessarily the same as frame_num. """ - cut_list = [] metric_keys = self._metric_keys _unused = '' - # Initialize last scene cut point at the beginning of the frames of interest. - if self.last_scene_cut is None: - self.last_scene_cut = frame_num - # We can only start detecting once we have a frame to compare with. if self.last_frame is not None: # Change in average of HSV (hsv), (h)ue only, (s)aturation only, (l)uminance only. @@ -126,15 +121,8 @@ def process_frame(self, frame_num, frame_img): self.last_hsv = curr_hsv - # We consider any frame over the threshold a new scene, but only if - # the minimum scene length has been reached (otherwise it is ignored). - if delta_hsv_avg >= self.threshold and ( - (frame_num - self.last_scene_cut) >= self.min_scene_len): - cut_list.append(frame_num) - self.last_scene_cut = frame_num - - if self.last_frame is not None and self.last_frame is not _unused: - del self.last_frame + if self.last_frame is not None and self.last_frame is not _unused: + del self.last_frame # If we have the next frame computed, don't copy the current frame # into last_frame since we won't use it on the next call anyways. @@ -144,7 +132,7 @@ def process_frame(self, frame_num, frame_img): else: self.last_frame = frame_img.copy() - return cut_list + return [] def get_content_val(self, frame_num): """ @@ -152,7 +140,7 @@ def get_content_val(self, frame_num): """ return self.stats_manager.get_metrics(frame_num, ['content_val'])[0] - def meta_post_process(self, video_manager): + def post_process(self, frame): """ After an initial run through the video to detect content change between each frame, we try to identify fast cuts as short peaks in the @@ -163,11 +151,11 @@ def meta_post_process(self, video_manager): more than a single frame. """ revised_cut_list = [] - _, start_timecode, end_timecode = video_manager.get_duration() + _, start_timecode, end_timecode = self.video_manager.get_duration() start_frame = start_timecode.get_frames() end_frame = end_timecode.get_frames() metric_keys = self._metric_keys - metathreshold = self.metathreshold + adaptive_threshold = self.adaptive_threshold if self.stats_manager is not None: for frame_num in range(start_frame + 3, end_frame - 2): @@ -198,16 +186,9 @@ def meta_post_process(self, video_manager): {metric_keys[4]: 0}) for frame_num in range(start_frame + 3, end_frame - 2): if (self.stats_manager.get_metrics( - frame_num, ['con_val_ratio'])[0] > metathreshold and + frame_num, ['con_val_ratio'])[0] > adaptive_threshold and self.stats_manager.get_metrics(frame_num, ['content_val'])[0] > 6): revised_cut_list.append(frame_num) return revised_cut_list return None - - #def post_process(self, frame_num): - # """ TODO: Based on the parameters passed to the ContentDetector constructor, - # ensure that the last scene meets the minimum length requirement, - # otherwise it should be merged with the previous scene. - # """ - # return [] diff --git a/scenedetect/scene_detector.py b/scenedetect/scene_detector.py index e039dcb4..6f3c50f8 100644 --- a/scenedetect/scene_detector.py +++ b/scenedetect/scene_detector.py @@ -37,7 +37,7 @@ class SceneDetector(object): - """ Base class to inheret from when implementing a scene detection algorithm. + """ Base class to inherit from when implementing a scene detection algorithm. This represents a "dense" scene detector, which returns a list of frames where the next scene/shot begins in a video. @@ -108,16 +108,6 @@ def post_process(self, frame_num): List[int]: List of frame numbers of cuts to be added to the cutting list. """ return [] - - def meta_post_process(self, video_manager): - # type: (video) -> List[int] - """Performs additional post processing based on values for the entire video - rather than frame by frame. - - Returns: - List[int]: revised list of cuts - """ - return None class SparseSceneDetector(SceneDetector): diff --git a/scenedetect/scene_manager.py b/scenedetect/scene_manager.py index b155ce3e..a652bd55 100644 --- a/scenedetect/scene_manager.py +++ b/scenedetect/scene_manager.py @@ -514,17 +514,6 @@ def _post_process(self, frame_num): for detector in self._detector_list: self._cutting_list += detector.post_process(frame_num) - def _meta_post_process(self, video_manager): - # type(int, VideoManager) -> None - """ Replaces the cut list with a revised one based on metaanalysis of the - entire video. - """ - for detector in self._detector_list: - #if isinstance(detector, SmartContentDetector) is True: - returned_cut_list = detector.meta_post_process(video_manager) - if returned_cut_list is not None: - self._cutting_list = returned_cut_list - def detect_scenes(self, frame_source, end_time=None, frame_skip=0, show_progress=True): # type: (VideoManager, Union[int, FrameTimecode], @@ -629,8 +618,6 @@ def detect_scenes(self, frame_source, end_time=None, frame_skip=0, self._post_process(curr_frame) num_frames = curr_frame - start_frame - - self._meta_post_process(frame_source) finally: From 361edb575003c5b16ef00f2526e20b121d96b6e0 Mon Sep 17 00:00:00 2001 From: Walter Schwenger Date: Mon, 2 Mar 2020 14:58:40 -0500 Subject: [PATCH 08/10] Implemented min_scene_len in AdaptiveContentDetector. --- scenedetect/cli/__init__.py | 18 +++- .../detectors/adaptive_content_detector.py | 100 +++++++++++------- 2 files changed, 75 insertions(+), 43 deletions(-) diff --git a/scenedetect/cli/__init__.py b/scenedetect/cli/__init__.py index 219d5dee..f41388d5 100644 --- a/scenedetect/cli/__init__.py +++ b/scenedetect/cli/__init__.py @@ -455,8 +455,18 @@ def detect_content_command(ctx, threshold, min_scene_len): #, intensity_cutoff): 'Minimum size/length of any scene. TIMECODE can be specified as exact' ' number of frames, a time in seconds followed by s, or a timecode in the' ' format HH:MM:SS or HH:MM:SS.nnn') +@click.option( + '--min-delta-hsv', '-d', metavar='VAL', + type=click.FLOAT, default=5.0, show_default=True, help= + 'Minimum threshold (float) that the content_val must exceed in order to register as a new scene.' + ' This is calculated the same way that the detect-content command calculates content_val.') +@click.option( + '--frame-window', '-w', metavar='VAL', + type=click.INT, default=2, show_default=True, help= + 'Number of frames before and after each frame to average together in' + ' order to detect deviations from the mean.') @click.pass_context -def adaptive_detect_content_command(ctx, threshold, min_scene_len): +def adaptive_detect_content_command(ctx, threshold, min_scene_len, min_delta_hsv, frame_window): """ Perform adaptive content detection algorithm on input video(s). adaptive-detect-content @@ -484,7 +494,11 @@ def adaptive_detect_content_command(ctx, threshold, min_scene_len): # Need to ensure that a detector is not added twice, or will cause # a frame metric key error when registering the detector. ctx.obj.add_detector(scenedetect.detectors.AdaptiveContentDetector( - video_manager=ctx.obj.video_manager, adaptive_threshold=threshold, min_scene_len=min_scene_len)) + video_manager=ctx.obj.video_manager, + adaptive_threshold=threshold, + min_scene_len=min_scene_len, + min_delta_hsv=min_delta_hsv, + window_width=frame_window)) @click.command('detect-threshold') diff --git a/scenedetect/detectors/adaptive_content_detector.py b/scenedetect/detectors/adaptive_content_detector.py index 18665788..755583fb 100644 --- a/scenedetect/detectors/adaptive_content_detector.py +++ b/scenedetect/detectors/adaptive_content_detector.py @@ -43,20 +43,18 @@ class AdaptiveContentDetector(SceneDetector): - """Detects fast cuts using changes in colour and intensity between frames. - - Since the difference between frames is used, unlike the ThresholdDetector, - only fast cuts are detected with this method. To detect slow fades between - content scenes still using HSV information, use the DissolveDetector. - - + """Detects cuts using HSV changes similar to ContentDetector, but with a + rolling average that can help mitigate false detections in situations such + as camera moves. """ - def __init__(self, video_manager=None, adaptive_threshold=3.0, min_scene_len=15): + def __init__(self, video_manager=None, adaptive_threshold=3.0, min_scene_len=15, min_delta_hsv=5.0, window_width=2): super(AdaptiveContentDetector, self).__init__() self.video_manager = video_manager self.min_scene_len = min_scene_len # minimum length of any given scene, in frames (int) or FrameTimecode self.adaptive_threshold = adaptive_threshold + self.min_delta_hsv = min_delta_hsv + self.window_width = window_width self.last_frame = None self.last_scene_cut = None self.last_hsv = None @@ -66,8 +64,8 @@ def __init__(self, video_manager=None, adaptive_threshold=3.0, min_scene_len=15) def process_frame(self, frame_num, frame_img): # type: (int, numpy.ndarray) -> List[int] - """ Similar to ThresholdDetector, but using the HSV colour space DIFFERENCE instead - of single-frame RGB/grayscale intensity (thus cannot detect slow fades with this method). + """ Similar to ContentDetector, but looking for frames in which the HSV difference is + significantly different than the neighboring frames. Arguments: frame_num (int): Frame number of frame that is being passed. @@ -77,8 +75,9 @@ def process_frame(self, frame_num, frame_img): (inhereted from the base SceneDetector class) returns True. Returns: - List[int]: List of frames where scene cuts have been detected. There may be 0 - or more frames in the list, and not necessarily the same as frame_num. + Empty list: The process_frame function for this detector does not register any cuts, + instead only calculating scene metrics that are used to detect cuts with the + post_process function. """ metric_keys = self._metric_keys @@ -150,45 +149,64 @@ def post_process(self, frame): could be fast camera movement or a change in lighting that lasts for more than a single frame. """ - revised_cut_list = [] + cut_list = [] _, start_timecode, end_timecode = self.video_manager.get_duration() start_frame = start_timecode.get_frames() end_frame = end_timecode.get_frames() metric_keys = self._metric_keys adaptive_threshold = self.adaptive_threshold + window_width = self.window_width + last_cut = None if self.stats_manager is not None: - for frame_num in range(start_frame + 3, end_frame - 2): - # If the `content-val` of the frame is more than - # `metathreshold` times the mean `content-val` of the + # Loop through the stats, building the con_val_ratio metric + for frame_num in range(start_frame + window_width + 1, end_frame - window_width): + # If the content-val of the frame is more than + # adaptive_threshold times the mean content_val of the # frames around it, then we mark it as a cut. - denominator = sum([ - self.get_content_val(frame_num - 2), - self.get_content_val(frame_num - 1), - self.get_content_val(frame_num + 1), - self.get_content_val(frame_num + 2) - ]) / 4 + denominator = 0 + for offset in range(-window_width, window_width + 1): + if offset == 0: + continue + else: + denominator += self.get_content_val(frame_num + offset) + + denominator = denominator / (2 * window_width) + if denominator != 0: + # store the calculated con_val_ratio in our metrics self.stats_manager.set_metrics( - frame_num, { - metric_keys[4]: - self.get_content_val(frame_num) / denominator - }) - elif denominator == 0 and self.get_content_val(frame_num) >= 6: - # avoid dividing by zero, setting con_val_ratio to - # a really high value - self.stats_manager.set_metrics(frame_num, - {metric_keys[4]: 99}) + frame_num, + {metric_keys[4]: self.get_content_val(frame_num) / denominator}) + + elif denominator == 0 and self.get_content_val(frame_num) >= self.min_delta_hsv: + # avoid dividing by zero, setting con_val_ratio to above the threshold + self.stats_manager.set_metrics(frame_num, {metric_keys[4]: adaptive_threshold + 1}) + else: - # avoid dividing by zero, setting con_val_ratio to zero - # if content_val is still very low - self.stats_manager.set_metrics(frame_num, - {metric_keys[4]: 0}) - for frame_num in range(start_frame + 3, end_frame - 2): + # avoid dividing by zero, setting con_val_ratio to zero if content_val is still very low + self.stats_manager.set_metrics(frame_num, {metric_keys[4]: 0}) + + # Loop through the frames again now that con_val_ratio has been calculated to detect + # cuts using con_val_ratio + for frame_num in range(start_frame + window_width + 1, end_frame - window_width): + # Check to see if con_val_ratio exceeds the adaptive_threshold as well as there + # being a large enough content_val to trigger a cut if (self.stats_manager.get_metrics( - frame_num, ['con_val_ratio'])[0] > adaptive_threshold and - self.stats_manager.get_metrics(frame_num, - ['content_val'])[0] > 6): - revised_cut_list.append(frame_num) - return revised_cut_list + frame_num, ['con_val_ratio'])[0] >= adaptive_threshold and + self.stats_manager.get_metrics( + frame_num, ['content_val'])[0] >= self.min_delta_hsv): + + if last_cut is None: + # No previously detected cuts + cut_list.append(frame_num) + last_cut = frame_num + elif (frame_num - last_cut) >= self.min_scene_len: + # Respect the min_scene_len parameter + cut_list.append(frame_num) + last_cut = frame_num + + return cut_list + + # Stats manager must be used for this detector return None From 3cc9685813f19ce75e29e41483d03653a80608ca Mon Sep 17 00:00:00 2001 From: wjs018 Date: Tue, 19 Jan 2021 23:26:26 -0500 Subject: [PATCH 09/10] Added ContentDetector inheritance to reduce code duplication --- .../detectors/adaptive_content_detector.py | 99 +++---------------- 1 file changed, 14 insertions(+), 85 deletions(-) diff --git a/scenedetect/detectors/adaptive_content_detector.py b/scenedetect/detectors/adaptive_content_detector.py index 755583fb..ec69db6b 100644 --- a/scenedetect/detectors/adaptive_content_detector.py +++ b/scenedetect/detectors/adaptive_content_detector.py @@ -6,7 +6,7 @@ # [ Github: https://github.com/Breakthrough/PySceneDetect/ ] # [ Documentation: http://pyscenedetect.readthedocs.org/ ] # -# Copyright (C) 2014-2019 Brandon Castellano . +# Copyright (C) 2014-2021 Brandon Castellano . # # PySceneDetect is licensed under the BSD 3-Clause License; see the included # LICENSE file, or visit one of the following pages for details: @@ -24,32 +24,32 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -""" Module: ``scenedetect.detectors.content_detector`` +""" Module: ``scenedetect.detectors.adaptive_content_detector`` -This module implements the :py:class:`ContentDetector`, which compares the -difference in content between adjacent frames against a set threshold/score, -which if exceeded, triggers a scene cut. +This module implements the :py:class:`AdaptiveContentDetector`, which compares the +difference in content between adjacent frames similar to `ContentDetector` except the +threshold isn't fixed, but is a rolling average of adjacent frame changes. This can +help mitigate false detections in situations such as fast camera motions. This detector is available from the command-line interface by using the -`detect-content` command. +`adaptive-detect-content` command. """ -# Third-Party Library Imports -import numpy -import cv2 - # PySceneDetect Library Imports -from scenedetect.scene_detector import SceneDetector +from scenedetect.detectors import ContentDetector -class AdaptiveContentDetector(SceneDetector): +class AdaptiveContentDetector(ContentDetector): """Detects cuts using HSV changes similar to ContentDetector, but with a rolling average that can help mitigate false detections in situations such as camera moves. """ - def __init__(self, video_manager=None, adaptive_threshold=3.0, min_scene_len=15, min_delta_hsv=5.0, window_width=2): - super(AdaptiveContentDetector, self).__init__() + def __init__(self, video_manager=None, adaptive_threshold=3.0, min_scene_len=15, + min_delta_hsv=5.0, window_width=2): + # Initialize ContentDetector with an impossibly high threshold + # so it does not trigger any cuts + super(AdaptiveContentDetector, self).__init__(threshold=300, min_scene_len=min_scene_len) self.video_manager = video_manager self.min_scene_len = min_scene_len # minimum length of any given scene, in frames (int) or FrameTimecode self.adaptive_threshold = adaptive_threshold @@ -62,77 +62,6 @@ def __init__(self, video_manager=None, adaptive_threshold=3.0, min_scene_len=15, 'delta_lum', 'con_val_ratio'] self.cli_name = 'adaptive-detect-content' - def process_frame(self, frame_num, frame_img): - # type: (int, numpy.ndarray) -> List[int] - """ Similar to ContentDetector, but looking for frames in which the HSV difference is - significantly different than the neighboring frames. - - Arguments: - frame_num (int): Frame number of frame that is being passed. - - frame_img (Optional[int]): Decoded frame image (numpy.ndarray) to perform scene - detection on. Can be None *only* if the self.is_processing_required() method - (inhereted from the base SceneDetector class) returns True. - - Returns: - Empty list: The process_frame function for this detector does not register any cuts, - instead only calculating scene metrics that are used to detect cuts with the - post_process function. - """ - - metric_keys = self._metric_keys - _unused = '' - - # We can only start detecting once we have a frame to compare with. - if self.last_frame is not None: - # Change in average of HSV (hsv), (h)ue only, (s)aturation only, (l)uminance only. - # These are refered to in a statsfile as their respective self._metric_keys string. - delta_hsv_avg, delta_h, delta_s, delta_v = 0.0, 0.0, 0.0, 0.0 - - if (self.stats_manager is not None and - self.stats_manager.metrics_exist(frame_num, metric_keys)): - delta_hsv_avg, delta_h, delta_s, delta_v = self.stats_manager.get_metrics( - frame_num, metric_keys)[:4] - - else: - num_pixels = frame_img.shape[0] * frame_img.shape[1] - curr_hsv = cv2.split(cv2.cvtColor(frame_img, cv2.COLOR_BGR2HSV)) - last_hsv = self.last_hsv - if not last_hsv: - last_hsv = cv2.split(cv2.cvtColor(self.last_frame, cv2.COLOR_BGR2HSV)) - - delta_hsv = [0, 0, 0, 0] - for i in range(3): - num_pixels = curr_hsv[i].shape[0] * curr_hsv[i].shape[1] - curr_hsv[i] = curr_hsv[i].astype(numpy.int32) - last_hsv[i] = last_hsv[i].astype(numpy.int32) - delta_hsv[i] = numpy.sum( - numpy.abs(curr_hsv[i] - last_hsv[i])) / float(num_pixels) - delta_hsv[3] = sum(delta_hsv[0:3]) / 3.0 - delta_h, delta_s, delta_v, delta_hsv_avg = delta_hsv - - if self.stats_manager is not None: - self.stats_manager.set_metrics(frame_num, { - metric_keys[0]: delta_hsv_avg, - metric_keys[1]: delta_h, - metric_keys[2]: delta_s, - metric_keys[3]: delta_v}) - - self.last_hsv = curr_hsv - - if self.last_frame is not None and self.last_frame is not _unused: - del self.last_frame - - # If we have the next frame computed, don't copy the current frame - # into last_frame since we won't use it on the next call anyways. - if (self.stats_manager is not None and - self.stats_manager.metrics_exist(frame_num+1, metric_keys)): - self.last_frame = _unused - else: - self.last_frame = frame_img.copy() - - return [] - def get_content_val(self, frame_num): """ Returns the average content change for a frame. From 30385a26a2006d58c76b0e20716efdb40d86a5fe Mon Sep 17 00:00:00 2001 From: wjs018 Date: Wed, 20 Jan 2021 22:02:42 -0500 Subject: [PATCH 10/10] Modified inheritance from ContentDetector to avoid hard coding values --- .../detectors/adaptive_content_detector.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/scenedetect/detectors/adaptive_content_detector.py b/scenedetect/detectors/adaptive_content_detector.py index ec69db6b..b02280f2 100644 --- a/scenedetect/detectors/adaptive_content_detector.py +++ b/scenedetect/detectors/adaptive_content_detector.py @@ -49,7 +49,7 @@ def __init__(self, video_manager=None, adaptive_threshold=3.0, min_scene_len=15, min_delta_hsv=5.0, window_width=2): # Initialize ContentDetector with an impossibly high threshold # so it does not trigger any cuts - super(AdaptiveContentDetector, self).__init__(threshold=300, min_scene_len=min_scene_len) + super(AdaptiveContentDetector, self).__init__() self.video_manager = video_manager self.min_scene_len = min_scene_len # minimum length of any given scene, in frames (int) or FrameTimecode self.adaptive_threshold = adaptive_threshold @@ -61,6 +61,29 @@ def __init__(self, video_manager=None, adaptive_threshold=3.0, min_scene_len=15, self._metric_keys = ['content_val', 'delta_hue', 'delta_sat', 'delta_lum', 'con_val_ratio'] self.cli_name = 'adaptive-detect-content' + + def process_frame(self, frame_num, frame_img): + # type: (int, numpy.ndarray) -> List[int] + """ Similar to ThresholdDetector, but using the HSV colour space DIFFERENCE instead + of single-frame RGB/grayscale intensity (thus cannot detect slow fades with this method). + + Arguments: + frame_num (int): Frame number of frame that is being passed. + + frame_img (Optional[int]): Decoded frame image (numpy.ndarray) to perform scene + detection on. Can be None *only* if the self.is_processing_required() method + (inhereted from the base SceneDetector class) returns True. + + Returns: + Empty list + """ + + # Call the process_frame function of ContentDetector but ignore any + # returned cuts + _ = super(AdaptiveContentDetector, self).process_frame( + frame_num=frame_num, frame_img=frame_img) + + return [] def get_content_val(self, frame_num): """