From 0ee9c2796698346e78c640565ec6a0dfb7a95151 Mon Sep 17 00:00:00 2001 From: Dave Merchant <131936113+D00E@users.noreply.github.com> Date: Tue, 14 Oct 2025 07:56:06 +0100 Subject: [PATCH] Merge pull request #27810 from D00E:known-foreground-mask 2025-10-14T05:53:31.5387050Z C:\GHA-OCV-1\_work\ci-gha-workflow\ci-gha-workflow\opencv\modules\imgcodecs\src\bitstrm.cpp(156,57): warning C4244: 'argument': conversion from 'int64_t' to 'ptrdiff_t', possible loss of data [C:\GHA-OCV-1\_work\ci-gha-workflow\ci-gha-workflow\build\modules\imgcodecs\opencv_imgcodecs.vcxproj] ### Pull Request Readiness Checklist Optional Known Foreground Mask for Background Subtractors #27810 See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake ### Description This adds an optional foreground input mask parameter to the MOG2 and KNN background subtractors, in line with issue https://github.com/opencv/opencv/issues/26476 4 tests are added under test_bgfg2.cpp: 2 for each subtractor type (1 with shadow detection and 1 without) A demo shows the feature with only 3 parameters and with a 4th optional foreground mask for both core subtractor types. Note: To patch contrib inheritance of the background subtraction class, empty apply method which throws a not implemented error is added to contrib subclasses. This is done to keep the overloaded apply function as pure virtual. Contrib PR to be made and linked shortly. Contrib Repo Paired Pull Request: https://github.com/opencv/opencv_contrib/pull/4017 --- .../include/opencv2/video/background_segm.hpp | 27 +++++ modules/video/src/bgfg_KNN.cpp | 36 +++++- modules/video/src/bgfg_gaussmix2.cpp | 35 +++++- modules/video/test/test_bgfg2.cpp | 108 ++++++++++++++++++ samples/python/background_subtractor_mask.py | 68 +++++++++++ 5 files changed, 268 insertions(+), 6 deletions(-) create mode 100644 modules/video/test/test_bgfg2.cpp create mode 100644 samples/python/background_subtractor_mask.py diff --git a/modules/video/include/opencv2/video/background_segm.hpp b/modules/video/include/opencv2/video/background_segm.hpp index e1dfa15a9a..73409f27d4 100644 --- a/modules/video/include/opencv2/video/background_segm.hpp +++ b/modules/video/include/opencv2/video/background_segm.hpp @@ -71,6 +71,21 @@ public: */ CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate=-1) = 0; + /** @brief Computes a foreground mask with known foreground mask input. + + @param image Next video frame. Floating point frame will be used without scaling and should be in range \f$[0,255]\f$. + @param fgmask The output foreground mask as an 8-bit binary image. + @param knownForegroundMask The mask for inputting already known foreground, allows model to ignore pixels. + @param learningRate The value between 0 and 1 that indicates how fast the background model is + learnt. Negative parameter value makes the algorithm to use some automatically chosen learning + rate. 0 means that the background model is not updated at all, 1 means that the background model + is completely reinitialized from the last frame. + + @note This method has a default virtual implementation that throws a "not impemented" error. + Foreground masking may not be supported by all background subtractors. + */ + CV_WRAP virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate=-1) = 0; + /** @brief Computes a background image. @param backgroundImage The output background image. @@ -206,6 +221,18 @@ public: is completely reinitialized from the last frame. */ CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE = 0; + + /** @brief Computes a foreground mask and skips known foreground in evaluation. + + @param image Next video frame. Floating point frame will be used without scaling and should be in range \f$[0,255]\f$. + @param fgmask The output foreground mask as an 8-bit binary image. + @param knownForegroundMask The mask for inputting already known foreground, allows model to ignore pixels. + @param learningRate The value between 0 and 1 that indicates how fast the background model is + learnt. Negative parameter value makes the algorithm to use some automatically chosen learning + rate. 0 means that the background model is not updated at all, 1 means that the background model + is completely reinitialized from the last frame. + */ + CV_WRAP virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE = 0; }; /** @brief Creates MOG2 Background Subtractor diff --git a/modules/video/src/bgfg_KNN.cpp b/modules/video/src/bgfg_KNN.cpp index 5ec2266921..cdf4eb3a51 100644 --- a/modules/video/src/bgfg_KNN.cpp +++ b/modules/video/src/bgfg_KNN.cpp @@ -132,6 +132,8 @@ public: //! the update operator void apply(InputArray image, OutputArray fgmask, double learningRate) CV_OVERRIDE; + void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate) CV_OVERRIDE; + //! computes a background image which are the mean of all background gaussians virtual void getBackgroundImage(OutputArray backgroundImage) const CV_OVERRIDE; @@ -526,7 +528,9 @@ public: int _nkNN, float _fTau, bool _bShadowDetection, - uchar _nShadowDetection) + uchar _nShadowDetection, + const Mat& _knownForegroundMask) + : knownForegroundMask(_knownForegroundMask) { src = &_src; dst = &_dst; @@ -587,6 +591,17 @@ public: m_nShortCounter, include ); + // Check that foreground mask exists + if (!knownForegroundMask.empty()) { + // If input mask states pixel is foreground + if (knownForegroundMask.at(y, x) > 0) + { + mask[x] = 255; // ensure output mask marks this pixel as FG + data += nchannels; + m_aModel += m_nN*3*ndata; + continue; + } + } switch (result) { case 0: @@ -626,6 +641,7 @@ public: int m_nkNN; bool m_bShadowDetection; uchar m_nShadowDetection; + const Mat& knownForegroundMask; }; #ifdef HAVE_OPENCL @@ -728,7 +744,12 @@ void BackgroundSubtractorKNNImpl::create_ocl_apply_kernel() #endif -void BackgroundSubtractorKNNImpl::apply(InputArray _image, OutputArray _fgmask, double learningRate) +// Base 3 version class +void BackgroundSubtractorKNNImpl::apply(InputArray _image, OutputArray _fgmask, double learningRate) { + apply(_image, noArray(), _fgmask, learningRate); +} + +void BackgroundSubtractorKNNImpl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double learningRate) { CV_INSTRUMENT_REGION(); @@ -757,6 +778,14 @@ void BackgroundSubtractorKNNImpl::apply(InputArray _image, OutputArray _fgmask, _fgmask.create( image.size(), CV_8U ); Mat fgmask = _fgmask.getMat(); + Mat knownForegroundMask = _knownForegroundMask.getMat(); + + if(!knownForegroundMask.empty()) + { + CV_Assert(knownForegroundMask.type() == CV_8UC1); + CV_Assert(knownForegroundMask.size() == image.size()); + } + ++nframes; learningRate = learningRate >= 0 && nframes > 1 ? learningRate : 1./std::min( 2*nframes, history ); CV_Assert(learningRate >= 0); @@ -791,7 +820,8 @@ void BackgroundSubtractorKNNImpl::apply(InputArray _image, OutputArray _fgmask, nkNN, fTau, bShadowDetection, - nShadowDetection), + nShadowDetection, + knownForegroundMask), image.total()/(double)(1 << 16)); nShortCounter++;//0,1,...,nShortUpdate-1 diff --git a/modules/video/src/bgfg_gaussmix2.cpp b/modules/video/src/bgfg_gaussmix2.cpp index 00a65fcf8f..48d8ee8286 100644 --- a/modules/video/src/bgfg_gaussmix2.cpp +++ b/modules/video/src/bgfg_gaussmix2.cpp @@ -178,6 +178,8 @@ public: //! the update operator void apply(InputArray image, OutputArray fgmask, double learningRate) CV_OVERRIDE; + void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate) CV_OVERRIDE; + //! computes a background image which are the mean of all background gaussians virtual void getBackgroundImage(OutputArray backgroundImage) const CV_OVERRIDE; @@ -546,7 +548,8 @@ public: float _Tb, float _TB, float _Tg, float _varInit, float _varMin, float _varMax, float _prune, float _tau, bool _detectShadows, - uchar _shadowVal) + uchar _shadowVal, const Mat& _knownForegroundMask) + : knownForegroundMask(_knownForegroundMask) { src = &_src; dst = &_dst; @@ -590,6 +593,18 @@ public: for( int x = 0; x < ncols; x++, data += nchannels, gmm += nmixtures, mean += nmixtures*nchannels ) { + + // Check that foreground mask exists + if (!knownForegroundMask.empty()) + { + // If input mask states pixel is foreground + if (knownForegroundMask.at(y, x) > 0) + { + mask[x] = 255; // ensure output mask marks this pixel as FG + continue; + } + } + //calculate distances to the modes (+ sort) //here we need to go in descending order!!! bool background = false;//return value -> true - the pixel classified as background @@ -766,6 +781,7 @@ public: bool detectShadows; uchar shadowVal; + const Mat& knownForegroundMask; }; #ifdef HAVE_OPENCL @@ -844,7 +860,12 @@ void BackgroundSubtractorMOG2Impl::create_ocl_apply_kernel() #endif -void BackgroundSubtractorMOG2Impl::apply(InputArray _image, OutputArray _fgmask, double learningRate) +// Base 3 version class +void BackgroundSubtractorMOG2Impl::apply(InputArray _image, OutputArray _fgmask, double learningRate) { + apply(_image, noArray(), _fgmask, learningRate); +} + +void BackgroundSubtractorMOG2Impl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double learningRate) { CV_INSTRUMENT_REGION(); @@ -867,6 +888,14 @@ void BackgroundSubtractorMOG2Impl::apply(InputArray _image, OutputArray _fgmask, _fgmask.create( image.size(), CV_8U ); Mat fgmask = _fgmask.getMat(); + Mat knownForegroundMask = _knownForegroundMask.getMat(); + + if(!knownForegroundMask.empty()) + { + CV_Assert(knownForegroundMask.type() == CV_8UC1); + CV_Assert(knownForegroundMask.size() == image.size()); + } + ++nframes; learningRate = learningRate >= 0 && nframes > 1 ? learningRate : 1./std::min( 2*nframes, history ); CV_Assert(learningRate >= 0); @@ -879,7 +908,7 @@ void BackgroundSubtractorMOG2Impl::apply(InputArray _image, OutputArray _fgmask, (float)varThreshold, backgroundRatio, varThresholdGen, fVarInit, fVarMin, fVarMax, float(-learningRate*fCT), fTau, - bShadowDetection, nShadowDetection), + bShadowDetection, nShadowDetection, knownForegroundMask), image.total()/(double)(1 << 16)); } diff --git a/modules/video/test/test_bgfg2.cpp b/modules/video/test/test_bgfg2.cpp new file mode 100644 index 0000000000..eda022abf4 --- /dev/null +++ b/modules/video/test/test_bgfg2.cpp @@ -0,0 +1,108 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" +#include "opencv2/video/background_segm.hpp" + +namespace opencv_test { namespace { + +using namespace cv; + +///////////////////////// MOG2 ////////////////////////////// +TEST(BackgroundSubtractorMOG2, KnownForegroundMaskShadowsTrue) +{ + Ptr mog2 = createBackgroundSubtractorMOG2(500, 16, true); + + //Black Frame + Mat input = Mat::zeros(480,640 , CV_8UC3); + + //White Rectangle + Mat knownFG = Mat::zeros(input.size(), CV_8U); + + rectangle(knownFG, Rect(3,3,5,5), Scalar(255,255,255), -1); + + Mat output; + mog2->apply(input, knownFG, output); + + for(int y = 3; y < 8; y++) + { + for (int x = 3; x < 8; x++){ + EXPECT_EQ(255,output.at(y,x)) << "Expected foreground at (" << x << "," << y << ")"; + } + } +} + +TEST(BackgroundSubtractorMOG2, KnownForegroundMaskShadowsFalse) +{ + Ptr mog2 = createBackgroundSubtractorMOG2(500, 16, false); + + //Black Frame + Mat input = Mat::zeros(480,640 , CV_8UC3); + + //White Rectangle + Mat knownFG = Mat::zeros(input.size(), CV_8U); + + rectangle(knownFG, Rect(3,3,5,5), Scalar(255,255,255), FILLED); + + Mat output; + mog2->apply(input, knownFG, output); + + for(int y = 3; y < 8; y++) + { + for (int x = 3; x < 8; x++){ + EXPECT_EQ(255,output.at(y,x)) << "Expected foreground at (" << x << "," << y << ")"; + } + } +} + +///////////////////////// KNN ////////////////////////////// + +TEST(BackgroundSubtractorKNN, KnownForegroundMaskShadowsTrue) +{ + Ptr knn = createBackgroundSubtractorKNN(500, 400.0, true); + + //Black Frame + Mat input = Mat::zeros(480,640 , CV_8UC3); + + //White Rectangle + Mat knownFG = Mat::zeros(input.size(), CV_8U); + + rectangle(knownFG, Rect(3,3,5,5), Scalar(255,255,255), FILLED); + + Mat output; + knn->apply(input, knownFG, output); + + for(int y = 3; y < 8; y++) + { + for (int x = 3; x < 8; x++){ + EXPECT_EQ(255,output.at(y,x)) << "Expected foreground at (" << x << "," << y << ")"; + } + } +} + +TEST(BackgroundSubtractorKNN, KnownForegroundMaskShadowsFalse) +{ + Ptr knn = createBackgroundSubtractorKNN(500, 400.0, false); + + //Black Frame + Mat input = Mat::zeros(480,640 , CV_8UC3); + + //White Rectangle + Mat knownFG = Mat::zeros(input.size(), CV_8U); + + rectangle(knownFG, Rect(3,3,5,5), Scalar(255,255,255), FILLED); + + Mat output; + knn->apply(input, knownFG, output); + + for(int y = 3; y < 8; y++) + { + for (int x = 3; x < 8; x++){ + EXPECT_EQ(255,output.at(y,x)) << "Expected foreground at (" << x << "," << y << ")"; + } + } +} + +}} // namespace +/* End of file. */ diff --git a/samples/python/background_subtractor_mask.py b/samples/python/background_subtractor_mask.py new file mode 100644 index 0000000000..5d4aae4701 --- /dev/null +++ b/samples/python/background_subtractor_mask.py @@ -0,0 +1,68 @@ + +''' +Showcases the use of background subtraction from a live video feed, +aswell as pass through of a known foreground parameter +''' + +# Python 2/3 compatibility +from __future__ import print_function + +import numpy as np +import cv2 as cv + +def main(): + cap = cv.VideoCapture(0) + if not cap.isOpened: + print("Capture source avaialable.") + exit() + + # Create background subtractor + mog2_bg_subtractor = cv.createBackgroundSubtractorMOG2(history=300, varThreshold=50, detectShadows=False) + knn_bg_subtractor = cv.createBackgroundSubtractorKNN(history=300, detectShadows=False) + + frame_count = 0 + # Allows for a frame buffer for the mask to learn pre known foreground + show_count = 10 + + while True: + ret, frame = cap.read() + if not ret: + break + + x = 100 + (frame_count % 10) * 3 + + frame = cv.resize(frame, (640, 480)) + aKnownForegroundMask = np.zeros(frame.shape[:2], dtype=np.uint8) + + # Allow for models to "settle"/learn + if frame_count > show_count: + cv.rectangle(aKnownForegroundMask, (x,200), (x+50,300), 255, -1) + cv.rectangle(aKnownForegroundMask, (540,180), (640,480), 255, -1) + + #MOG2 Subtraction + mog2_with_mask = mog2_bg_subtractor.apply(frame,knownForegroundMask=aKnownForegroundMask) + mog2_without_mask = mog2_bg_subtractor.apply(frame) + + #KNN Subtraction + knn_with_mask = knn_bg_subtractor.apply(frame,knownForegroundMask=aKnownForegroundMask) + knn_without_mask = knn_bg_subtractor.apply(frame) + + # Display the 3 parameter apply and the 4 parameter apply for both subtractors + cv.imshow("MOG2 With a Foreground Mask", mog2_with_mask) + cv.imshow("MOG2 Without a Foreground Mask", mog2_without_mask) + cv.imshow("KNN With a Foreground Mask", knn_with_mask) + cv.imshow("KNN Without a Foreground Mask", knn_without_mask) + + key = cv.waitKey(30) + if key == 27: # ESC + break + + frame_count += 1 + + cap.release() + cv.destroyAllWindows() + +if __name__ == '__main__': + print(__doc__) + main() + cv.destroyAllWindows()