Merge pull request #27810 from D00E:known-foreground-mask

2025-10-14T05:53:31.5387050Z C:\GHA-OCV-1\_work\ci-gha-workflow\ci-gha-workflow\opencv\modules\imgcodecs\src\bitstrm.cpp(156,57): warning C4244: 'argument': conversion from 'int64_t' to 'ptrdiff_t', possible loss of data [C:\GHA-OCV-1\_work\ci-gha-workflow\ci-gha-workflow\build\modules\imgcodecs\opencv_imgcodecs.vcxproj]

### Pull Request Readiness Checklist

Optional Known Foreground Mask for Background Subtractors #27810

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake

### Description
This adds an optional foreground input mask parameter to the MOG2 and KNN background subtractors, in line with issue https://github.com/opencv/opencv/issues/26476

4 tests are added under test_bgfg2.cpp:
2 for each subtractor type (1 with shadow detection and 1 without)
A demo shows the feature with only 3 parameters and with a 4th optional foreground mask for both core subtractor types.

Note: To patch contrib inheritance of the background subtraction class, empty apply method which throws a not implemented error is added to contrib subclasses. This is done to keep the overloaded apply function as pure virtual. Contrib PR to be made and linked shortly.  
Contrib Repo Paired Pull Request: https://github.com/opencv/opencv_contrib/pull/4017

modules/video/include/opencv2/video/background_segm.hpp

@@ -71,6 +71,21 @@ public:
      */
     CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate=-1) = 0;
 
+    /** @brief Computes a foreground mask with known foreground mask input.
+
+    @param image Next video frame. Floating point frame will be used without scaling and should be in range \f$[0,255]\f$.
+    @param fgmask The output foreground mask as an 8-bit binary image.
+    @param knownForegroundMask The mask for inputting already known foreground, allows model to ignore pixels.
+    @param learningRate The value between 0 and 1 that indicates how fast the background model is
+    learnt. Negative parameter value makes the algorithm to use some automatically chosen learning
+    rate. 0 means that the background model is not updated at all, 1 means that the background model
+    is completely reinitialized from the last frame.
+
+    @note This method has a default virtual implementation that throws a "not impemented" error.
+    Foreground masking may not be supported by all background subtractors.
+    */
+    CV_WRAP virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate=-1) = 0;
+
     /** @brief Computes a background image.
 
     @param backgroundImage The output background image.
@@ -206,6 +221,18 @@ public:
     is completely reinitialized from the last frame.
      */
     CV_WRAP virtual void apply(InputArray image, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE = 0;
+
+    /** @brief Computes a foreground mask and skips known foreground in evaluation.
+
+    @param image Next video frame. Floating point frame will be used without scaling and should be in range \f$[0,255]\f$.
+    @param fgmask The output foreground mask as an 8-bit binary image.
+    @param knownForegroundMask The mask for inputting already known foreground, allows model to ignore pixels.
+    @param learningRate The value between 0 and 1 that indicates how fast the background model is
+    learnt. Negative parameter value makes the algorithm to use some automatically chosen learning
+    rate. 0 means that the background model is not updated at all, 1 means that the background model
+    is completely reinitialized from the last frame.
+     */
+    CV_WRAP virtual void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate=-1) CV_OVERRIDE = 0;
 };
 
 /** @brief Creates MOG2 Background Subtractor

modules/video/src/bgfg_KNN.cpp

@@ -132,6 +132,8 @@ public:
     //! the update operator
     void apply(InputArray image, OutputArray fgmask, double learningRate) CV_OVERRIDE;
 
+    void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate) CV_OVERRIDE;
+
     //! computes a background image which are the mean of all background gaussians
     virtual void getBackgroundImage(OutputArray backgroundImage) const CV_OVERRIDE;
 
@@ -526,7 +528,9 @@ public:
                int _nkNN,
                float _fTau,
                bool _bShadowDetection,
-               uchar _nShadowDetection)
+               uchar _nShadowDetection,
+               const Mat& _knownForegroundMask)
+            :  knownForegroundMask(_knownForegroundMask)
     {
         src = &_src;
         dst = &_dst;
@@ -587,6 +591,17 @@ public:
                         m_nShortCounter,
                         include
                         );
+                // Check that foreground mask exists
+                if (!knownForegroundMask.empty()) {
+                    // If input mask states pixel is foreground
+                    if (knownForegroundMask.at<uchar>(y, x) > 0)
+                    {
+                        mask[x] = 255; // ensure output mask marks this pixel as FG
+                        data += nchannels;
+                        m_aModel += m_nN*3*ndata;
+                        continue;
+                    }
+                }
                 switch (result)
                 {
                     case 0:
@@ -626,6 +641,7 @@ public:
     int m_nkNN;
     bool m_bShadowDetection;
     uchar m_nShadowDetection;
+    const Mat& knownForegroundMask;
 };
 
 #ifdef HAVE_OPENCL
@@ -728,7 +744,12 @@ void BackgroundSubtractorKNNImpl::create_ocl_apply_kernel()
 
 #endif
 
-void BackgroundSubtractorKNNImpl::apply(InputArray _image, OutputArray _fgmask, double learningRate)
+// Base 3 version class
+void BackgroundSubtractorKNNImpl::apply(InputArray _image, OutputArray _fgmask, double learningRate) {
+    apply(_image,  noArray(), _fgmask, learningRate);
+}
+
+void BackgroundSubtractorKNNImpl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double learningRate)
 {
     CV_INSTRUMENT_REGION();
 
@@ -757,6 +778,14 @@ void BackgroundSubtractorKNNImpl::apply(InputArray _image, OutputArray _fgmask,
     _fgmask.create( image.size(), CV_8U );
     Mat fgmask = _fgmask.getMat();
 
+    Mat knownForegroundMask = _knownForegroundMask.getMat();
+
+    if(!knownForegroundMask.empty())
+    {
+    CV_Assert(knownForegroundMask.type() == CV_8UC1);
+    CV_Assert(knownForegroundMask.size() == image.size());
+    }
+
     ++nframes;
     learningRate = learningRate >= 0 && nframes > 1 ? learningRate : 1./std::min( 2*nframes, history );
     CV_Assert(learningRate >= 0);
@@ -791,7 +820,8 @@ void BackgroundSubtractorKNNImpl::apply(InputArray _image, OutputArray _fgmask,
                              nkNN,
                              fTau,
                              bShadowDetection,
-                             nShadowDetection),
+                             nShadowDetection,
+                             knownForegroundMask),
                              image.total()/(double)(1 << 16));
 
     nShortCounter++;//0,1,...,nShortUpdate-1

modules/video/src/bgfg_gaussmix2.cpp

@@ -178,6 +178,8 @@ public:
     //! the update operator
     void apply(InputArray image, OutputArray fgmask, double learningRate) CV_OVERRIDE;
 
+    void apply(InputArray image, InputArray knownForegroundMask, OutputArray fgmask, double learningRate) CV_OVERRIDE;
+
     //! computes a background image which are the mean of all background gaussians
     virtual void getBackgroundImage(OutputArray backgroundImage) const CV_OVERRIDE;
 
@@ -546,7 +548,8 @@ public:
                 float _Tb, float _TB, float _Tg,
                 float _varInit, float _varMin, float _varMax,
                 float _prune, float _tau, bool _detectShadows,
-                uchar _shadowVal)
+                uchar _shadowVal, const Mat& _knownForegroundMask)
+            :   knownForegroundMask(_knownForegroundMask)
     {
         src = &_src;
         dst = &_dst;
@@ -590,6 +593,18 @@ public:
 
             for( int x = 0; x < ncols; x++, data += nchannels, gmm += nmixtures, mean += nmixtures*nchannels )
             {
+
+                // Check that foreground mask exists
+                if (!knownForegroundMask.empty())
+                {
+                    // If input mask states pixel is foreground
+                    if (knownForegroundMask.at<uchar>(y, x) > 0)
+                    {
+                        mask[x] = 255; // ensure output mask marks this pixel as FG
+                        continue;
+                    }
+                }
+
                 //calculate distances to the modes (+ sort)
                 //here we need to go in descending order!!!
                 bool background = false;//return value -> true - the pixel classified as background
@@ -766,6 +781,7 @@ public:
 
     bool detectShadows;
     uchar shadowVal;
+    const Mat& knownForegroundMask;
 };
 
 #ifdef HAVE_OPENCL
@@ -844,7 +860,12 @@ void BackgroundSubtractorMOG2Impl::create_ocl_apply_kernel()
 
 #endif
 
-void BackgroundSubtractorMOG2Impl::apply(InputArray _image, OutputArray _fgmask, double learningRate)
+// Base 3 version class
+void BackgroundSubtractorMOG2Impl::apply(InputArray _image, OutputArray _fgmask, double learningRate) {
+    apply(_image,  noArray(), _fgmask, learningRate);
+}
+
+void BackgroundSubtractorMOG2Impl::apply(InputArray _image, InputArray _knownForegroundMask, OutputArray _fgmask, double learningRate)
 {
     CV_INSTRUMENT_REGION();
 
@@ -867,6 +888,14 @@ void BackgroundSubtractorMOG2Impl::apply(InputArray _image, OutputArray _fgmask,
     _fgmask.create( image.size(), CV_8U );
     Mat fgmask = _fgmask.getMat();
 
+    Mat knownForegroundMask = _knownForegroundMask.getMat();
+
+    if(!knownForegroundMask.empty())
+    {
+    CV_Assert(knownForegroundMask.type() == CV_8UC1);
+    CV_Assert(knownForegroundMask.size() == image.size());
+    }
+
     ++nframes;
     learningRate = learningRate >= 0 && nframes > 1 ? learningRate : 1./std::min( 2*nframes, history );
     CV_Assert(learningRate >= 0);
@@ -879,7 +908,7 @@ void BackgroundSubtractorMOG2Impl::apply(InputArray _image, OutputArray _fgmask,
                               (float)varThreshold,
                               backgroundRatio, varThresholdGen,
                               fVarInit, fVarMin, fVarMax, float(-learningRate*fCT), fTau,
-                              bShadowDetection, nShadowDetection),
+                              bShadowDetection, nShadowDetection, knownForegroundMask),
                               image.total()/(double)(1 << 16));
 }
 

modules/video/test/test_bgfg2.cpp

@@ -0,0 +1,108 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#include "test_precomp.hpp"
+#include "opencv2/video/background_segm.hpp"
+
+namespace opencv_test { namespace {
+
+using namespace cv;
+
+///////////////////////// MOG2 //////////////////////////////
+TEST(BackgroundSubtractorMOG2, KnownForegroundMaskShadowsTrue)
+{
+    Ptr<BackgroundSubtractorMOG2> mog2 = createBackgroundSubtractorMOG2(500, 16, true);
+
+    //Black Frame
+    Mat input = Mat::zeros(480,640 , CV_8UC3);
+
+    //White Rectangle
+    Mat knownFG = Mat::zeros(input.size(), CV_8U);
+
+    rectangle(knownFG, Rect(3,3,5,5), Scalar(255,255,255), -1);
+
+    Mat output;
+    mog2->apply(input, knownFG, output);
+
+    for(int y = 3; y < 8; y++)
+    {
+        for (int x = 3; x < 8; x++){
+            EXPECT_EQ(255,output.at<uchar>(y,x)) << "Expected foreground at (" << x << "," << y << ")";
+        }
+    }
+}
+
+TEST(BackgroundSubtractorMOG2, KnownForegroundMaskShadowsFalse)
+{
+    Ptr<BackgroundSubtractorMOG2> mog2 = createBackgroundSubtractorMOG2(500, 16, false);
+
+    //Black Frame
+    Mat input = Mat::zeros(480,640 , CV_8UC3);
+
+    //White Rectangle
+    Mat knownFG = Mat::zeros(input.size(), CV_8U);
+
+    rectangle(knownFG, Rect(3,3,5,5), Scalar(255,255,255), FILLED);
+
+    Mat output;
+    mog2->apply(input, knownFG, output);
+
+    for(int y = 3; y < 8; y++)
+    {
+        for (int x = 3; x < 8; x++){
+            EXPECT_EQ(255,output.at<uchar>(y,x)) << "Expected foreground at (" << x << "," << y << ")";
+        }
+    }
+}
+
+///////////////////////// KNN //////////////////////////////
+
+TEST(BackgroundSubtractorKNN, KnownForegroundMaskShadowsTrue)
+{
+    Ptr<BackgroundSubtractorKNN> knn = createBackgroundSubtractorKNN(500, 400.0, true);
+
+    //Black Frame
+    Mat input = Mat::zeros(480,640 , CV_8UC3);
+
+    //White Rectangle
+    Mat knownFG = Mat::zeros(input.size(), CV_8U);
+
+    rectangle(knownFG, Rect(3,3,5,5), Scalar(255,255,255), FILLED);
+
+    Mat output;
+    knn->apply(input, knownFG, output);
+
+    for(int y = 3; y < 8; y++)
+    {
+        for (int x = 3; x < 8; x++){
+            EXPECT_EQ(255,output.at<uchar>(y,x)) << "Expected foreground at (" << x << "," << y << ")";
+        }
+    }
+}
+
+TEST(BackgroundSubtractorKNN, KnownForegroundMaskShadowsFalse)
+{
+    Ptr<BackgroundSubtractorKNN> knn = createBackgroundSubtractorKNN(500, 400.0, false);
+
+    //Black Frame
+    Mat input = Mat::zeros(480,640 , CV_8UC3);
+
+    //White Rectangle
+    Mat knownFG = Mat::zeros(input.size(), CV_8U);
+
+    rectangle(knownFG, Rect(3,3,5,5), Scalar(255,255,255), FILLED);
+
+    Mat output;
+    knn->apply(input, knownFG, output);
+
+    for(int y = 3; y < 8; y++)
+    {
+        for (int x = 3; x < 8; x++){
+            EXPECT_EQ(255,output.at<uchar>(y,x)) << "Expected foreground at (" << x << "," << y << ")";
+        }
+    }
+}
+
+}} // namespace
+/* End of file. */

samples/python/background_subtractor_mask.py

@@ -0,0 +1,68 @@
+
+'''
+Showcases the use of background subtraction from a live video feed,
+aswell as pass through of a known foreground parameter
+'''
+
+# Python 2/3 compatibility
+from __future__ import print_function
+
+import numpy as np
+import cv2 as cv
+
+def main():
+    cap = cv.VideoCapture(0)
+    if not cap.isOpened:
+        print("Capture source avaialable.")
+        exit()
+
+    # Create background subtractor
+    mog2_bg_subtractor = cv.createBackgroundSubtractorMOG2(history=300, varThreshold=50, detectShadows=False)
+    knn_bg_subtractor = cv.createBackgroundSubtractorKNN(history=300, detectShadows=False)
+
+    frame_count = 0
+    # Allows for a frame buffer for the mask to learn pre known foreground
+    show_count = 10
+
+    while True:
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+        x = 100 + (frame_count % 10) * 3
+
+        frame = cv.resize(frame, (640, 480))
+        aKnownForegroundMask = np.zeros(frame.shape[:2], dtype=np.uint8)
+
+        # Allow for models to "settle"/learn
+        if frame_count > show_count:
+            cv.rectangle(aKnownForegroundMask, (x,200), (x+50,300), 255, -1)
+            cv.rectangle(aKnownForegroundMask, (540,180), (640,480), 255, -1)
+
+        #MOG2 Subtraction
+        mog2_with_mask = mog2_bg_subtractor.apply(frame,knownForegroundMask=aKnownForegroundMask)
+        mog2_without_mask = mog2_bg_subtractor.apply(frame)
+
+        #KNN Subtraction
+        knn_with_mask = knn_bg_subtractor.apply(frame,knownForegroundMask=aKnownForegroundMask)
+        knn_without_mask = knn_bg_subtractor.apply(frame)
+
+        # Display the 3 parameter apply and the 4 parameter apply for both subtractors
+        cv.imshow("MOG2 With a Foreground Mask", mog2_with_mask)
+        cv.imshow("MOG2 Without a Foreground Mask", mog2_without_mask)
+        cv.imshow("KNN With a Foreground Mask", knn_with_mask)
+        cv.imshow("KNN Without a Foreground Mask", knn_without_mask)
+
+        key = cv.waitKey(30)
+        if key == 27:  # ESC
+            break
+
+        frame_count += 1
+
+    cap.release()
+    cv.destroyAllWindows()
+
+if __name__ == '__main__':
+    print(__doc__)
+    main()
+    cv.destroyAllWindows()