mirror of
https://github.com/zebrajr/pytorch.git
synced 2025-12-07 12:21:27 +01:00
ca0ac3a74b
Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/72741
as titled.
Context:
This is useful in fast mitigating feature induced overfitting in the sense that we can do omni-transfer on a trained model and apply dropout with ratio = 1 on features resulting in overfitting. Directly removing the features would not be feasible on omni-transfer scenarios since the downstream FC sizes would change.
Experimental records:
https://fb.quip.com/npIkAgRc8jl9#temp:C:DWC050ceaba14424d23a78462c01
Doing dropout = 1 on selected features improves the eval NE over the next few hours (compared to v0 baseline) as is shown in the figures.
Test Plan:
```
buck test caffe2/caffe2/python/operator_test:dropout_op_test
```
Reviewed By: ustctf
Differential Revision: D34178732
fbshipit-source-id: 533feebe21bc582eefd756de397d5c7807c7438d
(cherry picked from commit 5dabf9c484)
56 lines
1.4 KiB
C++
56 lines
1.4 KiB
C++
#ifndef CAFFE2_OPERATORS_DROPOUT_OP_H_
|
|
#define CAFFE2_OPERATORS_DROPOUT_OP_H_
|
|
|
|
#include "caffe2/core/context.h"
|
|
#include "caffe2/core/logging.h"
|
|
#include "caffe2/core/operator.h"
|
|
#include "caffe2/utils/math.h"
|
|
|
|
namespace caffe2 {
|
|
|
|
template <typename T, class Context>
|
|
class DropoutOp final : public Operator<Context> {
|
|
public:
|
|
USE_OPERATOR_CONTEXT_FUNCTIONS;
|
|
template <class... Args>
|
|
explicit DropoutOp(Args&&... args)
|
|
: Operator<Context>(std::forward<Args>(args)...),
|
|
ratio_(this->template GetSingleArgument<float>("ratio", 0.5)),
|
|
is_test_(
|
|
this->template GetSingleArgument<int>(OpSchema::Arg_IsTest, 0)) {
|
|
CAFFE_ENFORCE_GE(ratio_, 0);
|
|
}
|
|
|
|
bool RunOnDevice() override;
|
|
|
|
protected:
|
|
float ratio_;
|
|
bool is_test_;
|
|
// Input: X; Output: Y, mask.
|
|
};
|
|
|
|
template <typename T, class Context>
|
|
class DropoutGradientOp final : public Operator<Context> {
|
|
public:
|
|
USE_OPERATOR_CONTEXT_FUNCTIONS;
|
|
template <class... Args>
|
|
explicit DropoutGradientOp(Args&&... args)
|
|
: Operator<Context>(std::forward<Args>(args)...),
|
|
ratio_(this->template GetSingleArgument<float>("ratio", 0.5)),
|
|
is_test_(
|
|
this->template GetSingleArgument<int>(OpSchema::Arg_IsTest, 0)) {
|
|
CAFFE_ENFORCE_GE(ratio_, 0);
|
|
}
|
|
|
|
bool RunOnDevice() override;
|
|
|
|
protected:
|
|
float ratio_;
|
|
bool is_test_;
|
|
// Input: dY, mask; Output: dX
|
|
};
|
|
|
|
} // namespace caffe2
|
|
|
|
#endif // CAFFE2_OPERATORS_DROPOUT_OP_H_
|