mirror of
https://github.com/zebrajr/pytorch.git
synced 2025-12-07 00:21:07 +01:00
3ca7c0ffdb
Summary: We need a way to figure get a complete list fo features that are used in training a model. One way to do this is to make it possible to get the list of features used in each Model Layer. Then once the model is complete we can go through the layers and aggregate the features. I've introduced a function to expose that information here, get_accessed_features, and implemented it in the FeatureSparseToDense layer to start with. I've tried to include the minimum amount of information to make this useful, while making it easy to integrate into the variety of model layers. This is, for example, why AccessedFeatures does not contain feature_names which is not always present in a model layer. I debated whether or not to include feature_type, but I think that's useful enough, and easy enough to figure out in a model layer, that it's worth including. Pull Request resolved: https://github.com/pytorch/pytorch/pull/23036 Test Plan: Added a unit test to verify the behavior of get_accessed_features in FeatureSparseToDense. aml_dper2-fblearner-flow-integration-tests failed due to a known issue D16355865 aml_dper3-fblearner-flow-integration-tests failed due to a known issue T47197113 I verified no tests in the integration tests failed to issues other than those known ones. DPER2 canaries: https://fburl.com/fblearner/1217voga Reviewed By: volkhin Differential Revision: D16365380 Pulled By: kevinwilfong fbshipit-source-id: 2dbb4d832628180336533f29f7d917cbad171950
2212 lines
80 KiB
Python
2212 lines
80 KiB
Python
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
from __future__ import unicode_literals
|
|
|
|
import hypothesis.strategies as st
|
|
import numpy as np
|
|
import numpy.testing as npt
|
|
import unittest
|
|
from hypothesis import given
|
|
|
|
import caffe2.python.hypothesis_test_util as hu
|
|
|
|
from caffe2.python import (
|
|
layer_model_instantiator,
|
|
core,
|
|
schema,
|
|
workspace,
|
|
)
|
|
from caffe2.python.layers.layers import (
|
|
AccessedFeatures,
|
|
get_key,
|
|
IdList,
|
|
InstantiationContext,
|
|
is_request_only_scalar,
|
|
set_request_only,
|
|
)
|
|
from caffe2.python.layers.tags import Tags
|
|
from caffe2.python.layer_test_util import (
|
|
LayersTestCase,
|
|
OpSpec,
|
|
)
|
|
import logging
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class TestLayers(LayersTestCase):
|
|
def testAddLoss(self):
|
|
input_record_LR = self.new_record(
|
|
schema.Struct(
|
|
('label', schema.Scalar((np.float64, (1, )))),
|
|
('logit', schema.Scalar((np.float32, (2, )))),
|
|
('weight', schema.Scalar((np.float64, (1, ))))
|
|
)
|
|
)
|
|
loss_LR = self.model.BatchLRLoss(input_record_LR)
|
|
|
|
self.model.add_loss(loss_LR)
|
|
assert 'unnamed' in self.model.loss
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, tuple())), self.model.loss.unnamed
|
|
)
|
|
self.assertEqual(loss_LR, self.model.loss.unnamed)
|
|
|
|
self.model.add_loss(loss_LR, 'addLoss')
|
|
assert 'addLoss' in self.model.loss
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, tuple())), self.model.loss.addLoss
|
|
)
|
|
self.assertEqual(loss_LR, self.model.loss.addLoss)
|
|
|
|
self.model.add_loss(
|
|
schema.Scalar(
|
|
dtype=np.float32, blob=core.BlobReference('loss_blob_1')
|
|
), 'addLoss'
|
|
)
|
|
assert 'addLoss_auto_0' in self.model.loss
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, tuple())), self.model.loss.addLoss_auto_0
|
|
)
|
|
assert core.BlobReference('loss_blob_1') in self.model.loss.field_blobs()
|
|
|
|
self.model.add_loss(
|
|
schema.Struct(
|
|
(
|
|
'structName', schema.Scalar(
|
|
dtype=np.float32,
|
|
blob=core.BlobReference('loss_blob_2')
|
|
)
|
|
)
|
|
), 'addLoss'
|
|
)
|
|
assert 'addLoss_auto_1' in self.model.loss
|
|
self.assertEqual(
|
|
schema.Struct(('structName', schema.Scalar((np.float32, tuple())))),
|
|
self.model.loss.addLoss_auto_1
|
|
)
|
|
assert core.BlobReference('loss_blob_2') in self.model.loss.field_blobs()
|
|
|
|
loss_in_tuple_0 = schema.Scalar(
|
|
dtype=np.float32, blob=core.BlobReference('loss_blob_in_tuple_0')
|
|
)
|
|
|
|
loss_in_tuple_1 = schema.Scalar(
|
|
dtype=np.float32, blob=core.BlobReference('loss_blob_in_tuple_1')
|
|
)
|
|
|
|
loss_tuple = schema.NamedTuple(
|
|
'loss_in_tuple', * [loss_in_tuple_0, loss_in_tuple_1]
|
|
)
|
|
self.model.add_loss(loss_tuple, 'addLoss')
|
|
assert 'addLoss_auto_2' in self.model.loss
|
|
self.assertEqual(
|
|
schema.Struct(
|
|
('loss_in_tuple_0', schema.Scalar((np.float32, tuple()))),
|
|
('loss_in_tuple_1', schema.Scalar((np.float32, tuple())))
|
|
), self.model.loss.addLoss_auto_2
|
|
)
|
|
assert core.BlobReference('loss_blob_in_tuple_0')\
|
|
in self.model.loss.field_blobs()
|
|
assert core.BlobReference('loss_blob_in_tuple_1')\
|
|
in self.model.loss.field_blobs()
|
|
|
|
def testFilterMetricSchema(self):
|
|
self.model.add_metric_field("a:b", schema.Scalar())
|
|
self.model.add_metric_field("a:c", schema.Scalar())
|
|
self.model.add_metric_field("d", schema.Scalar())
|
|
|
|
self.assertEqual(
|
|
self.model.metrics_schema,
|
|
schema.Struct(
|
|
("a", schema.Struct(
|
|
("b", schema.Scalar()),
|
|
("c", schema.Scalar()),
|
|
)),
|
|
("d", schema.Scalar()),
|
|
))
|
|
|
|
self.model.filter_metrics_schema({"a:b", "d"})
|
|
self.assertEqual(
|
|
self.model.metrics_schema,
|
|
schema.Struct(
|
|
("a", schema.Struct(
|
|
("b", schema.Scalar()),
|
|
)),
|
|
("d", schema.Scalar()),
|
|
))
|
|
|
|
def testAddOutputSchema(self):
|
|
# add the first field
|
|
self.model.add_output_schema('struct', schema.Struct())
|
|
expected_output_schema = schema.Struct(('struct', schema.Struct()))
|
|
self.assertEqual(
|
|
self.model.output_schema,
|
|
expected_output_schema,
|
|
)
|
|
|
|
# add the second field
|
|
self.model.add_output_schema('scalar', schema.Scalar(np.float64))
|
|
expected_output_schema = schema.Struct(
|
|
('struct', schema.Struct()),
|
|
('scalar', schema.Scalar(np.float64)),
|
|
)
|
|
self.assertEqual(
|
|
self.model.output_schema,
|
|
expected_output_schema,
|
|
)
|
|
|
|
# overwrite a field should raise
|
|
with self.assertRaises(AssertionError):
|
|
self.model.add_output_schema('scalar', schema.Struct())
|
|
|
|
def _test_net(self, net, ops_list):
|
|
"""
|
|
Helper function to assert the net contains some set of operations and
|
|
then to run the net.
|
|
|
|
Inputs:
|
|
net -- the network to test and run
|
|
ops_list -- the list of operation specifications to check for
|
|
in the net
|
|
"""
|
|
ops_output = self.assertNetContainOps(net, ops_list)
|
|
workspace.RunNetOnce(net)
|
|
return ops_output
|
|
|
|
def testFCWithoutBias(self):
|
|
output_dims = 2
|
|
fc_without_bias = self.model.FCWithoutBias(
|
|
self.model.input_feature_schema.float_features, output_dims)
|
|
self.model.output_schema = fc_without_bias
|
|
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (output_dims, ))),
|
|
fc_without_bias
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
init_ops = self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("UniformFill", None, None),
|
|
]
|
|
)
|
|
|
|
mat_mul_spec = OpSpec(
|
|
"MatMul",
|
|
[
|
|
self.model.input_feature_schema.float_features(),
|
|
init_ops[0].output[0],
|
|
],
|
|
fc_without_bias.field_blobs()
|
|
)
|
|
|
|
self.assertNetContainOps(train_net, [mat_mul_spec])
|
|
|
|
predict_net = self.get_predict_net()
|
|
self.assertNetContainOps(predict_net, [mat_mul_spec])
|
|
|
|
def testFCwithAxis2(self):
|
|
input_dim = 10
|
|
output_dim = 30
|
|
max_length = 20
|
|
input_record = self.new_record(
|
|
schema.Struct(
|
|
('history_sequence', schema.Scalar((np.float32, (max_length,
|
|
input_dim)))),
|
|
)
|
|
)
|
|
fc_out = self.model.FC(
|
|
input_record.history_sequence, output_dim,
|
|
axis=2)
|
|
self.model.output_schema = fc_out
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (max_length, output_dim))),
|
|
fc_out
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
def testSparseLookupSumPoolingWithEviction(self):
|
|
# Create test embedding table of 1 row
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('sparse', schema.Struct(
|
|
('sparse_feature_0', schema.ListWithEvicted(
|
|
schema.Scalar(np.int64,
|
|
metadata=schema.Metadata(categorical_limit=1)),)),)),
|
|
))
|
|
embedding_dim = 8
|
|
lengths_blob = record.sparse.sparse_feature_0.lengths.get()
|
|
values_blob = record.sparse.sparse_feature_0.items.get()
|
|
evicted_values_blob = record.sparse.sparse_feature_0._evicted_values.get()
|
|
lengths = np.array([1]).astype(np.int32)
|
|
values = np.array([0]).astype(np.int64)
|
|
# Need to reset row 0
|
|
evicted_values = np.array([0]).astype(np.int64)
|
|
workspace.FeedBlob(lengths_blob, lengths)
|
|
workspace.FeedBlob(values_blob, values)
|
|
workspace.FeedBlob(evicted_values_blob, evicted_values)
|
|
|
|
embedding_after_pooling = self.model.SparseLookup(
|
|
record.sparse.sparse_feature_0, [embedding_dim], 'Sum', weight_init=("ConstantFill", {"value": 1.0}))
|
|
|
|
self.model.output_schema = schema.Struct()
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (embedding_dim, ))),
|
|
embedding_after_pooling
|
|
)
|
|
train_init_net, train_net = self.get_training_nets()
|
|
workspace.RunNetOnce(train_init_net)
|
|
embedding_after_init = workspace.FetchBlob("sparse_lookup/w")
|
|
# Change row 0's value before reset
|
|
new_values = np.array([[2, 2, 2, 2, 2, 2, 2, 2]]).astype(np.float32)
|
|
workspace.FeedBlob("sparse_lookup/w", new_values)
|
|
workspace.RunNetOnce(train_net.Proto())
|
|
embedding_after_training = workspace.FetchBlob("sparse_lookup/w")
|
|
# Verify row 0's value does not change after reset
|
|
self.assertEquals(embedding_after_training.all(), embedding_after_init.all())
|
|
|
|
|
|
|
|
def testSparseLookupSumPooling(self):
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('sparse', schema.Struct(
|
|
('sparse_feature_0', schema.List(
|
|
schema.Scalar(np.int64,
|
|
metadata=schema.Metadata(categorical_limit=1000)))),
|
|
)),
|
|
))
|
|
embedding_dim = 64
|
|
embedding_after_pooling = self.model.SparseLookup(
|
|
record.sparse.sparse_feature_0, [embedding_dim], 'Sum')
|
|
self.model.output_schema = schema.Struct()
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (embedding_dim, ))),
|
|
embedding_after_pooling
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
init_ops = self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("UniformFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
]
|
|
)
|
|
sparse_lookup_op_spec = OpSpec(
|
|
'SparseLengthsSum',
|
|
[
|
|
init_ops[0].output[0],
|
|
record.sparse.sparse_feature_0.items(),
|
|
record.sparse.sparse_feature_0.lengths(),
|
|
],
|
|
[embedding_after_pooling()]
|
|
)
|
|
self.assertNetContainOps(train_net, [sparse_lookup_op_spec])
|
|
|
|
predict_net = self.get_predict_net()
|
|
self.assertNetContainOps(predict_net, [sparse_lookup_op_spec])
|
|
|
|
@given(
|
|
use_hashing=st.booleans(),
|
|
modulo=st.integers(min_value=100, max_value=200),
|
|
use_divide_mod=st.booleans(),
|
|
divisor=st.integers(min_value=10, max_value=20),
|
|
)
|
|
def testSparseFeatureHashIdList(self, use_hashing, modulo, use_divide_mod, divisor):
|
|
record = schema.NewRecord(
|
|
self.model.net,
|
|
schema.List(schema.Scalar(
|
|
np.int64,
|
|
metadata=schema.Metadata(categorical_limit=60000)
|
|
))
|
|
)
|
|
use_divide_mod = use_divide_mod if use_hashing is False else False
|
|
output_schema = self.model.SparseFeatureHash(
|
|
record,
|
|
modulo=modulo,
|
|
use_hashing=use_hashing,
|
|
use_divide_mod=use_divide_mod,
|
|
divisor=divisor,
|
|
)
|
|
|
|
self.model.output_schema = output_schema
|
|
|
|
self.assertEqual(len(self.model.layers), 1)
|
|
self.assertEqual(output_schema._items.metadata.categorical_limit,
|
|
modulo)
|
|
train_init_net, train_net = self.get_training_nets()
|
|
if use_divide_mod:
|
|
self.assertEqual(len(train_net.Proto().op), 3)
|
|
else:
|
|
self.assertEqual(len(train_net.Proto().op), 2)
|
|
|
|
@given(
|
|
use_hashing=st.booleans(),
|
|
modulo=st.integers(min_value=100, max_value=200),
|
|
)
|
|
def testSparseFeatureHashIdScoreList(self, use_hashing, modulo):
|
|
record = schema.NewRecord(self.model.net,
|
|
schema.Map(schema.Scalar(np.int64,
|
|
metadata=schema.Metadata(
|
|
categorical_limit=60000)),
|
|
np.float32))
|
|
|
|
output_schema = self.model.SparseFeatureHash(
|
|
record,
|
|
modulo=modulo,
|
|
use_hashing=use_hashing)
|
|
|
|
self.model.output_schema = output_schema
|
|
|
|
self.assertEqual(len(self.model.layers), 1)
|
|
self.assertEqual(output_schema._items.keys.metadata.categorical_limit,
|
|
modulo)
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
def testSparseLookupIncorrectPositionWeightedOnIdList(self):
|
|
'''
|
|
Currently the implementation of SparseLookup assumed input is id_score_list
|
|
when use PositionWeighted.
|
|
'''
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('sparse', schema.Struct(
|
|
('sparse_feature_0', schema.List(
|
|
schema.Scalar(np.int64,
|
|
metadata=schema.Metadata(categorical_limit=1000)))),
|
|
)),
|
|
))
|
|
|
|
embedding_dim = 64
|
|
with self.assertRaises(AssertionError):
|
|
self.model.SparseLookup(
|
|
record.sparse.sparse_feature_0, [embedding_dim], 'PositionWeighted')
|
|
|
|
def testSparseLookupPositionWeightedOnIdList(self):
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('sparse', schema.Struct(
|
|
('sparse_feature_0', schema.List(
|
|
schema.Scalar(np.int64,
|
|
metadata=schema.Metadata(categorical_limit=1000)))),
|
|
)),
|
|
))
|
|
|
|
# convert id_list to id_score_list with PositionWeighted layer
|
|
sparse_segment = record.sparse.sparse_feature_0
|
|
pos_w_layer = self.model.PositionWeighted(sparse_segment)
|
|
|
|
sparse_segment = schema.Map(
|
|
keys=get_key(sparse_segment),
|
|
values=pos_w_layer.position_weights,
|
|
lengths_blob=sparse_segment.lengths
|
|
)
|
|
|
|
embedding_dim = 64
|
|
embedding_after_pooling = self.model.SparseLookup(
|
|
sparse_segment, [embedding_dim], 'PositionWeighted')
|
|
self.model.output_schema = schema.Struct()
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (embedding_dim, ))),
|
|
embedding_after_pooling
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("ConstantFill", None, None), # position_weights/pos_w
|
|
OpSpec("UniformFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
]
|
|
)
|
|
self.assertNetContainOps(train_net, [
|
|
OpSpec("LengthsRangeFill", None, None),
|
|
OpSpec("Gather", None, None),
|
|
OpSpec("SparseLengthsWeightedSum", None, None),
|
|
])
|
|
|
|
predict_net = self.get_predict_net()
|
|
self.assertNetContainOps(predict_net, [
|
|
OpSpec("LengthsRangeFill", None, None),
|
|
OpSpec("Gather", None, None),
|
|
OpSpec("SparseLengthsWeightedSum", None, None),
|
|
])
|
|
|
|
def testSparseLookupPositionWeightedOnIdScoreList(self):
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('sparse', schema.Struct(
|
|
('id_score_list_0', schema.Map(
|
|
schema.Scalar(
|
|
np.int64,
|
|
metadata=schema.Metadata(
|
|
categorical_limit=1000
|
|
),
|
|
),
|
|
np.float32
|
|
)),
|
|
)),
|
|
))
|
|
|
|
embedding_dim = 64
|
|
embedding_after_pooling = self.model.SparseLookup(
|
|
record.sparse.id_score_list_0, [embedding_dim], 'PositionWeighted')
|
|
self.model.output_schema = schema.Struct()
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (embedding_dim, ))),
|
|
embedding_after_pooling
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
init_ops = self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("UniformFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
]
|
|
)
|
|
sparse_lookup_op_spec = OpSpec(
|
|
'SparseLengthsWeightedSum',
|
|
[
|
|
init_ops[0].output[0],
|
|
record.sparse.id_score_list_0.values(),
|
|
record.sparse.id_score_list_0.keys(),
|
|
record.sparse.id_score_list_0.lengths(),
|
|
],
|
|
[embedding_after_pooling()]
|
|
)
|
|
self.assertNetContainOps(train_net, [sparse_lookup_op_spec])
|
|
|
|
predict_net = self.get_predict_net()
|
|
self.assertNetContainOps(predict_net, [sparse_lookup_op_spec])
|
|
|
|
def testSparseLookupIncorrectRecencyWeightedOnIdList(self):
|
|
'''
|
|
Currently the implementation of SparseLookup assumed input is id_score_list
|
|
when use RecencyWeighted.
|
|
'''
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('sparse', schema.Struct(
|
|
('sparse_feature_0', schema.List(
|
|
schema.Scalar(np.int64,
|
|
metadata=schema.Metadata(categorical_limit=1000)))),
|
|
)),
|
|
))
|
|
|
|
embedding_dim = 64
|
|
with self.assertRaises(AssertionError):
|
|
self.model.SparseLookup(
|
|
record.sparse.sparse_feature_0, [embedding_dim], 'RecencyWeighted')
|
|
|
|
def testSparseLookupRecencyWeightedOnIdScoreList(self):
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('sparse', schema.Struct(
|
|
('id_score_list_0', schema.Map(
|
|
schema.Scalar(
|
|
np.int64,
|
|
metadata=schema.Metadata(
|
|
categorical_limit=1000
|
|
),
|
|
),
|
|
np.float32
|
|
)),
|
|
)),
|
|
))
|
|
|
|
embedding_dim = 64
|
|
embedding_after_pooling = self.model.SparseLookup(
|
|
record.sparse.id_score_list_0, [embedding_dim], 'RecencyWeighted')
|
|
self.model.output_schema = schema.Struct()
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (embedding_dim, ))),
|
|
embedding_after_pooling
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
init_ops = self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("UniformFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
]
|
|
)
|
|
sparse_lookup_op_spec = OpSpec(
|
|
'SparseLengthsWeightedSum',
|
|
[
|
|
init_ops[0].output[0],
|
|
record.sparse.id_score_list_0.values(),
|
|
record.sparse.id_score_list_0.keys(),
|
|
record.sparse.id_score_list_0.lengths(),
|
|
],
|
|
[embedding_after_pooling()]
|
|
)
|
|
self.assertNetContainOps(train_net, [sparse_lookup_op_spec])
|
|
|
|
predict_net = self.get_predict_net()
|
|
self.assertNetContainOps(predict_net, [sparse_lookup_op_spec])
|
|
|
|
def testPairwiseSimilarityWithAllEmbeddings(self):
|
|
embedding_dim = 64
|
|
N = 5
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('all_embeddings', schema.Scalar(
|
|
((np.float32, (N, embedding_dim)))
|
|
)),
|
|
))
|
|
current = self.model.PairwiseSimilarity(
|
|
record, N * N)
|
|
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (N * N, ))),
|
|
current
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
self.assertNetContainOps(train_init_net, [])
|
|
self.assertNetContainOps(train_net, [
|
|
OpSpec("BatchMatMul", None, None),
|
|
OpSpec("Flatten", None, None),
|
|
])
|
|
|
|
def testPairwiseSimilarityWithXandYEmbeddings(self):
|
|
embedding_dim = 64
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('x_embeddings', schema.Scalar(
|
|
((np.float32, (5, embedding_dim)))
|
|
)),
|
|
('y_embeddings', schema.Scalar(
|
|
((np.float32, (6, embedding_dim)))
|
|
)),
|
|
))
|
|
current = self.model.PairwiseSimilarity(
|
|
record, 5 * 6)
|
|
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (5 * 6, ))),
|
|
current
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
self.assertNetContainOps(train_init_net, [])
|
|
self.assertNetContainOps(train_net, [
|
|
OpSpec("BatchMatMul", None, None),
|
|
OpSpec("Flatten", None, None),
|
|
])
|
|
|
|
def testPairwiseSimilarityWithXandYEmbeddingsAndGather(self):
|
|
embedding_dim = 64
|
|
|
|
output_idx = [1, 3, 5]
|
|
output_idx_blob = self.model.add_global_constant(
|
|
str(self.model.net.NextScopedBlob('pairwise_dot_product_gather')),
|
|
output_idx,
|
|
dtype=np.int32,
|
|
)
|
|
indices_to_gather = schema.Scalar(
|
|
(np.int32, len(output_idx)),
|
|
output_idx_blob,
|
|
)
|
|
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('x_embeddings', schema.Scalar(
|
|
((np.float32, (5, embedding_dim)))
|
|
)),
|
|
('y_embeddings', schema.Scalar(
|
|
((np.float32, (6, embedding_dim)))
|
|
)),
|
|
('indices_to_gather', indices_to_gather),
|
|
))
|
|
current = self.model.PairwiseSimilarity(
|
|
record, len(output_idx))
|
|
|
|
# This assert is not necessary,
|
|
# output size is passed into PairwiseSimilarity
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (len(output_idx), ))),
|
|
current
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
self.assertNetContainOps(train_init_net, [])
|
|
self.assertNetContainOps(train_net, [
|
|
OpSpec("BatchMatMul", None, None),
|
|
OpSpec("Flatten", None, None),
|
|
OpSpec("BatchGather", None, None),
|
|
])
|
|
|
|
def testPairwiseSimilarityIncorrectInput(self):
|
|
embedding_dim = 64
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('x_embeddings', schema.Scalar(
|
|
((np.float32, (5, embedding_dim)))
|
|
)),
|
|
))
|
|
with self.assertRaises(AssertionError):
|
|
self.model.PairwiseSimilarity(
|
|
record, 25)
|
|
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('all_embeddings', schema.List(np.float32))
|
|
))
|
|
with self.assertRaises(AssertionError):
|
|
self.model.PairwiseSimilarity(
|
|
record, 25)
|
|
|
|
def testConcat(self):
|
|
embedding_dim = 64
|
|
input_record = self.new_record(schema.Struct(
|
|
('input1', schema.Scalar((np.float32, (embedding_dim, )))),
|
|
('input2', schema.Scalar((np.float32, (embedding_dim, )))),
|
|
('input3', schema.Scalar((np.float32, (embedding_dim, )))),
|
|
))
|
|
|
|
output = self.model.Concat(input_record)
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, ((len(input_record.fields) * embedding_dim, )))),
|
|
output
|
|
)
|
|
|
|
# Note that in Concat layer we assume first dimension is batch.
|
|
# so input is B * embedding_dim
|
|
# add_axis=1 make it B * 1 * embedding_dim
|
|
# concat on axis=1 make it B * N * embedding_dim
|
|
output = self.model.Concat(input_record, axis=1, add_axis=1)
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, ((len(input_record.fields), embedding_dim)))),
|
|
output
|
|
)
|
|
|
|
def testSamplingTrain(self):
|
|
output_dims = 1000
|
|
|
|
indices = self.new_record(schema.Scalar((np.int32, (10,))))
|
|
sampling_prob = self.new_record(schema.Scalar((np.float32, (10, ))))
|
|
|
|
sampled_fc = self.model.SamplingTrain(
|
|
schema.Struct(
|
|
('input', self.model.input_feature_schema.float_features),
|
|
('indices', indices),
|
|
('sampling_prob', sampling_prob),
|
|
),
|
|
"FC",
|
|
output_dims,
|
|
)
|
|
self.model.output_schema = sampled_fc
|
|
|
|
# Check that we don't add prediction layer into the model
|
|
self.assertEqual(1, len(self.model.layers))
|
|
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (output_dims, ))),
|
|
sampled_fc
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
init_ops = self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("UniformFill", None, None),
|
|
OpSpec("UniformFill", None, None),
|
|
]
|
|
)
|
|
|
|
sampled_fc_layer = self.model.layers[0]
|
|
|
|
gather_w_spec = OpSpec(
|
|
"Gather",
|
|
[
|
|
init_ops[0].output[0],
|
|
indices(),
|
|
],
|
|
[
|
|
sampled_fc_layer._prediction_layer.train_param_blobs[0]
|
|
]
|
|
)
|
|
gather_b_spec = OpSpec(
|
|
"Gather",
|
|
[
|
|
init_ops[1].output[0],
|
|
indices(),
|
|
],
|
|
[
|
|
sampled_fc_layer._prediction_layer.train_param_blobs[1]
|
|
]
|
|
)
|
|
train_fc_spec = OpSpec(
|
|
"FC",
|
|
[
|
|
self.model.input_feature_schema.float_features(),
|
|
] + sampled_fc_layer._prediction_layer.train_param_blobs,
|
|
sampled_fc.field_blobs()
|
|
)
|
|
log_spec = OpSpec("Log", [sampling_prob()], [None])
|
|
sub_spec = OpSpec(
|
|
"Sub",
|
|
[sampled_fc.field_blobs()[0], None],
|
|
sampled_fc.field_blobs()
|
|
)
|
|
|
|
train_ops = self.assertNetContainOps(
|
|
train_net,
|
|
[gather_w_spec, gather_b_spec, train_fc_spec, log_spec, sub_spec])
|
|
|
|
self.assertEqual(train_ops[3].output[0], train_ops[4].input[1])
|
|
|
|
predict_net = self.get_predict_net()
|
|
self.assertNetContainOps(
|
|
predict_net,
|
|
[
|
|
OpSpec(
|
|
"FC",
|
|
[
|
|
self.model.input_feature_schema.float_features(),
|
|
init_ops[0].output[0],
|
|
init_ops[1].output[0],
|
|
],
|
|
sampled_fc.field_blobs()
|
|
)
|
|
]
|
|
)
|
|
|
|
def testBatchLRLoss(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('label', schema.Scalar((np.float64, (1,)))),
|
|
('logit', schema.Scalar((np.float32, (2,)))),
|
|
('weight', schema.Scalar((np.float64, (1,))))
|
|
))
|
|
loss = self.model.BatchLRLoss(input_record)
|
|
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
|
|
|
|
def testBatchLRLossWithUncertainty(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('label', schema.Scalar((np.float64, (1,)))),
|
|
('logit', schema.Scalar((np.float32, (2,)))),
|
|
('weight', schema.Scalar((np.float64, (1,)))),
|
|
('log_variance', schema.Scalar((np.float64, (1,)))),
|
|
))
|
|
loss = self.model.BatchLRLoss(input_record)
|
|
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
|
|
|
|
def testMarginRankLoss(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('pos_prediction', schema.Scalar((np.float32, (1,)))),
|
|
('neg_prediction', schema.List(np.float32)),
|
|
))
|
|
pos_items = np.array([0.1, 0.2, 0.3], dtype=np.float32)
|
|
neg_lengths = np.array([1, 2, 3], dtype=np.int32)
|
|
neg_items = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6], dtype=np.float32)
|
|
schema.FeedRecord(
|
|
input_record,
|
|
[pos_items, neg_lengths, neg_items]
|
|
)
|
|
loss = self.model.MarginRankLoss(input_record)
|
|
self.run_train_net_forward_only()
|
|
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
|
|
|
|
def testBatchMSELoss(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('label', schema.Scalar((np.float64, (1,)))),
|
|
('prediction', schema.Scalar((np.float32, (2,)))),
|
|
))
|
|
loss = self.model.BatchMSELoss(input_record)
|
|
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
|
|
|
|
def testBatchHuberLoss(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('label', schema.Scalar((np.float32, (1,)))),
|
|
('prediction', schema.Scalar((np.float32, (2,)))),
|
|
))
|
|
loss = self.model.BatchHuberLoss(input_record)
|
|
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
|
|
|
|
def testBatchSigmoidCrossEntropyLoss(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('label', schema.Scalar((np.float32, (32,)))),
|
|
('prediction', schema.Scalar((np.float32, (32,))))
|
|
))
|
|
loss = self.model.BatchSigmoidCrossEntropyLoss(input_record)
|
|
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
|
|
|
|
def testBatchSoftmaxLoss(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('label', schema.Scalar((np.float32, tuple()))),
|
|
('prediction', schema.Scalar((np.float32, (32,))))
|
|
))
|
|
loss = self.model.BatchSoftmaxLoss(input_record)
|
|
self.assertEqual(schema.Struct(
|
|
('softmax', schema.Scalar((np.float32, (32,)))),
|
|
('loss', schema.Scalar(np.float32)),
|
|
), loss)
|
|
|
|
def testBatchSoftmaxLossWeight(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('label', schema.Scalar((np.float32, tuple()))),
|
|
('prediction', schema.Scalar((np.float32, (32,)))),
|
|
('weight', schema.Scalar((np.float64, (1,))))
|
|
))
|
|
loss = self.model.BatchSoftmaxLoss(input_record)
|
|
self.assertEqual(schema.Struct(
|
|
('softmax', schema.Scalar((np.float32, (32,)))),
|
|
('loss', schema.Scalar(np.float32)),
|
|
), loss)
|
|
|
|
@given(
|
|
X=hu.arrays(dims=[2, 5]),
|
|
)
|
|
def testBatchNormalization(self, X):
|
|
input_record = self.new_record(schema.Scalar((np.float32, (5,))))
|
|
schema.FeedRecord(input_record, [X])
|
|
bn_output = self.model.BatchNormalization(input_record)
|
|
self.assertEqual(schema.Scalar((np.float32, (5,))), bn_output)
|
|
self.model.output_schema = schema.Struct()
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
init_ops = self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("ConstantFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
]
|
|
)
|
|
|
|
input_blob = input_record.field_blobs()[0]
|
|
output_blob = bn_output.field_blobs()[0]
|
|
|
|
expand_dims_spec = OpSpec(
|
|
"ExpandDims",
|
|
[input_blob],
|
|
None,
|
|
)
|
|
|
|
train_bn_spec = OpSpec(
|
|
"SpatialBN",
|
|
[None, init_ops[0].output[0], init_ops[1].output[0],
|
|
init_ops[2].output[0], init_ops[3].output[0]],
|
|
[output_blob, init_ops[2].output[0], init_ops[3].output[0], None, None],
|
|
{'is_test': 0, 'order': 'NCHW', 'momentum': 0.9},
|
|
)
|
|
|
|
test_bn_spec = OpSpec(
|
|
"SpatialBN",
|
|
[None, init_ops[0].output[0], init_ops[1].output[0],
|
|
init_ops[2].output[0], init_ops[3].output[0]],
|
|
[output_blob],
|
|
{'is_test': 1, 'order': 'NCHW', 'momentum': 0.9},
|
|
)
|
|
|
|
squeeze_spec = OpSpec(
|
|
"Squeeze",
|
|
[output_blob],
|
|
[output_blob],
|
|
)
|
|
|
|
self.assertNetContainOps(
|
|
train_net,
|
|
[expand_dims_spec, train_bn_spec, squeeze_spec]
|
|
)
|
|
|
|
eval_net = self.get_eval_net()
|
|
|
|
self.assertNetContainOps(
|
|
eval_net,
|
|
[expand_dims_spec, test_bn_spec, squeeze_spec]
|
|
)
|
|
|
|
predict_net = self.get_predict_net()
|
|
|
|
self.assertNetContainOps(
|
|
predict_net,
|
|
[expand_dims_spec, test_bn_spec, squeeze_spec]
|
|
)
|
|
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
|
|
schema.FeedRecord(input_record, [X])
|
|
workspace.RunNetOnce(eval_net)
|
|
|
|
schema.FeedRecord(input_record, [X])
|
|
workspace.RunNetOnce(predict_net)
|
|
|
|
@given(
|
|
X=hu.arrays(dims=[2, 5, 6]),
|
|
use_layer_norm_op=st.booleans(),
|
|
)
|
|
def testLayerNormalization(self, X, use_layer_norm_op):
|
|
expect = (5, 6,)
|
|
if not use_layer_norm_op:
|
|
X = X.reshape(10, 6)
|
|
expect = (6,)
|
|
input_record = self.new_record(schema.Scalar((np.float32, expect)))
|
|
schema.FeedRecord(input_record, [X])
|
|
ln_output = self.model.LayerNormalization(
|
|
input_record, use_layer_norm_op=use_layer_norm_op
|
|
)
|
|
self.assertEqual(schema.Scalar((np.float32, expect)), ln_output)
|
|
self.model.output_schema = schema.Struct()
|
|
|
|
train_init_net, train_net = self.get_training_nets(add_constants=True)
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
|
|
@given(
|
|
X=hu.arrays(dims=[5, 2]),
|
|
num_to_collect=st.integers(min_value=1, max_value=10),
|
|
)
|
|
def testLastNWindowCollector(self, X, num_to_collect):
|
|
input_record = self.new_record(schema.Scalar(np.float32))
|
|
schema.FeedRecord(input_record, [X])
|
|
last_n = self.model.LastNWindowCollector(input_record, num_to_collect)
|
|
self.run_train_net_forward_only()
|
|
output_record = schema.FetchRecord(last_n.last_n)
|
|
start = max(0, 5 - num_to_collect)
|
|
npt.assert_array_equal(X[start:], output_record())
|
|
num_visited = schema.FetchRecord(last_n.num_visited)
|
|
npt.assert_array_equal([5], num_visited())
|
|
|
|
@given(
|
|
X=hu.arrays(dims=[5, 2]),
|
|
num_to_collect=st.integers(min_value=3, max_value=3),
|
|
)
|
|
def testReservoirSamplingWithID(self, X, num_to_collect):
|
|
ID = np.array([1, 2, 3, 1, 2], dtype=np.int64)
|
|
input_record = self.new_record(
|
|
schema.Struct(
|
|
('record', schema.Struct(
|
|
('dense', schema.Scalar()),
|
|
)),
|
|
('object_id', schema.Scalar(np.int64)),
|
|
)
|
|
)
|
|
schema.FeedRecord(input_record, [X, ID])
|
|
packed_record = self.model.PackRecords(
|
|
input_record.record, 1, fields=input_record.record.field_names())
|
|
reservoir_input = schema.Struct(
|
|
('data', packed_record),
|
|
('object_id', input_record.object_id),
|
|
)
|
|
reservoir = self.model.ReservoirSampling(
|
|
reservoir_input, num_to_collect)
|
|
self.model.output_schema = schema.Struct()
|
|
train_init_net, train_net = \
|
|
layer_model_instantiator.generate_training_nets_forward_only(
|
|
self.model)
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.CreateNet(train_net)
|
|
workspace.RunNet(train_net.Proto().name, num_iter=2)
|
|
num_visited = schema.FetchRecord(reservoir.num_visited)
|
|
npt.assert_array_equal([3], num_visited())
|
|
for param in self.model.params:
|
|
serialized = workspace.SerializeBlob(str(param))
|
|
workspace.DeserializeBlob(str(param), serialized)
|
|
ID = np.array([3, 5, 3, 3, 5], dtype=np.int64)
|
|
schema.FeedRecord(input_record.object_id, [ID])
|
|
workspace.RunNet(train_net.Proto().name, num_iter=2)
|
|
num_visited = schema.FetchRecord(reservoir.num_visited)
|
|
npt.assert_array_equal([2], num_visited())
|
|
|
|
def testUniformSampling(self):
|
|
input_record = self.new_record(schema.Scalar(np.int32))
|
|
input_array = np.array([3, 10, 11, 15, 20, 99], dtype=np.int32)
|
|
schema.FeedRecord(input_record, [input_array])
|
|
num_samples = 20
|
|
num_elements = 100
|
|
uniform_sampling_output = self.model.UniformSampling(
|
|
input_record, num_samples, num_elements)
|
|
self.model.loss = uniform_sampling_output
|
|
self.run_train_net()
|
|
samples = workspace.FetchBlob(uniform_sampling_output.samples())
|
|
sampling_prob = workspace.FetchBlob(
|
|
uniform_sampling_output.sampling_prob())
|
|
self.assertEqual(num_samples, len(samples))
|
|
np.testing.assert_array_equal(input_array, samples[:len(input_array)])
|
|
np.testing.assert_almost_equal(
|
|
np.array([float(num_samples) / num_elements] * num_samples,
|
|
dtype=np.float32),
|
|
sampling_prob
|
|
)
|
|
|
|
def testUniformSamplingWithIncorrectSampleSize(self):
|
|
input_record = self.new_record(schema.Scalar(np.int32))
|
|
num_samples = 200
|
|
num_elements = 100
|
|
with self.assertRaises(AssertionError):
|
|
self.model.UniformSampling(input_record, num_samples, num_elements)
|
|
|
|
def testGatherRecord(self):
|
|
indices = np.array([1, 3, 4], dtype=np.int32)
|
|
dense = np.array(list(range(20)), dtype=np.float32).reshape(10, 2)
|
|
lengths = np.array(list(range(10)), dtype=np.int32)
|
|
items = np.array(list(range(lengths.sum())), dtype=np.int64)
|
|
items_lengths = np.array(list(range(lengths.sum())), dtype=np.int32)
|
|
items_items = np.array(list(range(items_lengths.sum())), dtype=np.int64)
|
|
record = self.new_record(schema.Struct(
|
|
('dense', schema.Scalar(np.float32)),
|
|
('sparse', schema.Struct(
|
|
('list', schema.List(np.int64)),
|
|
('list_of_list', schema.List(schema.List(np.int64))),
|
|
)),
|
|
('empty_struct', schema.Struct())
|
|
))
|
|
indices_record = self.new_record(schema.Scalar(np.int32))
|
|
input_record = schema.Struct(
|
|
('indices', indices_record),
|
|
('record', record),
|
|
)
|
|
schema.FeedRecord(
|
|
input_record,
|
|
[indices, dense, lengths, items, lengths, items_lengths,
|
|
items_items])
|
|
gathered_record = self.model.GatherRecord(input_record)
|
|
self.assertTrue(schema.equal_schemas(gathered_record, record))
|
|
|
|
self.run_train_net_forward_only()
|
|
gathered_dense = workspace.FetchBlob(gathered_record.dense())
|
|
np.testing.assert_array_equal(
|
|
np.concatenate([dense[i:i + 1] for i in indices]), gathered_dense)
|
|
gathered_lengths = workspace.FetchBlob(
|
|
gathered_record.sparse.list.lengths())
|
|
np.testing.assert_array_equal(
|
|
np.concatenate([lengths[i:i + 1] for i in indices]),
|
|
gathered_lengths)
|
|
gathered_items = workspace.FetchBlob(
|
|
gathered_record.sparse.list.items())
|
|
offsets = lengths.cumsum() - lengths
|
|
np.testing.assert_array_equal(
|
|
np.concatenate([
|
|
items[offsets[i]: offsets[i] + lengths[i]]
|
|
for i in indices
|
|
]), gathered_items)
|
|
|
|
gathered_items_lengths = workspace.FetchBlob(
|
|
gathered_record.sparse.list_of_list.items.lengths())
|
|
np.testing.assert_array_equal(
|
|
np.concatenate([
|
|
items_lengths[offsets[i]: offsets[i] + lengths[i]]
|
|
for i in indices
|
|
]),
|
|
gathered_items_lengths
|
|
)
|
|
|
|
nested_offsets = []
|
|
nested_lengths = []
|
|
nested_offset = 0
|
|
j = 0
|
|
for l in lengths:
|
|
nested_offsets.append(nested_offset)
|
|
nested_length = 0
|
|
for _i in range(l):
|
|
nested_offset += items_lengths[j]
|
|
nested_length += items_lengths[j]
|
|
j += 1
|
|
nested_lengths.append(nested_length)
|
|
|
|
gathered_items_items = workspace.FetchBlob(
|
|
gathered_record.sparse.list_of_list.items.items())
|
|
np.testing.assert_array_equal(
|
|
np.concatenate([
|
|
items_items[nested_offsets[i]:
|
|
nested_offsets[i] + nested_lengths[i]]
|
|
for i in indices
|
|
]),
|
|
gathered_items_items
|
|
)
|
|
|
|
def testMapToRange(self):
|
|
input_record = self.new_record(schema.Scalar(np.int32))
|
|
indices_blob = self.model.MapToRange(input_record,
|
|
max_index=100).indices
|
|
self.model.output_schema = schema.Struct()
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
schema.FeedRecord(
|
|
input_record,
|
|
[np.array([10, 3, 20, 99, 15, 11, 3, 11], dtype=np.int32)]
|
|
)
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
indices = workspace.FetchBlob(indices_blob())
|
|
np.testing.assert_array_equal(
|
|
np.array([1, 2, 3, 4, 5, 6, 2, 6], dtype=np.int32),
|
|
indices
|
|
)
|
|
|
|
schema.FeedRecord(
|
|
input_record,
|
|
[np.array([10, 3, 23, 35, 60, 15, 10, 15], dtype=np.int32)]
|
|
)
|
|
workspace.RunNetOnce(train_net)
|
|
indices = workspace.FetchBlob(indices_blob())
|
|
np.testing.assert_array_equal(
|
|
np.array([1, 2, 7, 8, 9, 5, 1, 5], dtype=np.int32),
|
|
indices
|
|
)
|
|
|
|
eval_net = self.get_eval_net()
|
|
|
|
schema.FeedRecord(
|
|
input_record,
|
|
[np.array([10, 3, 23, 35, 60, 15, 200], dtype=np.int32)]
|
|
)
|
|
workspace.RunNetOnce(eval_net)
|
|
indices = workspace.FetchBlob(indices_blob())
|
|
np.testing.assert_array_equal(
|
|
np.array([1, 2, 7, 8, 9, 5, 0], dtype=np.int32),
|
|
indices
|
|
)
|
|
|
|
schema.FeedRecord(
|
|
input_record,
|
|
[np.array([10, 3, 23, 15, 101, 115], dtype=np.int32)]
|
|
)
|
|
workspace.RunNetOnce(eval_net)
|
|
indices = workspace.FetchBlob(indices_blob())
|
|
np.testing.assert_array_equal(
|
|
np.array([1, 2, 7, 5, 0, 0], dtype=np.int32),
|
|
indices
|
|
)
|
|
|
|
predict_net = self.get_predict_net()
|
|
|
|
schema.FeedRecord(
|
|
input_record,
|
|
[np.array([3, 3, 20, 23, 151, 35, 60, 15, 200], dtype=np.int32)]
|
|
)
|
|
workspace.RunNetOnce(predict_net)
|
|
indices = workspace.FetchBlob(indices_blob())
|
|
np.testing.assert_array_equal(
|
|
np.array([2, 2, 3, 7, 0, 8, 9, 5, 0], dtype=np.int32),
|
|
indices
|
|
)
|
|
|
|
def testSelectRecordByContext(self):
|
|
float_features = self.model.input_feature_schema.float_features
|
|
|
|
float_array = np.array([1.0, 2.0], dtype=np.float32)
|
|
|
|
schema.FeedRecord(float_features, [float_array])
|
|
|
|
with Tags(Tags.EXCLUDE_FROM_PREDICTION):
|
|
log_float_features = self.model.Log(float_features, 1)
|
|
joined = self.model.SelectRecordByContext(
|
|
schema.Struct(
|
|
(InstantiationContext.PREDICTION, float_features),
|
|
(InstantiationContext.TRAINING, log_float_features),
|
|
# TODO: TRAIN_ONLY layers are also generated in eval
|
|
(InstantiationContext.EVAL, log_float_features),
|
|
)
|
|
)
|
|
|
|
# model.output_schema has to a struct
|
|
self.model.output_schema = schema.Struct((
|
|
'joined', joined
|
|
))
|
|
predict_net = layer_model_instantiator.generate_predict_net(self.model)
|
|
workspace.RunNetOnce(predict_net)
|
|
predict_output = schema.FetchRecord(predict_net.output_record())
|
|
npt.assert_array_equal(float_array,
|
|
predict_output['joined']())
|
|
eval_net = layer_model_instantiator.generate_eval_net(self.model)
|
|
workspace.RunNetOnce(eval_net)
|
|
eval_output = schema.FetchRecord(eval_net.output_record())
|
|
npt.assert_array_equal(np.log(float_array),
|
|
eval_output['joined']())
|
|
_, train_net = (
|
|
layer_model_instantiator.generate_training_nets_forward_only(
|
|
self.model
|
|
)
|
|
)
|
|
workspace.RunNetOnce(train_net)
|
|
train_output = schema.FetchRecord(train_net.output_record())
|
|
npt.assert_array_equal(np.log(float_array),
|
|
train_output['joined']())
|
|
|
|
def testFunctionalLayer(self):
|
|
def normalize(net, in_record, out_record):
|
|
mean = net.ReduceFrontMean(in_record(), 1)
|
|
net.Sub(
|
|
[in_record(), mean],
|
|
out_record(),
|
|
broadcast=1)
|
|
normalized = self.model.Functional(
|
|
self.model.input_feature_schema.float_features, 1,
|
|
normalize, name="normalizer")
|
|
|
|
# Attach metadata to one of the outputs and use it in FC
|
|
normalized.set_type((np.float32, 32))
|
|
self.model.output_schema = self.model.FC(normalized, 2)
|
|
|
|
predict_net = layer_model_instantiator.generate_predict_net(
|
|
self.model)
|
|
ops = predict_net.Proto().op
|
|
assert len(ops) == 3
|
|
assert ops[0].type == "ReduceFrontMean"
|
|
assert ops[1].type == "Sub"
|
|
assert ops[2].type == "FC"
|
|
assert len(ops[0].input) == 1
|
|
assert ops[0].input[0] ==\
|
|
self.model.input_feature_schema.float_features()
|
|
assert len(ops[1].output) == 1
|
|
assert ops[1].output[0] in ops[2].input
|
|
|
|
def testFunctionalLayerHelper(self):
|
|
mean = self.model.ReduceFrontMean(
|
|
self.model.input_feature_schema.float_features, 1)
|
|
normalized = self.model.Sub(
|
|
schema.Tuple(
|
|
self.model.input_feature_schema.float_features, mean),
|
|
1, broadcast=1)
|
|
# Attach metadata to one of the outputs and use it in FC
|
|
normalized.set_type((np.float32, (32,)))
|
|
self.model.output_schema = self.model.FC(normalized, 2)
|
|
|
|
predict_net = layer_model_instantiator.generate_predict_net(
|
|
self.model)
|
|
ops = predict_net.Proto().op
|
|
assert len(ops) == 3
|
|
assert ops[0].type == "ReduceFrontMean"
|
|
assert ops[1].type == "Sub"
|
|
assert ops[2].type == "FC"
|
|
assert len(ops[0].input) == 1
|
|
assert ops[0].input[0] ==\
|
|
self.model.input_feature_schema.float_features()
|
|
assert len(ops[1].output) == 1
|
|
assert ops[1].output[0] in ops[2].input
|
|
|
|
def testFunctionalLayerHelperAutoInference(self):
|
|
softsign = self.model.Softsign(
|
|
schema.Tuple(self.model.input_feature_schema.float_features),
|
|
1)
|
|
assert softsign.field_type().base == np.float32
|
|
assert softsign.field_type().shape == (32,)
|
|
self.model.output_schema = self.model.FC(softsign, 2)
|
|
|
|
predict_net = layer_model_instantiator.generate_predict_net(
|
|
self.model)
|
|
ops = predict_net.Proto().op
|
|
assert len(ops) == 2
|
|
assert ops[0].type == "Softsign"
|
|
assert ops[1].type == "FC"
|
|
assert len(ops[0].input) == 1
|
|
assert ops[0].input[0] ==\
|
|
self.model.input_feature_schema.float_features()
|
|
assert len(ops[0].output) == 1
|
|
assert ops[0].output[0] in ops[1].input
|
|
|
|
def testHalfToFloatTypeInference(self):
|
|
input = self.new_record(schema.Scalar((np.float32, (32,))))
|
|
|
|
output = self.model.FloatToHalf(input, 1)
|
|
assert output.field_type().base == np.float16
|
|
assert output.field_type().shape == (32, )
|
|
|
|
output = self.model.HalfToFloat(output, 1)
|
|
assert output.field_type().base == np.float32
|
|
assert output.field_type().shape == (32, )
|
|
|
|
def testFunctionalLayerHelperAutoInferenceScalar(self):
|
|
loss = self.model.AveragedLoss(self.model.input_feature_schema, 1)
|
|
self.assertEqual(1, len(loss.field_types()))
|
|
self.assertEqual(np.float32, loss.field_types()[0].base)
|
|
self.assertEqual(tuple(), loss.field_types()[0].shape)
|
|
|
|
def testFunctionalLayerInputCoercion(self):
|
|
one = self.model.global_constants['ONE']
|
|
two = self.model.Add([one, one], 1)
|
|
self.model.loss = two
|
|
self.run_train_net()
|
|
data = workspace.FetchBlob(two.field_blobs()[0])
|
|
np.testing.assert_array_equal([2.0], data)
|
|
|
|
def testFunctionalLayerWithOutputNames(self):
|
|
k = 3
|
|
topk = self.model.TopK(
|
|
self.model.input_feature_schema,
|
|
output_names_or_num=['values', 'indices'],
|
|
k=k,
|
|
)
|
|
self.assertEqual(2, len(topk.field_types()))
|
|
self.assertEqual(np.float32, topk.field_types()[0].base)
|
|
self.assertEqual((k,), topk.field_types()[0].shape)
|
|
self.assertEqual(np.int32, topk.field_types()[1].base)
|
|
self.assertEqual((k,), topk.field_types()[1].shape)
|
|
self.assertEqual(['TopK/values', 'TopK/indices'], topk.field_blobs())
|
|
|
|
def testFunctionalLayerSameOperatorOutputNames(self):
|
|
Con1 = self.model.ConstantFill([], 1, value=1)
|
|
Con2 = self.model.ConstantFill([], 1, value=2)
|
|
self.assertNotEqual(str(Con1), str(Con2))
|
|
|
|
def testFunctionalLayerWithOutputDtypes(self):
|
|
loss = self.model.AveragedLoss(
|
|
self.model.input_feature_schema,
|
|
1,
|
|
output_dtypes=(np.float32, (1,)),
|
|
)
|
|
self.assertEqual(1, len(loss.field_types()))
|
|
self.assertEqual(np.float32, loss.field_types()[0].base)
|
|
self.assertEqual((1,), loss.field_types()[0].shape)
|
|
|
|
def testPropagateRequestOnly(self):
|
|
# test case when output is request only
|
|
input_record = self.new_record(schema.Struct(
|
|
('input1', schema.Scalar((np.float32, (32, )))),
|
|
('input2', schema.Scalar((np.float32, (64, )))),
|
|
('input3', schema.Scalar((np.float32, (16, )))),
|
|
))
|
|
|
|
set_request_only(input_record)
|
|
concat_output = self.model.Concat(input_record)
|
|
self.assertEqual(is_request_only_scalar(concat_output), True)
|
|
|
|
# test case when output is not request only
|
|
input_record2 = self.new_record(schema.Struct(
|
|
('input4', schema.Scalar((np.float32, (100, ))))
|
|
)) + input_record
|
|
|
|
concat_output2 = self.model.Concat(input_record2)
|
|
self.assertEqual(is_request_only_scalar(concat_output2), False)
|
|
|
|
def testSetRequestOnly(self):
|
|
input_record = schema.Scalar(np.int64)
|
|
schema.attach_metadata_to_scalars(
|
|
input_record,
|
|
schema.Metadata(
|
|
categorical_limit=100000000,
|
|
expected_value=99,
|
|
feature_specs=schema.FeatureSpec(
|
|
feature_ids=[1, 100, 1001]
|
|
)
|
|
)
|
|
)
|
|
|
|
set_request_only(input_record)
|
|
self.assertEqual(input_record.metadata.categorical_limit, 100000000)
|
|
self.assertEqual(input_record.metadata.expected_value, 99)
|
|
self.assertEqual(
|
|
input_record.metadata.feature_specs.feature_ids,
|
|
[1, 100, 1001]
|
|
)
|
|
|
|
@given(
|
|
X=hu.arrays(dims=[5, 5]), # Shape of X is irrelevant
|
|
dropout_for_eval=st.booleans(),
|
|
)
|
|
def testDropout(self, X, dropout_for_eval):
|
|
input_record = self.new_record(schema.Scalar((np.float32, (1,))))
|
|
schema.FeedRecord(input_record, [X])
|
|
d_output = self.model.Dropout(
|
|
input_record,
|
|
dropout_for_eval=dropout_for_eval
|
|
)
|
|
self.assertEqual(schema.Scalar((np.float32, (1,))), d_output)
|
|
self.model.output_schema = schema.Struct()
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
input_blob = input_record.field_blobs()[0]
|
|
output_blob = d_output.field_blobs()[0]
|
|
|
|
with_d_spec = OpSpec(
|
|
"Dropout",
|
|
[input_blob],
|
|
[output_blob, None],
|
|
{'is_test': 0, 'ratio': 0.5}
|
|
)
|
|
|
|
without_d_spec = OpSpec(
|
|
"Dropout",
|
|
[input_blob],
|
|
[output_blob, None],
|
|
{'is_test': 1, 'ratio': 0.5}
|
|
)
|
|
|
|
self.assertNetContainOps(
|
|
train_net,
|
|
[with_d_spec]
|
|
)
|
|
|
|
eval_net = self.get_eval_net()
|
|
predict_net = self.get_predict_net()
|
|
|
|
if dropout_for_eval:
|
|
self.assertNetContainOps(
|
|
eval_net,
|
|
[with_d_spec]
|
|
)
|
|
self.assertNetContainOps(
|
|
predict_net,
|
|
[with_d_spec]
|
|
)
|
|
else:
|
|
self.assertNetContainOps(
|
|
eval_net,
|
|
[without_d_spec]
|
|
)
|
|
self.assertNetContainOps(
|
|
predict_net,
|
|
[without_d_spec]
|
|
)
|
|
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
|
|
schema.FeedRecord(input_record, [X])
|
|
workspace.RunNetOnce(eval_net)
|
|
|
|
schema.FeedRecord(input_record, [X])
|
|
workspace.RunNetOnce(predict_net)
|
|
|
|
@given(
|
|
num_inputs=st.integers(1, 3),
|
|
batch_size=st.integers(5, 10)
|
|
)
|
|
def testMergeIdListsLayer(self, num_inputs, batch_size):
|
|
inputs = []
|
|
for _ in range(num_inputs):
|
|
lengths = np.random.randint(5, size=batch_size).astype(np.int32)
|
|
size = lengths.sum()
|
|
values = np.random.randint(1, 10, size=size).astype(np.int64)
|
|
inputs.append(lengths)
|
|
inputs.append(values)
|
|
input_schema = schema.Tuple(
|
|
*[schema.List(
|
|
schema.Scalar(dtype=np.int64, metadata=schema.Metadata(
|
|
categorical_limit=20
|
|
))) for _ in range(num_inputs)]
|
|
)
|
|
|
|
input_record = schema.NewRecord(self.model.net, input_schema)
|
|
schema.FeedRecord(input_record, inputs)
|
|
output_schema = self.model.MergeIdLists(input_record)
|
|
assert schema.equal_schemas(
|
|
output_schema, IdList,
|
|
check_field_names=False)
|
|
|
|
@given(
|
|
batch_size=st.integers(min_value=2, max_value=10),
|
|
input_dims=st.integers(min_value=5, max_value=10),
|
|
output_dims=st.integers(min_value=5, max_value=10),
|
|
bandwidth=st.floats(min_value=0.1, max_value=5),
|
|
)
|
|
def testRandomFourierFeatures(self, batch_size, input_dims, output_dims, bandwidth):
|
|
|
|
def _rff_hypothesis_test(rff_output, X, W, b, scale):
|
|
"""
|
|
Runs hypothesis test for Semi Random Features layer.
|
|
|
|
Inputs:
|
|
rff_output -- output of net after running random fourier features layer
|
|
X -- input data
|
|
W -- weight parameter from train_init_net
|
|
b -- bias parameter from train_init_net
|
|
scale -- value by which to scale the output vector
|
|
"""
|
|
output = workspace.FetchBlob(rff_output)
|
|
output_ref = scale * np.cos(np.dot(X, np.transpose(W)) + b)
|
|
npt.assert_allclose(output, output_ref, rtol=1e-3, atol=1e-3)
|
|
|
|
X = np.random.random((batch_size, input_dims)).astype(np.float32)
|
|
scale = np.sqrt(2.0 / output_dims)
|
|
input_record = self.new_record(schema.Scalar((np.float32, (input_dims,))))
|
|
schema.FeedRecord(input_record, [X])
|
|
input_blob = input_record.field_blobs()[0]
|
|
rff_output = self.model.RandomFourierFeatures(input_record,
|
|
output_dims,
|
|
bandwidth)
|
|
self.model.output_schema = schema.Struct()
|
|
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (output_dims, ))),
|
|
rff_output
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
# Init net assertions
|
|
init_ops_list = [
|
|
OpSpec("GaussianFill", None, None),
|
|
OpSpec("UniformFill", None, None),
|
|
]
|
|
init_ops = self._test_net(train_init_net, init_ops_list)
|
|
W = workspace.FetchBlob(self.model.layers[0].w)
|
|
b = workspace.FetchBlob(self.model.layers[0].b)
|
|
|
|
# Operation specifications
|
|
fc_spec = OpSpec("FC", [input_blob, init_ops[0].output[0],
|
|
init_ops[1].output[0]], None)
|
|
cosine_spec = OpSpec("Cos", None, None)
|
|
scale_spec = OpSpec("Scale", None, rff_output.field_blobs(),
|
|
{'scale': scale})
|
|
ops_list = [
|
|
fc_spec,
|
|
cosine_spec,
|
|
scale_spec
|
|
]
|
|
|
|
# Train net assertions
|
|
self._test_net(train_net, ops_list)
|
|
_rff_hypothesis_test(rff_output(), X, W, b, scale)
|
|
|
|
# Eval net assertions
|
|
eval_net = self.get_eval_net()
|
|
self._test_net(eval_net, ops_list)
|
|
_rff_hypothesis_test(rff_output(), X, W, b, scale)
|
|
|
|
# Predict net assertions
|
|
predict_net = self.get_predict_net()
|
|
self._test_net(predict_net, ops_list)
|
|
_rff_hypothesis_test(rff_output(), X, W, b, scale)
|
|
|
|
@given(
|
|
batch_size=st.integers(min_value=2, max_value=10),
|
|
input_dims=st.integers(min_value=5, max_value=10),
|
|
output_dims=st.integers(min_value=5, max_value=10),
|
|
s=st.integers(min_value=0, max_value=3),
|
|
scale=st.floats(min_value=0.1, max_value=5),
|
|
set_weight_as_global_constant=st.booleans()
|
|
)
|
|
def testArcCosineFeatureMap(self, batch_size, input_dims, output_dims, s, scale,
|
|
set_weight_as_global_constant):
|
|
|
|
def _arc_cosine_hypothesis_test(ac_output, X, W, b, s):
|
|
"""
|
|
Runs hypothesis test for Arc Cosine layer.
|
|
|
|
Inputs:
|
|
ac_output -- output of net after running arc cosine layer
|
|
X -- input data
|
|
W -- weight parameter from train_init_net
|
|
b -- bias parameter from train_init_net
|
|
s -- degree parameter
|
|
"""
|
|
# Get output from net
|
|
net_output = workspace.FetchBlob(ac_output)
|
|
|
|
# Computing output directly
|
|
x_rand = np.matmul(X, np.transpose(W)) + b
|
|
x_pow = np.power(x_rand, s)
|
|
if s > 0:
|
|
h_rand_features = np.piecewise(x_rand,
|
|
[x_rand <= 0, x_rand > 0],
|
|
[0, 1])
|
|
else:
|
|
h_rand_features = np.piecewise(x_rand,
|
|
[x_rand <= 0, x_rand > 0],
|
|
[0, lambda x: x / (1 + x)])
|
|
output_ref = np.multiply(x_pow, h_rand_features)
|
|
|
|
# Comparing net output and computed output
|
|
npt.assert_allclose(net_output, output_ref, rtol=1e-3, atol=1e-3)
|
|
|
|
X = np.random.normal(size=(batch_size, input_dims)).astype(np.float32)
|
|
input_record = self.new_record(schema.Scalar((np.float32, (input_dims,))))
|
|
schema.FeedRecord(input_record, [X])
|
|
input_blob = input_record.field_blobs()[0]
|
|
|
|
ac_output = self.model.ArcCosineFeatureMap(
|
|
input_record,
|
|
output_dims,
|
|
s=s,
|
|
scale=scale,
|
|
set_weight_as_global_constant=set_weight_as_global_constant
|
|
)
|
|
self.model.output_schema = schema.Struct()
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (output_dims, ))),
|
|
ac_output
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
# Run create_init_net to initialize the global constants, and W and b
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(self.model.create_init_net(name='init_net'))
|
|
|
|
if set_weight_as_global_constant:
|
|
W = workspace.FetchBlob(
|
|
self.model.global_constants['arc_cosine_feature_map_fixed_rand_W']
|
|
)
|
|
b = workspace.FetchBlob(
|
|
self.model.global_constants['arc_cosine_feature_map_fixed_rand_b']
|
|
)
|
|
else:
|
|
W = workspace.FetchBlob(self.model.layers[0].random_w)
|
|
b = workspace.FetchBlob(self.model.layers[0].random_b)
|
|
|
|
# Operation specifications
|
|
fc_spec = OpSpec("FC", [input_blob, None, None], None)
|
|
softsign_spec = OpSpec("Softsign", None, None)
|
|
relu_spec = OpSpec("Relu", None, None)
|
|
relu_spec_output = OpSpec("Relu", None, ac_output.field_blobs())
|
|
pow_spec = OpSpec("Pow", None, None, {'exponent': float(s - 1)})
|
|
mul_spec = OpSpec("Mul", None, ac_output.field_blobs())
|
|
|
|
if s == 0:
|
|
ops_list = [
|
|
fc_spec,
|
|
softsign_spec,
|
|
relu_spec_output,
|
|
]
|
|
elif s == 1:
|
|
ops_list = [
|
|
fc_spec,
|
|
relu_spec_output,
|
|
]
|
|
else:
|
|
ops_list = [
|
|
fc_spec,
|
|
relu_spec,
|
|
pow_spec,
|
|
mul_spec,
|
|
]
|
|
|
|
# Train net assertions
|
|
self._test_net(train_net, ops_list)
|
|
_arc_cosine_hypothesis_test(ac_output(), X, W, b, s)
|
|
|
|
# Eval net assertions
|
|
eval_net = self.get_eval_net()
|
|
self._test_net(eval_net, ops_list)
|
|
_arc_cosine_hypothesis_test(ac_output(), X, W, b, s)
|
|
|
|
# Predict net assertions
|
|
predict_net = self.get_predict_net()
|
|
self._test_net(predict_net, ops_list)
|
|
_arc_cosine_hypothesis_test(ac_output(), X, W, b, s)
|
|
|
|
@given(
|
|
batch_size=st.integers(min_value=2, max_value=10),
|
|
input_dims=st.integers(min_value=5, max_value=10),
|
|
output_dims=st.integers(min_value=5, max_value=10),
|
|
s=st.integers(min_value=0, max_value=3),
|
|
scale=st.floats(min_value=0.1, max_value=5),
|
|
set_weight_as_global_constant=st.booleans(),
|
|
use_struct_input=st.booleans(),
|
|
)
|
|
def testSemiRandomFeatures(self, batch_size, input_dims, output_dims, s, scale,
|
|
set_weight_as_global_constant, use_struct_input):
|
|
|
|
def _semi_random_hypothesis_test(srf_output, X_full, X_random, rand_w,
|
|
rand_b, s):
|
|
"""
|
|
Runs hypothesis test for Semi Random Features layer.
|
|
|
|
Inputs:
|
|
srf_output -- output of net after running semi random features layer
|
|
X_full -- full input data
|
|
X_random -- random-output input data
|
|
rand_w -- random-initialized weight parameter from train_init_net
|
|
rand_b -- random-initialized bias parameter from train_init_net
|
|
s -- degree parameter
|
|
|
|
"""
|
|
# Get output from net
|
|
net_output = workspace.FetchBlob(srf_output)
|
|
|
|
# Fetch learned parameter blobs
|
|
learned_w = workspace.FetchBlob(self.model.layers[0].learned_w)
|
|
learned_b = workspace.FetchBlob(self.model.layers[0].learned_b)
|
|
|
|
# Computing output directly
|
|
x_rand = np.matmul(X_random, np.transpose(rand_w)) + rand_b
|
|
x_learn = np.matmul(X_full, np.transpose(learned_w)) + learned_b
|
|
x_pow = np.power(x_rand, s)
|
|
if s > 0:
|
|
h_rand_features = np.piecewise(x_rand,
|
|
[x_rand <= 0, x_rand > 0],
|
|
[0, 1])
|
|
else:
|
|
h_rand_features = np.piecewise(x_rand,
|
|
[x_rand <= 0, x_rand > 0],
|
|
[0, lambda x: x / (1 + x)])
|
|
output_ref = np.multiply(np.multiply(x_pow, h_rand_features), x_learn)
|
|
|
|
# Comparing net output and computed output
|
|
npt.assert_allclose(net_output, output_ref, rtol=1e-3, atol=1e-3)
|
|
|
|
X_full = np.random.normal(size=(batch_size, input_dims)).astype(np.float32)
|
|
if use_struct_input:
|
|
X_random = np.random.normal(size=(batch_size, input_dims)).\
|
|
astype(np.float32)
|
|
input_data = [X_full, X_random]
|
|
input_record = self.new_record(schema.Struct(
|
|
('full', schema.Scalar(
|
|
(np.float32, (input_dims,))
|
|
)),
|
|
('random', schema.Scalar(
|
|
(np.float32, (input_dims,))
|
|
))
|
|
))
|
|
else:
|
|
X_random = X_full
|
|
input_data = [X_full]
|
|
input_record = self.new_record(schema.Scalar(
|
|
(np.float32, (input_dims,))
|
|
))
|
|
|
|
schema.FeedRecord(input_record, input_data)
|
|
srf_output = self.model.SemiRandomFeatures(
|
|
input_record,
|
|
output_dims,
|
|
s=s,
|
|
scale_random=scale,
|
|
scale_learned=scale,
|
|
set_weight_as_global_constant=set_weight_as_global_constant
|
|
)
|
|
|
|
self.model.output_schema = schema.Struct()
|
|
|
|
self.assertEqual(
|
|
schema.Struct(
|
|
('full', schema.Scalar(
|
|
(np.float32, (output_dims,))
|
|
)),
|
|
('random', schema.Scalar(
|
|
(np.float32, (output_dims,))
|
|
))
|
|
),
|
|
srf_output
|
|
)
|
|
|
|
init_ops_list = [
|
|
OpSpec("GaussianFill", None, None),
|
|
OpSpec("UniformFill", None, None),
|
|
OpSpec("GaussianFill", None, None),
|
|
OpSpec("UniformFill", None, None),
|
|
]
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
# Need to run to initialize the global constants for layer
|
|
workspace.RunNetOnce(self.model.create_init_net(name='init_net'))
|
|
|
|
if set_weight_as_global_constant:
|
|
# If weight params are global constants, they won't be in train_init_net
|
|
init_ops = self._test_net(train_init_net, init_ops_list[:2])
|
|
rand_w = workspace.FetchBlob(
|
|
self.model.global_constants['semi_random_features_fixed_rand_W']
|
|
)
|
|
rand_b = workspace.FetchBlob(
|
|
self.model.global_constants['semi_random_features_fixed_rand_b']
|
|
)
|
|
|
|
# Operation specifications
|
|
fc_random_spec = OpSpec("FC", [None, None, None], None)
|
|
fc_learned_spec = OpSpec("FC", [None, init_ops[0].output[0],
|
|
init_ops[1].output[0]], None)
|
|
else:
|
|
init_ops = self._test_net(train_init_net, init_ops_list)
|
|
rand_w = workspace.FetchBlob(self.model.layers[0].random_w)
|
|
rand_b = workspace.FetchBlob(self.model.layers[0].random_b)
|
|
|
|
# Operation specifications
|
|
fc_random_spec = OpSpec("FC", [None, init_ops[0].output[0],
|
|
init_ops[1].output[0]], None)
|
|
fc_learned_spec = OpSpec("FC", [None, init_ops[2].output[0],
|
|
init_ops[3].output[0]], None)
|
|
|
|
softsign_spec = OpSpec("Softsign", None, None)
|
|
relu_spec = OpSpec("Relu", None, None)
|
|
relu_output_spec = OpSpec("Relu", None, srf_output.random.field_blobs())
|
|
pow_spec = OpSpec("Pow", None, None, {'exponent': float(s - 1)})
|
|
mul_interim_spec = OpSpec("Mul", None, srf_output.random.field_blobs())
|
|
mul_spec = OpSpec("Mul", None, srf_output.full.field_blobs())
|
|
|
|
if s == 0:
|
|
ops_list = [
|
|
fc_learned_spec,
|
|
fc_random_spec,
|
|
softsign_spec,
|
|
relu_output_spec,
|
|
mul_spec,
|
|
]
|
|
elif s == 1:
|
|
ops_list = [
|
|
fc_learned_spec,
|
|
fc_random_spec,
|
|
relu_output_spec,
|
|
mul_spec,
|
|
]
|
|
else:
|
|
ops_list = [
|
|
fc_learned_spec,
|
|
fc_random_spec,
|
|
relu_spec,
|
|
pow_spec,
|
|
mul_interim_spec,
|
|
mul_spec,
|
|
]
|
|
|
|
# Train net assertions
|
|
self._test_net(train_net, ops_list)
|
|
_semi_random_hypothesis_test(srf_output.full(), X_full, X_random,
|
|
rand_w, rand_b, s)
|
|
|
|
# Eval net assertions
|
|
eval_net = self.get_eval_net()
|
|
self._test_net(eval_net, ops_list)
|
|
_semi_random_hypothesis_test(srf_output.full(), X_full, X_random,
|
|
rand_w, rand_b, s)
|
|
|
|
# Predict net assertions
|
|
predict_net = self.get_predict_net()
|
|
self._test_net(predict_net, ops_list)
|
|
_semi_random_hypothesis_test(srf_output.full(), X_full, X_random,
|
|
rand_w, rand_b, s)
|
|
|
|
def testConv(self):
|
|
batch_size = 50
|
|
H = 1
|
|
W = 10
|
|
C = 50
|
|
output_dims = 32
|
|
kernel_h = 1
|
|
kernel_w = 3
|
|
stride_h = 1
|
|
stride_w = 1
|
|
pad_t = 0
|
|
pad_b = 0
|
|
pad_r = None
|
|
pad_l = None
|
|
|
|
input_record = self.new_record(schema.Scalar((np.float32, (H, W, C))))
|
|
X = np.random.random((batch_size, H, W, C)).astype(np.float32)
|
|
schema.FeedRecord(input_record, [X])
|
|
conv = self.model.Conv(
|
|
input_record,
|
|
output_dims,
|
|
kernel_h=kernel_h,
|
|
kernel_w=kernel_w,
|
|
stride_h=stride_h,
|
|
stride_w=stride_w,
|
|
pad_t=pad_t,
|
|
pad_b=pad_b,
|
|
pad_r=pad_r,
|
|
pad_l=pad_l,
|
|
order='NHWC'
|
|
)
|
|
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (output_dims,))),
|
|
conv
|
|
)
|
|
|
|
self.run_train_net_forward_only()
|
|
output_record = schema.FetchRecord(conv)
|
|
# check the number of output channels is the same as input in this example
|
|
assert output_record.field_types()[0].shape == (H, W, output_dims)
|
|
assert output_record().shape == (batch_size, H, W, output_dims)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
# Init net assertions
|
|
init_ops = self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("XavierFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
]
|
|
)
|
|
conv_spec = OpSpec(
|
|
"Conv",
|
|
[
|
|
input_record.field_blobs()[0],
|
|
init_ops[0].output[0],
|
|
init_ops[1].output[0],
|
|
],
|
|
conv.field_blobs()
|
|
)
|
|
|
|
# Train net assertions
|
|
self.assertNetContainOps(train_net, [conv_spec])
|
|
|
|
# Predict net assertions
|
|
predict_net = self.get_predict_net()
|
|
self.assertNetContainOps(predict_net, [conv_spec])
|
|
|
|
# Eval net assertions
|
|
eval_net = self.get_eval_net()
|
|
self.assertNetContainOps(eval_net, [conv_spec])
|
|
|
|
@given(
|
|
num=st.integers(min_value=10, max_value=100),
|
|
feed_weight=st.booleans(),
|
|
use_inv_var_parameterization=st.booleans(),
|
|
use_log_barrier=st.booleans(),
|
|
enable_diagnose=st.booleans(),
|
|
**hu.gcs
|
|
)
|
|
def testAdaptiveWeight(
|
|
self, num, feed_weight, use_inv_var_parameterization, use_log_barrier,
|
|
enable_diagnose, gc, dc
|
|
):
|
|
input_record = self.new_record(schema.RawTuple(num))
|
|
data = np.random.random(num)
|
|
schema.FeedRecord(
|
|
input_record, [np.array(x).astype(np.float32) for x in data]
|
|
)
|
|
weights = np.random.random(num) if feed_weight else None
|
|
result = self.model.AdaptiveWeight(
|
|
input_record,
|
|
weights=weights,
|
|
estimation_method=(
|
|
'inv_var' if use_inv_var_parameterization else 'log_std'
|
|
),
|
|
pos_optim_method=(
|
|
'log_barrier' if use_log_barrier else 'pos_grad_proj'
|
|
),
|
|
enable_diagnose=enable_diagnose
|
|
)
|
|
train_init_net, train_net = self.get_training_nets(True)
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
result = workspace.FetchBlob(result())
|
|
if not feed_weight:
|
|
weights = np.array([1. / num for _ in range(num)])
|
|
expected = np.sum(weights * data + 0.5 * np.log(1. / 2. / weights))
|
|
npt.assert_allclose(expected, result, atol=1e-4, rtol=1e-4)
|
|
if enable_diagnose:
|
|
assert len(self.model.ad_hoc_plot_blobs) == num
|
|
reconst_weights_from_ad_hoc = np.array(
|
|
[workspace.FetchBlob(b) for b in self.model.ad_hoc_plot_blobs]
|
|
).flatten()
|
|
npt.assert_allclose(
|
|
reconst_weights_from_ad_hoc, weights, atol=1e-4, rtol=1e-4
|
|
)
|
|
else:
|
|
assert len(self.model.ad_hoc_plot_blobs) == 0
|
|
|
|
@given(num=st.integers(min_value=10, max_value=100), **hu.gcs)
|
|
def testConstantWeight(self, num, gc, dc):
|
|
input_record = self.new_record(schema.RawTuple(num))
|
|
data = np.random.random(num)
|
|
schema.FeedRecord(
|
|
input_record, [np.array(x).astype(np.float32) for x in data]
|
|
)
|
|
weights = np.random.random(num)
|
|
result = self.model.ConstantWeight(input_record, weights=weights)
|
|
train_init_net, train_net = self.get_training_nets(True)
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
result = workspace.FetchBlob(result())
|
|
expected = np.sum(weights * data)
|
|
npt.assert_allclose(expected, result, atol=1e-4, rtol=1e-4)
|
|
|
|
@given(**hu.gcs)
|
|
def testHomotopyWeight(self, gc, dc):
|
|
input_record = self.new_record(schema.RawTuple(2))
|
|
data = np.random.random(2)
|
|
schema.FeedRecord(
|
|
input_record, [np.array(x).astype(np.float32) for x in data]
|
|
)
|
|
# ensure: quad_life > 2 * half_life
|
|
half_life = int(np.random.random() * 1e2 + 1)
|
|
quad_life = int(np.random.random() * 1e3 + 2 * half_life + 1)
|
|
min_weight = np.random.random()
|
|
max_weight = np.random.random() + min_weight + 1e-5
|
|
result = self.model.HomotopyWeight(
|
|
input_record,
|
|
min_weight=min_weight,
|
|
max_weight=max_weight,
|
|
half_life=half_life,
|
|
quad_life=quad_life,
|
|
)
|
|
train_init_net, train_net = self.get_training_nets(True)
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.CreateNet(train_net)
|
|
workspace.RunNet(train_net.Name(), num_iter=half_life)
|
|
half_life_result = workspace.FetchBlob(result())
|
|
workspace.RunNet(train_net.Name(), num_iter=quad_life - half_life)
|
|
quad_life_result = workspace.FetchBlob(result())
|
|
|
|
alpha = (min_weight + max_weight) / 2.
|
|
beta = (min_weight + max_weight) / 2.
|
|
expected_half_life_result = alpha * data[0] + beta * data[1]
|
|
alpha = (3 * min_weight + max_weight) / 4.
|
|
beta = (min_weight + 3 * max_weight) / 4.
|
|
expected_quad_life_result = alpha * data[0] + beta * data[1]
|
|
npt.assert_allclose(
|
|
expected_half_life_result, half_life_result, atol=1e-2, rtol=1e-2
|
|
)
|
|
npt.assert_allclose(
|
|
expected_quad_life_result, quad_life_result, atol=1e-2, rtol=1e-2
|
|
)
|
|
|
|
def _testLabelSmooth(self, categories, binary_prob_label, bsz):
|
|
label = self.new_record(schema.Scalar((np.float32, (1, ))))
|
|
label_np = np.random.randint(categories, size=bsz).astype(np.float32)
|
|
schema.FeedRecord(label, [label_np])
|
|
smooth_matrix_shape = (
|
|
2 if binary_prob_label else (categories, categories)
|
|
)
|
|
smooth_matrix = np.random.random(smooth_matrix_shape)
|
|
smoothed_label = self.model.LabelSmooth(label, smooth_matrix)
|
|
train_init_net, train_net = self.get_training_nets(True)
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
smoothed_label_np = workspace.FetchBlob(smoothed_label())
|
|
if binary_prob_label:
|
|
expected = np.array(
|
|
[
|
|
smooth_matrix[0] if x == 0.0 else smooth_matrix[1]
|
|
for x in label_np
|
|
]
|
|
)
|
|
else:
|
|
expected = np.array([smooth_matrix[int(x)] for x in label_np])
|
|
npt.assert_allclose(expected, smoothed_label_np, atol=1e-4, rtol=1e-4)
|
|
|
|
@given(
|
|
categories=st.integers(min_value=2, max_value=10),
|
|
bsz=st.integers(min_value=10, max_value=100),
|
|
**hu.gcs
|
|
)
|
|
def testLabelSmoothForCategoricalLabel(self, categories, bsz, gc, dc):
|
|
self._testLabelSmooth(categories, False, bsz)
|
|
|
|
@given(
|
|
bsz=st.integers(min_value=10, max_value=100),
|
|
**hu.gcs
|
|
)
|
|
def testLabelSmoothForBinaryProbLabel(self, bsz, gc, dc):
|
|
self._testLabelSmooth(2, True, bsz)
|
|
|
|
@given(
|
|
num_inputs=st.integers(min_value=2, max_value=10),
|
|
batch_size=st.integers(min_value=2, max_value=10),
|
|
input_dim=st.integers(min_value=5, max_value=10),
|
|
seed=st.integers(1, 10),
|
|
)
|
|
def testBlobWeightedSum(self, num_inputs, batch_size, input_dim, seed):
|
|
|
|
def get_blob_weighted_sum():
|
|
weights = []
|
|
for i in range(num_inputs):
|
|
w_blob_name = 'blob_weighted_sum/w_{0}'.format(i)
|
|
assert workspace.HasBlob(w_blob_name), (
|
|
"cannot fine blob {}".format(w_blob_name)
|
|
)
|
|
w = workspace.FetchBlob(w_blob_name)
|
|
weights.append(w)
|
|
|
|
result = np.sum([
|
|
input_data[idx] * weights[idx] for idx in range(num_inputs)
|
|
], axis=0)
|
|
return result
|
|
|
|
np.random.seed(seed)
|
|
expected_output_schema = schema.Scalar((np.float32, (input_dim,)))
|
|
input_schema = schema.Tuple(
|
|
*[expected_output_schema for _ in range(num_inputs)]
|
|
)
|
|
input_data = [
|
|
np.random.random((batch_size, input_dim)).astype(np.float32)
|
|
for _ in range(num_inputs)
|
|
]
|
|
input_record = self.new_record(input_schema)
|
|
schema.FeedRecord(input_record, input_data)
|
|
|
|
# test output schema
|
|
ws_output = self.model.BlobWeightedSum(input_record)
|
|
self.assertEqual(len(self.model.layers), 1)
|
|
assert schema.equal_schemas(ws_output, expected_output_schema)
|
|
|
|
# test train net
|
|
train_init_net, train_net = self.get_training_nets()
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
output = workspace.FetchBlob(ws_output())
|
|
npt.assert_almost_equal(get_blob_weighted_sum(), output, decimal=5)
|
|
|
|
self.run_train_net_forward_only()
|
|
output = workspace.FetchBlob(ws_output())
|
|
npt.assert_almost_equal(get_blob_weighted_sum(), output, decimal=5)
|
|
|
|
# test eval net
|
|
eval_net = self.get_eval_net()
|
|
workspace.RunNetOnce(eval_net)
|
|
output = workspace.FetchBlob(ws_output())
|
|
npt.assert_almost_equal(get_blob_weighted_sum(), output, decimal=5)
|
|
|
|
# test pred net
|
|
pred_net = self.get_predict_net()
|
|
workspace.RunNetOnce(pred_net)
|
|
output = workspace.FetchBlob(ws_output())
|
|
npt.assert_almost_equal(get_blob_weighted_sum(), output, decimal=5)
|
|
|
|
def testFeatureSparseToDenseGetAccessedFeatures(self):
|
|
float_features_column = "float_features"
|
|
float_features_type = "FLOAT"
|
|
float_features_ids = [1, 2, 3]
|
|
|
|
id_list_features_column = "id_list_features"
|
|
id_list_features_type = "ID_LIST"
|
|
id_list_features_ids = [4, 5, 6]
|
|
|
|
id_score_list_features_column = "id_score_list_features"
|
|
id_score_list_features_type = "ID_SCORE_LIST"
|
|
id_score_list_features_ids = [7, 8 , 9]
|
|
|
|
feature_names = ["a", "b", "c"]
|
|
|
|
input_record = self.new_record(schema.Struct(
|
|
(float_features_column, schema.Map(np.int32, np.float32)),
|
|
(id_list_features_column,
|
|
schema.Map(np.int32, schema.List(np.int64))),
|
|
(id_score_list_features_column,
|
|
schema.Map(np.int32, schema.Map(np.int64, np.float32))),
|
|
))
|
|
|
|
input_specs = [
|
|
(
|
|
float_features_column,
|
|
schema.FeatureSpec(
|
|
feature_type=float_features_type,
|
|
feature_ids=float_features_ids,
|
|
feature_names=feature_names,
|
|
),
|
|
),
|
|
(
|
|
id_list_features_column,
|
|
schema.FeatureSpec(
|
|
feature_type=id_list_features_type,
|
|
feature_ids=id_list_features_ids,
|
|
feature_names=feature_names,
|
|
),
|
|
),
|
|
(
|
|
id_score_list_features_column,
|
|
schema.FeatureSpec(
|
|
feature_type=id_score_list_features_type,
|
|
feature_ids=id_score_list_features_ids,
|
|
feature_names=feature_names,
|
|
),
|
|
),
|
|
]
|
|
|
|
self.model.FeatureSparseToDense(input_record, input_specs)
|
|
|
|
expected_accessed_features = {
|
|
float_features_column: AccessedFeatures(
|
|
float_features_type, set(float_features_ids)),
|
|
id_list_features_column: AccessedFeatures(
|
|
id_list_features_type, set(id_list_features_ids)),
|
|
id_score_list_features_column: AccessedFeatures(
|
|
id_score_list_features_type, set(id_score_list_features_ids)),
|
|
}
|
|
|
|
self.assertEqual(len(self.model.layers), 1)
|
|
self.assertEqual(
|
|
self.model.layers[0].get_accessed_features(),
|
|
expected_accessed_features
|
|
)
|