Elias Ellison
9441c7a944
[JIT] add IR complexity tests ( #34918 )
...
Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/34918
I'm going to set this up as a benchmarking test that runs internally in FB, but soliciting reviews externally first.
I think that benchmarking complexity of our nn module & functional tests is useful because they are the building blocks of models, so they should be pretty representative of generic model complexity . This also separates out complexity benchmarking into tests that are easily debuggable given a regression, instead of a 50K node resnet graph.
For each test, i am testing the profiled graph with consistent shapes, and I am testing
- Number of If & loop statements
- Number of non-tensor nodes (outputs don't include tensor)
This is just a starting off point for testing IR complexity. Future plans could involve:
- adding resnet, or other models in the model repo
- benchmarking number of guards
Current output:
Functional tests:
```
('Name', 'Ifs/Loops', 'non-tensor ops')
('conv1d', 0, 0)
('conv2d', 0, 0)
('conv3d', 0, 0)
('conv_transpose1d', 0, 0)
('conv_transpose2d', 0, 0)
('conv_transpose3d', 0, 0)
('conv_tbc', 0, 0)
('avg_pool1d', 0, 0)
('avg_pool2d', 0, 0)
('avg_pool3d', 0, 0)
('fractional_max_pool2d', 0, 3)
('max_pool1d', 0, 0)
('max_pool1d', 0, 0)
('max_pool2d', 0, 0)
('max_pool2d', 0, 0)
('max_pool3d', 0, 0)
('max_unpool1d', 0, 12)
('max_unpool2d', 0, 22)
('max_unpool3d', 0, 33)
('lp_pool1d', 0, 0)
('lp_pool2d', 0, 0)
('adaptive_max_pool1d', 0, 0)
('adaptive_max_pool2d', 0, 6)
('adaptive_max_pool3d', 0, 9)
('adaptive_avg_pool1d', 0, 0)
('adaptive_avg_pool2d', 0, 6)
('adaptive_avg_pool3d', 0, 9)
('dropout', 0, 0)
('alpha_dropout', 0, 0)
('dropout2d', 0, 0)
('dropout3d', 0, 0)
('feature_alpha_dropout', 0, 0)
('threshold', 0, 0)
('threshold', 0, 0)
('relu', 0, 0)
('relu', 0, 0)
('glu', 0, 0)
('hardtanh', 0, 0)
('hardtanh', 0, 0)
('relu6', 0, 0)
('relu6', 0, 0)
('elu', 0, 0)
('elu', 0, 0)
('selu', 0, 0)
('selu', 0, 0)
('celu', 0, 0)
('celu', 0, 0)
('leaky_relu', 0, 0)
('leaky_relu', 0, 0)
('rrelu', 0, 0)
('rrelu', 0, 0)
('hardshrink', 0, 0)
('tanhshrink', 0, 0)
('softsign', 0, 0)
('softplus', 0, 0)
('softmin', 0, 0)
('softmax', 0, 0)
('softmax', 0, 0)
('tanh', 0, 1)
('sigmoid', 0, 1)
('log_softmax', 0, 0)
('linear', 0, 0)
('linear', 0, 0)
('bilinear', 0, 0)
('embedding', 0, 0)
('embedding_bag', 0, 0)
('batch_norm', 0, 0)
('instance_norm', 1, 6)
('layer_norm', 0, 0)
('layer_norm', 0, 0)
('layer_norm', 0, 0)
('layer_norm', 0, 0)
('group_norm', 3, 53)
('local_response_norm', 0, 0)
('nll_loss', 1, 5)
('poisson_nll_loss', 0, 0)
('poisson_nll_loss', 0, 0)
('kl_div', 0, 1)
('cross_entropy', 1, 5)
('binary_cross_entropy_with_logits', 0, 0)
('smooth_l1_loss', 1, 1)
('l1_loss', 1, 1)
('mse_loss', 1, 1)
('smooth_l1_loss', 1, 1)
('l1_loss', 1, 1)
('mse_loss', 1, 1)
('margin_ranking_loss', 0, 0)
('hinge_embedding_loss', 0, 0)
('soft_margin_loss', 0, 0)
('multilabel_soft_margin_loss', 0, 1)
('cosine_embedding_loss', 0, 0)
('pixel_shuffle', 0, 0)
('affine_grid', 3, 14)
('pad', 0, 0)
('pairwise_distance', 0, 0)
('pdist', 0, 0)
('cosine_similarity', 0, 0)
('triplet_margin_loss', 0, 0)
('normalize', 0, 0)
('unfold', 0, 0)
('fold', 0, 0)
('grid_sample', 0, 1)
('gumbel_softmax', 0, 0)
('gumbel_softmax', 0, 0)
('multilabel_margin_loss', 0, 0)
('multi_margin_loss', 0, 0)
('binary_cross_entropy', 1, 5)
('binary_cross_entropy', 1, 5)
('ctc_loss', 0, 0)
('upsample', 13, 71)
('upsample', 13, 71)
('interpolate', 14, 71)
('interpolate', 13, 70)
('interpolate', 14, 71)
('interpolate', 14, 71)
('interpolate', 13, 70)
('interpolate', 14, 71)
('interpolate', 14, 71)
('interpolate', 13, 70)
('interpolate', 14, 71)
('interpolate', 14, 71)
('interpolate', 13, 70)
('interpolate', 14, 71)
('interpolate', 14, 60)
('interpolate', 13, 58)
('interpolate', 14, 60)
('interpolate', 14, 60)
('interpolate', 13, 58)
('interpolate', 14, 60)
('interpolate', 14, 60)
('interpolate', 13, 58)
('interpolate', 14, 60)
('interpolate', 13, 82)
('interpolate', 14, 82)
('interpolate', 14, 82)
('interpolate', 13, 82)
('interpolate', 14, 82)
('interpolate', 14, 82)
('interpolate', 13, 82)
('interpolate', 14, 82)
('interpolate', 14, 71)
('interpolate', 14, 71)
('interpolate', 15, 106)
('interpolate', 14, 73)
('interpolate', 15, 106)
('interpolate', 14, 73)
('interpolate', 15, 92)
('interpolate', 14, 60)
('interpolate', 15, 94)
('interpolate', 14, 62)
('interpolate', 15, 116)
('interpolate', 14, 82)
('interpolate', 15, 118)
('interpolate', 14, 84)
```
nn module tests:
```
('Name', 'Ifs/Loops', 'non-tensor ops')
('test_nn_Linear', 0, 0)
('test_nn_Linear_no_bias', 0, 0)
('test_nn_Threshold_threshold_value', 0, 0)
('test_nn_Threshold_large_value', 0, 0)
('test_nn_ReLU', 0, 0)
('test_nn_ReLU6', 0, 0)
('test_nn_RReLU', 0, 0)
('test_nn_RReLU_with_up_down', 0, 0)
('test_nn_Hardtanh', 0, 0)
('test_nn_Sigmoid', 0, 0)
('test_nn_Tanh', 0, 0)
('test_nn_Flatten', 0, 0)
('test_nn_Softmax', 0, 0)
('test_nn_Softmax2d', 0, 0)
('test_nn_LogSoftmax', 0, 0)
('test_nn_LogSoftmax_multiparam', 0, 0)
('test_nn_ELU', 0, 0)
('test_nn_Hardshrink', 0, 0)
('test_nn_LeakyReLU', 0, 0)
('test_nn_LeakyReLU_with_negval', 0, 0)
('test_nn_LogSigmoid', 0, 0)
('test_nn_Softplus', 0, 0)
('test_nn_Softplus_beta', 0, 0)
('test_nn_Softplus_beta_threshold', 0, 0)
('test_nn_Softshrink', 0, 0)
('test_nn_Softshrink_lambda', 0, 0)
('test_nn_PReLU_1d', 0, 0)
('test_nn_PReLU_1d_multiparam', 0, 0)
('test_nn_PReLU_2d', 0, 0)
('test_nn_PReLU_2d_multiparam', 0, 0)
('test_nn_PReLU_3d', 0, 0)
('test_nn_PReLU_3d_multiparam', 0, 0)
('test_nn_Softsign', 0, 0)
('test_nn_Softmin', 0, 0)
('test_nn_Softmin_multidim', 0, 0)
('test_nn_Tanhshrink', 0, 0)
('test_nn_FractionalMaxPool2d_ratio', 0, 7)
('test_nn_FractionalMaxPool2d_size', 0, 0)
('test_nn_FractionalMaxPool3d_ratio', 0, 10)
('test_nn_FractionalMaxPool3d_size', 0, 0)
('test_nn_FractionalMaxPool3d_asymsize', 0, 0)
('test_nn_BatchNorm1d_affine', 2, 3)
('test_nn_BatchNorm1d_3d_input', 3, 9)
('test_nn_BatchNorm1d_affine_simple_average', 2, 5)
('test_nn_BatchNorm1d_not_affine', 2, 3)
('test_nn_BatchNorm1d_not_tracking_stats', 0, 0)
('test_nn_BatchNorm1d_3d_input_not_affine', 3, 9)
('test_nn_BatchNorm1d_zero_batch', 3, 9)
('test_nn_BatchNorm2d', 3, 13)
('test_nn_BatchNorm2d_2d_simple_average', 3, 15)
('test_nn_BatchNorm2d_momentum', 3, 13)
('test_nn_BatchNorm2d_not_affine', 3, 13)
('test_nn_BatchNorm2d_not_tracking_stats', 1, 10)
('test_nn_BatchNorm2d_zero_batch', 3, 13)
('test_nn_BatchNorm3d', 3, 17)
('test_nn_BatchNorm3d_3d_simple_average', 3, 19)
('test_nn_BatchNorm3d_momentum', 3, 17)
('test_nn_BatchNorm3d_not_affine', 3, 17)
('test_nn_BatchNorm3d_not_tracking_stats', 1, 14)
('test_nn_BatchNorm3d_zero_batch', 3, 17)
('test_nn_InstanceNorm1d', 1, 6)
('test_nn_InstanceNorm1d_tracking_stats', 1, 6)
('test_nn_InstanceNorm2d', 1, 10)
('test_nn_InstanceNorm2d_tracking_stats', 1, 10)
('test_nn_InstanceNorm3d', 1, 14)
('test_nn_InstanceNorm3d_tracking_stats', 1, 14)
('test_nn_LayerNorm_1d_elementwise_affine', 0, 0)
('test_nn_LayerNorm_1d_no_elementwise_affine', 0, 0)
('test_nn_LayerNorm_3d_elementwise_affine', 0, 0)
('test_nn_LayerNorm_3d_no_elementwise_affine', 0, 0)
('test_nn_LayerNorm_1d_empty_elementwise_affine', 0, 0)
('test_nn_GroupNorm_1d_affine', 3, 53)
('test_nn_GroupNorm_1d_no_affine_IN', 3, 53)
('test_nn_GroupNorm_1d_no_affine_LN', 3, 53)
('test_nn_GroupNorm_2d_affine', 3, 53)
('test_nn_GroupNorm_2d_no_affine_IN', 3, 53)
('test_nn_GroupNorm_2d_no_affine_LN', 3, 53)
('test_nn_Conv1d', 0, 0)
('test_nn_Conv1d_stride', 0, 0)
('test_nn_Conv1d_pad1', 0, 0)
('test_nn_Conv1d_pad2', 0, 0)
('test_nn_Conv1d_pad1size1', 0, 0)
('test_nn_Conv1d_pad2size1', 0, 0)
('test_nn_Conv1d_zero_batch', 0, 0)
('test_nn_Conv1d_dilated', 0, 0)
('test_nn_Conv1d_groups', 0, 0)
('test_nn_ConvTranspose1d', 0, 0)
('test_nn_ConvTranspose1d_no_bias', 0, 0)
('test_nn_ConvTranspose1d_dilated', 0, 0)
('test_nn_ConvTranspose1d_groups', 0, 0)
('test_nn_MaxPool1d', 0, 0)
('test_nn_MaxPool1d_stride', 0, 0)
('test_nn_Conv2d', 0, 0)
('test_nn_Conv2d_strided', 0, 0)
('test_nn_Conv2d_padding', 0, 0)
('test_nn_Conv2d_dilated', 0, 0)
('test_nn_Conv2d_no_bias', 0, 0)
('test_nn_Conv2d_zero_batch', 0, 0)
('test_nn_Conv2d_groups', 0, 0)
('test_nn_Conv2d_groups_thnn', 0, 0)
('test_nn_ConvTranspose2d', 0, 0)
('test_nn_ConvTranspose2d_dilated', 0, 0)
('test_nn_ConvTranspose2d_no_bias', 0, 0)
('test_nn_ConvTranspose2d_groups', 0, 0)
('test_nn_Conv2d_depthwise', 0, 0)
('test_nn_Conv2d_depthwise_with_multiplier', 0, 0)
('test_nn_Conv2d_depthwise_strided', 0, 0)
('test_nn_Conv2d_depthwise_padded', 0, 0)
('test_nn_Conv2d_depthwise_dilated', 0, 0)
('test_nn_MaxPool2d', 0, 0)
('test_nn_AvgPool1d', 0, 0)
('test_nn_AvgPool1d_stride', 0, 0)
('test_nn_AvgPool1d_stride_pad', 0, 0)
('test_nn_AvgPool2d', 0, 0)
('test_nn_AvgPool2d_stride', 0, 0)
('test_nn_AvgPool2d_stride_pad', 0, 0)
('test_nn_AvgPool2d_divisor', 0, 0)
('test_nn_AvgPool2d_divisor_stride', 0, 0)
('test_nn_AvgPool2d_divisor_stride_pad', 0, 0)
('test_nn_LPPool2d', 0, 0)
('test_nn_LPPool2d_norm', 0, 0)
('test_nn_LPPool1d_norm', 0, 0)
('test_nn_LPPool1d', 0, 0)
('test_nn_LocalResponseNorm_1d', 0, 0)
('test_nn_LocalResponseNorm_2d_uneven_pad', 0, 0)
('test_nn_LocalResponseNorm_3d_custom_params', 0, 0)
('test_nn_ReflectionPad1d', 0, 0)
('test_nn_ReflectionPad2d', 0, 0)
('test_nn_ReplicationPad1d', 0, 0)
('test_nn_ReplicationPad2d', 0, 0)
('test_nn_ZeroPad2d', 0, 0)
('test_nn_ZeroPad2d_negative_dims', 0, 0)
('test_nn_ConstantPad1d', 0, 0)
('test_nn_ConstantPad2d', 0, 0)
('test_nn_ConstantPad3d', 0, 0)
('test_nn_Conv3d', 0, 0)
('test_nn_Conv3d_no_bias', 0, 0)
('test_nn_Conv3d_stride', 0, 0)
('test_nn_Conv3d_stride_padding', 0, 0)
('test_nn_Conv3d_zero_batch', 0, 0)
('test_nn_Conv3d_groups', 0, 0)
('test_nn_Conv3d_dilated', 0, 0)
('test_nn_Conv3d_dilated_strided', 0, 0)
('test_nn_ConvTranspose3d', 0, 0)
('test_nn_ConvTranspose3d_dilated', 0, 0)
('test_nn_MaxPool3d', 0, 0)
('test_nn_MaxPool3d_stride', 0, 0)
('test_nn_MaxPool3d_stride_padding', 0, 0)
('test_nn_AvgPool3d', 0, 0)
('test_nn_AvgPool3d_stride', 0, 0)
('test_nn_AvgPool3d_stride_pad', 0, 0)
('test_nn_AvgPool3d_stride_pad_gpu_fixedkw_output', 0, 0)
('test_nn_AvgPool3d_stride_pad_gpu_general_output', 0, 0)
('test_nn_AvgPool3d_stride1_pad0_gpu_input', 0, 0)
('test_nn_AvgPool3d_stride_pad_gpu_input_nooverlap', 0, 0)
('test_nn_AvgPool3d_divisor', 0, 0)
('test_nn_AvgPool3d_divisor_stride', 0, 0)
('test_nn_AvgPool3d_divisor_stride_pad', 0, 0)
('test_nn_AvgPool3d_divisor_stride_pad_gpu_fixedkw_output', 0, 0)
('test_nn_AvgPool3d_divisor_stride_pad_gpu_general_output', 0, 0)
('test_nn_AvgPool3d_divisor_stride1_pad0_gpu_input', 0, 0)
('test_nn_AvgPool3d_divisor_stride_pad_gpu_input_nooverlap', 0, 0)
('test_nn_ReplicationPad3d', 0, 0)
('test_nn_Embedding', 0, 0)
('test_nn_EmbeddingBag_mean', 0, 2)
('test_nn_EmbeddingBag_sum', 0, 2)
('test_nn_EmbeddingBag_max', 0, 2)
('test_nn_EmbeddingBag_sparse', 0, 2)
('test_nn_Embedding_sparse', 0, 0)
('test_nn_PixelShuffle', 0, 0)
('test_nn_AdaptiveMaxPool1d', 0, 0)
('test_nn_AdaptiveMaxPool2d_single', 0, 6)
('test_nn_AdaptiveMaxPool2d_tuple', 0, 6)
('test_nn_AdaptiveMaxPool3d_single', 0, 9)
('test_nn_AdaptiveMaxPool3d_tuple', 0, 9)
('test_nn_AdaptiveMaxPool3d_single_nonatomic', 0, 9)
('test_nn_AdaptiveMaxPool3d_tuple_nonatomic', 0, 9)
('test_nn_AdaptiveAvgPool1d', 0, 0)
('test_nn_AdaptiveAvgPool1d_one_output', 0, 0)
('test_nn_AdaptiveAvgPool2d_single', 0, 6)
('test_nn_AdaptiveAvgPool2d_single_1x1output', 0, 6)
('test_nn_AdaptiveAvgPool2d_tuple', 0, 6)
('test_nn_AdaptiveAvgPool3d_single', 0, 9)
('test_nn_AdaptiveAvgPool3d_tuple', 0, 9)
('test_nn_SELU', 0, 0)
('test_nn_SELU_scalar', 0, 0)
('test_nn_CELU', 0, 0)
('test_nn_CELU_scalar', 0, 0)
('test_nn_GLU', 0, 0)
('test_nn_GLU_dim', 0, 0)
('test_nn_GELU_scalar', 0, 0)
('test_nn_GELU', 0, 0)
('test_nn_Unfold', 0, 0)
('test_nn_Fold', 0, 0)
('test_nn_Unfold_int_input', 0, 0)
('test_nn_Fold_int_input', 0, 0)
('test_nn_Threshold_threshold_value_scalar', 0, 0)
('test_nn_ReLU_scalar', 0, 0)
('test_nn_ReLU6_scalar', 0, 0)
('test_nn_RReLU_with_up_down_scalar', 0, 0)
('test_nn_Hardtanh_scalar', 0, 0)
('test_nn_Sigmoid_scalar', 0, 0)
('test_nn_Tanh_scalar', 0, 0)
('test_nn_Softmax_scalar', 0, 0)
('test_nn_LogSoftmax_multiparam_scalar', 0, 0)
('test_nn_ELU_scalar', 0, 0)
('test_nn_Hardshrink_scalar', 0, 0)
('test_nn_LeakyReLU_with_negval_scalar', 0, 0)
('test_nn_LogSigmoid_scalar', 0, 0)
('test_nn_Softplus_beta_threshold_scalar', 0, 0)
('test_nn_Softshrink_lambda_scalar', 0, 0)
('test_nn_PReLU_scalar', 0, 0)
('test_nn_Softsign_scalar', 0, 0)
('test_nn_Softmin_scalar', 0, 0)
('test_nn_Tanhshrink_scalar', 0, 0)
('test_nn_Conv1d_reflect_stride2_pad2', 3, 14)
('test_nn_Conv2d_reflect_stride2_pad2', 3, 14)
('test_nn_Conv1d_circular_stride2_pad2', 5, 31)
('test_nn_Conv2d_circular_stride2_pad2', 5, 31)
('test_nn_Conv3d_circular_stride2_pad2', 5, 31)
('test_nn_Conv1d_replicate_stride2_pad2', 3, 14)
('test_nn_Conv2d_replicate_stride2_pad2', 3, 14)
('test_nn_Conv3d_replicate_stride2_pad2', 3, 14)
('test_nn_Conv1d_zeros_stride2_pad2', 0, 0)
('test_nn_Conv2d_zeros_stride2_pad2', 0, 0)
('test_nn_Conv3d_zeros_stride2_pad2', 0, 0)
('test_nn_Bilinear', 0, 0)
('test_nn_RNNCell', 3, 14)
('test_nn_LSTMCell', 5, 22)
('test_nn_GRUCell', 3, 14)
('test_nn_MultiheadAttention', 40, 160)
('test_nn_Transformer', 128, 499)
```
Test Plan: Imported from OSS
Differential Revision: D20539336
Pulled By: eellison
fbshipit-source-id: 14ac00a7b2b029b9e57f6131dd45426b0101941a
2020-03-23 11:59:11 -07:00
