This is an unofficial PyTorch implementation of InceptionTime (Fawaz, 2019) created by Ignacio Oguiza.

References:

class InceptionModulePlus[source]

InceptionModulePlus(ni, nf, ks=40, bottleneck=True, padding='same', coord=False, separable=False, dilation=1, stride=1, conv_dropout=0.0, sa=False, se=None, norm='Batch', zero_norm=False, bn_1st=True, act=ReLU, act_kwargs={}) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

class InceptionBlockPlus[source]

InceptionBlockPlus(ni, nf, residual=True, depth=6, coord=False, norm='Batch', zero_norm=False, act=ReLU, act_kwargs={}, sa=False, se=None, keep_prob=1.0, ks=40, bottleneck=True, padding='same', separable=False, dilation=1, stride=1, conv_dropout=0.0, bn_1st=True) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

class InceptionTimePlus[source]

InceptionTimePlus(c_in, c_out, seq_len=None, nf=32, nb_filters=None, depth=6, stoch_depth=1.0, flatten=False, concat_pool=False, fc_dropout=0.0, bn=False, y_range=None, custom_head=None, ks=40, bottleneck=True, padding='same', coord=False, separable=False, dilation=1, stride=1, conv_dropout=0.0, sa=False, se=None, norm='Batch', zero_norm=False, bn_1st=True, act=ReLU, act_kwargs={}) :: Sequential

A sequential container. Modules will be added to it in the order they are passed in the constructor. Alternatively, an ordered dict of modules can also be passed in.

To make it easier to understand, here is a small example::

# Example of using Sequential
model = nn.Sequential(
          nn.Conv2d(1,20,5),
          nn.ReLU(),
          nn.Conv2d(20,64,5),
          nn.ReLU()
        )

# Example of using Sequential with OrderedDict
model = nn.Sequential(OrderedDict([
          ('conv1', nn.Conv2d(1,20,5)),
          ('relu1', nn.ReLU()),
          ('conv2', nn.Conv2d(20,64,5)),
          ('relu2', nn.ReLU())
        ]))

class InCoordTime[source]

InCoordTime(c_in, c_out, seq_len=None, nf=32, nb_filters=None, depth=6, stoch_depth=1.0, flatten=False, concat_pool=False, fc_dropout=0.0, bn=False, y_range=None, custom_head=None, ks=40, bottleneck=True, padding='same', coord=False, separable=False, dilation=1, stride=1, conv_dropout=0.0, sa=False, se=None, norm='Batch', zero_norm=False, bn_1st=True, act=ReLU, act_kwargs={}) :: InceptionTimePlus

A sequential container. Modules will be added to it in the order they are passed in the constructor. Alternatively, an ordered dict of modules can also be passed in.

To make it easier to understand, here is a small example::

# Example of using Sequential
model = nn.Sequential(
          nn.Conv2d(1,20,5),
          nn.ReLU(),
          nn.Conv2d(20,64,5),
          nn.ReLU()
        )

# Example of using Sequential with OrderedDict
model = nn.Sequential(OrderedDict([
          ('conv1', nn.Conv2d(1,20,5)),
          ('relu1', nn.ReLU()),
          ('conv2', nn.Conv2d(20,64,5)),
          ('relu2', nn.ReLU())
        ]))

class XCoordTime[source]

XCoordTime(c_in, c_out, seq_len=None, nf=32, nb_filters=None, depth=6, stoch_depth=1.0, flatten=False, concat_pool=False, fc_dropout=0.0, bn=False, y_range=None, custom_head=None, ks=40, bottleneck=True, padding='same', coord=False, separable=False, dilation=1, stride=1, conv_dropout=0.0, sa=False, se=None, norm='Batch', zero_norm=False, bn_1st=True, act=ReLU, act_kwargs={}) :: InceptionTimePlus

A sequential container. Modules will be added to it in the order they are passed in the constructor. Alternatively, an ordered dict of modules can also be passed in.

To make it easier to understand, here is a small example::

# Example of using Sequential
model = nn.Sequential(
          nn.Conv2d(1,20,5),
          nn.ReLU(),
          nn.Conv2d(20,64,5),
          nn.ReLU()
        )

# Example of using Sequential with OrderedDict
model = nn.Sequential(OrderedDict([
          ('conv1', nn.Conv2d(1,20,5)),
          ('relu1', nn.ReLU()),
          ('conv2', nn.Conv2d(20,64,5)),
          ('relu2', nn.ReLU())
        ]))
bs = 16
n_vars = 3
seq_len = 51
c_out = 2
xb = torch.rand(bs, n_vars, seq_len)

test_eq(InceptionTimePlus(n_vars,c_out)(xb).shape, [bs, c_out])
test_eq(InceptionTimePlus(n_vars,c_out,concat_pool=True)(xb).shape, [bs, c_out])
test_eq(InceptionTimePlus(n_vars,c_out, bottleneck=False)(xb).shape, [bs, c_out])
test_eq(InceptionTimePlus(n_vars,c_out, residual=False)(xb).shape, [bs, c_out])
test_eq(InceptionTimePlus(n_vars,c_out, conv_dropout=.5)(xb).shape, [bs, c_out])
test_eq(InceptionTimePlus(n_vars,c_out, stoch_depth=.5)(xb).shape, [bs, c_out])
test_eq(InceptionTimePlus(n_vars, c_out, seq_len=seq_len, zero_norm=True, flatten=True)(xb).shape, [bs, c_out])
test_eq(InceptionTimePlus(n_vars,c_out, coord=True, separable=True, 
                          norm='Instance', zero_norm=True, bn_1st=False, fc_dropout=.5, sa=True, se=True, act=nn.PReLU, act_kwargs={})(xb).shape, [bs, c_out])
test_eq(InceptionTimePlus(n_vars,c_out, coord=True, separable=True,
                          norm='Instance', zero_norm=True, bn_1st=False, act=nn.PReLU, act_kwargs={})(xb).shape, [bs, c_out])
test_eq(total_params(InceptionTimePlus(3, 2))[0], 455490)
test_eq(total_params(InceptionTimePlus(6, 2, **{'coord': True, 'separable': True, 'zero_norm': True}))[0], 77204)
test_eq(total_params(InceptionTimePlus(3, 2, ks=40))[0], total_params(InceptionTimePlus(3, 2, ks=[9, 19, 39]))[0])

bs = 16
n_vars = 3
seq_len = 51
c_out = 2
xb = torch.rand(bs, n_vars, seq_len)

model = InceptionTimePlus(n_vars, c_out)
model(xb).shape
test_eq(model[0](xb), model.backbone(xb))
test_eq(model[1](model[0](xb)), model.head(model[0](xb)))
test_eq(model[1].state_dict().keys(), model.head.state_dict().keys())
test_eq(len(ts_splitter(model)), 2)

test_eq(check_bias(InceptionTimePlus(2,3, zero_norm=True), is_conv)[0].sum(), 0)
test_eq(check_weight(InceptionTimePlus(2,3, zero_norm=True), is_bn)[0].sum(), 6)
test_eq(check_weight(InceptionTimePlus(2,3), is_bn)[0], np.array([1., 1., 1., 1., 1., 1., 1., 1.]))

for i in range(10): InceptionTimePlus(n_vars,c_out,stoch_depth=0.8,depth=9,zero_norm=True)(xb)

net = InceptionTimePlus(2,3,**{'coord': True, 'separable': True, 'zero_norm': True})
test_eq(check_weight(net, is_bn)[0], np.array([1., 1., 0., 1., 1., 0., 1., 1.]))
net

InceptionTimePlus(
  (backbone): Sequential(
    (0): InceptionBlockPlus(
      (inception): ModuleList(
        (0): InceptionModulePlus(
          (bottleneck): ConvBlock(
            (0): AddCoords1d()
            (1): Conv1d(3, 32, kernel_size=(1,), stride=(1,), bias=False)
          )
          (convs): ModuleList(
            (0): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(39,), stride=(1,), padding=(19,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
            (1): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(19,), stride=(1,), padding=(9,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
            (2): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(9,), stride=(1,), padding=(4,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
          )
          (mp_conv): Sequential(
            (0): MaxPool1d(kernel_size=3, stride=1, padding=1, dilation=1, ceil_mode=False)
            (1): ConvBlock(
              (0): AddCoords1d()
              (1): Conv1d(3, 32, kernel_size=(1,), stride=(1,), bias=False)
            )
          )
          (concat): Concat(dim=1)
          (norm): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (act): ReLU()
        )
        (1): InceptionModulePlus(
          (bottleneck): ConvBlock(
            (0): AddCoords1d()
            (1): Conv1d(129, 32, kernel_size=(1,), stride=(1,), bias=False)
          )
          (convs): ModuleList(
            (0): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(39,), stride=(1,), padding=(19,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
            (1): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(19,), stride=(1,), padding=(9,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
            (2): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(9,), stride=(1,), padding=(4,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
          )
          (mp_conv): Sequential(
            (0): MaxPool1d(kernel_size=3, stride=1, padding=1, dilation=1, ceil_mode=False)
            (1): ConvBlock(
              (0): AddCoords1d()
              (1): Conv1d(129, 32, kernel_size=(1,), stride=(1,), bias=False)
            )
          )
          (concat): Concat(dim=1)
          (norm): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (act): ReLU()
        )
        (2): InceptionModulePlus(
          (bottleneck): ConvBlock(
            (0): AddCoords1d()
            (1): Conv1d(129, 32, kernel_size=(1,), stride=(1,), bias=False)
          )
          (convs): ModuleList(
            (0): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(39,), stride=(1,), padding=(19,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
            (1): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(19,), stride=(1,), padding=(9,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
            (2): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(9,), stride=(1,), padding=(4,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
          )
          (mp_conv): Sequential(
            (0): MaxPool1d(kernel_size=3, stride=1, padding=1, dilation=1, ceil_mode=False)
            (1): ConvBlock(
              (0): AddCoords1d()
              (1): Conv1d(129, 32, kernel_size=(1,), stride=(1,), bias=False)
            )
          )
          (concat): Concat(dim=1)
          (norm): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        )
        (3): InceptionModulePlus(
          (bottleneck): ConvBlock(
            (0): AddCoords1d()
            (1): Conv1d(129, 32, kernel_size=(1,), stride=(1,), bias=False)
          )
          (convs): ModuleList(
            (0): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(39,), stride=(1,), padding=(19,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
            (1): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(19,), stride=(1,), padding=(9,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
            (2): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(9,), stride=(1,), padding=(4,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
          )
          (mp_conv): Sequential(
            (0): MaxPool1d(kernel_size=3, stride=1, padding=1, dilation=1, ceil_mode=False)
            (1): ConvBlock(
              (0): AddCoords1d()
              (1): Conv1d(129, 32, kernel_size=(1,), stride=(1,), bias=False)
            )
          )
          (concat): Concat(dim=1)
          (norm): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (act): ReLU()
        )
        (4): InceptionModulePlus(
          (bottleneck): ConvBlock(
            (0): AddCoords1d()
            (1): Conv1d(129, 32, kernel_size=(1,), stride=(1,), bias=False)
          )
          (convs): ModuleList(
            (0): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(39,), stride=(1,), padding=(19,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
            (1): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(19,), stride=(1,), padding=(9,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
            (2): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(9,), stride=(1,), padding=(4,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
          )
          (mp_conv): Sequential(
            (0): MaxPool1d(kernel_size=3, stride=1, padding=1, dilation=1, ceil_mode=False)
            (1): ConvBlock(
              (0): AddCoords1d()
              (1): Conv1d(129, 32, kernel_size=(1,), stride=(1,), bias=False)
            )
          )
          (concat): Concat(dim=1)
          (norm): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (act): ReLU()
        )
        (5): InceptionModulePlus(
          (bottleneck): ConvBlock(
            (0): AddCoords1d()
            (1): Conv1d(129, 32, kernel_size=(1,), stride=(1,), bias=False)
          )
          (convs): ModuleList(
            (0): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(39,), stride=(1,), padding=(19,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
            (1): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(19,), stride=(1,), padding=(9,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
            (2): ConvBlock(
              (0): AddCoords1d()
              (1): SeparableConv1d(
                (depthwise_conv): Conv1d(33, 33, kernel_size=(9,), stride=(1,), padding=(4,), groups=33, bias=False)
                (pointwise_conv): Conv1d(33, 32, kernel_size=(1,), stride=(1,), bias=False)
              )
            )
          )
          (mp_conv): Sequential(
            (0): MaxPool1d(kernel_size=3, stride=1, padding=1, dilation=1, ceil_mode=False)
            (1): ConvBlock(
              (0): AddCoords1d()
              (1): Conv1d(129, 32, kernel_size=(1,), stride=(1,), bias=False)
            )
          )
          (concat): Concat(dim=1)
          (norm): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        )
      )
      (shortcut): ModuleList(
        (0): ConvBlock(
          (0): AddCoords1d()
          (1): Conv1d(3, 128, kernel_size=(1,), stride=(1,), bias=False)
          (2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        )
        (1): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (act): ModuleList(
        (0): ReLU()
        (1): ReLU()
      )
      (add): Add
    )
  )
  (head): Sequential(
    (0): Sequential(
      (0): GAP1d(
        (gap): AdaptiveAvgPool1d(output_size=1)
        (flatten): Flatten(full=False)
      )
      (1): LinBnDrop(
        (0): Linear(in_features=128, out_features=3, bias=True)
      )
    )
  )
)

class MultiInceptionTimePlus[source]

MultiInceptionTimePlus(feat_list, c_out, seq_len=None, nf=32, nb_filters=None, depth=6, stoch_depth=1.0, flatten=False, concat_pool=False, fc_dropout=0.0, bn=False, y_range=None, custom_head=None, device=None) :: Sequential

Class that allows you to create a model with multiple branches of InceptionTimePlus.

bs = 16
n_vars = 3
seq_len = 51
c_out = 2
xb = torch.rand(bs, n_vars, seq_len)

test_eq(count_parameters(MultiInceptionTimePlus([1,1,1], c_out)) > count_parameters(MultiInceptionTimePlus(3, c_out)), True)
test_eq(MultiInceptionTimePlus([1,1,1], c_out)(xb).shape, MultiInceptionTimePlus(3, c_out)(xb).shape)

bs = 16
n_vars = 3
seq_len = 12
c_out = 10
xb = torch.rand(bs, n_vars, seq_len)
new_head = partial(conv_lin_3d_head, d=(5,2))
net = MultiInceptionTimePlus(n_vars, c_out, seq_len, custom_head=new_head)
print(net(xb).shape)
net.head

torch.Size([16, 5, 2])

Sequential(
  (0): create_conv_lin_3d_head(
    (0): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (1): Conv1d(128, 5, kernel_size=(1,), stride=(1,), bias=False)
    (2): Transpose(-1, -2)
    (3): BatchNorm1d(12, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (4): Transpose(-1, -2)
    (5): Linear(in_features=12, out_features=2, bias=False)
  )
)
bs = 16
n_vars = 6
seq_len = 12
c_out = 2
xb = torch.rand(bs, n_vars, seq_len)
net = MultiInceptionTimePlus([1,2,3], c_out, seq_len)
print(net(xb).shape)
net.head

torch.Size([16, 2])

Sequential(
  (0): Sequential(
    (0): GAP1d(
      (gap): AdaptiveAvgPool1d(output_size=1)
      (flatten): Flatten(full=False)
    )
    (1): LinBnDrop(
      (0): Linear(in_features=384, out_features=2, bias=True)
    )
  )
)
net = build_ts_model(MultiInceptionTimePlus, [1,2,3], c_out, seq_len)
xb.shape

torch.Size([16, 6, 12])
bs = 16
n_vars = 3
seq_len = 12
c_out = 2
xb = torch.rand(bs, n_vars, seq_len)
net = MultiInceptionTimePlus(n_vars, c_out)
change_model_head(net, create_pool_plus_head, concat_pool=False)
print(net(xb).shape)
net.head

torch.Size([16, 2])

Sequential(
  (0): AdaptiveAvgPool1d(output_size=1)
  (1): Flatten(full=False)
  (2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (3): Linear(in_features=128, out_features=512, bias=False)
  (4): ReLU(inplace=True)
  (5): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (6): Linear(in_features=512, out_features=2, bias=False)
)
bs = 8
c_in = 7  # aka channels, features, variables, dimensions
c_out = 2
seq_len = 10
xb2 = torch.randn(bs, c_in, seq_len)
model1 = MultiInceptionTimePlus([2, 5], c_out, seq_len, )
model2 = MultiInceptionTimePlus([[0,2,5], [0,1,3,4,6]], c_out, seq_len)
test_eq(model1(xb2).shape, (bs, c_out))
test_eq(model1(xb2).shape, model2(xb2).shape)

from tsai.data.all import *
from tsai.learner import ts_learner
dsid = 'NATOPS'
X, y, splits = get_UCR_data(dsid, split_data=False)
tfms  = [None, [Categorize()]]
batch_tfms = TSStandardize()
ts_dls = get_ts_dls(X, y, splits=splits, tfms=tfms, batch_tfms=batch_tfms)
learn = ts_learner(ts_dls, InceptionTimePlus)
xb,yb=first(learn.dls.train)
test_eq(learn.model(xb).shape, (ts_dls.bs, ts_dls.c))
p1 = count_parameters(learn.model)
learn.freeze()
p2 = count_parameters(learn.model)
assert p1 > p2 > 0
p1, p2

(460038, 2822)
from tsai.data.all import *
# from tsai.learner import ts_learner
dsid = 'NATOPS'
X, y, splits = get_UCR_data(dsid, split_data=False)
tfms  = [None, [Categorize()]]
batch_tfms = TSStandardize()
ts_dls = get_ts_dls(X, y, splits=splits, tfms=tfms, batch_tfms=batch_tfms)
learn = ts_learner(ts_dls, MultiInceptionTimePlus, c_in=[4, 15, 5])
xb,yb=first(learn.dls.train)
test_eq(learn.model(xb).shape, (ts_dls.bs, ts_dls.c))
p1 = count_parameters(learn.model)
learn.freeze()
p2 = count_parameters(learn.model)
assert p1 > p2 > 0
p1, p2

(1370886, 8454)