OiO.lk Community platform!

Oio.lk is an excellent forum for developers, providing a wide range of resources, discussions, and support for those in the developer community. Join oio.lk today to connect with like-minded professionals, share insights, and stay updated on the latest trends and technologies in the development field.
  You need to log in or register to access the solved answers to this problem.
  • You have reached the maximum number of guest views allowed
  • Please register below to remove this limitation

bert model showing TypeError: Layer input_spec must be an instance of InputSpec. Got: InputSpec(shape=(None, 55, 768), ndim=3)

  • Thread starter Thread starter moin
  • Start date Start date
M

moin

Guest
I am trying to use bert pretrained model for intent classification. here is my code in jupyter notebok.

Code:
class DataPreparation:
    
    text_column = "text"
    label_column = "intent"

    def __init__(self, train, test, tokenizer: FullTokenizer, classes, max_seq_len=192):
        self.tokenizer = tokenizer
        self.max_seq_len = 0
        self.classes = classes

        ((self.train_x, self.train_y), (self.test_x, self.test_y)) = map(self.prepare_data, [train, test])

        print("max seq_len", self.max_seq_len)
        self.max_seq_len = min(self.max_seq_len, max_seq_len)
        self.train_x, self.test_x = map(self.data_padding, [self.train_x, self.test_x])

    def prepare_data(self, df):
        x, y = [], []

        for _, row in tqdm(df.iterrows()):
            text, label = row[DataPreparation.text_column], row[DataPreparation.label_column]
            tokens = self.tokenizer.tokenize(text)
            tokens = ["[CLS]"] + tokens + ["[SEP]"]
            token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
            self.max_seq_len = max(self.max_seq_len, len(token_ids))
            x.append(token_ids)
            y.append(self.classes.index(label))

        return np.array(x), np.array(y)

    def data_padding(self, ids):
        x = []
        for input_ids in ids:
            input_ids = input_ids[:min(len(input_ids), self.max_seq_len - 2)]
            input_ids = input_ids + [0] * (self.max_seq_len - len(input_ids))
            x.append(np.array(input_ids))
        return np.array(x)

tokenizer = FullTokenizer(vocab_file=os.path.join(bert_ckpt_dir, "vocab.txt"))

def model_defination(max_seq_len, bert_ckpt_file):
    
    with tf.io.gfile.GFile(bert_config_file, "r") as reader:
        bc = StockBertConfig.from_json_string(reader.read())
        bert_params = map_stock_config_to_params(bc)
        bert_params.adapter_size = None
        bert = BertModelLayer.from_params(bert_params, name="bert")
        
    input_ids = keras.layers.Input(shape=(max_seq_len, ), dtype='int32',name="input_ids")
    bert_output = bert(input_ids)

    print("bert shape", bert_output.shape)

    cls_out = keras.layers.Lambda(lambda seq: seq[:, 0, :])(bert_output)
    cls_out = keras.layers.Dropout(0.5)(cls_out)
    logits = keras.layers.Dense(units=768, activation="tanh")(cls_out)
    logits = keras.layers.Dropout(0.5)(logits)
    logits = keras.layers.Dense(units=len(classes), activation="softmax")(logits)

    model = keras.Model(inputs=input_ids, outputs=logits)
    model.build(input_shape=(None, max_seq_len))

    load_stock_weights(bert, bert_ckpt_file)

    return model

classes = train.intent.unique().tolist()

data = DataPreparation(train, test, tokenizer, classes, max_seq_len=128)

data.train_x.shape

data.train_y[0]

model = model_defination(data.max_seq_len, bert_ckpt_file)

Now when I am trying to call the function, I am getting error. The parameter values have max_seq_len = 55, bert_ckpt_file = bert checkpoint file.

when i create the model i am getting the below error:

Code:
TypeError                                 Traceback (most recent call last)
<ipython-input-17-af3e534b3882> in <module>
----> 1 model = model_defination(data.max_seq_len, bert_ckpt_file)

<ipython-input-16-a83a622dafe3> in model_defination(max_seq_len, bert_ckpt_file)
      9     input_ids = keras.layers.Input(shape=(max_seq_len, ), dtype='int32',name="input_ids")
     10     #input_spec = tf.keras.layers.InputSpec(ndim=3)
---> 11     bert_output = bert(input_ids)
     12 
     13     print("bert shape", bert_output.shape)

~\Anaconda3\lib\site-packages\keras\engine\base_layer.py in __call__(self, *args, **kwargs)
    974     # >> model = tf.keras.Model(inputs, outputs)
    975     if _in_functional_construction_mode(self, inputs, args, kwargs, input_list):
--> 976       return self._functional_construction_call(inputs, args, kwargs,
    977                                                 input_list)
    978 

~\Anaconda3\lib\site-packages\keras\engine\base_layer.py in _functional_construction_call(self, inputs, args, kwargs, input_list)
   1112         layer=self, inputs=inputs, build_graph=True, training=training_value):
   1113       # Check input assumptions set after layer building, e.g. input shape.
-> 1114       outputs = self._keras_tensor_symbolic_call(
   1115           inputs, input_masks, args, kwargs)
   1116 

~\Anaconda3\lib\site-packages\keras\engine\base_layer.py in _keras_tensor_symbolic_call(self, inputs, input_masks, args, kwargs)
    846       return tf.nest.map_structure(keras_tensor.KerasTensor, output_signature)
    847     else:
--> 848       return self._infer_output_signature(inputs, args, kwargs, input_masks)
    849 
    850   def _infer_output_signature(self, inputs, args, kwargs, input_masks):

~\Anaconda3\lib\site-packages\keras\engine\base_layer.py in _infer_output_signature(self, inputs, args, kwargs, input_masks)
    886           self._maybe_build(inputs)
    887           inputs = self._maybe_cast_inputs(inputs)
--> 888           outputs = call_fn(inputs, *args, **kwargs)
    889 
    890         self._handle_activity_regularization(inputs, outputs)

~\Anaconda3\lib\site-packages\tensorflow\python\autograph\impl\api.py in wrapper(*args, **kwargs)
    693       except Exception as e:  # pylint:disable=broad-except
    694         if hasattr(e, 'ag_error_metadata'):
--> 695           raise e.ag_error_metadata.to_exception(e)
    696         else:
    697           raise

TypeError: in user code:

    C:\Users\kamrul.moin\Anaconda3\lib\site-packages\bert\model.py:80 call  *
        output           = self.encoders_layer(embedding_output, mask=mask, training=training)
    C:\Users\kamrul.moin\Anaconda3\lib\site-packages\keras\engine\base_layer.py:1030 __call__  **
        self._maybe_build(inputs)
    C:\Users\kamrul.moin\Anaconda3\lib\site-packages\keras\engine\base_layer.py:2659 _maybe_build
        self.build(input_shapes)  # pylint:disable=not-callable
    C:\Users\kamrul.moin\Anaconda3\lib\site-packages\bert\transformer.py:209 build
        self.input_spec = keras.layers.InputSpec(shape=input_shape)
    C:\Users\kamrul.moin\Anaconda3\lib\site-packages\keras\engine\base_layer.py:2777 __setattr__
        super(tf.__internal__.tracking.AutoTrackable, self).__setattr__(name, value)  # pylint: disable=bad-super-call
    C:\Users\kamrul.moin\Anaconda3\lib\site-packages\tensorflow\python\training\tracking\base.py:530 _method_wrapper
        result = method(self, *args, **kwargs)
    C:\Users\kamrul.moin\Anaconda3\lib\site-packages\keras\engine\base_layer.py:1296 input_spec
        raise TypeError('Layer input_spec must be an instance of InputSpec. '

    TypeError: Layer input_spec must be an instance of InputSpec. Got: InputSpec(shape=(None, 55, 768), ndim=3)
<p>I am trying to use bert pretrained model for intent classification. here is my code in jupyter notebok.</p>
<pre><code>class DataPreparation:

text_column = "text"
label_column = "intent"

def __init__(self, train, test, tokenizer: FullTokenizer, classes, max_seq_len=192):
self.tokenizer = tokenizer
self.max_seq_len = 0
self.classes = classes

((self.train_x, self.train_y), (self.test_x, self.test_y)) = map(self.prepare_data, [train, test])

print("max seq_len", self.max_seq_len)
self.max_seq_len = min(self.max_seq_len, max_seq_len)
self.train_x, self.test_x = map(self.data_padding, [self.train_x, self.test_x])

def prepare_data(self, df):
x, y = [], []

for _, row in tqdm(df.iterrows()):
text, label = row[DataPreparation.text_column], row[DataPreparation.label_column]
tokens = self.tokenizer.tokenize(text)
tokens = ["[CLS]"] + tokens + ["[SEP]"]
token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
self.max_seq_len = max(self.max_seq_len, len(token_ids))
x.append(token_ids)
y.append(self.classes.index(label))

return np.array(x), np.array(y)

def data_padding(self, ids):
x = []
for input_ids in ids:
input_ids = input_ids[:min(len(input_ids), self.max_seq_len - 2)]
input_ids = input_ids + [0] * (self.max_seq_len - len(input_ids))
x.append(np.array(input_ids))
return np.array(x)

tokenizer = FullTokenizer(vocab_file=os.path.join(bert_ckpt_dir, "vocab.txt"))

def model_defination(max_seq_len, bert_ckpt_file):

with tf.io.gfile.GFile(bert_config_file, "r") as reader:
bc = StockBertConfig.from_json_string(reader.read())
bert_params = map_stock_config_to_params(bc)
bert_params.adapter_size = None
bert = BertModelLayer.from_params(bert_params, name="bert")

input_ids = keras.layers.Input(shape=(max_seq_len, ), dtype='int32',name="input_ids")
bert_output = bert(input_ids)

print("bert shape", bert_output.shape)

cls_out = keras.layers.Lambda(lambda seq: seq[:, 0, :])(bert_output)
cls_out = keras.layers.Dropout(0.5)(cls_out)
logits = keras.layers.Dense(units=768, activation="tanh")(cls_out)
logits = keras.layers.Dropout(0.5)(logits)
logits = keras.layers.Dense(units=len(classes), activation="softmax")(logits)

model = keras.Model(inputs=input_ids, outputs=logits)
model.build(input_shape=(None, max_seq_len))

load_stock_weights(bert, bert_ckpt_file)

return model

classes = train.intent.unique().tolist()

data = DataPreparation(train, test, tokenizer, classes, max_seq_len=128)

data.train_x.shape

data.train_y[0]

model = model_defination(data.max_seq_len, bert_ckpt_file)
</code></pre>
<p>Now when I am trying to call the function, I am getting error. The parameter values have max_seq_len = 55, bert_ckpt_file = bert checkpoint file.</p>
<p>when i create the model i am getting the below error:</p>
<pre><code>TypeError Traceback (most recent call last)
<ipython-input-17-af3e534b3882> in <module>
----> 1 model = model_defination(data.max_seq_len, bert_ckpt_file)

<ipython-input-16-a83a622dafe3> in model_defination(max_seq_len, bert_ckpt_file)
9 input_ids = keras.layers.Input(shape=(max_seq_len, ), dtype='int32',name="input_ids")
10 #input_spec = tf.keras.layers.InputSpec(ndim=3)
---> 11 bert_output = bert(input_ids)
12
13 print("bert shape", bert_output.shape)

~\Anaconda3\lib\site-packages\keras\engine\base_layer.py in __call__(self, *args, **kwargs)
974 # >> model = tf.keras.Model(inputs, outputs)
975 if _in_functional_construction_mode(self, inputs, args, kwargs, input_list):
--> 976 return self._functional_construction_call(inputs, args, kwargs,
977 input_list)
978

~\Anaconda3\lib\site-packages\keras\engine\base_layer.py in _functional_construction_call(self, inputs, args, kwargs, input_list)
1112 layer=self, inputs=inputs, build_graph=True, training=training_value):
1113 # Check input assumptions set after layer building, e.g. input shape.
-> 1114 outputs = self._keras_tensor_symbolic_call(
1115 inputs, input_masks, args, kwargs)
1116

~\Anaconda3\lib\site-packages\keras\engine\base_layer.py in _keras_tensor_symbolic_call(self, inputs, input_masks, args, kwargs)
846 return tf.nest.map_structure(keras_tensor.KerasTensor, output_signature)
847 else:
--> 848 return self._infer_output_signature(inputs, args, kwargs, input_masks)
849
850 def _infer_output_signature(self, inputs, args, kwargs, input_masks):

~\Anaconda3\lib\site-packages\keras\engine\base_layer.py in _infer_output_signature(self, inputs, args, kwargs, input_masks)
886 self._maybe_build(inputs)
887 inputs = self._maybe_cast_inputs(inputs)
--> 888 outputs = call_fn(inputs, *args, **kwargs)
889
890 self._handle_activity_regularization(inputs, outputs)

~\Anaconda3\lib\site-packages\tensorflow\python\autograph\impl\api.py in wrapper(*args, **kwargs)
693 except Exception as e: # pylint:disable=broad-except
694 if hasattr(e, 'ag_error_metadata'):
--> 695 raise e.ag_error_metadata.to_exception(e)
696 else:
697 raise

TypeError: in user code:

C:\Users\kamrul.moin\Anaconda3\lib\site-packages\bert\model.py:80 call *
output = self.encoders_layer(embedding_output, mask=mask, training=training)
C:\Users\kamrul.moin\Anaconda3\lib\site-packages\keras\engine\base_layer.py:1030 __call__ **
self._maybe_build(inputs)
C:\Users\kamrul.moin\Anaconda3\lib\site-packages\keras\engine\base_layer.py:2659 _maybe_build
self.build(input_shapes) # pylint:disable=not-callable
C:\Users\kamrul.moin\Anaconda3\lib\site-packages\bert\transformer.py:209 build
self.input_spec = keras.layers.InputSpec(shape=input_shape)
C:\Users\kamrul.moin\Anaconda3\lib\site-packages\keras\engine\base_layer.py:2777 __setattr__
super(tf.__internal__.tracking.AutoTrackable, self).__setattr__(name, value) # pylint: disable=bad-super-call
C:\Users\kamrul.moin\Anaconda3\lib\site-packages\tensorflow\python\training\tracking\base.py:530 _method_wrapper
result = method(self, *args, **kwargs)
C:\Users\kamrul.moin\Anaconda3\lib\site-packages\keras\engine\base_layer.py:1296 input_spec
raise TypeError('Layer input_spec must be an instance of InputSpec. '

TypeError: Layer input_spec must be an instance of InputSpec. Got: InputSpec(shape=(None, 55, 768), ndim=3)
</code></pre>
 
Top