ValueError in the first epoch ending of tensorflow training
Question:
im trying to read and solve all the francois’s “deep learning with python” book (page 253 chapter 7), and there i found this arquitecture
import keras
from keras import layers
from keras.datasets import imdb
from keras.preprocessing import sequence
max_features = 2000
max_len = 500
(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
x_train = sequence.pad_sequences(x_train, maxlen=max_len)
x_test = sequence.pad_sequences(x_test, maxlen=max_len)
model = keras.models.Sequential()
model.add(layers.Embedding(max_features, 128, input_length=max_len, name='embed'))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.MaxPooling1D(5))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dense(1))
model.summary()
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
callbacks = [
keras.callbacks.TensorBoard(
log_dir='my_log_dir',
histogram_freq=1,
embeddings_freq=1,
)
]
history = model.fit(x_train, y_train,
epochs=20,
batch_size=128,
validation_split=0.2,
callbacks=callbacks)
now when i try to train in the ending of the first epoch i get this error(actually it freze on the last step of the epoch i thing in the validation proces)
Instructions for updating:
Deprecated in favor of operator or tf.math.divide.
Train on 20000 samples, validate on 5000 samples
Epoch 1/20
20000/20000 [==============================] - 52s 3ms/step - loss: 0.8421 - acc: 0.5126 - val_loss: 0.6876 - val_acc: 0.5532
ValueError Traceback (most recent call last)
<ipython-input-5-1a0c8832065d> in <module>
1 callbacks = [keras.callbacks.TensorBoard(log_dir='my_log_dir', histogram_freq=1, embeddings_freq=1,)]
----> 2 history = model.fit(x_train, y_train, epochs=20, batch_size=512, validation_split=0.2, callbacks=callbacks)
~Anaconda3envstensorflowlibsite-packageskerasenginetraining.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
1037 initial_epoch=initial_epoch,
1038 steps_per_epoch=steps_per_epoch,
-> 1039 validation_steps=validation_steps)
1040
1041 def evaluate(self, x=None, y=None,
~Anaconda3envstensorflowlibsite-packageskerasenginetraining_arrays.py in fit_loop(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)
215 for l, o in zip(out_labels, val_outs):
216 epoch_logs['val_' + l] = o
--> 217 callbacks.on_epoch_end(epoch, epoch_logs)
218 if callback_model.stop_training:
219 break
~Anaconda3envstensorflowlibsite-packageskerascallbacks.py in on_epoch_end(self, epoch, logs)
77 logs = logs or {}
78 for callback in self.callbacks:
---> 79 callback.on_epoch_end(epoch, logs)
80
81 def on_batch_begin(self, batch, logs=None):
~Anaconda3envstensorflowlibsite-packageskerascallbacks.py in on_epoch_end(self, epoch, logs)
913 "provided, and cannot be a generator.")
914 if self.embeddings_data is None and self.embeddings_freq:
--> 915 raise ValueError("To visualize embeddings, embeddings_data must "
916 "be provided.")
917 if self.validation_data and self.histogram_freq:
ValueError: To visualize embeddings, embeddings_data must be provided.
i cant figure out what could be the error U.U where do i need to set the embeddings_data? thank you in advance guys!
edit:
now changing the callback to
callbacks = [keras.callbacks.TensorBoard(log_dir='my_log_dir', histogram_freq=1, embeddings_freq=1, embeddings_data=x_test)]
i get this new error
WARNING:tensorflow:From C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonopsmath_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
WARNING:tensorflow:From C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonopsmath_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Deprecated in favor of operator or tf.math.divide.
Train on 17500 samples, validate on 7500 samples
Epoch 1/20
---------------------------------------------------------------------------
ResourceExhaustedError Traceback (most recent call last)
~Anaconda3envstensorflow_glibsite-packagestensorflowpythonclientsession.py in _do_call(self, fn, *args)
1333 try:
-> 1334 return fn(*args)
1335 except errors.OpError as e:
~Anaconda3envstensorflow_glibsite-packagestensorflowpythonclientsession.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
1318 return self._call_tf_sessionrun(
-> 1319 options, feed_dict, fetch_list, target_list, run_metadata)
1320
~Anaconda3envstensorflow_glibsite-packagestensorflowpythonclientsession.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
1406 self._session, options, feed_dict, fetch_list, target_list,
-> 1407 run_metadata)
1408
ResourceExhaustedError: OOM when allocating tensor of shape [25000,64000] and type float
[[{{node zeros}}]]
During handling of the above exception, another exception occurred:
ResourceExhaustedError Traceback (most recent call last)
<ipython-input-2-d7e702b62530> in <module>
3 callbacks = [keras.callbacks.TensorBoard(log_dir='my_log_dir', histogram_freq=1, embeddings_freq=1, embeddings_data=x_test)]
4
----> 5 history = model.fit(x_train, y_train, epochs=20, batch_size=128, validation_split=0.3, callbacks=callbacks)
~Anaconda3envstensorflow_glibsite-packageskerasenginetraining.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
1037 initial_epoch=initial_epoch,
1038 steps_per_epoch=steps_per_epoch,
-> 1039 validation_steps=validation_steps)
1040
1041 def evaluate(self, x=None, y=None,
~Anaconda3envstensorflow_glibsite-packageskerasenginetraining_arrays.py in fit_loop(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)
197 ins_batch[i] = ins_batch[i].toarray()
198
--> 199 outs = f(ins_batch)
200 outs = to_list(outs)
201 for l, o in zip(out_labels, outs):
~Anaconda3envstensorflow_glibsite-packageskerasbackendtensorflow_backend.py in __call__(self, inputs)
2695
2696 def __call__(self, inputs):
-> 2697 if hasattr(get_session(), '_make_callable_from_options'):
2698 if py_any(is_sparse(x) for x in self.inputs):
2699 if py_any(is_tensor(x) for x in inputs):
~Anaconda3envstensorflow_glibsite-packageskerasbackendtensorflow_backend.py in get_session()
204 v._keras_initialized = True
205 if uninitialized_vars:
--> 206 session.run(tf.variables_initializer(uninitialized_vars))
207 # hack for list_devices() function.
208 # list_devices() function is not available under tensorflow r1.3.
~Anaconda3envstensorflow_glibsite-packagestensorflowpythonclientsession.py in run(self, fetches, feed_dict, options, run_metadata)
927 try:
928 result = self._run(None, fetches, feed_dict, options_ptr,
--> 929 run_metadata_ptr)
930 if run_metadata:
931 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
~Anaconda3envstensorflow_glibsite-packagestensorflowpythonclientsession.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
1150 if final_fetches or final_targets or (handle and feed_dict_tensor):
1151 results = self._do_run(handle, final_targets, final_fetches,
-> 1152 feed_dict_tensor, options, run_metadata)
1153 else:
1154 results = []
~Anaconda3envstensorflow_glibsite-packagestensorflowpythonclientsession.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1326 if handle is None:
1327 return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1328 run_metadata)
1329 else:
1330 return self._do_call(_prun_fn, handle, feeds, fetches)
~Anaconda3envstensorflow_glibsite-packagestensorflowpythonclientsession.py in _do_call(self, fn, *args)
1346 pass
1347 message = error_interpolation.interpolate(message, self._graph)
-> 1348 raise type(e)(node_def, op, message)
1349
1350 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor of shape [25000,64000] and type float
[[node zeros (defined at C:UsersmimusAnaconda3envstensorflow_glibsite-packageskerascallbacks.py:880) ]]
Caused by op 'zeros', defined at:
File "C:UsersmimusAnaconda3envstensorflow_glibrunpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "C:UsersmimusAnaconda3envstensorflow_glibrunpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestraitletsconfigapplication.py", line 658, in launch_instance
app.start()
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesipykernelkernelapp.py", line 505, in start
self.io_loop.start()
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadoplatformasyncio.py", line 148, in start
self.asyncio_loop.run_forever()
File "C:UsersmimusAnaconda3envstensorflow_glibasynciobase_events.py", line 539, in run_forever
self._run_once()
File "C:UsersmimusAnaconda3envstensorflow_glibasynciobase_events.py", line 1775, in _run_once
handle._run()
File "C:UsersmimusAnaconda3envstensorflow_glibasyncioevents.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadoioloop.py", line 690, in <lambda>
lambda f: self._run_callback(functools.partial(callback, future))
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadoioloop.py", line 743, in _run_callback
ret = callback()
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadogen.py", line 781, in inner
self.run()
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadogen.py", line 742, in run
yielded = self.gen.send(value)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesipykernelkernelbase.py", line 365, in process_one
yield gen.maybe_future(dispatch(*args))
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadogen.py", line 209, in wrapper
yielded = next(result)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesipykernelkernelbase.py", line 272, in dispatch_shell
yield gen.maybe_future(handler(stream, idents, msg))
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadogen.py", line 209, in wrapper
yielded = next(result)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesipykernelkernelbase.py", line 542, in execute_request
user_expressions, allow_stdin,
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadogen.py", line 209, in wrapper
yielded = next(result)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesipykernelipkernel.py", line 294, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesipykernelzmqshell.py", line 536, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesIPythoncoreinteractiveshell.py", line 2848, in run_cell
raw_cell, store_history, silent, shell_futures)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesIPythoncoreinteractiveshell.py", line 2874, in _run_cell
return runner(coro)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesIPythoncoreasync_helpers.py", line 67, in _pseudo_sync_runner
coro.send(None)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesIPythoncoreinteractiveshell.py", line 3049, in run_cell_async
interactivity=interactivity, compiler=compiler, result=result)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesIPythoncoreinteractiveshell.py", line 3214, in run_ast_nodes
if (yield from self.run_code(code, result)):
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesIPythoncoreinteractiveshell.py", line 3296, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-2-d7e702b62530>", line 5, in <module>
history = model.fit(x_train, y_train, epochs=20, batch_size=128, validation_split=0.3, callbacks=callbacks)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packageskerasenginetraining.py", line 1039, in fit
validation_steps=validation_steps)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packageskerasenginetraining_arrays.py", line 117, in fit_loop
callbacks.set_model(callback_model)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packageskerascallbacks.py", line 54, in set_model
callback.set_model(model)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packageskerascallbacks.py", line 880, in set_model
embedding = tf.Variable(tf.zeros(shape),
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonopsarray_ops.py", line 1817, in zeros
output = fill(shape, constant(zero, dtype=dtype), name=name)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonopsgen_array_ops.py", line 3367, in fill
"Fill", dims=dims, value=value, name=name)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonframeworkop_def_library.py", line 788, in _apply_op_helper
op_def=op_def)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonutildeprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonframeworkops.py", line 3300, in create_op
op_def=op_def)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonframeworkops.py", line 1801, in __init__
self._traceback = tf_stack.extract_stack()
ResourceExhaustedError (see above for traceback): OOM when allocating tensor of shape [25000,64000] and type float
[[node zeros (defined at C:UsersmimusAnaconda3envstensorflow_glibsite-packageskerascallbacks.py:880) ]]
Answers:
Generating embedding visualization requires additional data, so you should pass some data that can be used to produce the visualization, generally this is the test set:
keras.callbacks.TensorBoard(
log_dir='my_log_dir',
histogram_freq=1,
embeddings_freq=1,
embeddings_data=x_test
)
There are also ways to pass metadata such as class labels to improve the visualization.
im trying to read and solve all the francois’s “deep learning with python” book (page 253 chapter 7), and there i found this arquitecture
import keras
from keras import layers
from keras.datasets import imdb
from keras.preprocessing import sequence
max_features = 2000
max_len = 500
(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
x_train = sequence.pad_sequences(x_train, maxlen=max_len)
x_test = sequence.pad_sequences(x_test, maxlen=max_len)
model = keras.models.Sequential()
model.add(layers.Embedding(max_features, 128, input_length=max_len, name='embed'))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.MaxPooling1D(5))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dense(1))
model.summary()
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
callbacks = [
keras.callbacks.TensorBoard(
log_dir='my_log_dir',
histogram_freq=1,
embeddings_freq=1,
)
]
history = model.fit(x_train, y_train,
epochs=20,
batch_size=128,
validation_split=0.2,
callbacks=callbacks)
now when i try to train in the ending of the first epoch i get this error(actually it freze on the last step of the epoch i thing in the validation proces)
Instructions for updating:
Deprecated in favor of operator or tf.math.divide.
Train on 20000 samples, validate on 5000 samples
Epoch 1/20
20000/20000 [==============================] - 52s 3ms/step - loss: 0.8421 - acc: 0.5126 - val_loss: 0.6876 - val_acc: 0.5532
ValueError Traceback (most recent call last)
<ipython-input-5-1a0c8832065d> in <module>
1 callbacks = [keras.callbacks.TensorBoard(log_dir='my_log_dir', histogram_freq=1, embeddings_freq=1,)]
----> 2 history = model.fit(x_train, y_train, epochs=20, batch_size=512, validation_split=0.2, callbacks=callbacks)
~Anaconda3envstensorflowlibsite-packageskerasenginetraining.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
1037 initial_epoch=initial_epoch,
1038 steps_per_epoch=steps_per_epoch,
-> 1039 validation_steps=validation_steps)
1040
1041 def evaluate(self, x=None, y=None,
~Anaconda3envstensorflowlibsite-packageskerasenginetraining_arrays.py in fit_loop(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)
215 for l, o in zip(out_labels, val_outs):
216 epoch_logs['val_' + l] = o
--> 217 callbacks.on_epoch_end(epoch, epoch_logs)
218 if callback_model.stop_training:
219 break
~Anaconda3envstensorflowlibsite-packageskerascallbacks.py in on_epoch_end(self, epoch, logs)
77 logs = logs or {}
78 for callback in self.callbacks:
---> 79 callback.on_epoch_end(epoch, logs)
80
81 def on_batch_begin(self, batch, logs=None):
~Anaconda3envstensorflowlibsite-packageskerascallbacks.py in on_epoch_end(self, epoch, logs)
913 "provided, and cannot be a generator.")
914 if self.embeddings_data is None and self.embeddings_freq:
--> 915 raise ValueError("To visualize embeddings, embeddings_data must "
916 "be provided.")
917 if self.validation_data and self.histogram_freq:
ValueError: To visualize embeddings, embeddings_data must be provided.
i cant figure out what could be the error U.U where do i need to set the embeddings_data? thank you in advance guys!
edit:
now changing the callback to
callbacks = [keras.callbacks.TensorBoard(log_dir='my_log_dir', histogram_freq=1, embeddings_freq=1, embeddings_data=x_test)]
i get this new error
WARNING:tensorflow:From C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonopsmath_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
WARNING:tensorflow:From C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonopsmath_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Deprecated in favor of operator or tf.math.divide.
Train on 17500 samples, validate on 7500 samples
Epoch 1/20
---------------------------------------------------------------------------
ResourceExhaustedError Traceback (most recent call last)
~Anaconda3envstensorflow_glibsite-packagestensorflowpythonclientsession.py in _do_call(self, fn, *args)
1333 try:
-> 1334 return fn(*args)
1335 except errors.OpError as e:
~Anaconda3envstensorflow_glibsite-packagestensorflowpythonclientsession.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
1318 return self._call_tf_sessionrun(
-> 1319 options, feed_dict, fetch_list, target_list, run_metadata)
1320
~Anaconda3envstensorflow_glibsite-packagestensorflowpythonclientsession.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
1406 self._session, options, feed_dict, fetch_list, target_list,
-> 1407 run_metadata)
1408
ResourceExhaustedError: OOM when allocating tensor of shape [25000,64000] and type float
[[{{node zeros}}]]
During handling of the above exception, another exception occurred:
ResourceExhaustedError Traceback (most recent call last)
<ipython-input-2-d7e702b62530> in <module>
3 callbacks = [keras.callbacks.TensorBoard(log_dir='my_log_dir', histogram_freq=1, embeddings_freq=1, embeddings_data=x_test)]
4
----> 5 history = model.fit(x_train, y_train, epochs=20, batch_size=128, validation_split=0.3, callbacks=callbacks)
~Anaconda3envstensorflow_glibsite-packageskerasenginetraining.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
1037 initial_epoch=initial_epoch,
1038 steps_per_epoch=steps_per_epoch,
-> 1039 validation_steps=validation_steps)
1040
1041 def evaluate(self, x=None, y=None,
~Anaconda3envstensorflow_glibsite-packageskerasenginetraining_arrays.py in fit_loop(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)
197 ins_batch[i] = ins_batch[i].toarray()
198
--> 199 outs = f(ins_batch)
200 outs = to_list(outs)
201 for l, o in zip(out_labels, outs):
~Anaconda3envstensorflow_glibsite-packageskerasbackendtensorflow_backend.py in __call__(self, inputs)
2695
2696 def __call__(self, inputs):
-> 2697 if hasattr(get_session(), '_make_callable_from_options'):
2698 if py_any(is_sparse(x) for x in self.inputs):
2699 if py_any(is_tensor(x) for x in inputs):
~Anaconda3envstensorflow_glibsite-packageskerasbackendtensorflow_backend.py in get_session()
204 v._keras_initialized = True
205 if uninitialized_vars:
--> 206 session.run(tf.variables_initializer(uninitialized_vars))
207 # hack for list_devices() function.
208 # list_devices() function is not available under tensorflow r1.3.
~Anaconda3envstensorflow_glibsite-packagestensorflowpythonclientsession.py in run(self, fetches, feed_dict, options, run_metadata)
927 try:
928 result = self._run(None, fetches, feed_dict, options_ptr,
--> 929 run_metadata_ptr)
930 if run_metadata:
931 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
~Anaconda3envstensorflow_glibsite-packagestensorflowpythonclientsession.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
1150 if final_fetches or final_targets or (handle and feed_dict_tensor):
1151 results = self._do_run(handle, final_targets, final_fetches,
-> 1152 feed_dict_tensor, options, run_metadata)
1153 else:
1154 results = []
~Anaconda3envstensorflow_glibsite-packagestensorflowpythonclientsession.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1326 if handle is None:
1327 return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1328 run_metadata)
1329 else:
1330 return self._do_call(_prun_fn, handle, feeds, fetches)
~Anaconda3envstensorflow_glibsite-packagestensorflowpythonclientsession.py in _do_call(self, fn, *args)
1346 pass
1347 message = error_interpolation.interpolate(message, self._graph)
-> 1348 raise type(e)(node_def, op, message)
1349
1350 def _extend_graph(self):
ResourceExhaustedError: OOM when allocating tensor of shape [25000,64000] and type float
[[node zeros (defined at C:UsersmimusAnaconda3envstensorflow_glibsite-packageskerascallbacks.py:880) ]]
Caused by op 'zeros', defined at:
File "C:UsersmimusAnaconda3envstensorflow_glibrunpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "C:UsersmimusAnaconda3envstensorflow_glibrunpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestraitletsconfigapplication.py", line 658, in launch_instance
app.start()
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesipykernelkernelapp.py", line 505, in start
self.io_loop.start()
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadoplatformasyncio.py", line 148, in start
self.asyncio_loop.run_forever()
File "C:UsersmimusAnaconda3envstensorflow_glibasynciobase_events.py", line 539, in run_forever
self._run_once()
File "C:UsersmimusAnaconda3envstensorflow_glibasynciobase_events.py", line 1775, in _run_once
handle._run()
File "C:UsersmimusAnaconda3envstensorflow_glibasyncioevents.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadoioloop.py", line 690, in <lambda>
lambda f: self._run_callback(functools.partial(callback, future))
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadoioloop.py", line 743, in _run_callback
ret = callback()
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadogen.py", line 781, in inner
self.run()
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadogen.py", line 742, in run
yielded = self.gen.send(value)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesipykernelkernelbase.py", line 365, in process_one
yield gen.maybe_future(dispatch(*args))
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadogen.py", line 209, in wrapper
yielded = next(result)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesipykernelkernelbase.py", line 272, in dispatch_shell
yield gen.maybe_future(handler(stream, idents, msg))
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadogen.py", line 209, in wrapper
yielded = next(result)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesipykernelkernelbase.py", line 542, in execute_request
user_expressions, allow_stdin,
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestornadogen.py", line 209, in wrapper
yielded = next(result)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesipykernelipkernel.py", line 294, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesipykernelzmqshell.py", line 536, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesIPythoncoreinteractiveshell.py", line 2848, in run_cell
raw_cell, store_history, silent, shell_futures)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesIPythoncoreinteractiveshell.py", line 2874, in _run_cell
return runner(coro)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesIPythoncoreasync_helpers.py", line 67, in _pseudo_sync_runner
coro.send(None)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesIPythoncoreinteractiveshell.py", line 3049, in run_cell_async
interactivity=interactivity, compiler=compiler, result=result)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesIPythoncoreinteractiveshell.py", line 3214, in run_ast_nodes
if (yield from self.run_code(code, result)):
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagesIPythoncoreinteractiveshell.py", line 3296, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-2-d7e702b62530>", line 5, in <module>
history = model.fit(x_train, y_train, epochs=20, batch_size=128, validation_split=0.3, callbacks=callbacks)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packageskerasenginetraining.py", line 1039, in fit
validation_steps=validation_steps)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packageskerasenginetraining_arrays.py", line 117, in fit_loop
callbacks.set_model(callback_model)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packageskerascallbacks.py", line 54, in set_model
callback.set_model(model)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packageskerascallbacks.py", line 880, in set_model
embedding = tf.Variable(tf.zeros(shape),
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonopsarray_ops.py", line 1817, in zeros
output = fill(shape, constant(zero, dtype=dtype), name=name)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonopsgen_array_ops.py", line 3367, in fill
"Fill", dims=dims, value=value, name=name)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonframeworkop_def_library.py", line 788, in _apply_op_helper
op_def=op_def)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonutildeprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonframeworkops.py", line 3300, in create_op
op_def=op_def)
File "C:UsersmimusAnaconda3envstensorflow_glibsite-packagestensorflowpythonframeworkops.py", line 1801, in __init__
self._traceback = tf_stack.extract_stack()
ResourceExhaustedError (see above for traceback): OOM when allocating tensor of shape [25000,64000] and type float
[[node zeros (defined at C:UsersmimusAnaconda3envstensorflow_glibsite-packageskerascallbacks.py:880) ]]
Generating embedding visualization requires additional data, so you should pass some data that can be used to produce the visualization, generally this is the test set:
keras.callbacks.TensorBoard(
log_dir='my_log_dir',
histogram_freq=1,
embeddings_freq=1,
embeddings_data=x_test
)
There are also ways to pass metadata such as class labels to improve the visualization.