Epoch 1/100 2021-12-16 11:23:30.105041: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:112] Plugin optimizer for device_type GPU is enabled. --------------------------------------------------------------------------- InvalidArgumentError Traceback (most recent call last) /var/folders/6t/c7tmm8qx2ql5c8ttm1vtc8000000gn/T/ipykernel_76112/581801063.py in <module> 20 21 with tf.device('/gpu:0'): ---> 22 model.fit( 23 train_dataset, 24 validation_data=validation_dataset, ~/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/utils/traceback_utils.py in error_handler(*args, **kwargs) 65 except Exception as e: # pylint: disable=broad-except 66 filtered_tb = _process_traceback_frames(e.__traceback__) ---> 67 raise e.with_traceback(filtered_tb) from None 68 finally: 69 del filtered_tb ~/miniforge3/envs/tensorflow/lib/python3.9/site-packages/tensorflow/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name) 56 try: 57 ctx.ensure_initialized() ---> 58 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name, 59 inputs, attrs, num_outputs) 60 except core._NotOkStatusException as e: InvalidArgumentError: 2 root error(s) found. (0) INVALID_ARGUMENT: input must be 4-dimensional[2,63,63,31,64] [[node 3dcnn/batch_normalization/FusedBatchNormV3 (defined at /Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/layers/normalization/batch_normalization.py:589) ]] [[3dcnn/batch_normalization/FusedBatchNormV3/_20]] (1) INVALID_ARGUMENT: input must be 4-dimensional[2,63,63,31,64] [[node 3dcnn/batch_normalization/FusedBatchNormV3 (defined at /Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/layers/normalization/batch_normalization.py:589) ]] 0 successful operations. 0 derived errors ignored. [Op:__inference_train_function_5848] Errors may have originated from an input operation. Input Source operations connected to node 3dcnn/batch_normalization/FusedBatchNormV3: In[0] 3dcnn/max_pooling3d/MaxPool3D (defined at /Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/layers/pooling.py:699) In[1] 3dcnn/batch_normalization/ReadVariableOp: In[2] 3dcnn/batch_normalization/ReadVariableOp_1: In[3] 3dcnn/batch_normalization/FusedBatchNormV3/ReadVariableOp: In[4] 3dcnn/batch_normalization/FusedBatchNormV3/ReadVariableOp_1: Operation defined at: (most recent call last) >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/runpy.py", line 197, in _run_module_as_main >>> return _run_code(code, main_globals, None, >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/runpy.py", line 87, in _run_code >>> exec(code, run_globals) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel_launcher.py", line 16, in <module> >>> app.launch_new_instance() >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/traitlets/config/application.py", line 846, in launch_instance >>> app.start() >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel/kernelapp.py", line 677, in start >>> self.io_loop.start() >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/tornado/platform/asyncio.py", line 199, in start >>> self.asyncio_loop.run_forever() >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/asyncio/base_events.py", line 596, in run_forever >>> self._run_once() >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/asyncio/base_events.py", line 1890, in _run_once >>> handle._run() >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/asyncio/events.py", line 80, in _run >>> self._context.run(self._callback, *self._args) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel/kernelbase.py", line 457, in dispatch_queue >>> await self.process_one() >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel/kernelbase.py", line 446, in process_one >>> await dispatch(*args) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel/kernelbase.py", line 353, in dispatch_shell >>> await result >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel/kernelbase.py", line 648, in execute_request >>> reply_content = await reply_content >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel/ipkernel.py", line 353, in do_execute >>> res = shell.run_cell(code, store_history=store_history, silent=silent) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel/zmqshell.py", line 533, in run_cell >>> return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/IPython/core/interactiveshell.py", line 2914, in run_cell >>> result = self._run_cell( >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/IPython/core/interactiveshell.py", line 2960, in _run_cell >>> return runner(coro) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/IPython/core/async_helpers.py", line 78, in _pseudo_sync_runner >>> coro.send(None) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/IPython/core/interactiveshell.py", line 3185, in run_cell_async >>> has_raised = await self.run_ast_nodes(code_ast.body, cell_name, >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/IPython/core/interactiveshell.py", line 3377, in run_ast_nodes >>> if (await self.run_code(code, result, async_=asy)): >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/IPython/core/interactiveshell.py", line 3457, in run_code >>> exec(code_obj, self.user_global_ns, self.user_ns) >>> >>> File "/var/folders/6t/c7tmm8qx2ql5c8ttm1vtc8000000gn/T/ipykernel_76112/581801063.py", line 22, in <module> >>> model.fit( >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 64, in error_handler >>> return fn(*args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/training.py", line 1216, in fit >>> tmp_logs = self.train_function(iterator) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/training.py", line 878, in train_function >>> return step_function(self, iterator) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/training.py", line 867, in step_function >>> outputs = model.distribute_strategy.run(run_step, args=(data,)) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/training.py", line 860, in run_step >>> outputs = model.train_step(data) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/training.py", line 808, in train_step >>> y_pred = self(x, training=True) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 64, in error_handler >>> return fn(*args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/base_layer.py", line 1083, in __call__ >>> outputs = call_fn(inputs, *args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 92, in error_handler >>> return fn(*args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/functional.py", line 451, in call >>> return self._run_internal_graph( >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/functional.py", line 589, in _run_internal_graph >>> outputs = node.layer(*args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 64, in error_handler >>> return fn(*args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/base_layer.py", line 1083, in __call__ >>> outputs = call_fn(inputs, *args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 92, in error_handler >>> return fn(*args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/layers/normalization/batch_normalization.py", line 767, in call >>> outputs = self._fused_batch_norm(inputs, training=training) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/layers/normalization/batch_normalization.py", line 623, in _fused_batch_norm >>> output, mean, variance = control_flow_util.smart_cond( >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/utils/control_flow_util.py", line 105, in smart_cond >>> return tf.__internal__.smart_cond.smart_cond( >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/layers/normalization/batch_normalization.py", line 589, in _fused_batch_norm_training >>> return tf.compat.v1.nn.fused_batch_norm( >>> Input Source operations connected to node 3dcnn/batch_normalization/FusedBatchNormV3: In[0] 3dcnn/max_pooling3d/MaxPool3D (defined at /Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/layers/pooling.py:699) In[1] 3dcnn/batch_normalization/ReadVariableOp: In[2] 3dcnn/batch_normalization/ReadVariableOp_1: In[3] 3dcnn/batch_normalization/FusedBatchNormV3/ReadVariableOp: In[4] 3dcnn/batch_normalization/FusedBatchNormV3/ReadVariableOp_1: Operation defined at: (most recent call last) >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/runpy.py", line 197, in _run_module_as_main >>> return _run_code(code, main_globals, None, >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/runpy.py", line 87, in _run_code >>> exec(code, run_globals) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel_launcher.py", line 16, in <module> >>> app.launch_new_instance() >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/traitlets/config/application.py", line 846, in launch_instance >>> app.start() >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel/kernelapp.py", line 677, in start >>> self.io_loop.start() >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/tornado/platform/asyncio.py", line 199, in start >>> self.asyncio_loop.run_forever() >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/asyncio/base_events.py", line 596, in run_forever >>> self._run_once() >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/asyncio/base_events.py", line 1890, in _run_once >>> handle._run() >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/asyncio/events.py", line 80, in _run >>> self._context.run(self._callback, *self._args) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel/kernelbase.py", line 457, in dispatch_queue >>> await self.process_one() >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel/kernelbase.py", line 446, in process_one >>> await dispatch(*args) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel/kernelbase.py", line 353, in dispatch_shell >>> await result >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel/kernelbase.py", line 648, in execute_request >>> reply_content = await reply_content >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel/ipkernel.py", line 353, in do_execute >>> res = shell.run_cell(code, store_history=store_history, silent=silent) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/ipykernel/zmqshell.py", line 533, in run_cell >>> return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/IPython/core/interactiveshell.py", line 2914, in run_cell >>> result = self._run_cell( >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/IPython/core/interactiveshell.py", line 2960, in _run_cell >>> return runner(coro) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/IPython/core/async_helpers.py", line 78, in _pseudo_sync_runner >>> coro.send(None) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/IPython/core/interactiveshell.py", line 3185, in run_cell_async >>> has_raised = await self.run_ast_nodes(code_ast.body, cell_name, >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/IPython/core/interactiveshell.py", line 3377, in run_ast_nodes >>> if (await self.run_code(code, result, async_=asy)): >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/IPython/core/interactiveshell.py", line 3457, in run_code >>> exec(code_obj, self.user_global_ns, self.user_ns) >>> >>> File "/var/folders/6t/c7tmm8qx2ql5c8ttm1vtc8000000gn/T/ipykernel_76112/581801063.py", line 22, in <module> >>> model.fit( >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 64, in error_handler >>> return fn(*args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/training.py", line 1216, in fit >>> tmp_logs = self.train_function(iterator) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/training.py", line 878, in train_function >>> return step_function(self, iterator) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/training.py", line 867, in step_function >>> outputs = model.distribute_strategy.run(run_step, args=(data,)) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/training.py", line 860, in run_step >>> outputs = model.train_step(data) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/training.py", line 808, in train_step >>> y_pred = self(x, training=True) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 64, in error_handler >>> return fn(*args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/base_layer.py", line 1083, in __call__ >>> outputs = call_fn(inputs, *args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 92, in error_handler >>> return fn(*args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/functional.py", line 451, in call >>> return self._run_internal_graph( >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/functional.py", line 589, in _run_internal_graph >>> outputs = node.layer(*args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 64, in error_handler >>> return fn(*args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/engine/base_layer.py", line 1083, in __call__ >>> outputs = call_fn(inputs, *args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 92, in error_handler >>> return fn(*args, **kwargs) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/layers/normalization/batch_normalization.py", line 767, in call >>> outputs = self._fused_batch_norm(inputs, training=training) >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/layers/normalization/batch_normalization.py", line 623, in _fused_batch_norm >>> output, mean, variance = control_flow_util.smart_cond( >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/utils/control_flow_util.py", line 105, in smart_cond >>> return tf.__internal__.smart_cond.smart_cond( >>> >>> File "/Users/michaelxcc/miniforge3/envs/tensorflow/lib/python3.9/site-packages/keras/layers/normalization/batch_normalization.py", line 589, in _fused_batch_norm_training >>> return tf.compat.v1.nn.fused_batch_norm( >>> Function call stack: train_function -> train_function