Author
name : Janghoo Lee
github : https://github.com/ProtossDragoon
contact : dlwkdgn1@naver.com
published date : November, 2021
ThridParty
if 'COLAB_TPU_ADDR' in os.environ: # Check TPU
assert 'COLAB_TPU_ADDR' in os.environ, 'Missing TPU.'
tf_master = 'grpc://{}'.format(os.environ['COLAB_TPU_ADDR'])
TPU_ADDRESS = tf_master
resolver = tf.distribute.cluster_resolver.TPUClusterResolver(TPU_ADDRESS)
tf.config.experimental_connect_to_cluster(resolver) # initialize the colab tpu
tf.tpu.experimental.initialize_tpu_system(resolver) # https://colab.research.google.com/github/tensorflow/tpu/blob/master/tools/colab/keras_mnist_tpu.ipynb?hl=ko&authuser=2#scrollTo=Hd5zB1G7Y9-7
TRAINING_PARALLEL_STRATEGY = tf.distribute.TPUStrategy(resolver) # Choose distribution strategy for parallel processing.
tpus = tf.config.list_logical_devices('TPU')
print(f'total {len(tpus)} of TPU devices: {tpus}')
USE_TPU = True
USE_GPU = False
else:
USE_TPU = False
print('TPU Not found')
TRAINING_PARALLEL_STRATEGY = None
device_name = tf.test.gpu_device_name()
if not device_name:
USE_GPU = False
print('GPU device not found.')
else:
USE_GPU = True
!nvidia-smi -L
gpus = tf.config.list_logical_devices('GPU')
print(f'total {len(gpus)} GPU devices: {gpus}')
if USE_TPU:
CURRENT_DEVICE = 'tpu'
elif USE_GPU:
CURRENT_DEVICE = 'gpu'
else:
CURRENT_DEVICE = 'cpu'
Plain Text
복사
INFO:tensorflow:Deallocate tpu buffers before initializing tpu system.
Plain Text
복사
INFO:tensorflow:Deallocate tpu buffers before initializing tpu system.
Plain Text
복사
WARNING:tensorflow:TPU system grpc://10.25.231.146:8470 has already been initialized. Reinitializing the TPU can cause previously created variables on TPU to be lost.
Plain Text
복사
WARNING:tensorflow:TPU system grpc://10.25.231.146:8470 has already been initialized. Reinitializing the TPU can cause previously created variables on TPU to be lost.
Plain Text
복사
INFO:tensorflow:Initializing the TPU system: grpc://10.25.231.146:8470
Plain Text
복사
INFO:tensorflow:Initializing the TPU system: grpc://10.25.231.146:8470
Plain Text
복사
INFO:tensorflow:Finished initializing TPU system.
Plain Text
복사
INFO:tensorflow:Finished initializing TPU system.
Plain Text
복사
INFO:tensorflow:Found TPU system:
Plain Text
복사
INFO:tensorflow:Found TPU system:
Plain Text
복사
INFO:tensorflow:*** Num TPU Cores: 8
Plain Text
복사
INFO:tensorflow:*** Num TPU Cores: 8
Plain Text
복사
INFO:tensorflow:*** Num TPU Workers: 1
Plain Text
복사
INFO:tensorflow:*** Num TPU Workers: 1
Plain Text
복사
INFO:tensorflow:*** Num TPU Cores Per Worker: 8
Plain Text
복사
INFO:tensorflow:*** Num TPU Cores Per Worker: 8
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)
Plain Text
복사
INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)
Plain Text
복사
total 8 of TPU devices: [LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:0', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:1', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:2', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:3', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:4', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:5', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:6', device_type='TPU'), LogicalDevice(name='/job:worker/replica:0/task:0/device:TPU:7', device_type='TPU')]
Plain Text
복사
def make_dummy_data(
image_input_hw:tuple,
mask_input_hw:tuple,
class_n:int,
data_n:int=50,
):
_default_channel_n = 3
image_input_shape = [data_n] + list(image_input_hw) + [_default_channel_n]
mask_input_shape = [data_n] + list(mask_input_hw) + [class_n]
return np.zeros(image_input_shape), np.zeros(mask_input_shape)
Plain Text
복사
# tf.data.Dataset 객체 생성을 도와주는 함수
def make_tensorflow_dataset(
batched_images,
batched_masks,
batch_size:int=10,
):
"""Helper function to create tf.data.Dataset object
"""
def assert_valid_shape(
batched_images_shape,
batched_masks_shape,
):
"""Shape assertion function
"""
assert len(batched_images_shape) == 4
assert len(batched_masks_shape) == 4
assert batched_images_shape[0] == batched_masks_shape[0]
assert batched_images_shape[1] == batched_masks_shape[1]
assert batched_images_shape[2] == batched_masks_shape[2]
assert_valid_shape(batched_images.shape, batched_masks.shape)
images_tf = tf.data.Dataset.from_tensor_slices(batched_images)
masks_tf = tf.data.Dataset.from_tensor_slices(batched_masks)
dataset = tf.data.Dataset.zip((images_tf, masks_tf))
dataset = dataset.shuffle(buffer_size=100).batch(batch_size, drop_remainder=True)
return dataset
Plain Text
복사
# tf.data.Dataset 객체 생성.
# TensorFlow 의 모델에 입력할 때에는 이 객체를 주로 사용.
tf_dataset = make_tensorflow_dataset(im, ma)
# 데이터 1개만 꺼내보기
def get_a_batch(tf_dataset):
debug_iter = iter(tf_dataset)
_im, _ma = batch = next(debug_iter)
return batch
im, ma = get_a_batch(tf_dataset)
print(f'{type(im)} typed image tensor: {im.shape}')
print(f'{type(ma)} typed mask tensor: {ma.shape}')
Plain Text
복사
def auto_tpu(device='cpu'):
"""Automatically open context manager
If your colab environment is on 'tpu'
"""
def decorator(fn):
def wrapper(*args, **kwargs):
s = time.time()
if device == 'tpu':
with TRAINING_PARALLEL_STRATEGY.scope():
ret = fn(*args, **kwargs)
else:
ret = fn(*args, **kwargs)
e = time.time()
print(f'device: {repr(device)}, time elapse: {e-s:.3} second(s)')
return ret
return wrapper
return decorator
Plain Text
복사
tf.keras.backend.clear_session()
import segmentation_models as sm
sm.set_framework('tf.keras')
@auto_tpu(device=CURRENT_DEVICE)
def create_segmentation_model(class_n):
model = sm.Unet(
'vgg16',
classes=class_n,
activation='softmax',
encoder_weights=None,
encoder_freeze=False,
)
return model
model = create_segmentation_model(class_n)
model.summary()
Plain Text
복사
device: 'tpu', time elapse: 2.28 second(s)
Model: "model"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_1 (InputLayer) [(None, None, None, 0 []
3)]
block1_conv1 (Conv2D) (None, None, None, 1792 ['input_1[0][0]']
64)
block1_conv2 (Conv2D) (None, None, None, 36928 ['block1_conv1[0][0]']
64)
block1_pool (MaxPooling2D) (None, None, None, 0 ['block1_conv2[0][0]']
64)
block2_conv1 (Conv2D) (None, None, None, 73856 ['block1_pool[0][0]']
128)
block2_conv2 (Conv2D) (None, None, None, 147584 ['block2_conv1[0][0]']
128)
block2_pool (MaxPooling2D) (None, None, None, 0 ['block2_conv2[0][0]']
128)
block3_conv1 (Conv2D) (None, None, None, 295168 ['block2_pool[0][0]']
256)
block3_conv2 (Conv2D) (None, None, None, 590080 ['block3_conv1[0][0]']
256)
block3_conv3 (Conv2D) (None, None, None, 590080 ['block3_conv2[0][0]']
256)
block3_pool (MaxPooling2D) (None, None, None, 0 ['block3_conv3[0][0]']
256)
block4_conv1 (Conv2D) (None, None, None, 1180160 ['block3_pool[0][0]']
512)
block4_conv2 (Conv2D) (None, None, None, 2359808 ['block4_conv1[0][0]']
512)
block4_conv3 (Conv2D) (None, None, None, 2359808 ['block4_conv2[0][0]']
512)
block4_pool (MaxPooling2D) (None, None, None, 0 ['block4_conv3[0][0]']
512)
block5_conv1 (Conv2D) (None, None, None, 2359808 ['block4_pool[0][0]']
512)
block5_conv2 (Conv2D) (None, None, None, 2359808 ['block5_conv1[0][0]']
512)
block5_conv3 (Conv2D) (None, None, None, 2359808 ['block5_conv2[0][0]']
512)
block5_pool (MaxPooling2D) (None, None, None, 0 ['block5_conv3[0][0]']
512)
center_block1_conv (Conv2D) (None, None, None, 2359296 ['block5_pool[0][0]']
512)
center_block1_bn (BatchNormali (None, None, None, 2048 ['center_block1_conv[0][0]']
zation) 512)
center_block1_relu (Activation (None, None, None, 0 ['center_block1_bn[0][0]']
) 512)
center_block2_conv (Conv2D) (None, None, None, 2359296 ['center_block1_relu[0][0]']
512)
center_block2_bn (BatchNormali (None, None, None, 2048 ['center_block2_conv[0][0]']
zation) 512)
center_block2_relu (Activation (None, None, None, 0 ['center_block2_bn[0][0]']
) 512)
decoder_stage0_upsampling (UpS (None, None, None, 0 ['center_block2_relu[0][0]']
ampling2D) 512)
decoder_stage0_concat (Concate (None, None, None, 0 ['decoder_stage0_upsampling[0][0]
nate) 1024) ',
'block5_conv3[0][0]']
decoder_stage0a_conv (Conv2D) (None, None, None, 2359296 ['decoder_stage0_concat[0][0]']
256)
decoder_stage0a_bn (BatchNorma (None, None, None, 1024 ['decoder_stage0a_conv[0][0]']
lization) 256)
decoder_stage0a_relu (Activati (None, None, None, 0 ['decoder_stage0a_bn[0][0]']
on) 256)
decoder_stage0b_conv (Conv2D) (None, None, None, 589824 ['decoder_stage0a_relu[0][0]']
256)
decoder_stage0b_bn (BatchNorma (None, None, None, 1024 ['decoder_stage0b_conv[0][0]']
lization) 256)
decoder_stage0b_relu (Activati (None, None, None, 0 ['decoder_stage0b_bn[0][0]']
on) 256)
decoder_stage1_upsampling (UpS (None, None, None, 0 ['decoder_stage0b_relu[0][0]']
ampling2D) 256)
decoder_stage1_concat (Concate (None, None, None, 0 ['decoder_stage1_upsampling[0][0]
nate) 768) ',
'block4_conv3[0][0]']
decoder_stage1a_conv (Conv2D) (None, None, None, 884736 ['decoder_stage1_concat[0][0]']
128)
decoder_stage1a_bn (BatchNorma (None, None, None, 512 ['decoder_stage1a_conv[0][0]']
lization) 128)
decoder_stage1a_relu (Activati (None, None, None, 0 ['decoder_stage1a_bn[0][0]']
on) 128)
decoder_stage1b_conv (Conv2D) (None, None, None, 147456 ['decoder_stage1a_relu[0][0]']
128)
decoder_stage1b_bn (BatchNorma (None, None, None, 512 ['decoder_stage1b_conv[0][0]']
lization) 128)
decoder_stage1b_relu (Activati (None, None, None, 0 ['decoder_stage1b_bn[0][0]']
on) 128)
decoder_stage2_upsampling (UpS (None, None, None, 0 ['decoder_stage1b_relu[0][0]']
ampling2D) 128)
decoder_stage2_concat (Concate (None, None, None, 0 ['decoder_stage2_upsampling[0][0]
nate) 384) ',
'block3_conv3[0][0]']
decoder_stage2a_conv (Conv2D) (None, None, None, 221184 ['decoder_stage2_concat[0][0]']
64)
decoder_stage2a_bn (BatchNorma (None, None, None, 256 ['decoder_stage2a_conv[0][0]']
lization) 64)
decoder_stage2a_relu (Activati (None, None, None, 0 ['decoder_stage2a_bn[0][0]']
on) 64)
decoder_stage2b_conv (Conv2D) (None, None, None, 36864 ['decoder_stage2a_relu[0][0]']
64)
decoder_stage2b_bn (BatchNorma (None, None, None, 256 ['decoder_stage2b_conv[0][0]']
lization) 64)
decoder_stage2b_relu (Activati (None, None, None, 0 ['decoder_stage2b_bn[0][0]']
on) 64)
decoder_stage3_upsampling (UpS (None, None, None, 0 ['decoder_stage2b_relu[0][0]']
ampling2D) 64)
decoder_stage3_concat (Concate (None, None, None, 0 ['decoder_stage3_upsampling[0][0]
nate) 192) ',
'block2_conv2[0][0]']
decoder_stage3a_conv (Conv2D) (None, None, None, 55296 ['decoder_stage3_concat[0][0]']
32)
decoder_stage3a_bn (BatchNorma (None, None, None, 128 ['decoder_stage3a_conv[0][0]']
lization) 32)
decoder_stage3a_relu (Activati (None, None, None, 0 ['decoder_stage3a_bn[0][0]']
on) 32)
decoder_stage3b_conv (Conv2D) (None, None, None, 9216 ['decoder_stage3a_relu[0][0]']
32)
decoder_stage3b_bn (BatchNorma (None, None, None, 128 ['decoder_stage3b_conv[0][0]']
lization) 32)
decoder_stage3b_relu (Activati (None, None, None, 0 ['decoder_stage3b_bn[0][0]']
on) 32)
decoder_stage4_upsampling (UpS (None, None, None, 0 ['decoder_stage3b_relu[0][0]']
ampling2D) 32)
decoder_stage4a_conv (Conv2D) (None, None, None, 4608 ['decoder_stage4_upsampling[0][0]
16) ']
decoder_stage4a_bn (BatchNorma (None, None, None, 64 ['decoder_stage4a_conv[0][0]']
lization) 16)
decoder_stage4a_relu (Activati (None, None, None, 0 ['decoder_stage4a_bn[0][0]']
on) 16)
decoder_stage4b_conv (Conv2D) (None, None, None, 2304 ['decoder_stage4a_relu[0][0]']
16)
decoder_stage4b_bn (BatchNorma (None, None, None, 64 ['decoder_stage4b_conv[0][0]']
lization) 16)
decoder_stage4b_relu (Activati (None, None, None, 0 ['decoder_stage4b_bn[0][0]']
on) 16)
final_conv (Conv2D) (None, None, None, 1450 ['decoder_stage4b_relu[0][0]']
10)
softmax (Activation) (None, None, None, 0 ['final_conv[0][0]']
10)
==================================================================================================
Total params: 23,753,578
Trainable params: 23,749,546
Non-trainable params: 4,032
__________________________________________________________________________________________________
Plain Text
복사
def get_loss(class_n):
if class_n == 1:
return sm.losses.BinaryFocalLoss()
else:
return sm.losses.CategoricalFocalLoss()
def get_metrics():
return sm.metrics.IOUScore(threshold=0.5)
@auto_tpu(device=CURRENT_DEVICE)
def run(model):
model.compile('adam', get_loss(class_n), get_metrics())
model.fit(tf_dataset)
# training start
run(model)
Plain Text
복사
tf.keras.backend.clear_session()
@auto_tpu(device=CURRENT_DEVICE)
def create_augmentation_model(
image_input_hw,
mask_input_hw,
class_n:int
):
_default_channel_n = 3
# runtime augmentation pipeline
seq = tf.keras.Sequential(
[
tf.keras.layers.RandomFlip("horizontal"),
tf.keras.layers.RandomRotation(0.02),
],
name='sequential_augmentation_layers'
)
image_input_shape = list(image_input_hw) + [_default_channel_n]
mask_input_shape = list(image_input_hw) + [class_n]
x_im = tf.keras.Input(shape=image_input_shape)
x_ma = tf.keras.Input(shape=mask_input_shape)
return tf.keras.Model(
inputs=[x_im, x_ma],
outputs=[seq(x_im), seq(x_ma)],
name='sequential_augmentation_model'
)
aug_model = create_augmentation_model(
image_input_hw,
mask_input_hw,
class_n
)
aug_model.summary()
Plain Text
복사
device: 'tpu', time elapse: 0.664 second(s)
Model: "sequential_augmentation_model"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_1 (InputLayer) [(None, 224, 224, 3 0 []
)]
input_2 (InputLayer) [(None, 224, 224, 1 0 []
0)]
sequential_augmentation_layers (None, 224, 224, No 0 ['input_1[0][0]',
(Sequential) ne) 'input_2[0][0]']
==================================================================================================
Total params: 0
Trainable params: 0
Non-trainable params: 0
__________________________________________________________________________________________________
Plain Text
복사
class AugConcatedSegModel(tf.keras.Model):
def __init__(
self,
inputs=None,
outputs=None,
augmentation_model=None,
**kwargs
):
super().__init__(inputs=inputs, outputs=outputs, **kwargs)
self.augmentation_model = augmentation_model
def train_step(self, data):
im, ma = data
im, ma = self.augmentation_model((im, ma))
with tf.GradientTape() as tape:
ma_pred = self(im, training=True) # Forward pass
# Compute the loss value
# (the loss function is configured in `compile()`)
loss = self.compiled_loss(ma, ma_pred, regularization_losses=self.losses)
# Compute gradients
trainable_vars = self.trainable_variables
gradients = tape.gradient(loss, trainable_vars)
# Update weights
self.optimizer.apply_gradients(zip(gradients, trainable_vars))
# Update metrics (includes the metric that tracks the loss)
self.compiled_metrics.update_state(ma, ma_pred)
# Return a dict mapping metric names to current value
return {m.name: m.result() for m in self.metrics}
Plain Text
복사
tf.keras.backend.clear_session()
def new_concatenated_model(
image_input_hw,
mask_input_hw,
class_n
):
seg_model = create_segmentation_model(class_n)
aug_model = create_augmentation_model(
image_input_hw, mask_input_hw, class_n)
_default_channel_n = 3
image_input_shape = list(image_input_hw) + [_default_channel_n]
@auto_tpu(device=CURRENT_DEVICE)
def create():
im = seg_model.input
model = AugConcatedSegModel(
inputs=im,
outputs=seg_model(im),
augmentation_model=aug_model,
name='seg_model_train_with_aug'
)
return model
model = create()
return model
new_seg_model = new_concatenated_model(
image_input_hw,
mask_input_hw,
class_n
)
new_seg_model.summary()
Plain Text
복사
device: 'tpu', time elapse: 2.3 second(s)
device: 'tpu', time elapse: 0.645 second(s)
device: 'tpu', time elapse: 0.5 second(s)
Model: "seg_model_train_with_aug"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_1 (InputLayer) [(None, None, None, 3)] 0
model (Functional) (None, None, None, 10) 23753578
sequential_augmentation_mod [(None, 224, 224, 3), 0
el (Functional) (None, 224, 224, 10)]
=================================================================
Total params: 23,753,578
Trainable params: 23,749,546
Non-trainable params: 4,032
_________________________________________________________________
Plain Text
복사
---------------------------------------------------------------------------
Plain Text
복사
InvalidArgumentError Traceback (most recent call last)
Plain Text
복사
<ipython-input-277-07b57a78fcc0> in <module>() 1 # 학습 시작----> 2 run(new_seg_model)
Plain Text
복사
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py in _numpy(self) 1115 return self._numpy_internal() 1116 except core._NotOkStatusException as e: # pylint: disable=protected-access-> 1117 raise core._status_to_exception(e) from None # pylint: disable=protected-access 1118 1119 @property
Plain Text
복사
InvalidArgumentError: 9 root error(s) found.
(0) INVALID_ARGUMENT: {{function_node __inference_train_function_692915}} Input 0 to node `sequential_augmentation_model/sequential_augmentation_layers/random_flip/stateless_random_flip_left_right/stateless_random_uniform/StatelessRandomUniformV2` with op StatelessRandomUniformV2 must be a compile-time constant.
XLA compilation requires that operator arguments that represent shapes or dimensions be evaluated to concrete values at compile time. This error means that a shape or dimension argument could not be evaluated at compile time, usually because the value of the argument depends on a parameter to the computation, on a variable, or on a stateful operation such as a random number generator.
[[{{node sequential_augmentation_model/sequential_augmentation_layers/random_flip/stateless_random_flip_left_right/stateless_random_uniform/StatelessRandomUniformV2}}]]
[[TPUReplicate/_compile/_1646634736830564460/_4]]
(1) INVALID_ARGUMENT: {{function_node __inference_train_function_692915}} Input 0 to node `sequential_augmentation_model/sequential_augmentation_layers/random_flip/stateless_random_flip_left_right/stateless_random_uniform/StatelessRandomUniformV2` with op StatelessRandomUniformV2 must be a compile-time constant.
XLA compilation requires that operator arguments that represent shapes or dimensions be evaluated to concrete values at compile time. This error means that a shape or dimension argument could not be evaluated at compile time, usually because the value of the argument depends on a parameter to the computation, on a variable, or on a stateful operation such as a random number generator.
[[{{node sequential_augmentation_model/sequential_augmentation_layers/random_flip/stateless_random_flip_left_right/stateless_random_uniform/StatelessRandomUniformV2}}]]
[[TPUReplicate/_compile/_1646634736830564460/_4]]
[[tpu_compile_succeeded_assert/_5094882425795608634/_5/_47]]
(2) INVALID_ARGUMENT: {{function_node __inference_train_function_692915}} Input 0 to node `sequential_augmentation_model/sequential_augmentation_layers/random_flip/stateless_random_flip_left_right/stateless_random_uniform/StatelessRandomUniformV2` with op StatelessRandomUniformV2 must be a compile-time constant.
XLA compilation requires that operator arguments that represent shapes or dimensions be evaluated to concrete values at compile time. This error means that a shape or dimension argument could not be evaluated at compile time, usually because the value of the argument depends on a parameter to the computation, on a variable, or on a stateful operation such as a random number generator.
[[{{node sequential_augmentation_model/sequential_augmentation_layers/random_flip/stateless_random_flip_left_right/stateless_random_uniform/StatelessRandomUniformV2}}]]
[[TPUReplicate/_compile/_1646634736830564460/_4]]
[[tpu_compile_succeeded_assert/_5094882425795608634/_5/_159]]
(3) INVALID_ARGUMENT: {{function_node __inference_train_function_692915}} Input 0 to node `sequential_augmentation_model/sequential_a ... [truncated]
Plain Text
복사
---------------------------------------------------------------------------
Plain Text
복사
InvalidArgumentError Traceback (most recent call last)
Plain Text
복사
<ipython-input-278-8ff9c68a21c5> in <module>() 1 # 학습 시작 2 tf.config.set_soft_device_placement(True)----> 3 run(new_seg_model)
Plain Text
복사
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py in _numpy(self) 1115 return self._numpy_internal() 1116 except core._NotOkStatusException as e: # pylint: disable=protected-access-> 1117 raise core._status_to_exception(e) from None # pylint: disable=protected-access 1118 1119 @property
Plain Text
복사
InvalidArgumentError: 9 root error(s) found.
(0) INVALID_ARGUMENT: {{function_node __inference_train_function_705179}} Input 0 to node `sequential_augmentation_model/sequential_augmentation_layers/random_flip/stateless_random_flip_left_right/stateless_random_uniform/StatelessRandomUniformV2` with op StatelessRandomUniformV2 must be a compile-time constant.
XLA compilation requires that operator arguments that represent shapes or dimensions be evaluated to concrete values at compile time. This error means that a shape or dimension argument could not be evaluated at compile time, usually because the value of the argument depends on a parameter to the computation, on a variable, or on a stateful operation such as a random number generator.
[[{{node sequential_augmentation_model/sequential_augmentation_layers/random_flip/stateless_random_flip_left_right/stateless_random_uniform/StatelessRandomUniformV2}}]]
[[TPUReplicate/_compile/_6928292766130233489/_4]]
(1) INVALID_ARGUMENT: {{function_node __inference_train_function_705179}} Input 0 to node `sequential_augmentation_model/sequential_augmentation_layers/random_flip/stateless_random_flip_left_right/stateless_random_uniform/StatelessRandomUniformV2` with op StatelessRandomUniformV2 must be a compile-time constant.
XLA compilation requires that operator arguments that represent shapes or dimensions be evaluated to concrete values at compile time. This error means that a shape or dimension argument could not be evaluated at compile time, usually because the value of the argument depends on a parameter to the computation, on a variable, or on a stateful operation such as a random number generator.
[[{{node sequential_augmentation_model/sequential_augmentation_layers/random_flip/stateless_random_flip_left_right/stateless_random_uniform/StatelessRandomUniformV2}}]]
[[TPUReplicate/_compile/_6928292766130233489/_4]]
[[tpu_compile_succeeded_assert/_14898046470848909582/_5/_79]]
(2) INVALID_ARGUMENT: {{function_node __inference_train_function_705179}} Input 0 to node `sequential_augmentation_model/sequential_augmentation_layers/random_flip/stateless_random_flip_left_right/stateless_random_uniform/StatelessRandomUniformV2` with op StatelessRandomUniformV2 must be a compile-time constant.
XLA compilation requires that operator arguments that represent shapes or dimensions be evaluated to concrete values at compile time. This error means that a shape or dimension argument could not be evaluated at compile time, usually because the value of the argument depends on a parameter to the computation, on a variable, or on a stateful operation such as a random number generator.
[[{{node sequential_augmentation_model/sequential_augmentation_layers/random_flip/stateless_random_flip_left_right/stateless_random_uniform/StatelessRandomUniformV2}}]]
[[TPUReplicate/_compile/_6928292766130233489/_4]]
[[tpu_compile_succeeded_assert/_14898046470848909582/_5/_47]]
(3) INVALID_ARGUMENT: {{function_node __inference_train_function_705179}} Input 0 to node `sequential_augmentation_model/sequential_ ... [truncated]
Plain Text
복사
#FIXME : 실험중. 이게 진짜 TPU 잘못인가..? Random 때문인가?
tf.keras.backend.clear_session()
@auto_tpu(device=CURRENT_DEVICE)
def create_augmentation_model(
image_input_hw,
mask_input_hw,
class_n:int
):
_default_channel_n = 3
# 내가 추가하고 싶은 runtime augmentation pipeline
im_seq = tf.keras.Sequential(
[
tf.keras.layers.Conv2D(_default_channel_n, (3,3), padding='same'),
tf.keras.layers.Conv2D(_default_channel_n, (3,3), padding='same'),
],
name='sequential_image_augmentation_layers_debug'
)
ma_seq = tf.keras.Sequential(
[
tf.keras.layers.Conv2D(class_n, (3,3), padding='same'),
tf.keras.layers.Conv2D(class_n, (3,3), padding='same'),
],
name='sequential_mask_augmentation_layers_debug'
)
image_input_shape = list(image_input_hw) + [_default_channel_n]
mask_input_shape = list(image_input_hw) + [class_n]
x_im = tf.keras.Input(shape=image_input_shape)
x_ma = tf.keras.Input(shape=mask_input_shape)
return tf.keras.Model(
inputs=[x_im, x_ma],
outputs=[im_seq(x_im), ma_seq(x_ma)],
name='sequential_augmentation_model_debug'
)
aug_model = create_augmentation_model(
image_input_hw,
mask_input_hw,
class_n
)
aug_model.summary()
Plain Text
복사
tf.keras.backend.clear_session()
def test_time(fn, device='cpu'):
def model_creation():
backbone_model = tf.keras.applications.VGG16(
input_shape=(224, 224, 3),
include_top=False,
weights=None,
)
return backbone_model
if device == 'tpu':
with TRAINING_PARALLEL_STRATEGY.scope():
backbone_model = model_creation()
else:
backbone_model = model_creation()
_s = time.time()
fn(backbone_model, _d, device=device)
_e = time.time()
print(f'device:{repr(device)}, caching... time elapsed:{_e-_s}')
s = time.time()
fn(backbone_model, d, device=device)
e = time.time()
print(f'device:{repr(device)}, time elapsed:{e-s}')
_d = np.zeros([10, 224, 224, 3], dtype=np.uint8)
d = np.ones([10, 224, 224, 3], dtype=np.uint8)
def run(model, image_data, device):
if device == 'tpu':
with TRAINING_PARALLEL_STRATEGY.scope():
model(image_data)
else:
model(image_data)
test_time(run, device=CURRENT_DEVICE)
Plain Text
복사