Tensorflow,ensemble learning, feed multiple datasets of different image size into Keras Model

I have an image classification problem for which I’d like to combine different models
I’ve trained.

for example I have two models:

mobilenet_v2_100_96    = tf.keras.models.load_model("saved_model_mobilenet_v2_100_96")
mobilenet_v2_100_224    = tf.keras.models.load_model("saved_model_mobilenet_v2_100_224")

After which I lock the layers of the models and combine them into an array called "models" and create a new model to combine these like so:

ensemble_visible = [model.input for model in models]
ensemble_outputs = [model.output for model in models]

merge = tf.keras.layers.concatenate(ensemble_outputs)
merge = tf.keras.layers.Dense(200, activation='relu')(merge)
output = tf.keras.layers.Dense(200, activation='sigmoid')(merge)
model = tf.keras.models.Model(inputs=ensemble_visible, outputs=ensemble_outputs)

model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=0.005, momentum=0.9),
                loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=0.1),
                metrics=['accuracy'])

As you can see I need two differently sized inputs in order to make it work.

I obtain my two datasets through the tf.keras.preprocessing.image_dataset_from_directory function.

Now I need a way to combine these two datasets. I made sure to set the seed value of both calls the same, so the images should be in the same order.

I’ve tried splitting the datasets via this tf_unzip (https://stackoverflow.com/a/68661507/20617581) function. And combining them like this:

model_inputs = tf.data.Dataset.zip(({"inputmobilenet_v2_100_96":train_x_96, "inputmobilenet_v2_100_224":train_x_224}, train_y_96))

When I run the model.fit function however the model is not able to learn above 1% which is way less than the original models. If I run the same model using only one input everything works as expected.

My complete code is following:

import tensorflow as tf
from utils.model import Model
from tqdm import tqdm
import numpy as np
import pandas as pd
import tensorflow_datasets as tfds

def tfdata_unzip(
    tfdata: tf.data.Dataset,
    *,
    recursive: bool=False,
    eager_numpy: bool=False,
    num_parallel_calls: int=tf.data.AUTOTUNE,
):
    """
    Unzip a zipped tf.data pipeline.

    Args:
        tfdata: the :py:class:`tf.data.Dataset`
            to unzip.

        recursive: Set to ``True`` to recursively unzip
            multiple layers of zipped pipelines.
            Defaults to ``False``.

        eager_numpy: Set this to ``True`` to return
            Python lists of primitive types or
            :py:class:`numpy.array` objects. Defaults
            to ``False``.

        num_parallel_calls: The level of parallelism to
            each time we ``map()`` over a
            :py:class:`tf.data.Dataset`.

    Returns:
        Returns a Python list of either
             :py:class:`tf.data.Dataset` or NumPy
             arrays.
    """
    if isinstance(tfdata.element_spec, tf.TensorSpec):
        if eager_numpy:
            return list(tfdata.as_numpy_iterator())
        return tfdata
        
    
    def tfdata_map(i: int) -> list:
        return tfdata.map(
            lambda *cols: cols[i],
            deterministic=True,
            num_parallel_calls=num_parallel_calls,
        )

    if isinstance(tfdata.element_spec, tuple):
        num_columns = len(tfdata.element_spec)
        if recursive:
            return [
                tfdata_unzip(
                    tfdata_map(i),
                    recursive=recursive,
                    eager_numpy=eager_numpy,
                    num_parallel_calls=num_parallel_calls,
                )
                for i in range(num_columns)
            ]
        else:
            return [
                tfdata_map(i)
                for i in range(num_columns)
            ]

    raise ValueError(
        "Unknown tf.data.Dataset element_spec: " +
        str(tfdata.element_spec)
    )

print("GPU is", "available" if tf.config.list_physical_devices('GPU') else "NOT AVAILABLE")

models = []

# mobilenet_v2_050_160    = tf.keras.models.load_model("saved_model_mobilenet_v2_050_160")
mobilenet_v2_100_96    = tf.keras.models.load_model("saved_model_mobilenet_v2_100_96")
mobilenet_v2_100_224    = tf.keras.models.load_model("saved_model_mobilenet_v2_100_224")

# models.append(mobilenet_v2_050_160)
models.append(mobilenet_v2_100_96)
models.append(mobilenet_v2_100_224)

for i, model in enumerate(models):
    for layer in model.layers:
        layer.trainable = False
        layer._name = layer.name + str(i)

    model.input._name = "input_" + str(i)
    model.input.type_spec._name = "input_" + str(i)
    # model.summary()


ensemble_visible = [model.input for model in models]
ensemble_outputs = [model.output for model in models]



merge = tf.keras.layers.concatenate(ensemble_outputs)
merge = tf.keras.layers.Dense(200, activation='relu')(merge)
output = tf.keras.layers.Dense(200, activation='sigmoid')(merge)
model = tf.keras.models.Model(inputs=ensemble_visible, outputs=output)

model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=0.005, momentum=0.9),
                loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=0.1),
                metrics=['accuracy'])


def build_dataset(image_size):
    return tf.keras.preprocessing.image_dataset_from_directory(
        "img",
        validation_split=.20,
        subset="both",
        label_mode="categorical",
        # Seed needs to provided when using validation_split and shuffle = True.
        # A fixed seed is used so that the validation set is stable across runs.
        seed=123,
        image_size=image_size,
        batch_size=16
    )

batch_size = 16

def gen_datasets(image_size):
    train_ds, val_ds = build_dataset(image_size)
    class_names = tuple(train_ds.class_names)
    train_size = train_ds.cardinality().numpy()
    train_ds = train_ds.unbatch().batch(batch_size)
    train_ds = train_ds.repeat()

    normalization_layer = tf.keras.layers.Rescaling(1. / 255)
    preprocessing_model = tf.keras.Sequential([normalization_layer])
    do_data_augmentation = False  # @param {type:"boolean"}
    if do_data_augmentation:
        preprocessing_model.add(tf.keras.layers.RandomRotation(40))
        preprocessing_model.add(tf.keras.layers.RandomTranslation(0, 0.2))
        preprocessing_model.add(tf.keras.layers.RandomTranslation(0.2, 0))
    preprocessing_model.add(tf.keras.layers.RandomZoom(0.2, 0.2))
    preprocessing_model.add(tf.keras.layers.RandomFlip(mode="horizontal"))
    train_ds = train_ds.map(lambda images, labels: (preprocessing_model(images), labels))

    valid_size = val_ds.cardinality().numpy()
    val_ds = val_ds.unbatch().batch(batch_size) #self.batch_size)
    val_ds = val_ds.map(lambda images, labels:
                        (normalization_layer(images), labels))

    return train_ds, val_ds, train_size, valid_size, class_names


train_ds_96, val_ds_96, train_size, valid_size, class_names = gen_datasets([96,96])
train_ds_224, val_ds_224, train_size, valid_size, class_names = gen_datasets([224,224])

print("aquired data")

train_x_96, train_y_96 = tfdata_unzip(train_ds_96)
train_x_224, train_y_224 = tfdata_unzip(train_ds_224)

val_x_96, val_y_96 = tfdata_unzip(val_ds_96)
val_x_224, val_y_224 = tfdata_unzip(val_ds_224)

model.summary()

model_inputs = tf.data.Dataset.zip(({"inputmobilenet_v2_100_96":train_x_96, "inputmobilenet_v2_100_224":train_x_224}, train_y_96))
model_vals = tf.data.Dataset.zip(({"inputmobilenet_v2_100_96":val_x_96, "inputmobilenet_v2_100_224":val_x_224}, val_y_96))

steps_per_epoch = train_size // batch_size
validation_steps = valid_size // batch_size
hist = model.fit(
    model_inputs,
    epochs=1,
    steps_per_epoch=steps_per_epoch,
    validation_data=model_vals,
    validation_steps=validation_steps).history

model.save("joined_model")

What i’ve tried so far

I’ve tried:

splitting the dataset with:

def fit_generator(dataset,len):
  df = tfds.as_numpy(dataset)
  X_ret = np.array([])
  Y_ret = np.array([])

  for a,b in tqdm(df, total=len):
    np.append(X_ret,a)
    np.append(Y_ret,b)

  return X_ret, Y_ret

which was to slow to handle the amount of data.

I’ve tried using a generator, but it was not accepted by the model.fit function like this:

def fit_generator(dataset,len):
  df = tfds.as_numpy(dataset)
  X_ret = np.array(e[0] for e in df)
  Y_ret = np.array(e[1] for e in df)

  return X_ret, Y_ret

Lastly i’ve tried the answer provided in the problem description tf_unzip (https://stackoverflow.com/a/68661507/20617581)

But the model does not learn in any significant way.