tenorflow tf.data.experimental.rejection_resample не достигает целевого распределения даже при начальном значении

Question

Я пытаюсь проверить выборку отклонения с помощью следующего кода.Вот результат, который я получил:

target_dist [0.5, 0.5] 
initial distribution [0.8333333333333334, 0.16666666666666666]
result counts [1500, 600] 
final dist 0.7142857142857143 0.2857142857142857

Окончательное распределение не отражает целевое распределение, которое я установил.

Есть идеи?

import tensorflow as tf
import numpy as np
# everything is based on tensorflow 2.0
tf.random.set_seed(2342)


def map2label(sample):
    return tf.cast(tf.math.equal(sample, 2), tf.int32)

np_data = np.array([0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2])
target_dist = [0.5, 0.5]
init_dist = [(np_data.shape[0]-3)/np_data.shape[0], 3/np_data.shape[0]]

dataset = tf.data.Dataset.from_tensor_slices(np_data)
rej = tf.data.experimental.rejection_resample(map2label, target_dist, init_dist, 2342)   # set seed explicitly
dataset = dataset.apply(rej)

bucket_counts = [0, 0]
for i in range(100):
    for data in dataset:
        class_id, data_content = data
        bucket_counts[class_id.numpy()] += 1

print("This is your target_dist", target_dist, "This is your initial distribution", init_dist)
print("This is your result counts", bucket_counts,
      "This is your final dist", bucket_counts[0] / np.sum(bucket_counts), bucket_counts[1] / np.sum(bucket_counts))

Answer 1

Это скорее обходной путь, чем решение для решения проблемы бракованной выборки.Я использовал свой собственный алгоритм занижения выборки, а не выборку отклонения.Это работает по-другому, однако, это служит цели.Я оставлю этот вопрос здесь, так как выборка отклонения будет более эффективной в некоторых ситуациях, поэтому она по-прежнему полезна для других.

# everything below is tensorflow 2.0
import tensorflow as tf
import tensorflow_probability as tfp
import numpy as np
from termcolor import colored

seed = 1341
tf.random.set_seed(seed)
np.random.seed(seed)

# generate data and distribution
init_dist = [0.9, 0.1]
num_per_class = np.random.multinomial(10000, init_dist)
np_data = np.hstack((np.zeros(num_per_class[0], dtype=np.int32),
                     np.ones(num_per_class[1], dtype=np.int32)))
np.random.shuffle(np_data)
batch_size = 40
epochs = 1
target_dist = np.array([0.5, 0.5], dtype=np.float32)
number_of_positives = np.sum(np_data)
number_of_negatives = np_data.shape[0] - number_of_positives
number_of_examples = np_data.shape[0]
all_dataset = tf.data.Dataset.from_tensor_slices(np_data)

def filter(dist, seed=seed):
    # create a uniform distribution
    tf.random.set_seed(seed)
    normal = tfp.distributions.Uniform(low=0, high=1.0)
    def _filter(sample):
        # explanation: uniform distribution has 60% chance larger than 0.4
        # so if you want to draw a sample at 60% chance, your tocken
        # needs to be larger than (1 - 60%)
        tocken = normal.sample()
        return tf.cast(tocken > (1 - dist[tf.cast(sample, tf.int32)]), tf.bool)
    return _filter

# Undersampling
drawing_prob = np.min(init_dist) / np.array(init_dist)
dist = tf.constant(drawing_prob, tf.float32)
method2_dataset = all_dataset.filter(filter(dist))
method2_dataset = method2_dataset.batch(batch_size)

# Verify dataset is balanced

def compute_distribution(dataset, epochs, batch_size):
    data_receives = []
    # count stats to verify both batch distribution and total distribution
    bucket_counts = [0, 0]
    batch_counts = []
    for epoch in range(epochs):
        data_receives.append([])
        for data_contents in dataset:
            batch_count = [0, 0]
            for j in range(batch_size):
                try:
                    data_content = data_contents.numpy()[j]
                    data_receives[epoch].append(data_content)
                    bucket_counts[data_content] += 1
                    batch_count[data_content] += 1
                except IndexError:
                    break
            batch_counts.append(batch_count)

    # Batch distribution
    for bid, batch in enumerate(batch_counts):
        print("batch %03d, batch count [%02d, %02d], 
               batch distribution [%02.2f, %02.2f]"
               % (bid, batch[0], batch[1], batch[0] / np.sum(batch),
                  batch[1] / np.sum(batch)))

    # Total distribution
    print(colored("result counts [%d %d] 
                   final distribution [%02.2f, %02.2f]"
          % (bucket_counts[0], bucket_counts[1],
          bucket_counts[0] / np.sum(bucket_counts), 
          bucket_counts[1] / np.sum(bucket_counts)),
          "green"))

    # check if each epoch produces the same order
    for i in range(len(data_receives)-1):
        if not np.array_equal(np.array(data_receives[i]),
                              np.array(data_receives[i+1])):
            raise ValueError("Order not preserved.")
    if epochs > 1:
        print(colored("Yes, order is preserved.", "green"))

compute_distribution(method2_dataset, epochs, batch_size)