ImageProcess/doc_clean_up/dataset.py

from torch.utils.data import Dataset
from PIL import Image
from torchvision import transforms
from typing import List, Tuple
import imgaug.augmenters as iaa
import numpy as np
from sklearn.model_selection import train_test_split

class UnNormalize(object):
    def __init__(self, mean, std):
        self.mean = mean
        self.std = std

    def __call__(self, tensor):
        """
        Args:
            tensor (Tensor): Tensor image of size (C, H, W) to be normalized.
        Returns:
            Tensor: Normalized image.
        """
        for t, m, s in zip(tensor, self.mean, self.std):
            t.mul_(s).add_(m)
            # The normalize code -> t.sub_(m).div_(s)
        return tensor

class DocCleanDataset(Dataset):

    @staticmethod
    def prepareDataset(dataset:str, shuffle=True):
        # imgs_dir = "dataset/raw_data/imgs_Trainblocks"
        with open(f"{dataset}/train_block_names.txt") as train_block_names_file:
            image_names = train_block_names_file.read().splitlines()
            train_img_names, eval_img_names, _, _ = train_test_split(
                image_names, image_names, test_size=0.2, random_state=1, shuffle=shuffle)
            return train_img_names, eval_img_names, dataset

    def __init__(self, img_names: List[str], imgs_dir: str, normalized_tuple: Tuple[List[float], List[float]] = None, dev=False, img_aug=False):
        if dev:
            num = int(len(img_names) * 0.01)
            img_names = img_names[0:num]
        self.img_names = img_names
        self.imgs_dir = imgs_dir
        if normalized_tuple:
            mean, std = normalized_tuple
            self.normalized = transforms.Compose([
                transforms.ToTensor(), 
                transforms.Normalize(mean=mean, std=std)
            ])
            self.aug_seq = iaa.Sometimes(0.7, iaa.OneOf([
                    iaa.SaltAndPepper(p=(0.0, 0.05)),
                    iaa.imgcorruptlike.MotionBlur(severity=2),
                    iaa.SigmoidContrast(gain=(3, 10), cutoff=(0.4, 0.6)),
                    iaa.imgcorruptlike.JpegCompression(severity=2),
                    iaa.GammaContrast((0.5, 2.0)),
                    iaa.LogContrast(gain=(0.5, 0.9)),
                    iaa.GaussianBlur(sigma=(0, 1)),
                    iaa.imgcorruptlike.SpeckleNoise(severity=1),
                    iaa.AdditiveGaussianNoise(scale=(0.03*255, 0.2*255), per_channel=True),
                    iaa.Add((-20, 20), per_channel=0.5),
                    iaa.AddToBrightness((-30, 30))
                ]))
        self.img_aug = img_aug
        self.toTensor = transforms.ToTensor()

    def __len__(self):
        return len(self.img_names)

    def __getitem__(self, index):
        img = Image.open(f"{self.imgs_dir}/{self.img_names[index]}")
        gt = Image.open(f"{self.imgs_dir}/gt{self.img_names[index]}")
        if hasattr(self, 'normalized'):
            img_np = np.array(img)
            if self.img_aug == True:
                img_np = self.aug_seq.augment_images([np.array(img)])[0]
            normalized_img = self.normalized(img_np)
            img = self.toTensor(img_np)
        else:
            img = self.toTensor(img)
            normalized_img = img
        return img, normalized_img, self.toTensor(gt)