2023年7月3日发(作者：)

使⽤pytorch⾃定义DataSet，以加载图像数据集为例，实现⼀些骚操作使⽤pytorch⾃定义DataSet，以加载图像数据集为例，实现⼀些骚操作总共分为四步构造⼀个my_dataset类，继承⾃t重写__getitem__ 和__len__ 类函数建⽴两个函数find_classes、has_file_allowed_extension，直接从这copy过去建⽴my_make_dataset函数⽤来构造(path,lable)对⼀、构造⼀个my_dataset类，继承⾃t⼆、 重写__getitem__ 和__len__ 类函数要构造Dataset的⼦类，就必须要实现两个⽅法：getitem_(self, index)：根据index来返回数据集中标号为index的元素及其标签。len_(self)：返回数据集的长度。class my_dataset(Dataset): def __init__(self,root_original, root_cdtfed, transform=None): super(my_dataset, self).__init__() orm = transform _original = root_original _cdtfed = root_cdtfed al_imgs = [] _imgs = []

#add (img_path, label) to lists al_imgs = my_make_dataset(root_original, class_to_idx=None, extensions=('.jpg', '.png'), is_valid_file=None) _imgs = my_make_dataset(root_original, class_to_idx=None, extensions=('.jpg', '.png'), is_valid_file=None) # super(my_dataset, self).__init__() def __getitem__(self, index): #这个⽅法是必须要有的，⽤于按照索引读取每个元素的具体内容 fn1, label1 = al_imgs[index] #fn是图⽚path #fn和label分别获得imgs[index]也即是刚才每⾏中word[0]和word[1]的信息 fn2, label2 = _imgs[index] img1 = (fn1).convert('RGB') #按照path读⼊图⽚from PIL import Image #

按照路径读取图⽚ img2 = (fn2).convert('RGB') #按照path读⼊图⽚from PIL import Image #

按照路径读取图⽚

if orm is not None: img1 = orm(img1) #是否进⾏transform img2 = orm(img2) #是否进⾏transform img_list = [img1, img2] label = label1 name = fn1 return img_list,label,name #return很关键，return回哪些内容，那么我们在训练时循环读取每个batch时，就能获得哪些内容

def __len__(self): #这个函数也必须要写，它返回的是数据集的长度，也就是多少张图⽚，要和loader的长度作区分 return len(al_imgs)三、建⽴两个函数find_classes、has_file_allowed_extension，直接从这copy过去def find_classes(directory: str) -> Tuple[List[str], Dict[str, int]]: """Finds the class folders in a dataset. See :class:`DatasetFolder` for details. """ classes = sorted( for entry in r(directory) if _dir()) if not classes: raise FileNotFoundError(f"Couldn't find any class folder in {directory}.") class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)} return classes, class_to_idxdef has_file_allowed_extension(filename: str, extensions: Tuple[str, ...]) -> bool: """Checks if a file is an allowed extension. Args: filename (string): path to a file extensions (tuple of strings): extensions to consider (lowercase) Returns: bool: True if the filename ends with one of given extensions """ return ().endswith(extensions)建⽴my_make_dataset函数⽤来构造(path,lable)对def my_make_dataset( directory: str, class_to_idx: Optional[Dict[str, int]] = None, extensions: Optional[Tuple[str, ...]] = None, is_valid_file: Optional[Callable[[str], bool]] = None,) -> List[Tuple[str, int]]: """Generates a list of samples of a form (path_to_sample, class). See :class:`DatasetFolder` for details. Note: The class_to_idx parameter is here optional and will use the logic of the ``find_classes`` function by default. """ directory = user(directory) if class_to_idx is None: _, class_to_idx = find_classes(directory) elif not class_to_idx: raise ValueError("'class_to_index' must have at least one entry to collect any samples.") both_none = extensions is None and is_valid_file is None both_something = extensions is not None and is_valid_file is not None if both_none or both_something: raise ValueError("Both extensions and is_valid_file cannot be None or not None at the same time") if extensions is not None: def is_valid_file(x: str) -> bool: return has_file_allowed_extension(x, cast(Tuple[str, ...], extensions)) is_valid_file = cast(Callable[[str], bool], is_valid_file) instances = [] available_classes = set() for target_class in sorted(class_to_()): class_index = class_to_idx[target_class] target_dir = (directory, target_class) if not (target_dir): continue for root, _, fnames in sorted((target_dir, followlinks=True)): for fname in sorted(fnames): if is_valid_file(fname): path = (root, fname) # item = path, [int(cl) for cl in target_('_')] item = path, target_class (item) if target_class not in available_classes: available_(target_class) empty_classes = set(class_to_()) - available_classes if empty_classes: msg = f"Found no valid file for the classes {', '.join(sorted(empty_classes))}. " if extensions is not None: msg += f"Supported extensions are: {', '.join(extensions)}" raise FileNotFoundError(msg) return instances #instance:[item:(path, int(class_name)), ]附录：完整代码我这⾥传⼊两个root_dir，因为我要⽤⼀个dataset加载两个数据集，分别放在data1和data2⾥class my_dataset(Dataset): def __init__(self,root_original, root_cdtfed, transform=None): super(my_dataset, self).__init__() orm = transform orm = transform _original = root_original _cdtfed = root_cdtfed al_imgs = [] _imgs = []

#add (img_path, label) to lists al_imgs = my_make_dataset(root_original, class_to_idx=None, extensions=('.jpg', '.png'), is_valid_file=None) _imgs = my_make_dataset(root_original, class_to_idx=None, extensions=('.jpg', '.png'), is_valid_file=None) # super(my_dataset, self).__init__() def __getitem__(self, index): #这个⽅法是必须要有的，⽤于按照索引读取每个元素的具体内容 fn1, label1 = al_imgs[index] #fn是图⽚path #fn和label分别获得imgs[index]也即是刚才每⾏中word[0]和word[1]的信息 fn2, label2 = _imgs[index] img1 = (fn1).convert('RGB') #按照path读⼊图⽚from PIL import Image #

按照路径读取图⽚ img2 = (fn2).convert('RGB') #按照path读⼊图⽚from PIL import Image #

按照路径读取图⽚

if orm is not None: img1 = orm(img1) #是否进⾏transform img2 = orm(img2) #是否进⾏transform img_list = [img1, img2] label = label1 name = fn1 return img_list,label,name #return很关键，return回哪些内容，那么我们在训练时循环读取每个batch时，就能获得哪些内容

def __len__(self): #这个函数也必须要写，它返回的是数据集的长度，也就是多少张图⽚，要和loader的长度作区分 return len(al_imgs)def find_classes(directory: str) -> Tuple[List[str], Dict[str, int]]: """Finds the class folders in a dataset. See :class:`DatasetFolder` for details. """ classes = sorted( for entry in r(directory) if _dir()) if not classes: raise FileNotFoundError(f"Couldn't find any class folder in {directory}.") class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)} return classes, class_to_idxdef has_file_allowed_extension(filename: str, extensions: Tuple[str, ...]) -> bool: """Checks if a file is an allowed extension. Args: filename (string): path to a file extensions (tuple of strings): extensions to consider (lowercase) Returns: bool: True if the filename ends with one of given extensions """ return ().endswith(extensions)def my_make_dataset( directory: str, class_to_idx: Optional[Dict[str, int]] = None, extensions: Optional[Tuple[str, ...]] = None, is_valid_file: Optional[Callable[[str], bool]] = None,) -> List[Tuple[str, int]]: """Generates a list of samples of a form (path_to_sample, class). See :class:`DatasetFolder` for details. Note: The class_to_idx parameter is here optional and will use the logic of the ``find_classes`` function by default. """ directory = user(directory) if class_to_idx is None: _, class_to_idx = find_classes(directory) elif not class_to_idx: raise ValueError("'class_to_index' must have at least one entry to collect any samples.") both_none = extensions is None and is_valid_file is None both_something = extensions is not None and is_valid_file is not None if both_none or both_something: raise ValueError("Both extensions and is_valid_file cannot be None or not None at the same time") if extensions is not None: def is_valid_file(x: str) -> bool: return has_file_allowed_extension(x, cast(Tuple[str, ...], extensions)) is_valid_file = cast(Callable[[str], bool], is_valid_file) instances = [] available_classes = set() for target_class in sorted(class_to_()): class_index = class_to_idx[target_class] target_dir = (directory, target_class) if not (target_dir): continue for root, _, fnames in sorted((target_dir, followlinks=True)): for fname in sorted(fnames): if is_valid_file(fname): path = (root, fname) # item = path, [int(cl) for cl in target_('_')] item = path, target_class (item) if target_class not in available_classes: available_(target_class) empty_classes = set(class_to_()) - available_classes if empty_classes: msg = f"Found no valid file for the classes {', '.join(sorted(empty_classes))}. " if extensions is not None: msg += f"Supported extensions are: {', '.join(extensions)}" raise FileNotFoundError(msg) return instances #instance:[item:(path, int(class_name)), ]