1.2 committed in Nov 20

.vscode/settings.json

@@ -1,3 +1,3 @@
 {
-    "python.pythonPath": "C:\\Users\\atari\\Miniconda3\\envs\\chouse\\python.exe"
+    "python.pythonPath": "C:\\Users\\atari\\miniconda3\\envs\\chouse\\python.exe"
 }

cfgs/README.txt

@@ -0,0 +1,2 @@
+This file descripts how to make .lcfg file(learning configuration).
+.lcfg file has 10 pramas: {role} {bbname} {dsversion} {dsname} {epoch} {batchsize} {optmizer} {learningloss} {lossfunction} {saveperiod}

cfgs/middle_MobileNetV2_201118.lcfg

@@ -0,0 +1 @@
+middle_201118 MobileNetV2 201120 box 1000 64 RMSprop 0.001 CrossEntropy 100

datasetmaker.py

@@ -0,0 +1,50 @@
+from torch.utils.data import Dataset, DataLoader
+from torchvision import transforms, datasets
+import os
+import sys
+
+class TrainDataClass(Dataset):
+    def __init__(self, image_path, transform=None):
+        super(TrainDataClass, self).__init__()
+        self.data = datasets.ImageFolder(os.path.join(image_path,'train'), transform)
+    
+    def __getitem__(self,idx):
+        x,y = self.data[idx]
+        return x,y
+    def __len__(self):
+        return len(self.data)
+
+class TestDataClass(Dataset):
+    def __init__(self, image_path, transform=None):
+        super(TestDataClass, self).__init__()
+        self.data = datasets.ImageFolder(os.path.join(image_path,'test'), transform)
+    
+    def __getitem__(self,idx):
+        x,y = self.data[idx]
+        return x,y
+    def __len__(self):
+        return len(self.data)
+
+
+
+
+#if __name__ == "__main__":
+    #NOTE: we don't have to transform because we already shrink raw images as same as MoblieNetV2's input size
+    #TODO: PLEASE ERASE WHEN YOU REFRACORING!!
+    #dataset_root = 'C:\\Users\\atari\\workspace\\chouse_train\\datasets\\201118\\middle'
+    #classes = tuple(os.listdir(os.path.join(dataset_root,'train')))
+#
+    #tsfm=transforms.Compose([
+    #    transforms.ToTensor()
+    #])
+#
+    #train = TrainDataClass(dataset_root, tsfm)
+    #trainloader = DataLoader(train, batch_size=512, shuffle=True)
+#
+    #test = TestDataClass(dataset_root, tsfm)
+    #testloader = DataLoader(test, batch_size=512, shuffle=True)
+#
+#
+    #dataiter = iter(trainloader)
+    #images,labels = dataiter.next()
+    #print('GroundTruth: ', ' '.join('%5s' % classes[labels[j]] for j in range(512)))

learning_cofigurations/sample_shape_mobilenetv2_201118.lcfg

@@ -1,2 +0,0 @@
-{ModelName} {Epoch} {batch_size} {Optmizer} {Loss_rate} {} {} {save_period}
-MobileNetV2 5000 100 

namechanger_for_middle.py

@@ -0,0 +1,32 @@
+import torch
+import torchvision.models
+import sys
+import pandas as pd
+import os
+
+
+abs_path = os.path.dirname(os.path.abspath(__file__))
+
+#증식 하기 전의 데이터셋의 폴더 경로(상대경로)를 적어주세요
+ds_root = 'raw_ds_201118_1.0'
+
+
+
+print(abs_path)
+for root, dirs, files in os.walk(ds_root, topdown = True):
+    if '_a' in root:
+        if 'train' in root:
+            pass
+        elif 'validation' in root:
+            pass
+        else:            
+            prefix_id = root[-2:]            
+            #print(prefix_id)
+        for file_name in files:
+            path = os.path.join(abs_path,root)
+            print(path, file_name)
+            new_file_name = prefix_id + '_' + file_name
+            if prefix_id in file_name:
+                pass
+            else:
+                os.rename(os.path.join(path+'\\'+file_name),os.path.join(path+'\\'+new_file_name))

recorder.py

@@ -1,5 +0,0 @@
-import os
-import sys
-import pandas
-import torch
-import torchvision

tmp_lcfg/box_moblienetv2_10_201120.lcfg

@@ -0,0 +1 @@
+box_moblienetv2_10_201120 moblienetv2 201120 box 100 32 SGD 0.001 MultiLabelMarginLoss 1

tmp_lcfg/box_moblienetv2_11_201120.lcfg

@@ -0,0 +1 @@
+box_moblienetv2_11_201120 moblienetv2 201120 box 100 32 RMSprop 0.001 MultiLabelMarginLoss 1

tmp_lcfg/box_moblienetv2_12_201120.lcfg

@@ -0,0 +1 @@
+box_moblienetv2_12_201120 moblienetv2 201120 box 100 32 Adam 0.001 MultiLabelMarginLoss 1

tmp_lcfg/box_moblienetv2_7_201120.lcfg

@@ -0,0 +1 @@
+box_moblienetv2_7_201120 moblienetv2 201120 box 100 32 SGD 0.001 CrossEntropyLoss 1

tmp_lcfg/box_moblienetv2_8_201120.lcfg

@@ -0,0 +1 @@
+box_moblienetv2_8_201120 moblienetv2 201120 box 100 32 RMSprop 0.001 CrossEntropyLoss 1

tmp_lcfg/box_moblienetv2_9_201120.lcfg

@@ -0,0 +1 @@
+box_moblienetv2_9_201120 moblienetv2 201120 box 100 32 Adam 0.001 CrossEntropyLoss 1

tmp_lcfg/box_resnet50_49_201120.lcfg

@@ -0,0 +1 @@
+box_resnet50_49_201120 resnet50 201120 box 100 32 SGD 0.001 CrossEntropyLoss 1

tmp_lcfg/box_resnet50_50_201120.lcfg

@@ -0,0 +1 @@
+box_resnet50_50_201120 resnet50 201120 box 100 32 RMSprop 0.001 CrossEntropyLoss 1

tmp_lcfg/box_resnet50_51_201120.lcfg

@@ -0,0 +1 @@
+box_resnet50_51_201120 resnet50 201120 box 100 32 Adam 0.001 CrossEntropyLoss 1

tmp_lcfg/box_resnet50_52_201120.lcfg

@@ -0,0 +1 @@
+box_resnet50_52_201120 resnet50 201120 box 100 32 SGD 0.001 MultiLabelMarginLoss 1

tmp_lcfg/box_resnet50_53_201120.lcfg

@@ -0,0 +1 @@
+box_resnet50_53_201120 resnet50 201120 box 100 32 RMSprop 0.001 MultiLabelMarginLoss 1

tmp_lcfg/box_resnet50_54_201120.lcfg

@@ -0,0 +1 @@
+box_resnet50_54_201120 resnet50 201120 box 100 32 Adam 0.001 MultiLabelMarginLoss 1

tmp_lcfg/capitallima_moblienetv2_13_201120.lcfg

@@ -0,0 +1 @@
+capitallima_moblienetv2_13_201120 moblienetv2 201120 capitallima 100 32 SGD 0.001 CrossEntropyLoss 1

tmp_lcfg/capitallima_moblienetv2_14_201120.lcfg

@@ -0,0 +1 @@
+capitallima_moblienetv2_14_201120 moblienetv2 201120 capitallima 100 32 RMSprop 0.001 CrossEntropyLoss 1

tmp_lcfg/capitallima_moblienetv2_15_201120.lcfg

@@ -0,0 +1 @@
+capitallima_moblienetv2_15_201120 moblienetv2 201120 capitallima 100 32 Adam 0.001 CrossEntropyLoss 1

tmp_lcfg/capitallima_moblienetv2_16_201120.lcfg

@@ -0,0 +1 @@
+capitallima_moblienetv2_16_201120 moblienetv2 201120 capitallima 100 32 SGD 0.001 MultiLabelMarginLoss 1

tmp_lcfg/capitallima_moblienetv2_17_201120.lcfg

@@ -0,0 +1 @@
+capitallima_moblienetv2_17_201120 moblienetv2 201120 capitallima 100 32 RMSprop 0.001 MultiLabelMarginLoss 1

tmp_lcfg/capitallima_moblienetv2_18_201120.lcfg

@@ -0,0 +1 @@
+capitallima_moblienetv2_18_201120 moblienetv2 201120 capitallima 100 32 Adam 0.001 MultiLabelMarginLoss 1

tmp_lcfg/capitallima_resnet50_55_201120.lcfg

@@ -0,0 +1 @@
+capitallima_resnet50_55_201120 resnet50 201120 capitallima 100 32 SGD 0.001 CrossEntropyLoss 1

tmp_lcfg/capitallima_resnet50_56_201120.lcfg

@@ -0,0 +1 @@
+capitallima_resnet50_56_201120 resnet50 201120 capitallima 100 32 RMSprop 0.001 CrossEntropyLoss 1

tmp_lcfg/capitallima_resnet50_57_201120.lcfg

@@ -0,0 +1 @@
+capitallima_resnet50_57_201120 resnet50 201120 capitallima 100 32 Adam 0.001 CrossEntropyLoss 1

tmp_lcfg/capitallima_resnet50_58_201120.lcfg

@@ -0,0 +1 @@
+capitallima_resnet50_58_201120 resnet50 201120 capitallima 100 32 SGD 0.001 MultiLabelMarginLoss 1

tmp_lcfg/capitallima_resnet50_59_201120.lcfg

@@ -0,0 +1 @@
+capitallima_resnet50_59_201120 resnet50 201120 capitallima 100 32 RMSprop 0.001 MultiLabelMarginLoss 1

tmp_lcfg/capitallima_resnet50_60_201120.lcfg

@@ -0,0 +1 @@
+capitallima_resnet50_60_201120 resnet50 201120 capitallima 100 32 Adam 0.001 MultiLabelMarginLoss 1

tmp_lcfg/carexterior_moblienetv2_19_201120.lcfg

@@ -0,0 +1 @@
+carexterior_moblienetv2_19_201120 moblienetv2 201120 carexterior 100 32 SGD 0.001 CrossEntropyLoss 1

tmp_lcfg/carexterior_moblienetv2_20_201120.lcfg

@@ -0,0 +1 @@
+carexterior_moblienetv2_20_201120 moblienetv2 201120 carexterior 100 32 RMSprop 0.001 CrossEntropyLoss 1

tmp_lcfg/carexterior_moblienetv2_21_201120.lcfg

@@ -0,0 +1 @@
+carexterior_moblienetv2_21_201120 moblienetv2 201120 carexterior 100 32 Adam 0.001 CrossEntropyLoss 1

tmp_lcfg/carexterior_moblienetv2_22_201120.lcfg

@@ -0,0 +1 @@
+carexterior_moblienetv2_22_201120 moblienetv2 201120 carexterior 100 32 SGD 0.001 MultiLabelMarginLoss 1

tmp_lcfg/carexterior_moblienetv2_23_201120.lcfg

@@ -0,0 +1 @@
+carexterior_moblienetv2_23_201120 moblienetv2 201120 carexterior 100 32 RMSprop 0.001 MultiLabelMarginLoss 1

tmp_lcfg/carexterior_moblienetv2_24_201120.lcfg

@@ -0,0 +1 @@
+carexterior_moblienetv2_24_201120 moblienetv2 201120 carexterior 100 32 Adam 0.001 MultiLabelMarginLoss 1

tmp_lcfg/carexterior_resnet50_61_201120.lcfg

@@ -0,0 +1 @@
+carexterior_resnet50_61_201120 resnet50 201120 carexterior 100 32 SGD 0.001 CrossEntropyLoss 1

tmp_lcfg/carexterior_resnet50_62_201120.lcfg

@@ -0,0 +1 @@
+carexterior_resnet50_62_201120 resnet50 201120 carexterior 100 32 RMSprop 0.001 CrossEntropyLoss 1

tmp_lcfg/carexterior_resnet50_63_201120.lcfg

@@ -0,0 +1 @@
+carexterior_resnet50_63_201120 resnet50 201120 carexterior 100 32 Adam 0.001 CrossEntropyLoss 1

tmp_lcfg/carexterior_resnet50_64_201120.lcfg

@@ -0,0 +1 @@
+carexterior_resnet50_64_201120 resnet50 201120 carexterior 100 32 SGD 0.001 MultiLabelMarginLoss 1

tmp_lcfg/carexterior_resnet50_65_201120.lcfg

@@ -0,0 +1 @@
+carexterior_resnet50_65_201120 resnet50 201120 carexterior 100 32 RMSprop 0.001 MultiLabelMarginLoss 1

tmp_lcfg/carexterior_resnet50_66_201120.lcfg

@@ -0,0 +1 @@
+carexterior_resnet50_66_201120 resnet50 201120 carexterior 100 32 Adam 0.001 MultiLabelMarginLoss 1

tmp_lcfg/carlamp_moblienetv2_25_201120.lcfg

@@ -0,0 +1 @@
+carlamp_moblienetv2_25_201120 moblienetv2 201120 carlamp 100 32 SGD 0.001 CrossEntropyLoss 1

tmp_lcfg/carlamp_moblienetv2_26_201120.lcfg

@@ -0,0 +1 @@
+carlamp_moblienetv2_26_201120 moblienetv2 201120 carlamp 100 32 RMSprop 0.001 CrossEntropyLoss 1

tmp_lcfg/carlamp_moblienetv2_27_201120.lcfg

@@ -0,0 +1 @@
+carlamp_moblienetv2_27_201120 moblienetv2 201120 carlamp 100 32 Adam 0.001 CrossEntropyLoss 1

tmp_lcfg/carlamp_moblienetv2_28_201120.lcfg

@@ -0,0 +1 @@
+carlamp_moblienetv2_28_201120 moblienetv2 201120 carlamp 100 32 SGD 0.001 MultiLabelMarginLoss 1

tmp_lcfg/carlamp_moblienetv2_29_201120.lcfg

@@ -0,0 +1 @@
+carlamp_moblienetv2_29_201120 moblienetv2 201120 carlamp 100 32 RMSprop 0.001 MultiLabelMarginLoss 1

tmp_lcfg/carlamp_moblienetv2_30_201120.lcfg

@@ -0,0 +1 @@
+carlamp_moblienetv2_30_201120 moblienetv2 201120 carlamp 100 32 Adam 0.001 MultiLabelMarginLoss 1

tmp_lcfg/carlamp_resnet50_67_201120.lcfg

@@ -0,0 +1 @@
+carlamp_resnet50_67_201120 resnet50 201120 carlamp 100 32 SGD 0.001 CrossEntropyLoss 1

tmp_lcfg/carlamp_resnet50_68_201120.lcfg

@@ -0,0 +1 @@
+carlamp_resnet50_68_201120 resnet50 201120 carlamp 100 32 RMSprop 0.001 CrossEntropyLoss 1

tmp_lcfg/carlamp_resnet50_69_201120.lcfg

@@ -0,0 +1 @@
+carlamp_resnet50_69_201120 resnet50 201120 carlamp 100 32 Adam 0.001 CrossEntropyLoss 1

tmp_lcfg/carlamp_resnet50_70_201120.lcfg

@@ -0,0 +1 @@
+carlamp_resnet50_70_201120 resnet50 201120 carlamp 100 32 SGD 0.001 MultiLabelMarginLoss 1

tmp_lcfg/carlamp_resnet50_71_201120.lcfg

@@ -0,0 +1 @@
+carlamp_resnet50_71_201120 resnet50 201120 carlamp 100 32 RMSprop 0.001 MultiLabelMarginLoss 1

tmp_lcfg/carlamp_resnet50_72_201120.lcfg

@@ -0,0 +1 @@
+carlamp_resnet50_72_201120 resnet50 201120 carlamp 100 32 Adam 0.001 MultiLabelMarginLoss 1

tmp_lcfg/cosmeticbottle_moblienetv2_31_201120.lcfg

@@ -0,0 +1 @@
+cosmeticbottle_moblienetv2_31_201120 moblienetv2 201120 cosmeticbottle 100 32 SGD 0.001 CrossEntropyLoss 1

tmp_lcfg/cosmeticbottle_moblienetv2_32_201120.lcfg

@@ -0,0 +1 @@
+cosmeticbottle_moblienetv2_32_201120 moblienetv2 201120 cosmeticbottle 100 32 RMSprop 0.001 CrossEntropyLoss 1

tmp_lcfg/cosmeticbottle_moblienetv2_33_201120.lcfg

@@ -0,0 +1 @@
+cosmeticbottle_moblienetv2_33_201120 moblienetv2 201120 cosmeticbottle 100 32 Adam 0.001 CrossEntropyLoss 1

tmp_lcfg/cosmeticbottle_moblienetv2_34_201120.lcfg

@@ -0,0 +1 @@
+cosmeticbottle_moblienetv2_34_201120 moblienetv2 201120 cosmeticbottle 100 32 SGD 0.001 MultiLabelMarginLoss 1

tmp_lcfg/cosmeticbottle_moblienetv2_35_201120.lcfg

@@ -0,0 +1 @@
+cosmeticbottle_moblienetv2_35_201120 moblienetv2 201120 cosmeticbottle 100 32 RMSprop 0.001 MultiLabelMarginLoss 1

tmp_lcfg/cosmeticbottle_moblienetv2_36_201120.lcfg

@@ -0,0 +1 @@
+cosmeticbottle_moblienetv2_36_201120 moblienetv2 201120 cosmeticbottle 100 32 Adam 0.001 MultiLabelMarginLoss 1

tmp_lcfg/cosmeticbottle_resnet50_73_201120.lcfg

@@ -0,0 +1 @@
+cosmeticbottle_resnet50_73_201120 resnet50 201120 cosmeticbottle 100 32 SGD 0.001 CrossEntropyLoss 1

tmp_lcfg/cosmeticbottle_resnet50_74_201120.lcfg

@@ -0,0 +1 @@
+cosmeticbottle_resnet50_74_201120 resnet50 201120 cosmeticbottle 100 32 RMSprop 0.001 CrossEntropyLoss 1

tmp_lcfg/cosmeticbottle_resnet50_75_201120.lcfg

@@ -0,0 +1 @@
+cosmeticbottle_resnet50_75_201120 resnet50 201120 cosmeticbottle 100 32 Adam 0.001 CrossEntropyLoss 1

tmp_lcfg/cosmeticbottle_resnet50_76_201120.lcfg

@@ -0,0 +1 @@
+cosmeticbottle_resnet50_76_201120 resnet50 201120 cosmeticbottle 100 32 SGD 0.001 MultiLabelMarginLoss 1

tmp_lcfg/cosmeticbottle_resnet50_77_201120.lcfg

@@ -0,0 +1 @@
+cosmeticbottle_resnet50_77_201120 resnet50 201120 cosmeticbottle 100 32 RMSprop 0.001 MultiLabelMarginLoss 1

tmp_lcfg/cosmeticbottle_resnet50_78_201120.lcfg

@@ -0,0 +1 @@
+cosmeticbottle_resnet50_78_201120 resnet50 201120 cosmeticbottle 100 32 Adam 0.001 MultiLabelMarginLoss 1

tmp_lcfg/lcfgmaker.py

@@ -0,0 +1,33 @@
+
+import os
+
+bbnames = ['moblienetv2','resnet50']
+dsversion = 201120
+dsnames = ['middle', 'box', 'capitallima', 'carexterior', 'carlamp', 'cosmeticbottle', 'slotted']
+epoch = 100
+saveperiod = 1
+lossfunctions = ['CrossEntropyLoss', 'MultiLabelMarginLoss']
+batchsizes = ['32','64']
+optimizers = ['SGD', 'RMSprop', 'Adam']
+learningloss = 0.001
+
+
+count = 1
+
+for bbname in bbnames:
+    for dsname in dsnames:
+        for lossfunction in lossfunctions:
+            for optimizer in optimizers:
+                role = dsname+"_"+ bbname + "_" + str(count) +'_'+ str(dsversion)
+                current_dir = os.path.join(os.getcwd(),role)
+                f  = open(current_dir + '.lcfg','w')
+                if dsname == 'middle':
+                    f.write(role +' ' +bbname +' '+ str(dsversion) +' '+ str(dsname) +' '+ str(epoch) +' '+ str(batchsizes[1]) +' '+ optimizer +' '+ str(learningloss) +' '+ lossfunction +' '+ str(saveperiod))
+                    f.close()
+                else:
+                    f.write(role +' ' +bbname +' '+ str(dsversion) +' '+ str(dsname) +' '+ str(epoch) +' '+ str(batchsizes[0]) +' '+ optimizer +' '+ str(learningloss) +' '+ lossfunction +' '+ str(saveperiod))
+                    f.close()
+
+                print(current_dir)
+                count += 1
+    

tmp_lcfg/middle_moblienetv2_1_201120.lcfg

@@ -0,0 +1 @@
+middle_moblienetv2_1_201120 moblienetv2 201120 middle 100 64 SGD 0.001 CrossEntropyLoss 1

tmp_lcfg/middle_moblienetv2_2_201120.lcfg

@@ -0,0 +1 @@
+middle_moblienetv2_2_201120 moblienetv2 201120 middle 100 64 RMSprop 0.001 CrossEntropyLoss 1

tmp_lcfg/middle_moblienetv2_3_201120.lcfg

@@ -0,0 +1 @@
+middle_moblienetv2_3_201120 moblienetv2 201120 middle 100 64 Adam 0.001 CrossEntropyLoss 1

tmp_lcfg/middle_moblienetv2_4_201120.lcfg

@@ -0,0 +1 @@
+middle_moblienetv2_4_201120 moblienetv2 201120 middle 100 64 SGD 0.001 MultiLabelMarginLoss 1

tmp_lcfg/middle_moblienetv2_5_201120.lcfg

@@ -0,0 +1 @@
+middle_moblienetv2_5_201120 moblienetv2 201120 middle 100 64 RMSprop 0.001 MultiLabelMarginLoss 1

tmp_lcfg/middle_moblienetv2_6_201120.lcfg

@@ -0,0 +1 @@
+middle_moblienetv2_6_201120 moblienetv2 201120 middle 100 64 Adam 0.001 MultiLabelMarginLoss 1

tmp_lcfg/middle_resnet50_43_201120.lcfg

@@ -0,0 +1 @@
+middle_resnet50_43_201120 resnet50 201120 middle 100 64 SGD 0.001 CrossEntropyLoss 1

tmp_lcfg/middle_resnet50_44_201120.lcfg

@@ -0,0 +1 @@
+middle_resnet50_44_201120 resnet50 201120 middle 100 64 RMSprop 0.001 CrossEntropyLoss 1

tmp_lcfg/middle_resnet50_45_201120.lcfg

@@ -0,0 +1 @@
+middle_resnet50_45_201120 resnet50 201120 middle 100 64 Adam 0.001 CrossEntropyLoss 1

tmp_lcfg/middle_resnet50_46_201120.lcfg

@@ -0,0 +1 @@
+middle_resnet50_46_201120 resnet50 201120 middle 100 64 SGD 0.001 MultiLabelMarginLoss 1

tmp_lcfg/middle_resnet50_47_201120.lcfg

@@ -0,0 +1 @@
+middle_resnet50_47_201120 resnet50 201120 middle 100 64 RMSprop 0.001 MultiLabelMarginLoss 1

tmp_lcfg/middle_resnet50_48_201120.lcfg

@@ -0,0 +1 @@
+middle_resnet50_48_201120 resnet50 201120 middle 100 64 Adam 0.001 MultiLabelMarginLoss 1

tmp_lcfg/slotted_moblienetv2_37_201120.lcfg

@@ -0,0 +1 @@
+slotted_moblienetv2_37_201120 moblienetv2 201120 slotted 100 32 SGD 0.001 CrossEntropyLoss 1

tmp_lcfg/slotted_moblienetv2_38_201120.lcfg

@@ -0,0 +1 @@
+slotted_moblienetv2_38_201120 moblienetv2 201120 slotted 100 32 RMSprop 0.001 CrossEntropyLoss 1

tmp_lcfg/slotted_moblienetv2_39_201120.lcfg

@@ -0,0 +1 @@
+slotted_moblienetv2_39_201120 moblienetv2 201120 slotted 100 32 Adam 0.001 CrossEntropyLoss 1

tmp_lcfg/slotted_moblienetv2_40_201120.lcfg

@@ -0,0 +1 @@
+slotted_moblienetv2_40_201120 moblienetv2 201120 slotted 100 32 SGD 0.001 MultiLabelMarginLoss 1

tmp_lcfg/slotted_moblienetv2_41_201120.lcfg

@@ -0,0 +1 @@
+slotted_moblienetv2_41_201120 moblienetv2 201120 slotted 100 32 RMSprop 0.001 MultiLabelMarginLoss 1

tmp_lcfg/slotted_moblienetv2_42_201120.lcfg

@@ -0,0 +1 @@
+slotted_moblienetv2_42_201120 moblienetv2 201120 slotted 100 32 Adam 0.001 MultiLabelMarginLoss 1

tmp_lcfg/slotted_resnet50_79_201120.lcfg

@@ -0,0 +1 @@
+slotted_resnet50_79_201120 resnet50 201120 slotted 100 32 SGD 0.001 CrossEntropyLoss 1

tmp_lcfg/slotted_resnet50_80_201120.lcfg

@@ -0,0 +1 @@
+slotted_resnet50_80_201120 resnet50 201120 slotted 100 32 RMSprop 0.001 CrossEntropyLoss 1

tmp_lcfg/slotted_resnet50_81_201120.lcfg

@@ -0,0 +1 @@
+slotted_resnet50_81_201120 resnet50 201120 slotted 100 32 Adam 0.001 CrossEntropyLoss 1

tmp_lcfg/slotted_resnet50_82_201120.lcfg

@@ -0,0 +1 @@
+slotted_resnet50_82_201120 resnet50 201120 slotted 100 32 SGD 0.001 MultiLabelMarginLoss 1

tmp_lcfg/slotted_resnet50_83_201120.lcfg

@@ -0,0 +1 @@
+slotted_resnet50_83_201120 resnet50 201120 slotted 100 32 RMSprop 0.001 MultiLabelMarginLoss 1

tmp_lcfg/slotted_resnet50_84_201120.lcfg

@@ -0,0 +1 @@
+slotted_resnet50_84_201120 resnet50 201120 slotted 100 32 Adam 0.001 MultiLabelMarginLoss 1

train_model_1.0.py

@@ -1,91 +0,0 @@
-import argparse
-import torch
-import torchvision.models
-import sys
-import pandas
-
-
-
-'''
-filename:train_model_1.0.py
-create date: 11/17/2020 Tue
-Author: Jeong Geol Kim
-Contact: atarib4816@gmail.com
-Description: This .py file will train, save weight and save Results with parameters. parameters will input by .txt file.
-eg) train_model_1.0.py -r train_parameter.lcfg
-guidelines for train_parameter.txt
-filename format: partofarchitecture_backbone_yymmdd.lcfg
-eg) shapesort_mobilenetv2_201118.txt
-'''
-
-
-def Params_Set():
-    '''
-    Description: read options from .lcfg and initialize learning options.
-    You should write 
-    '''
-    HELP_TEXT = "how to use .py" #TODO: write description more precisely
-    N_OF_PARAMS = 3 #from now on, .txt file has 3 pramas(backbone Name, iterations, save period)
-
-    parser = argparse.ArgumentParser(description = HELP_TEXT)
-    parser.add_argument("-r", required = True, help = '.lcfg file contains with learning configures')
-    args = parser.parse_args()
-
-    f = open(args.r, 'r')    
-    p_string = f.readline()
-    f.close()
-    params = p_string.split()
-
-    if str(args.r[-4:]) != 'lcfg':
-        print ("Error: {} is wrong filename extension. Learning configures must written in .lcfg file")
-        return 
-    elif len(params) != N_OF_PARAMS: 
-        print("Error: {} is broken. please check learning configures.".format(args.r))
-        return
-    else:
-        print("---Learning Options---")
-        print("Backbone Name: {}".format(params[0]))
-        print("Number of iterations: {}".format(params[1]))
-        print("Period of saving weight: {}".format(params[2]))
-        return params, args.r
-
-
-def Preprocess():
-    pass
-
-
-
-def Model_Construct():
-    pass
-
-def Train():
-    pass
-
-def Save_Weight():
-    pass
-
-
-def Result():
-    pass
-
-
-
-
-if __name__ == "__main__":
-    #get all params from option files
-    params, name_of_lcfg = Params_Set()
-
-    Preprocess()
-
-    Model_Construct()
-
-    Train()   
-
-    Save_Weight()
-
-    Result()
-
-
-
-
-

train_model_1.2.py

@@ -0,0 +1,257 @@
+import torch
+import torchvision.models
+import sys
+import pandas as pd
+import os
+from datasetmaker import TrainDataClass, TestDataClass
+from torch.utils.data import Dataset, DataLoader
+from torchvision import transforms, datasets
+from torch import optim
+import torch.nn as nn
+import torch.nn.functional as F
+
+weight_dir_root = 'weights_backup'
+acc_dir_root = 'accs'
+
+
+'''
+filename:train_model_1.0.py
+create date: 11/17/2020 Tue
+Author: Jeong Geol Kim
+Contact: atarib4816@gmail.com
+Description: This .py file will train, save weight and save Results with parameters. parameters will input by .txt file.
+filename format: partofarchitecture_backbone_yymmdd.lcfg
+eg) shapesort_mobilenetv2_201118.txt
+'''
+
+
+def Read_Config(lcfg_name):
+    '''
+    Description: read options from .lcfg and initialize learning options.
+    .lcfg file has 10 pramas: {role} {bbname} {dsversion} {dsname} {epoch} {batchsize} {optmizer} {learningloss} {lossfunction} {saveperiod}
+    '''       
+    N_OF_PARAMS = 10
+    dict_key = ['role', 'bbname', 'dsversion', 'dsname', 'epoch', 'batchsize', 'optimizer', 'learningloss', 'lossfunction', 'saveperiod']
+
+    f = open(lcfg_name, 'r')
+    p_string = f.readline()
+    f.close()
+    params = p_string.split()
+
+    if len(params) != N_OF_PARAMS: 
+        print("Error: {} is broken. please check learning configures. Or you have to change constant N_OF_PARMAS.".format(lcfg_name))
+        return
+    else:
+        print("----Learning Options----")
+        print("Role of Classifier: {}".format(params[0]))
+        print("Backbone Name: {}".format(params[1]))
+        print("Dataset version: {}".format(params[2]))
+        print("Dataset target: {}".format(params[3]))
+        print("Number of iterations: {}".format(params[4]))
+        print("Images per Batch: {}".format(params[5]))
+        print("Optimizer Name: {}".format(params[6]))
+        print("learning loss: {}".format(params[7]))
+        print("Loss function Name: {}".format(params[8]))
+        print("Period of saving weight: {}".format(params[9]))
+        parmas_dictionary = dict(zip(dict_key, params))
+        
+        return parmas_dictionary
+
+
+def Preprocess(params, lcfg_name):
+    dataset_root = os.path.join(os.getcwd(),'datasets',params['dsversion'],params['dsname'])
+    classes = tuple(os.listdir(os.path.join(dataset_root,'train')))
+
+    #NOTE: we don't have to transform because we already shrink raw images as same as MoblieNetV2's input size
+    tsfm=transforms.Compose([
+        transforms.ToTensor()
+    ])
+
+    train = TrainDataClass(dataset_root, tsfm)
+    trainloader = DataLoader(train, batch_size=int(params['batchsize']), shuffle=True)
+
+    test = TestDataClass(dataset_root, tsfm)
+    testloader = DataLoader(test, batch_size=int(params['batchsize']), shuffle=True)
+
+    return classes, trainloader, testloader
+
+
+
+
+def Model_Construct(params, lcfg_name):
+    if params['bbname'] == 'moblienetv2':
+        model = torch.hub.load('pytorch/vision', 'mobilenet_v2', pretrained=True)
+        return model
+    if params['bbname'] == 'resnet50':
+        model = torch.hub.load('pytorch/vision', 'resnet50', pretrained=True)
+        return model
+    
+
+def Train(params, lcfg_name, model, trainLoader, testLoader, criterion, optimizer):
+    params = params
+    lcfg_name = lcfg_name
+    model = model
+    trainLoader = trainLoader
+    criterion = criterion
+    optimizer = optimizer
+    weight_dir_root = 'weights_backup'
+    acc_dir_root = 'accs'
+
+    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+    #print(torch.cuda.device_count())
+    print(device)
+    model = model.to(device)
+                
+    for epoch in range(int(params['epoch'])):    
+        running_loss = 0.0
+        for i, data in enumerate(trainLoader, 0):
+            # [inputs, labels]의 목록인 data로부터 입력을 받은 후;
+            inputs, labels = data[0].to(device), data[1].to(device)
+
+            # 변화도(Gradient) 매개변수를 0으로 만들고
+            optimizer.zero_grad()
+
+            # 순전파 + 역전파 + 최적화를 한 후
+            outputs = model(inputs).to(device)
+            loss = criterion(outputs, labels)
+            loss.backward()
+            optimizer.step()
+
+            # 통계를 출력합니다.
+            running_loss += loss.item()
+            #print(i)
+            if i % 50 == 49:    # print every 50 mini-batches
+                print('[%d, %5d] loss: %.3f' %
+                      (epoch + 1, i + 1, running_loss / 2000))                
+                running_loss = 0.0
+        correct = 0
+        total = 0
+        with torch.no_grad():
+            for data in testLoader:
+                inputs, labels = data[0].to(device), data[1].to(device)
+                outputs = model(inputs).to(device)
+                _, predicted = torch.max(outputs.data, 1)
+                total += labels.size(0)
+                correct += (predicted == labels).sum().item()
+        accuracy = (100 * correct / total)
+        print('Accuracy: {0}'.format(accuracy))
+
+        acc_name = os.path.join(os.getcwd(),acc_dir_root,lcfg_name + '.csv')
+        weight_name = os.path.join(os.getcwd(), weight_dir_root,lcfg_name,params['role']+'_'+str(epoch+1)+'.pth')
+
+        if epoch % int(params['saveperiod']) == (int(params['saveperiod']) - 1):
+            params['savedat'] = epoch + 1
+            params['accuracy'] = accuracy
+            cols = params.keys()
+            #NOTE: dictionary needs extend
+            if not os.path.isfile(acc_name):                
+                df = pd.DataFrame(index = range(0), columns = cols)
+                row = list(params.values())
+                
+                df = df.append(pd.Series(row, index=cols), ignore_index= True)
+                df.to_csv(acc_name)
+            else:
+                df = pd.read_csv(acc_name, index_col = 0)                
+                row = list(params.values())                
+                df = df.append(pd.Series(row, index=cols), ignore_index= True)
+                df.to_csv(acc_name)
+
+            if not os.path.isfile(weight_name):
+                torch.save(model.state_dict(),weight_name)
+        
+
+
+        #if epoch % int(params['saveperiod']) == (int(params['saveperiod']) - 1):            
+        #    torch.save(model.state_dict(), os.path.join(os.getcwd(), weight_dir_root,lcfg_name))
+
+    print('Finished Training')
+    return model
+
+
+'''
+def Test(model, testLoader):
+    correct = 0
+    total = 0
+    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+    with torch.no_grad():
+        for data in testLoader:
+            inputs, labels = data[0].to(device), data[1].to(device)
+            outputs = model(inputs).to(device)
+            _, predicted = torch.max(outputs.data, 1)
+            total += labels.size(0)
+            correct += (predicted == labels).sum().item()
+    accuracy = (100 * correct / total)
+    print('Accuracy: {}'.format(accuracy))
+    return accuracy
+'''
+
+
+
+def Save_Weight(params, lcfg_name):
+    DEFAULT_DIR = 'weights_backup'
+    pass
+
+
+def Save_Result(params, lcfg_name):
+    DEFAULT_DIR = 'acc_save'
+    result_file = os.path.join(os.getcwd(),DEFAULT_DIR,lcfg_name) + '.csv'
+    
+    if os.path.isfile(result_file):
+        df = pd.read_csv(result_file)
+    else:
+        cols = ['backbone', 'dataset', 'epoch', 'batchsize', 'optimizer', 'learningloss', 'lossfunction', 'saveat', 'accuracy', 'precision', 'recall']
+        #row = []
+        df = pd.DataFrame(params, columns = cols)
+        print(df)
+        df.to_csv(result_file)
+
+    pass
+
+def Set_Optimizer(params, model):    
+    if params['optimizer'] == 'RMSprop':
+        optimizer = optim.RMSprop(model.parameters(), lr = float(params['learningloss']), momentum=0.9)
+        return optimizer
+    elif params['optimizer'] == 'SGD':
+        optimizer = optim.SGD(model.parameters(), lr = float(params['learningloss']), momentum=0.9)
+        return optimizer
+    elif params['optimizer'] == 'Adam':
+        optimizer = optim.Adam(model.parameters(), lr = float(params['learningloss']))
+        return optimizer
+        
+        
+    
+
+
+
+
+if __name__ == "__main__":    
+    for root, dirs, files in os.walk('.', topdown = True):
+        for filename in files:
+            if filename.endswith(('lcfg')):                                
+                lcfg_path = os.path.join(root, filename)
+                lcfg_name = os.path.splitext(filename)[0]
+                
+                params = Read_Config(lcfg_path)
+                #print(params)
+                if not os.path.exists(os.path.join(os.getcwd(),weight_dir_root,lcfg_name)):
+                    os.makedirs(os.path.join(os.getcwd(),weight_dir_root,lcfg_name), exist_ok=True)
+                else:
+                    pass
+
+                classes, trainLoader, testLoader = Preprocess(params, lcfg_name)
+                #print(classes)
+
+                model = Model_Construct(params, lcfg_name)
+
+                #print(model.eval())
+                if params['lossfunction'] == 'CrossEntropyLoss':
+                    criterion = nn.CrossEntropyLoss()
+                else:
+                    criterion = nn.MultiLabelMarginLoss()
+
+                optimizer = Set_Optimizer(params, model)
+                
+                model = Train(params, lcfg_name, model, trainLoader, testLoader, criterion, optimizer)               
+            else:
+                pass

weights_backup/sample_shape_mobilenetv2_201118/100_sample_shape_mobilenetv2_201118.rtf

@@ -1 +0,0 @@
-{\rtf1}

weights_backup/sample_shape_mobilenetv2_201118/200_sample_shape_mobilenetv2_201118 - Copy.rtf

@@ -1 +0,0 @@
-{\rtf1}