添加文档切边项目

.gitignore

@@ -27,6 +27,14 @@ __pycache__/
 /output/
 /inference_model/
 /output_inference/
+/Dewarp/convert_result/
+/Dewarp/datasets/
+/Dewarp/outputs/
+/Dewarp/test_data/
+/Dewarp/test_imgs/
+/Dewarp/imgs_result/
+/Dewarp/video/
+/Dewarp/video_frame/
 /parts/
 /sdist/
 /var/

Dewarp/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Dewarp/corner_data_generator.py

@@ -0,0 +1,68 @@
+''' Document Localization using Recursive CNN
+ Maintainer : Khurram Javed
+ Email : kjaved@ualberta.ca '''
+
+import os
+import cv2
+import numpy as np
+
+import dataprocessor
+from utils import utils
+
+
+def args_processor():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-i", "--input-dir", help="Path to data files (Extract images using video_to_image.py first")
+    parser.add_argument("-o", "--output-dir", help="Directory to store results")
+    parser.add_argument("--dataset", default="smartdoc", help="'smartdoc' or 'selfcollected' dataset")
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = args_processor()
+    input_directory = args.input_dir
+    if not os.path.isdir(args.output_dir):
+        os.mkdir(args.output_dir)
+    import csv
+
+    # Dataset iterator
+    if args.dataset=="smartdoc":
+        dataset_test = dataprocessor.dataset.SmartDocDirectories(input_directory)
+    elif args.dataset=="selfcollected":
+        dataset_test = dataprocessor.dataset.SelfCollectedDataset(input_directory)
+    else:
+        print ("Incorrect dataset type; please choose between smartdoc or selfcollected")
+        assert(False)
+    with open(os.path.join(args.output_dir, 'gt.csv'), 'a', newline='') as csvfile:
+        spamwriter = csv.writer(csvfile, delimiter=',',
+                                quotechar='|')
+        # Counter for file naming
+        counter = 0
+        for data_elem in dataset_test.myData:
+
+            img_path = data_elem[0]
+            target = data_elem[1].reshape((4, 2))
+            img = cv2.imread(img_path)
+            if args.dataset=="selfcollected":
+                target = target / (img.shape[1], img.shape[0])
+                target = target * (1920, 1920)
+                img = cv2.resize(img, (1920, 1920))
+            corner_cords = target
+
+            for angle in range(0, 90, 1):
+                print(angle)
+                img_rotate, gt_rotate = utils.rotate(img, corner_cords, angle)
+                for random_crop in range(0, 1):
+                    img_list, gt_list = utils.get_corners(img_rotate, gt_rotate)
+                    for a in range(0, 4):
+                        counter += 1
+                        f_name = str(counter).zfill(8)
+                        print(gt_list[a])
+                        gt_store = list(np.array(gt_list[a]) / (300, 300))
+                        img_store = cv2.resize(img_list[a], (64, 64))
+                        # cv2.circle(img_store, tuple(list((np.array(gt_store)*64).astype(int))), 2, (255, 0, 0), 2)
+
+                        cv2.imwrite(os.path.join(args.output_dir, f_name + ".jpg"),
+                                    img_store, [int(cv2.IMWRITE_JPEG_QUALITY), 80])
+                        spamwriter.writerow((f_name + ".jpg", tuple(gt_store)))

Dewarp/csv_convert.py

@@ -0,0 +1,40 @@
+import argparse
+import csv
+import os
+from tqdm import tqdm
+
+
+def run(csv_path, save_folder):
+    if not os.path.exists(csv_path):
+        print(csv_path, 'not exist')
+        return
+    if not os.path.exists(save_folder):
+        os.makedirs(save_folder)
+    csv_file = csv.reader(open(csv_path, 'r'))
+    cnt = 1
+    for line in tqdm(csv_file):
+        # 文件名
+        name = line[0].split('/')[-1]
+        print(name)
+        if cnt != 1:
+            with open(save_folder + '/' + name + '.csv', 'a', encoding='utf-8') as fp:
+            # 打印四个点信息
+                ll = line[7:]
+                height = int(ll[4].split(':')[-1])
+                width = int(ll[5][16:-1])
+                # print(height*width)
+                for i in range(0, 24, 6):
+                    x = float(ll[i].split(':')[-1])
+                    y = float(ll[i+1].split(':')[-1])
+                    text = str(width*x/100) + ' ' + str(height*y/100) + '\n'
+                    fp.write(text)
+            fp.close()
+        cnt += 1
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--csv_path', type=str, default='my_data_1.0.1.csv')
+    parser.add_argument('--save_folder', type=str, default='./convert_result')
+    args = parser.parse_args()
+    run(args.csv_path, args.save_folder)

Dewarp/data.csv

@@ -0,0 +1,4 @@
+470.909090909091 511.27272727272725
+1905.0 568.0909090909091
+373.18181818181824 2618.090909090909
+2066.3636363636365 2638.5454545454545

Dewarp/data_augmentor/augmentData.py

@@ -0,0 +1,67 @@
+import os
+
+import cv2
+import numpy as np
+
+import utils
+
+
+def argsProcessor():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-i", "--dataPath", help="DataPath")
+    parser.add_argument("-o", "--outputFiles", help="outputFiles", default="bar")
+    return parser.parse_args()
+
+args = argsProcessor()
+
+output_dir = args.outputFiles
+if (not os.path.isdir(output_dir)):
+    os.mkdir(output_dir)
+
+dir = args.dataPath
+import csv
+
+with open(output_dir+"/gt.csv", 'a') as csvfile:
+    spamwriter_1 = csv.writer(csvfile, delimiter=',',
+                                quotechar='|', quoting=csv.QUOTE_MINIMAL)
+    for image in os.listdir(dir):
+        if image.endswith("jpg") or image.endswith("JPG"):
+            if os.path.isfile(dir+"/"+image+".csv"):
+                with open(dir+"/"+image+ ".csv", 'r') as csvfile:
+                    spamwriter = csv.reader(csvfile, delimiter=' ',
+                                            quotechar='|', quoting=csv.QUOTE_MINIMAL)
+                    img = cv2.imread(dir +"/"+ image)
+                    print (image)
+                    gt= []
+                    for row in spamwriter:
+                        gt.append(row)
+                        # img = cv2.circle(img, (int(float(row[0])), int(float(row[1]))), 2,(255,0,0),90)
+                    gt =np.array(gt).astype(np.float32)
+                    gt = gt / (img.shape[1], img.shape[0])
+                    gt = gt * (1080, 1080)
+                    img = cv2.resize(img, (1080, 1080))
+
+
+                    print (gt)
+
+                    for angle in range(0,271,90):
+                        img_rotate, gt_rotate = utils.rotate(img, gt, angle)
+                        for random_crop in range(0,16):
+                            img_crop, gt_crop = utils.random_crop(img_rotate, gt_rotate)
+                            mah_size = img_crop.shape
+                            img_crop = cv2.resize(img_crop, (64, 64))
+                            gt_crop = np.array(gt_crop)
+
+                            # gt_crop = gt_crop*(1.0 / mah_size[1],1.0 / mah_size[0])
+
+                            # for a in range(0,4):
+                            # no=0
+                            # for a in range(0,4):
+                            #     no+=1
+                            #     cv2.circle(img_crop, tuple(((gt_crop[a]*64).astype(int))), 2,(255-no*60,no*60,0),9)
+                            # # # cv2.imwrite("asda.jpg", img)
+
+                            cv2.imwrite(output_dir + "/" +str(angle)+str(random_crop)+ image, img_crop)
+                            spamwriter_1.writerow((str(angle)+str(random_crop)+ image, tuple(list(gt_crop))))
+

Dewarp/data_augmentor/cornerData.py

@@ -0,0 +1,63 @@
+import os
+
+import cv2
+import numpy as np
+
+import utils
+
+def argsProcessor():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-i", "--dataPath", help="DataPath")
+    parser.add_argument("-o", "--outputFiles", help="outputFiles", default="bar")
+    return parser.parse_args()
+
+args = argsProcessor()
+
+output_dir = args.outputFiles
+if (not os.path.isdir(output_dir)):
+    os.mkdir(output_dir)
+
+dir = args.dataPath
+import csv
+
+with open(output_dir+"/gt.csv", 'a') as csvfile:
+    spamwriter_1 = csv.writer(csvfile, delimiter=',',
+                                quotechar='|', quoting=csv.QUOTE_MINIMAL)
+    for image in os.listdir(dir):
+        if image.endswith("jpg"):
+            if os.path.isfile(dir+"/"+image+".csv"):
+                with open(dir+"/"+image+ ".csv", 'r') as csvfile:
+                    spamwriter = csv.reader(csvfile, delimiter=' ',
+                                            quotechar='|', quoting=csv.QUOTE_MINIMAL)
+                    img = cv2.imread(dir +"/"+ image)
+                    print (image)
+                    gt= []
+                    for row in spamwriter:
+                        gt.append(row)
+                        # img = cv2.circle(img, (int(float(row[0])), int(float(row[1]))), 2,(255,0,0),90)
+                    gt =np.array(gt).astype(np.float32)
+
+
+                    # print gt
+                    gt = gt / (img.shape[1], img.shape[0])
+
+                    gt = gt * (1080, 1080)
+
+                    img = cv2.resize(img, ( 1080,1080))
+                    # for a in range(0,4):
+                    #     img = cv2.circle(img, tuple((gt[a].astype(int))), 2, (255, 0, 0), 9)
+                    # cv2.imwrite("asda.jpg", img)
+                    # 0/0
+                    for angle in range(0,271,90):
+                        img_rotate, gt_rotate = utils.rotate(img, gt, angle)
+                        for random_crop in range(0,16):
+                            img_list, gt_list = utils.getCorners(img_rotate, gt_rotate)
+                            for a in range(0,4):
+                                print (gt_list[a])
+                                gt_store = list(np.array(gt_list[a])/(300,300))
+                                img_store = cv2.resize(img_list[a], (64,64))
+                                print (tuple(list(np.array(gt_store)*64)))
+                                # cv2.circle(img_store, tuple(list((np.array(gt_store)*64).astype(int))), 2, (255, 0, 0), 2)
+                                cv2.imwrite( output_dir+"/"+image + str(angle) +str(random_crop) + str(a) +".jpg", img_store)
+                                spamwriter_1.writerow(( image + str(angle) +str(random_crop) + str(a) +".jpg", tuple(gt_store)))

Dewarp/data_augmentor/label.py

@@ -0,0 +1,32 @@
+import os
+
+import matplotlib.image as mpimg
+import matplotlib.pyplot as plt
+
+current_file = None
+
+
+def onclick(event):
+    if event.dblclick:
+        print('button=%d, x=%d, y=%d, xdata=%f, ydata=%f' %
+              (event.button, event.x, event.y, event.xdata, event.ydata))
+        import csv
+        with open(current_file + ".csv", 'a') as csvfile:
+            spamwriter = csv.writer(csvfile, delimiter=' ',
+                                    quotechar='|', quoting=csv.QUOTE_MINIMAL)
+            spamwriter.writerow([str(event.xdata), str(event.ydata)])
+
+
+dir = "../data1/"
+for image in os.listdir(dir):
+    if image.endswith("jpg") or image.endswith("JPG"):
+        if os.path.isfile(dir + image + ".csv"):
+            pass
+        else:
+            fig = plt.figure()
+            cid = fig.canvas.mpl_connect('button_press_event', onclick)
+            print(dir + image)
+            current_file = dir + image
+            img = mpimg.imread(dir + image)
+            plt.imshow(img)
+            plt.show()

Dewarp/data_augmentor/testData.py

@@ -0,0 +1,21 @@
+import os
+import numpy as np
+import cv2
+import csv
+
+dir = "../data1/"
+for image in os.listdir(dir):
+    if image.endswith("jpg") or image.endswith("JPG"):
+        if os.path.isfile(dir+image+".csv"):
+            with open(dir+image+ ".csv", 'r') as csvfile:
+                spamwriter = csv.reader(csvfile, delimiter=' ',
+                                        quotechar='|', quoting=csv.QUOTE_MINIMAL)
+                img = cv2.imread(dir + image)
+                no = 0
+                for row in spamwriter:
+                    no+=1
+                    print (row)
+                    img = cv2.circle(img, (int(float(row[0])), int(float(row[1]))), 2,(255-no*60,no*60,0),90)
+                img = cv2.resize(img, (300,300))
+                cv2.imshow("a",img)
+                cv2.waitKey(0)

Dewarp/dataprocessor/__init__.py

@@ -0,0 +1,4 @@
+from dataprocessor.datasetfactory import *
+from dataprocessor.dataloaders import *
+from dataprocessor.loaderfactory import *
+from dataprocessor.dataset import *

Dewarp/dataprocessor/dataloaders.py

@@ -0,0 +1,117 @@
+''' Document Localization using Recursive CNN
+ Maintainer : Khurram Javed
+ Email : kjaved@ualberta.ca '''
+
+import logging
+
+import PIL
+import torch.utils.data as td
+import tqdm
+from PIL import Image
+
+logger = logging.getLogger('iCARL')
+
+
+class HddLoader(td.Dataset):
+    def __init__(self, data, transform=None, cuda=False):
+        self.data = data
+
+        self.transform = transform
+        self.cuda = cuda
+        self.len = len(data[0])
+
+    def __len__(self):
+        return self.len
+
+    def __getitem__(self, index):
+        '''
+        Replacing this with a more efficient implemnetation selection; removing c
+        :param index: 
+        :return: 
+        '''
+        assert (index < len(self.data[0]))
+        assert (index < self.len)
+        img = Image.open(self.data[0][index])
+        target = self.data[1][index]
+        if self.transform is not None:
+            img = self.transform(img)
+
+        return img, target
+
+class RamLoader(td.Dataset):
+    def __init__(self, data, transform=None, cuda=False):
+        self.data = data
+
+        self.transform = transform
+        self.cuda = cuda
+        self.len = len(data[0])
+        self.loadInRam()
+
+    def loadInRam(self):
+        self.loaded_data = []
+        logger.info("Loading data in RAM")
+        for i in tqdm.tqdm(self.data[0]):
+            img = Image.open(i)
+            if self.transform is not None:
+                img = self.transform(img)
+            self.loaded_data.append(img)
+
+    def __len__(self):
+        return self.len
+
+    def __getitem__(self, index):
+        '''
+        Replacing this with a more efficient implemnetation selection; removing c
+        :param index: 
+        :return: 
+        '''
+        assert (index < len(self.data[0]))
+        assert (index < self.len)
+        target = self.data[1][index]
+        img = self.loaded_data[index]
+        return img, target
+
+
+
+class SingleFolderLoaderResized(td.Dataset):
+    '''
+    This loader class decodes all the images into tensors; this removes the decoding time.
+    '''
+
+    def __init__(self, data, transform=None, cuda=False):
+
+        self.data = data
+
+        self.transform = transform
+        self.cuda = cuda
+        self.len = len(data)
+        self.decodeImages()
+
+    def decodeImages(self):
+        self.loaded_data = []
+        logger.info("Resizing Images")
+        for i in tqdm.tqdm(self.data):
+            i = i[0]
+            img = Image.open(i)
+            img = img.resize((32, 32), PIL.Image.ANTIALIAS)
+            img.save(i)
+
+    def __len__(self):
+        return self.len
+
+    def __getitem__(self, index):
+        '''
+        Replacing this with a more efficient implemnetation selection; removing c
+        :param index: 
+        :return: 
+        '''
+        assert (index < len(self.data))
+        assert (index < self.len)
+
+        img = Image.open(self.data[index][0])
+        target = self.data[index][1]
+        if self.transform is not None:
+            img = self.transform(img)
+
+        return img, target
+

Dewarp/dataprocessor/dataset.py

@@ -0,0 +1,207 @@
+''' Document Localization using Recursive CNN
+ Maintainer : Khurram Javed
+ Email : kjaved@ualberta.ca '''
+
+import csv
+import logging
+import os
+import xml.etree.ElementTree as ET
+
+import numpy as np
+from torchvision import transforms
+
+import utils.utils as utils
+
+# To incdude a new Dataset, inherit from Dataset and add all the Dataset specific parameters here.
+# Goal : Remove any data specific parameters from the rest of the code
+
+logger = logging.getLogger('iCARL')
+
+
+class Dataset():
+    '''
+    Base class to reprenent a Dataset
+    '''
+
+    def __init__(self, name):
+        self.name = name
+        self.data = []
+        self.labels = []
+
+
+class SmartDoc(Dataset):
+    '''
+    Class to include MNIST specific details
+    '''
+
+    def __init__(self, directory="data"):
+        super().__init__("smartdoc")
+        self.data = []
+        self.labels = []
+        for d in directory:
+            self.directory = d
+            self.train_transform = transforms.Compose([transforms.Resize([32, 32]),
+                                                       transforms.ColorJitter(1.5, 1.5, 0.9, 0.5),
+                                                       transforms.ToTensor()])
+
+            self.test_transform = transforms.Compose([transforms.Resize([32, 32]),
+                                                      transforms.ToTensor()])
+
+            logger.info("Pass train/test data paths here")
+
+            self.classes_list = {}
+
+            file_names = []
+            print (self.directory, "gt.csv")
+            with open(os.path.join(self.directory, "gt.csv"), 'r') as csvfile:
+                spamreader = csv.reader(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
+                import ast
+                for row in spamreader:
+                    file_names.append(row[0])
+                    self.data.append(os.path.join(self.directory, row[0]))
+                    test = row[1].replace("array", "")
+                    self.labels.append((ast.literal_eval(test)))
+        self.labels = np.array(self.labels)
+
+        self.labels = np.reshape(self.labels, (-1, 8))
+        logger.debug("Ground Truth Shape: %s", str(self.labels.shape))
+        logger.debug("Data shape %s", str(len(self.data)))
+
+        self.myData = [self.data, self.labels]
+
+
+class SmartDocDirectories(Dataset):
+    '''
+    Class to include MNIST specific details
+    '''
+
+    def __init__(self, directory="data"):
+        super().__init__("smartdoc")
+        self.data = []
+        self.labels = []
+
+        for folder in os.listdir(directory):
+            if (os.path.isdir(directory + "/" + folder)):
+                for file in os.listdir(directory + "/" + folder):
+                    images_dir = directory + "/" + folder + "/" + file
+                    if (os.path.isdir(images_dir)):
+
+                        list_gt = []
+                        tree = ET.parse(images_dir + "/" + file + ".gt")
+                        root = tree.getroot()
+                        for a in root.iter("frame"):
+                            list_gt.append(a)
+
+                        im_no = 0
+                        for image in os.listdir(images_dir):
+                            if image.endswith(".jpg"):
+                                # print(im_no)
+                                im_no += 1
+
+                                # Now we have opened the file and GT. Write code to create multiple files and scale gt
+                                list_of_points = {}
+
+                                # img = cv2.imread(images_dir + "/" + image)
+                                self.data.append(os.path.join(images_dir, image))
+
+                                for point in list_gt[int(float(image[0:-4])) - 1].iter("point"):
+                                    myDict = point.attrib
+
+                                    list_of_points[myDict["name"]] = (
+                                        int(float(myDict['x'])), int(float(myDict['y'])))
+
+                                ground_truth = np.asarray(
+                                    (list_of_points["tl"], list_of_points["tr"], list_of_points["br"],
+                                     list_of_points["bl"]))
+                                ground_truth = utils.sort_gt(ground_truth)
+                                self.labels.append(ground_truth)
+
+        self.labels = np.array(self.labels)
+
+        self.labels = np.reshape(self.labels, (-1, 8))
+        logger.debug("Ground Truth Shape: %s", str(self.labels.shape))
+        logger.debug("Data shape %s", str(len(self.data)))
+
+        self.myData = []
+        for a in range(len(self.data)):
+            self.myData.append([self.data[a], self.labels[a]])
+
+class SelfCollectedDataset(Dataset):
+    '''
+    Class to include MNIST specific details
+    '''
+
+    def __init__(self, directory="data"):
+        super().__init__("smartdoc")
+        self.data = []
+        self.labels = []
+
+        for image in os.listdir(directory):
+            # print (image)
+            if image.endswith("jpg") or image.endswith("JPG"):
+                if os.path.isfile(os.path.join(directory, image + ".csv")):
+                    with open(os.path.join(directory, image + ".csv"), 'r') as csvfile:
+                        spamwriter = csv.reader(csvfile, delimiter=' ',
+                                                quotechar='|', quoting=csv.QUOTE_MINIMAL)
+
+                        img_path = os.path.join(directory, image)
+
+                        gt = []
+                        for row in spamwriter:
+                            gt.append(row)
+                        gt = np.array(gt).astype(np.float32)
+                        ground_truth = utils.sort_gt(gt)
+                        self.labels.append(ground_truth)
+                        self.data.append(img_path)
+
+        self.labels = np.array(self.labels)
+
+        self.labels = np.reshape(self.labels, (-1, 8))
+        logger.debug("Ground Truth Shape: %s", str(self.labels.shape))
+        logger.debug("Data shape %s", str(len(self.data)))
+
+        self.myData = []
+        for a in range(len(self.data)):
+            self.myData.append([self.data[a], self.labels[a]])
+
+
+
+
+class SmartDocCorner(Dataset):
+    '''
+    Class to include MNIST specific details
+    '''
+
+    def __init__(self, directory="data"):
+        super().__init__("smartdoc")
+        self.data = []
+        self.labels = []
+        for d in directory:
+            self.directory = d
+            self.train_transform = transforms.Compose([transforms.Resize([32, 32]),
+                                                       transforms.ColorJitter(0.5, 0.5, 0.5, 0.5),
+                                                       transforms.ToTensor()])
+
+            self.test_transform = transforms.Compose([transforms.Resize([32, 32]),
+                                                      transforms.ToTensor()])
+
+            logger.info("Pass train/test data paths here")
+
+            self.classes_list = {}
+
+            file_names = []
+            with open(os.path.join(self.directory, "gt.csv"), 'r') as csvfile:
+                spamreader = csv.reader(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
+                import ast
+                for row in spamreader:
+                    file_names.append(row[0])
+                    self.data.append(os.path.join(self.directory, row[0]))
+                    test = row[1].replace("array", "")
+                    self.labels.append((ast.literal_eval(test)))
+        self.labels = np.array(self.labels)
+
+        self.labels = np.reshape(self.labels, (-1, 2))
+        logger.debug("Ground Truth Shape: %s", str(self.labels.shape))
+        logger.debug("Data shape %s", str(len(self.data)))
+
+        self.myData = [self.data, self.labels]

Dewarp/dataprocessor/datasetfactory.py

@@ -0,0 +1,19 @@
+''' Document Localization using Recursive CNN
+ Maintainer : Khurram Javed
+ Email : kjaved@ualberta.ca '''
+
+import dataprocessor.dataset as data
+import torchvision
+
+class DatasetFactory:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_dataset(directory, type="document"):
+        if type=="document":
+            return data.SmartDoc(directory)
+        elif type =="corner":
+            return data.SmartDocCorner(directory)
+        elif type=="CIFAR":
+            return torchvision.datasets.CIFAR100(root='./data', train=True, download=True, transform=torchvision.transforms.ToTensor())

Dewarp/dataprocessor/loaderfactory.py

@@ -0,0 +1,21 @@
+''' Document Localization using Recursive CNN
+ Maintainer : Khurram Javed
+ Email : kjaved@ualberta.ca '''
+
+
+
+import dataprocessor.dataloaders as loader
+
+
+class LoaderFactory:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_loader(type, data, transform=None, cuda=False):
+        if type=="hdd":
+            return loader.HddLoader(data, transform=transform,
+                                    cuda=cuda)
+        elif type =="ram":
+            return loader.RamLoader(data, transform=transform,
+                                    cuda=cuda)

Dewarp/dataset_intro.md

@@ -0,0 +1,24 @@
+### 图像切边增强数据集
+
+| **数据对象**                                   |            **倾斜角度**            |    **场景**     |             **备注**             | **举例**                                               |
+|:-------------------------------------------|:------------------------------:|:-------------:|:------------------------------:|------------------------------------------------------| 
+| <font color="#006600">浅色、四角点完整</font>      |               无                | 办公桌、地毯、沙发、椅子等 |          办公桌上的简历、票据等           | <img src="test_data/001.jpg" width=64px;height:64px> |
+| <font color="#006600">**深色、四角点完整**</font>  |               无                |               |             书本的封面              |                                                      |                                       |
+| <font color="#dd0000">浅色、四角点不完整</font>     |               无                |               | （上两角点、下两角点、左两角点、右两角点）之一未出现在图像中 | <img src="test_data/002.jpg" width=64px;height:64px> |
+| <font color="#dd0000">浅色、四角点不完整</font>     |               无                |               |          三个角点未出现在图像中           | <img src="test_data/003.jpg" width=64px;height:64px> |
+| <font color="#dd0000">浅色、四角点不完整</font>     |               无                |               |           四角点未出现在图像中           | <img src="test_data/004.jpg" width=64px;height:64px> |
+| <font color="#dd0000">**深色、四角点不完整**</font> |               无                |               | （上两角点、下两角点、左两角点、右两角点）之一未出现在图像中 |                                                      |
+| <font color="#dd0000">**深色、四角点不完整**</font> |               无                |               |          三个角点未出现在图像中           |                                                      |
+| <font color="#dd0000">**深色、四角点不完整**</font> |               无                |               |           四角点未出现在图像中           |                                                      |
+| <font color="#006600">浅色、四角点完整</font>      | <font color="#150CFF">有</font> |               |                                |                                                      |
+| <font color="#006600">**深色、四角点完整**</font>  | <font color="#150CFF">有</font> |               |                                |                                                      |
+| <font color="#dd0000">浅色、四角点不完整</font>     | <font color="#150CFF">有</font> |               |            一个角点不在图中            | <img src="test_data/005.jpg" width=64px;height:64px> |
+| <font color="#dd0000">浅色、四角点不完整</font>     | <font color="#150CFF">有</font> |               |            两个角点不在图中            | <img src="test_data/006.jpg" width=64px;height:64px> |
+| <font color="#dd0000">浅色、四角点不完整</font>     | <font color="#150CFF">有</font> |               |            三个角点不在图中            | <img src="test_data/007.jpg" width=64px;height:64px> |
+| <font color="#dd0000">浅色、四角点不完整</font>     | <font color="#150CFF">有</font> |               |            四个角点不在图中            | <img src="test_data/008.jpg" width=64px;height:64px> |
+| <font color="#dd0000">**深色、四角点不完整**</font> | <font color="#150CFF">有</font> |               |            一个角点不在图中            |                                                      |
+| <font color="#dd0000">**深色、四角点不完整**</font> | <font color="#150CFF">有</font> |               |            两个角点不在图中            |                                                      |
+| <font color="#dd0000">**深色、四角点不完整**</font> | <font color="#150CFF">有</font> |               |            三个角点不在图中            |                                                      |
+| <font color="#dd0000">**深色、四角点不完整**</font> | <font color="#150CFF">有</font> |               |            四个角点不在图中            |                                                      |
+
+

Dewarp/demo.py

@@ -0,0 +1,72 @@
+''' Document Localization using Recursive CNN
+ Maintainer : Khurram Javed
+ Email : kjaved@ualberta.ca '''
+import math
+
+import cv2
+import numpy as np
+
+import evaluation
+
+
+def args_processor():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-i", "--imagePath", default="../058.jpg", help="Path to the document image")
+    parser.add_argument("-o", "--outputPath", default="../output.jpg", help="Path to store the result")
+    parser.add_argument("-rf", "--retainFactor", help="Floating point in range (0,1) specifying retain factor",
+                        default="0.85")
+    parser.add_argument("-cm", "--cornerModel", help="Model for corner point refinement",
+                        default="../cornerModelWell")
+    parser.add_argument("-dm", "--documentModel", help="Model for document corners detection",
+                        default="../documentModelWell")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = args_processor()
+
+    corners_extractor = evaluation.corner_extractor.GetCorners(args.documentModel)
+    corner_refiner = evaluation.corner_refiner.corner_finder(args.cornerModel)
+
+    img = cv2.imread(args.imagePath)
+
+    oImg = img
+
+    extracted_corners = corners_extractor.get(oImg)
+    corner_address = []
+    # Refine the detected corners using corner refiner
+    image_name = 0
+    for corner in extracted_corners:
+        image_name += 1
+        corner_img = corner[0]
+        refined_corner = np.array(corner_refiner.get_location(corner_img, 0.85))
+
+        # Converting from local co-ordinate to global co-ordinates of the image
+        refined_corner[0] += corner[1]
+        refined_corner[1] += corner[2]
+
+        # Final results
+        corner_address.append(refined_corner)
+    print(corner_address)
+    width = int(math.sqrt(math.pow(corner_address[1][0]-corner_address[0][0], 2) + math.pow(corner_address[1][1]-corner_address[0][1], 2)))
+    height = int(math.sqrt(math.pow(corner_address[2][0]-corner_address[1][0], 2) + math.pow(corner_address[2][1]-corner_address[1][1], 2)))
+    print(width, height)
+    # if width > height:
+    #     t = width
+    #     width = height
+    #     height = t
+    # print(width, height)
+    pts1 = np.float32([[corner_address[0][0], corner_address[0][1]], [corner_address[1][0], corner_address[1][1]], [corner_address[3][0], corner_address[3][1]], [corner_address[2][0], corner_address[2][1]]])
+    pts2 = np.float32([[0, 0], [width, 0], [0, height], [width, height]])
+    M = cv2.getPerspectiveTransform(pts1, pts2)
+    warped = cv2.warpPerspective(img, M, (width, height))
+    # cv2.imshow('ww', warped)
+    # cv2.waitKey()
+    # cv2.destroyAllWindows()
+    cv2.imwrite(str(args.outputPath).split('.')[0] + '_warped.png', warped)
+
+    for a in range(0, len(extracted_corners)):
+        cv2.line(oImg, tuple(corner_address[a % 4]), tuple(corner_address[(a + 1) % 4]), (255, 0, 0), 4)
+
+    cv2.imwrite(args.outputPath, oImg)

Dewarp/document_data_generator.py

@@ -0,0 +1,72 @@
+import os
+from tqdm import tqdm
+
+import cv2
+import numpy as np
+import utils
+import dataprocessor
+
+def args_processor():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-i", "--input-dir", help="Path to data files (Extract images using video_to_image.py first")
+    parser.add_argument("-o", "--output-dir", help="Directory to store results")
+    parser.add_argument("--dataset", default="smartdoc", help="'smartdoc' or 'selfcollected' dataset")
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    if __name__ == '__main__':
+        args = args_processor()
+        input_directory = args.input_dir
+        if not os.path.isdir(args.output_dir):
+            os.mkdir(args.output_dir)
+        import csv
+
+
+        # Dataset iterator
+        if args.dataset == "smartdoc":
+            dataset_test = dataprocessor.dataset.SmartDocDirectories(input_directory)
+        elif args.dataset == "selfcollected":
+            dataset_test = dataprocessor.dataset.SelfCollectedDataset(input_directory)
+        else:
+            print("Incorrect dataset type; please choose between smartdoc or selfcollected")
+            assert (False)
+        with open(os.path.join(args.output_dir, 'gt.csv'), 'a', newline='') as csvfile:
+            spamwriter = csv.writer(csvfile, delimiter=',',
+                                    quotechar='|')
+            # Counter for file naming
+            counter = 0
+            for data_elem in tqdm(dataset_test.myData):
+
+                img_path = data_elem[0]
+                target = data_elem[1].reshape((4, 2))
+                img = cv2.imread(img_path)
+
+                if args.dataset == "selfcollected":
+                    target = target / (img.shape[1], img.shape[0])
+                    target = target * (1920, 1920)
+                    img = cv2.resize(img, (1920, 1920))
+
+                corner_cords = target
+
+                for angle in range(0, 271, 90):
+                    img_rotate, gt_rotate = utils.utils.rotate(img, corner_cords, angle)
+                    for random_crop in range(0, 16):
+                        counter += 1
+                        f_name = str(counter).zfill(8)
+
+                        img_crop, gt_crop = utils.utils.random_crop(img_rotate, gt_rotate)
+                        mah_size = img_crop.shape
+                        img_crop = cv2.resize(img_crop, (64, 64))
+                        gt_crop = np.array(gt_crop)
+
+                        # no=0
+                        # for a in range(0,4):
+                        #     no+=1
+                        #     cv2.circle(img_crop, tuple(((gt_crop[a]*64).astype(int))), 2,(255-no*60,no*60,0),9)
+                        # # cv2.imwrite("asda.jpg", img)
+
+                        cv2.imwrite(os.path.join(args.output_dir, f_name+".jpg"), img_crop)
+                        spamwriter.writerow((f_name+".jpg", tuple(list(gt_crop))))
+                        print(tuple(list(gt_crop)))

Dewarp/evaluate.py

@@ -0,0 +1,69 @@
+''' Document Localization using Recursive CNN
+ Maintainer : Khurram Javed
+ Email : kjaved@ualberta.ca '''
+
+import argparse
+import time
+
+import numpy as np
+import torch
+from PIL import Image
+
+import dataprocessor
+import evaluation
+
+from utils import utils
+
+parser = argparse.ArgumentParser(description='iCarl2.0')
+
+parser.add_argument("-i", "--data-dir", default="/Users/khurramjaved96/bg5",
+                    help="input Directory of test data")
+
+args = parser.parse_args()
+args.cuda = torch.cuda.is_available()
+if __name__ == '__main__':
+    corners_extractor = evaluation.corner_extractor.GetCorners("../documentModelWell")
+    corner_refiner = evaluation.corner_refiner.corner_finder("../cornerModelWell")
+    test_set_dir = args.data_dir
+    iou_results = []
+    my_results = []
+    dataset_test = dataprocessor.dataset.SmartDocDirectories(test_set_dir)
+    for data_elem in dataset_test.myData:
+
+        img_path = data_elem[0]
+        # print(img_path)
+        target = data_elem[1].reshape((4, 2))
+        img_array = np.array(Image.open(img_path))
+        computation_start_time = time.clock()
+        extracted_corners = corners_extractor.get(img_array)
+        temp_time = time.clock()
+        corner_address = []
+        # Refine the detected corners using corner refiner
+        counter=0
+        for corner in extracted_corners:
+            counter+=1
+            corner_img = corner[0]
+            refined_corner = np.array(corner_refiner.get_location(corner_img, 0.85))
+
+            # Converting from local co-ordinate to global co-ordinate of the image
+            refined_corner[0] += corner[1]
+            refined_corner[1] += corner[2]
+
+            # Final results
+            corner_address.append(refined_corner)
+        computation_end_time = time.clock()
+        print("TOTAL TIME : ", computation_end_time - computation_start_time)
+        r2 = utils.intersection_with_corection_smart_doc_implementation(target, np.array(corner_address), img_array)
+        r3 = utils.intersection_with_corection(target, np.array(corner_address), img_array)
+
+        if r3 - r2 > 0.1:
+            print ("Image Name", img_path)
+            print ("Prediction", np.array(corner_address), target)
+            0/0
+        assert (r2 > 0 and r2 < 1)
+        iou_results.append(r2)
+        my_results.append(r3)
+        print("MEAN CORRECTED JI: ", np.mean(np.array(iou_results)))
+        print("MEAN CORRECTED MY: ", np.mean(np.array(my_results)))
+
+    print(np.mean(np.array(iou_results)))

Dewarp/evaluation/__init__.py

@@ -0,0 +1,6 @@
+''' Document Localization using Recursive CNN
+ Maintainer : Khurram Javed
+ Email : kjaved@ualberta.ca '''
+
+import evaluation.corner_extractor
+import evaluation.corner_refiner

Dewarp/evaluation/corner_extractor.py

@@ -0,0 +1,72 @@
+''' Document Localization using Recursive CNN
+ Maintainer : Khurram Javed
+ Email : kjaved@ualberta.ca '''
+
+import numpy as np
+import torch
+from PIL import Image
+from torchvision import transforms
+
+import model
+
+
+class GetCorners:
+    def __init__(self, checkpoint_dir):
+        self.model = model.ModelFactory.get_model("resnet", 'document')
+        self.model.load_state_dict(torch.load(checkpoint_dir, map_location='cpu'))
+        if torch.cuda.is_available():
+            self.model.cuda()
+        self.model.eval()
+
+    def get(self, pil_image):
+        with torch.no_grad():
+            image_array = np.copy(pil_image)
+            pil_image = Image.fromarray(pil_image)
+            test_transform = transforms.Compose([transforms.Resize([32, 32]),
+                                                 transforms.ToTensor()])
+            img_temp = test_transform(pil_image)
+
+            img_temp = img_temp.unsqueeze(0)
+            if torch.cuda.is_available():
+                img_temp = img_temp.cuda()
+
+            model_prediction = self.model(img_temp).cpu().data.numpy()[0]
+
+            model_prediction = np.array(model_prediction)
+
+            x_cords = model_prediction[[0, 2, 4, 6]]
+            y_cords = model_prediction[[1, 3, 5, 7]]
+
+            x_cords = x_cords * image_array.shape[1]
+            y_cords = y_cords * image_array.shape[0]
+
+            # Extract the four corners of the image. Read "Region Extractor" in Section III of the paper for an explanation.
+
+            top_left = image_array[
+                       max(0, int(2 * y_cords[0] - (y_cords[3] + y_cords[0]) / 2)):int((y_cords[3] + y_cords[0]) / 2),
+                       max(0, int(2 * x_cords[0] - (x_cords[1] + x_cords[0]) / 2)):int((x_cords[1] + x_cords[0]) / 2)]
+
+            top_right = image_array[
+                        max(0, int(2 * y_cords[1] - (y_cords[1] + y_cords[2]) / 2)):int((y_cords[1] + y_cords[2]) / 2),
+                        int((x_cords[1] + x_cords[0]) / 2):min(image_array.shape[1] - 1,
+                                                               int(x_cords[1] + (x_cords[1] - x_cords[0]) / 2))]
+
+            bottom_right = image_array[int((y_cords[1] + y_cords[2]) / 2):min(image_array.shape[0] - 1, int(
+                y_cords[2] + (y_cords[2] - y_cords[1]) / 2)),
+                           int((x_cords[2] + x_cords[3]) / 2):min(image_array.shape[1] - 1,
+                                                                  int(x_cords[2] + (x_cords[2] - x_cords[3]) / 2))]
+
+            bottom_left = image_array[int((y_cords[0] + y_cords[3]) / 2):min(image_array.shape[0] - 1, int(
+                y_cords[3] + (y_cords[3] - y_cords[0]) / 2)),
+                          max(0, int(2 * x_cords[3] - (x_cords[2] + x_cords[3]) / 2)):int(
+                              (x_cords[3] + x_cords[2]) / 2)]
+
+            top_left = (top_left, max(0, int(2 * x_cords[0] - (x_cords[1] + x_cords[0]) / 2)),
+                        max(0, int(2 * y_cords[0] - (y_cords[3] + y_cords[0]) / 2)))
+            top_right = (
+            top_right, int((x_cords[1] + x_cords[0]) / 2), max(0, int(2 * y_cords[1] - (y_cords[1] + y_cords[2]) / 2)))
+            bottom_right = (bottom_right, int((x_cords[2] + x_cords[3]) / 2), int((y_cords[1] + y_cords[2]) / 2))
+            bottom_left = (bottom_left, max(0, int(2 * x_cords[3] - (x_cords[2] + x_cords[3]) / 2)),
+                           int((y_cords[0] + y_cords[3]) / 2))
+
+            return top_left, top_right, bottom_right, bottom_left

Dewarp/evaluation/corner_refiner.py

@@ -0,0 +1,84 @@
+''' Document Localization using Recursive CNN
+ Maintainer : Khurram Javed
+ Email : kjaved@ualberta.ca '''
+
+import numpy as np
+import torch
+from PIL import Image
+from torchvision import transforms
+
+import model
+
+
+class corner_finder():
+    def __init__(self, CHECKPOINT_DIR):
+
+        self.model = model.ModelFactory.get_model("resnet", "corner")
+        self.model.load_state_dict(torch.load(CHECKPOINT_DIR, map_location='cpu'))
+        if torch.cuda.is_available():
+            self.model.cuda()
+        self.model.eval()
+
+    def get_location(self, img, retainFactor=0.85):
+        with torch.no_grad():
+            ans_x = 0.0
+            ans_y = 0.0
+
+            o_img = np.copy(img)
+
+            y = [0, 0]
+            x_start = 0
+            y_start = 0
+            up_scale_factor = (img.shape[1], img.shape[0])
+
+            myImage = np.copy(o_img)
+
+            test_transform = transforms.Compose([transforms.Resize([32, 32]),
+                                                 transforms.ToTensor()])
+
+            CROP_FRAC = retainFactor
+            while (myImage.shape[0] > 10 and myImage.shape[1] > 10):
+
+                img_temp = Image.fromarray(myImage)
+                img_temp = test_transform(img_temp)
+                img_temp = img_temp.unsqueeze(0)
+
+                if torch.cuda.is_available():
+                    img_temp = img_temp.cuda()
+                response = self.model(img_temp).cpu().data.numpy()
+                response = response[0]
+
+                response_up = response
+
+                response_up = response_up * up_scale_factor
+                y = response_up + (x_start, y_start)
+                x_loc = int(y[0])
+                y_loc = int(y[1])
+
+                if x_loc > myImage.shape[1] / 2:
+                    start_x = min(x_loc + int(round(myImage.shape[1] * CROP_FRAC / 2)), myImage.shape[1]) - int(round(
+                        myImage.shape[1] * CROP_FRAC))
+                else:
+                    start_x = max(x_loc - int(myImage.shape[1] * CROP_FRAC / 2), 0)
+                if y_loc > myImage.shape[0] / 2:
+                    start_y = min(y_loc + int(myImage.shape[0] * CROP_FRAC / 2), myImage.shape[0]) - int(
+                        myImage.shape[0] * CROP_FRAC)
+                else:
+                    start_y = max(y_loc - int(myImage.shape[0] * CROP_FRAC / 2), 0)
+
+                ans_x += start_x
+                ans_y += start_y
+
+                myImage = myImage[start_y:start_y + int(myImage.shape[0] * CROP_FRAC),
+                          start_x:start_x + int(myImage.shape[1] * CROP_FRAC)]
+                img = img[start_y:start_y + int(img.shape[0] * CROP_FRAC),
+                      start_x:start_x + int(img.shape[1] * CROP_FRAC)]
+                up_scale_factor = (img.shape[1], img.shape[0])
+
+            ans_x += y[0]
+            ans_y += y[1]
+            return (int(round(ans_x)), int(round(ans_y)))
+
+
+if __name__ == "__main__":
+    pass

Dewarp/experiment/__init__.py

@@ -0,0 +1 @@
+from experiment.experiment import *

Dewarp/experiment/experiment.py

@@ -0,0 +1,53 @@
+''' Incremental-Classifier Learning 
+ Authors : Khurram Javed, Muhammad Talha Paracha
+ Maintainer : Khurram Javed
+ Lab : TUKL-SEECS R&D Lab
+ Email : 14besekjaved@seecs.edu.pk '''
+
+import json
+import os
+import subprocess
+
+
+class experiment:
+    '''
+    Class to store results of any experiment 
+    '''
+
+    def __init__(self, name, args, output_dir="../"):
+        # self.gitHash = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode("utf-8")
+        # print(self.gitHash)
+        if not args is None:
+            self.name = name
+            self.params = vars(args)
+            self.results = {}
+            self.dir = output_dir
+
+            import datetime
+            now = datetime.datetime.now()
+            rootFolder = str(now.day) + str(now.month) + str(now.year)
+            if not os.path.exists(output_dir + rootFolder):
+                os.makedirs(output_dir + rootFolder)
+            self.name = rootFolder + "/" + self.name
+            ver = 0
+
+            while os.path.exists(output_dir + self.name + "_" + str(ver)):
+                ver += 1
+
+            os.makedirs(output_dir + self.name + "_" + str(ver))
+            self.path = output_dir + self.name + "_" + str(ver) + "/" + name
+
+            self.results["Temp Results"] = [[1, 2, 3, 4], [5, 6, 2, 6]]
+
+    def store_json(self):
+        with open(self.path + "JSONDump.txt", 'w') as outfile:
+            json.dump(json.dumps(self.__dict__), outfile)
+
+
+import argparse
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='iCarl2.0')
+    args = parser.parse_args()
+    e = experiment("TestExperiment", args)
+    e.store_json()

Dewarp/json_to_csv.py

@@ -0,0 +1,28 @@
+import argparse
+import json
+import os
+
+
+def work(json_fold):
+    json_list = os.listdir(json_fold)
+    for json_path in json_list:
+        path = json_fold + '/' + json_path
+        # 处理json文件
+        if str(path).endswith('.json'):
+            with open(path, "r", encoding="utf-8") as f:
+                content = json.load(f)
+            # 将 JSON 数据写入 CSV 文件
+            csv_path = str(path).replace('json', 'jpg') + '.csv'
+            with open(csv_path, mode='w', newline='') as csv_file:
+                # 使用 DictWriter 而不是 csv.DictWriter，因为需要将 JSON 中的键作为列名
+                for point in content['shapes']:
+                    line = str(point['points'][0][0]) + ' ' + str(point['points'][0][1]) + '\n'
+                    print(line)
+                    csv_file.write(line)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--json_fold', type=str, default='./my_dataset_origin')
+    args = parser.parse_args()
+    work(args.json_fold)

Dewarp/model/__init__.py

@@ -0,0 +1 @@
+from model.modelfactory import *

Dewarp/model/cornerModel.py

@@ -0,0 +1,78 @@
+# Reference : Taken from https://github.com/kuangliu/pytorch-cifar
+
+# License
+# MIT License
+#
+# Copyright (c) 2017 liukuang
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+'''MobileNet in PyTorch.
+See the paper "MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications"
+for more details.
+'''
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class Block(nn.Module):
+    '''Depthwise conv + Pointwise conv'''
+    def __init__(self, in_planes, out_planes, stride=1):
+        super(Block, self).__init__()
+        self.conv1 = nn.Conv2d(in_planes, in_planes, kernel_size=3, stride=stride, padding=1, groups=in_planes, bias=False)
+        self.bn1 = nn.BatchNorm2d(in_planes)
+        self.conv2 = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False)
+        self.bn2 = nn.BatchNorm2d(out_planes)
+
+    def forward(self, x):
+        out = F.relu(self.bn1(self.conv1(x)))
+        out = F.relu(self.bn2(self.conv2(out)))
+        return out
+
+
+class MobileNet(nn.Module):
+    # (128,2) means conv planes=128, conv stride=2, by default conv stride=1
+    cfg = [64, (128,2), 128, (256,2), 256, (512,2), 512, 512, 512, 512, 512, (1024,2), 1024]
+
+    def __init__(self, num_classes=10):
+        super(MobileNet, self).__init__()
+        self.conv1 = nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(32)
+        self.layers = self._make_layers(in_planes=32)
+        self.linear = nn.Linear(1024, num_classes)
+
+    def _make_layers(self, in_planes):
+        layers = []
+        for x in self.cfg:
+            out_planes = x if isinstance(x, int) else x[0]
+            stride = 1 if isinstance(x, int) else x[1]
+            layers.append(Block(in_planes, out_planes, stride))
+            in_planes = out_planes
+        return nn.Sequential(*layers)
+
+    def forward(self, x, pretrain=False):
+        out = F.relu(self.bn1(self.conv1(x)))
+        out = self.layers(out)
+        out = F.avg_pool2d(out, 2)
+        out = out.view(out.size(0), -1)
+        out = self.linear(out)
+        return out
+

Dewarp/model/modelfactory.py

@@ -0,0 +1,37 @@
+''' Document Localization using Recursive CNN
+ Maintainer : Khurram Javed
+ Email : kjaved@ualberta.ca '''
+
+import model.resnet32 as resnet
+import model.cornerModel as tm
+import torchvision.models as models
+
+class ModelFactory():
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_model(model_type, dataset):
+        if model_type == "resnet":
+            if dataset == 'document':
+                return resnet.resnet20(8)
+            elif dataset == 'corner':
+                return resnet.resnet20(2)
+        if model_type == "resnet8":
+            if dataset == 'document':
+                return resnet.resnet8(8)
+            elif dataset == 'corner':
+                return resnet.resnet8(2)
+        elif model_type == 'shallow':
+            if dataset == 'document':
+                return tm.MobileNet(8)
+            elif dataset == 'corner':
+                return tm.MobileNet(2)
+        elif model_type =="squeeze":
+            if dataset == 'document':
+                return models.squeezenet1_1(True)
+            elif dataset == 'corner':
+                return models.squeezenet1_1(True)
+        else:
+            print("Unsupported model; either implement the model in model/ModelFactory or choose a different model")
+            assert (False)

Dewarp/model/res_utils.py

@@ -0,0 +1,37 @@
+import torch
+import torch.nn as nn
+
+
+class DownsampleA(nn.Module):
+    def __init__(self, nIn, nOut, stride):
+        super(DownsampleA, self).__init__()
+        assert stride == 2
+        self.avg = nn.AvgPool2d(kernel_size=1, stride=stride)
+
+    def forward(self, x):
+        x = self.avg(x)
+        return torch.cat((x, x.mul(0)), 1)
+
+
+class DownsampleC(nn.Module):
+    def __init__(self, nIn, nOut, stride):
+        super(DownsampleC, self).__init__()
+        assert stride != 1 or nIn != nOut
+        self.conv = nn.Conv2d(nIn, nOut, kernel_size=1, stride=stride, padding=0, bias=False)
+
+    def forward(self, x):
+        x = self.conv(x)
+        return x
+
+
+class DownsampleD(nn.Module):
+    def __init__(self, nIn, nOut, stride):
+        super(DownsampleD, self).__init__()
+        assert stride == 2
+        self.conv = nn.Conv2d(nIn, nOut, kernel_size=2, stride=stride, padding=0, bias=False)
+        self.bn = nn.BatchNorm2d(nOut)
+
+    def forward(self, x):
+        x = self.conv(x)
+        x = self.bn(x)
+        return x

Dewarp/model/resnet32.py

@@ -0,0 +1,184 @@
+# This is someone elses implementation of resnet optimized for CIFAR; I can't seem to find the repository again to reference the work.
+# I will keep on looking.
+import math
+
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn import init
+
+from .res_utils import DownsampleA
+
+
+class ResNetBasicblock(nn.Module):
+    expansion = 1
+    """
+    RexNet basicblock (https://github.com/facebook/fb.resnet.torch/blob/master/models/resnet.lua)
+    """
+
+    def __init__(self, inplanes, planes, stride=1, downsample=None):
+        super(ResNetBasicblock, self).__init__()
+
+        self.conv_a = nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
+        self.bn_a = nn.BatchNorm2d(planes)
+
+        self.conv_b = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
+        self.bn_b = nn.BatchNorm2d(planes)
+
+        self.downsample = downsample
+        self.featureSize = 64
+
+    def forward(self, x):
+        residual = x
+
+        basicblock = self.conv_a(x)
+        basicblock = self.bn_a(basicblock)
+        basicblock = F.relu(basicblock, inplace=True)
+
+        basicblock = self.conv_b(basicblock)
+        basicblock = self.bn_b(basicblock)
+
+        if self.downsample is not None:
+            residual = self.downsample(x)
+
+        return F.relu(residual + basicblock, inplace=True)
+
+
+class CifarResNet(nn.Module):
+    """
+    ResNet optimized for the Cifar Dataset, as specified in
+    https://arxiv.org/abs/1512.03385.pdf
+    """
+
+    def __init__(self, block, depth, num_classes, channels=3):
+        """ Constructor
+        Args:
+          depth: number of layers.
+          num_classes: number of classes
+          base_width: base width
+        """
+        super(CifarResNet, self).__init__()
+
+        self.featureSize = 64
+        # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model
+        assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110'
+        layer_blocks = (depth - 2) // 6
+
+        self.num_classes = num_classes
+
+        self.conv_1_3x3 = nn.Conv2d(channels, 16, kernel_size=3, stride=1, padding=1, bias=False)
+        self.bn_1 = nn.BatchNorm2d(16)
+
+        self.inplanes = 16
+        self.stage_1 = self._make_layer(block, 16, layer_blocks, 1)
+        self.stage_2 = self._make_layer(block, 32, layer_blocks, 2)
+        self.stage_3 = self._make_layer(block, 64, layer_blocks, 2)
+        self.avgpool = nn.AvgPool2d(8)
+        self.fc = nn.Linear(64 * block.expansion, num_classes)
+        self.fc2 = nn.Linear(64 * block.expansion, 100)
+
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
+                m.weight.data.normal_(0, math.sqrt(2. / n))
+                # m.bias.data.zero_()
+            elif isinstance(m, nn.BatchNorm2d):
+                m.weight.data.fill_(1)
+                m.bias.data.zero_()
+            elif isinstance(m, nn.Linear):
+                init.kaiming_normal(m.weight)
+                m.bias.data.zero_()
+
+    def _make_layer(self, block, planes, blocks, stride=1):
+        downsample = None
+        if stride != 1 or self.inplanes != planes * block.expansion:
+            downsample = DownsampleA(self.inplanes, planes * block.expansion, stride)
+
+        layers = []
+        layers.append(block(self.inplanes, planes, stride, downsample))
+        self.inplanes = planes * block.expansion
+        for i in range(1, blocks):
+            layers.append(block(self.inplanes, planes))
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x, pretrain=False):
+
+        x = self.conv_1_3x3(x)
+        x = F.relu(self.bn_1(x), inplace=True)
+        x = self.stage_1(x)
+        x = self.stage_2(x)
+        x = self.stage_3(x)
+        x = self.avgpool(x)
+        x = x.view(x.size(0), -1)
+        if pretrain:
+            return self.fc2(x)
+        x = self.fc(x)
+        return x
+
+
+def resnet20(num_classes=10):
+    """Constructs a ResNet-20 model for CIFAR-10 (by default)
+    Args:
+      num_classes (uint): number of classes
+    """
+    model = CifarResNet(ResNetBasicblock, 20, num_classes)
+    return model
+
+
+def resnet8(num_classes=10):
+    """Constructs a ResNet-20 model for CIFAR-10 (by default)
+    Args:
+      num_classes (uint): number of classes
+    """
+    model = CifarResNet(ResNetBasicblock, 8, num_classes, 3)
+    return model
+
+
+def resnet20mnist(num_classes=10):
+    """Constructs a ResNet-20 model for CIFAR-10 (by default)
+    Args:
+      num_classes (uint): number of classes
+    """
+    model = CifarResNet(ResNetBasicblock, 20, num_classes, 1)
+    return model
+
+
+def resnet32mnist(num_classes=10, channels=1):
+    model = CifarResNet(ResNetBasicblock, 32, num_classes, channels)
+    return model
+
+
+def resnet32(num_classes=10):
+    """Constructs a ResNet-32 model for CIFAR-10 (by default)
+    Args:
+      num_classes (uint): number of classes
+    """
+    model = CifarResNet(ResNetBasicblock, 32, num_classes)
+    return model
+
+
+def resnet44(num_classes=10):
+    """Constructs a ResNet-44 model for CIFAR-10 (by default)
+    Args:
+      num_classes (uint): number of classes
+    """
+    model = CifarResNet(ResNetBasicblock, 44, num_classes)
+    return model
+
+
+def resnet56(num_classes=10):
+    """Constructs a ResNet-56 model for CIFAR-10 (by default)
+    Args:
+      num_classes (uint): number of classes
+    """
+    model = CifarResNet(ResNetBasicblock, 56, num_classes)
+    return model
+
+
+def resnet110(num_classes=10):
+    """Constructs a ResNet-110 model for CIFAR-10 (by default)
+    Args:
+      num_classes (uint): number of classes
+    """
+    model = CifarResNet(ResNetBasicblock, 110, num_classes)
+    return model

Dewarp/plotter/__init__.py

@@ -0,0 +1 @@
+from plotter.plotter import *

Dewarp/plotter/plotter.py

@@ -0,0 +1,88 @@
+''' Incremental-Classifier Learning 
+ Authors : Khurram Javed, Muhammad Talha Paracha
+ Maintainer : Khurram Javed
+ Lab : TUKL-SEECS R&D Lab
+ Email : 14besekjaved@seecs.edu.pk '''
+
+import matplotlib
+import matplotlib.pyplot as plt
+
+plt.switch_backend('agg')
+
+MEDIUM_SIZE = 18
+
+font = {'family': 'sans-serif',
+        'weight': 'bold'}
+
+matplotlib.rc('xtick', labelsize=MEDIUM_SIZE)
+matplotlib.rc('ytick', labelsize=MEDIUM_SIZE)
+plt.rc('axes', labelsize=MEDIUM_SIZE)  # fontsize of the x and y labels
+
+# matplotlib.rc('font', **font)
+from matplotlib import rcParams
+
+rcParams.update({'figure.autolayout': True})
+
+
+class Plotter():
+    def __init__(self):
+        import itertools
+        # plt.figure(figsize=(12, 9))
+        self.marker = itertools.cycle(('o', '+', "v", "^", "8", '.', '*'))
+        self.handles = []
+        self.lines = itertools.cycle(('--', '-.', '-', ':'))
+
+    def plot(self, x, y, xLabel="Number of Classes", yLabel="Accuracy %", legend="none", title=None, error=None):
+        self.x = x
+        self.y = y
+        plt.grid(color='0.89', linestyle='--', linewidth=1.0)
+        if error is None:
+            l, = plt.plot(x, y, linestyle=next(self.lines), marker=next(self.marker), label=legend, linewidth=3.0)
+        else:
+            l = plt.errorbar(x, y, yerr=error, capsize=4.0, capthick=2.0, linestyle=next(self.lines),
+                             marker=next(self.marker), label=legend, linewidth=3.0)
+
+        self.handles.append(l)
+        self.x_label = xLabel
+        self.y_label = yLabel
+        if title is not None:
+            plt.title(title)
+
+    def save_fig(self, path, xticks=105, title=None, yStart=0, xRange=0, yRange=10):
+        if title is not None:
+            plt.title(title)
+        plt.legend(handles=self.handles)
+        plt.ylim((yStart, 100 + 0.2))
+        plt.xlim((0, xticks + .2))
+        plt.ylabel(self.y_label)
+        plt.xlabel(self.x_label)
+        plt.yticks(list(range(yStart, 101, yRange)))
+        print(list(range(yStart, 105, yRange)))
+        plt.xticks(list(range(0, xticks + 1, xRange + int(xticks / 10))))
+        plt.savefig(path + ".eps", format='eps')
+        plt.gcf().clear()
+
+    def save_fig2(self, path, xticks=105):
+        plt.legend(handles=self.handles)
+        plt.xlabel("Memory Budget")
+        plt.ylabel("Average Incremental Accuracy")
+        plt.savefig(path + ".jpg")
+        plt.gcf().clear()
+
+    def plotMatrix(self, epoch, path, img):
+
+        plt.imshow(img, cmap='plasma', interpolation='nearest')
+        plt.colorbar()
+        plt.savefig(path + str(epoch) + ".svg", format='svg')
+        plt.gcf().clear()
+
+    def saveImage(self, img, path, epoch):
+        from PIL import Image
+        im = Image.fromarray(img)
+        im.save(path + str(epoch) + ".jpg")
+
+
+if __name__ == "__main__":
+    pl = Plotter()
+    pl.plot([1, 2, 3, 4], [2, 3, 6, 2])
+    pl.save_fig("test.jpg")

Dewarp/pytorch2onnx.py

@@ -0,0 +1,10 @@
+import torch
+import model
+
+
+myModel = model.ModelFactory.get_model('resnet', 'document')
+myModel.load_state_dict(torch.load('outputs/doc552023/doc_0505_0/doc_0505document_resnet.pth'))
+myModel.eval()
+
+dummy_input = torch.randn(1, 3, 32, 32)
+torch.onnx.export(myModel, dummy_input, "document_1.0.0.onnx", do_constant_folding=False)

Dewarp/pytorch2paddle.py

@@ -0,0 +1,19 @@
+import torch
+import numpy as np
+
+# 构建输入
+import model
+
+input_data = np.random.rand(1, 3, 32, 32).astype("float32")
+# 获取PyTorch Module
+
+myModel = model.ModelFactory.get_model('resnet', 'corner')
+myModel.load_state_dict(torch.load('outputs/corner552023/corner_0505_1/corner_0505corner_resnet.pth'))
+# 设置为eval模式
+myModel.eval()
+# 进行转换
+from x2paddle.convert import pytorch2paddle
+pytorch2paddle(myModel,
+               save_dir="pd_model_trace1",
+               jit_type="trace",
+               input_examples=[torch.tensor(input_data)])

Dewarp/smartdoc_data_processor/video_to_image.py

@@ -0,0 +1,41 @@
+import os
+
+
+def argsProcessor():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-d", "--dataPath", help="path to main data folder")
+    parser.add_argument("-o", "--outputPath", help="output data")
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = argsProcessor()
+    dir = args.dataPath
+    output = args.outputPath
+    print(dir)
+    if not os.path.isdir(output):
+        os.mkdir(output)
+
+    for folder in os.listdir(dir):
+        if os.path.isdir(dir + "/" + folder):
+            dir_temp = dir + '/' + folder + "/"
+            for file in os.listdir(dir_temp):
+                print(file)
+                from subprocess import call
+
+                if file.endswith(".mp4"):
+                    call("mkdir " + output + folder, shell=True)
+                    if os.path.isdir(output + folder + "/" + file):
+                        print("Folder already exist")
+                    else:
+                        call("cd " + output + folder + " && mkdir " + file, shell=True)
+                        call("ls", shell=True)
+
+                        location = dir + folder + "/" + file
+                        gt_address = "cp " + location[
+                                             0:-4] + ".gt.xml " + output + folder + "/" + file + "/" + file + ".gt"
+                        call(gt_address, shell=True)
+                        command = "ffmpeg -i " + location + " " + output + folder + "/" + file + "/%3d.jpg"
+                        print(command)
+                        call(command, shell=True)

Dewarp/train_model.py

@@ -0,0 +1,157 @@
+''' Document Localization using Recursive CNN
+ Maintainer : Khurram Javed
+ Email : kjaved@ualberta.ca '''
+
+from __future__ import print_function
+
+import argparse
+
+import torch
+import torch.utils.data as td
+
+import dataprocessor
+import experiment as ex
+import model
+import trainer
+import utils
+
+parser = argparse.ArgumentParser(description='Recursive-CNNs')
+parser.add_argument('--batch-size', type=int, default=32, metavar='N',
+                    help='input batch size for training (default: 32)')
+parser.add_argument('--eval_interval', type=int, default=5)
+parser.add_argument('--lr', type=float, default=0.005, metavar='LR',
+                    help='learning rate (default: 0.005)')
+parser.add_argument('--schedule', type=int, nargs='+', default=[10, 20, 30],
+                    help='Decrease learning rate at these epochs.')
+parser.add_argument('--gammas', type=float, nargs='+', default=[0.2, 0.2, 0.2],
+                    help='LR is multiplied by gamma[k] on schedule[k], number of gammas should be equal to schedule')
+parser.add_argument('--momentum', type=float, default=0.9, metavar='M',
+                    help='SGD momentum (default: 0.9)')
+parser.add_argument('--no-cuda', action='store_true', default=False,
+                    help='disables CUDA training')
+parser.add_argument('--pretrain', action='store_true', default=False,
+                    help='Pretrain the model on CIFAR dataset?')
+parser.add_argument('--load-ram', action='store_true', default=False,
+                    help='Load data in ram: TODO : Remove this')
+parser.add_argument('--debug', action='store_true', default=True,
+                    help='Debug messages')
+parser.add_argument('--seed', type=int, default=2323,
+                    help='Seeds values to be used')
+parser.add_argument('--log-interval', type=int, default=5, metavar='N',
+                    help='how many batches to wait before logging training status')
+parser.add_argument('--model-type', default="resnet",
+                    help='model type to be used. Example : resnet32, resnet20, densenet, test')
+parser.add_argument('--name', default="noname",
+                    help='Name of the experiment')
+parser.add_argument('--output-dir', default="./",
+                    help='Directory to store the results; a new folder "DDMMYYYY" will be created '
+                         'in the specified directory to save the results.')
+parser.add_argument('--decay', type=float, default=0.00001, help='Weight decay (L2 penalty).')
+parser.add_argument('--epochs', type=int, default=100, help='Number of epochs for trianing')
+parser.add_argument('--dataset', default="document", help='Dataset to be used; example document, corner')
+parser.add_argument('--loader', default="hdd", 
+                    help='Loader to load data; hdd for reading from the hdd and ram for loading all data in the memory')
+parser.add_argument("-i", "--data-dirs", nargs='+', default="/Users/khurramjaved96/documentTest64",
+                    help="input Directory of train data")
+parser.add_argument("-v", "--validation-dirs", nargs='+', default="/Users/khurramjaved96/documentTest64",
+                    help="input Directory of val data")
+
+args = parser.parse_args()
+
+# Define an experiment.
+my_experiment = ex.experiment(args.name, args, args.output_dir)
+
+# Add logging support
+logger = utils.utils.setup_logger(my_experiment.path)
+
+args.cuda = not args.no_cuda and torch.cuda.is_available()
+
+dataset = dataprocessor.DatasetFactory.get_dataset(args.data_dirs, args.dataset)
+
+dataset_val = dataprocessor.DatasetFactory.get_dataset(args.validation_dirs, args.dataset)
+
+# Fix the seed.
+seed = args.seed
+torch.manual_seed(seed)
+if args.cuda:
+    torch.cuda.manual_seed(seed)
+
+train_dataset_loader = dataprocessor.LoaderFactory.get_loader(args.loader, dataset.myData,
+                                                              transform=dataset.train_transform,
+                                                              cuda=args.cuda)
+# Loader used for training data
+val_dataset_loader = dataprocessor.LoaderFactory.get_loader(args.loader, dataset_val.myData,
+                                                            transform=dataset.test_transform,
+                                                            cuda=args.cuda)
+kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
+
+# Iterator to iterate over training data.
+train_iterator = torch.utils.data.DataLoader(train_dataset_loader,
+                                             batch_size=args.batch_size, shuffle=True, **kwargs)
+# Iterator to iterate over training data.
+val_iterator = torch.utils.data.DataLoader(val_dataset_loader,
+                                           batch_size=args.batch_size, shuffle=True, **kwargs)
+
+# Get the required model
+myModel = model.ModelFactory.get_model(args.model_type, args.dataset)
+if args.cuda:
+    myModel.cuda()
+
+# Should I pretrain the model on CIFAR?
+if args.pretrain:
+    trainset = dataprocessor.DatasetFactory.get_dataset(None, "CIFAR")
+    train_iterator_cifar = torch.utils.data.DataLoader(trainset, batch_size=32, shuffle=True, num_workers=2)
+
+    # Define the optimizer used in the experiment
+    cifar_optimizer = torch.optim.SGD(myModel.parameters(), args.lr, momentum=args.momentum,
+                                      weight_decay=args.decay, nesterov=True)
+
+    # Trainer object used for training
+    cifar_trainer = trainer.CIFARTrainer(train_iterator_cifar, myModel, args.cuda, cifar_optimizer)
+
+    for epoch in range(0, 70):
+        logger.info("Epoch : %d", epoch)
+        cifar_trainer.update_lr(epoch, [30, 45, 60], args.gammas)
+        cifar_trainer.train(epoch)
+
+    # Freeze the model
+    counter = 0
+    for name, param in myModel.named_parameters():
+        # Getting the length of total layers so I can freeze x% of layers
+        gen_len = sum(1 for _ in myModel.parameters())
+        if counter < int(gen_len * 0.5):
+            param.requires_grad = False
+            logger.warning(name)
+        else:
+            logger.info(name)
+        counter += 1
+
+# Define the optimizer used in the experiment
+optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, myModel.parameters()), args.lr,
+                            momentum=args.momentum,
+                            weight_decay=args.decay, nesterov=True)
+
+# Trainer object used for training
+my_trainer = trainer.Trainer(train_iterator, myModel, args.cuda, optimizer)
+
+# Evaluator
+my_eval = trainer.EvaluatorFactory.get_evaluator("rmse", args.cuda)
+# Running epochs_class epochs
+cnt = 1
+max_loss = 100
+for epoch in range(0, args.epochs):
+    logger.info("Epoch : %d", epoch)
+    my_trainer.update_lr(epoch, args.schedule, args.gammas)
+    l = my_trainer.train(epoch)
+    loss = float(l)
+    if loss < max_loss:
+        torch.save(myModel.state_dict(), my_experiment.path + "_" + args.dataset + "_" + args.model_type+ "_best.pth")
+        logger.info("best_model saved, avg_loss is %s", l)
+        max_loss = loss
+    logger.info("the best model's avg_loss is %s", l)
+    if cnt%args.eval_interval == 0:
+        my_eval.evaluate(my_trainer.model, val_iterator)
+    cnt += 1
+torch.save(myModel.state_dict(), my_experiment.path + "_" + args.dataset + "_" + args.model_type+ "_final.pth")
+# torch.save(myModel, my_experiment.path + args.dataset + "_" + args.model_type+ ".pth")
+my_experiment.store_json()

Dewarp/train_seg_model.py

@@ -0,0 +1,149 @@
+## Document Localization using Recursive CNN
+## Maintainer : Khurram Javed
+## Email : kjaved@ualberta.ca
+
+''' Document Localization using Recursive CNN
+ Maintainer : Khurram Javed
+ Email : kjaved@ualberta.ca '''
+
+from __future__ import print_function
+
+import argparse
+
+import torch
+import torch.utils.data as td
+
+import dataprocessor
+import experiment as ex
+import model
+import trainer
+import utils
+
+parser = argparse.ArgumentParser(description='iCarl2.0')
+parser.add_argument('--batch-size', type=int, default=32, metavar='N',
+                    help='input batch size for training (default: 64)')
+parser.add_argument('--lr', type=float, default=0.005, metavar='LR',
+                    help='learning rate (default: 2.0)')
+parser.add_argument('--schedule', type=int, nargs='+', default=[10, 20, 30],
+                    help='Decrease learning rate at these epochs.')
+parser.add_argument('--gammas', type=float, nargs='+', default=[0.2, 0.2, 0.2],
+                    help='LR is multiplied by gamma on schedule, number of gammas should be equal to schedule')
+parser.add_argument('--momentum', type=float, default=0.9, metavar='M',
+                    help='SGD momentum (default: 0.9)')
+parser.add_argument('--no-cuda', action='store_true', default=False,
+                    help='disables CUDA training')
+parser.add_argument('--pretrain', action='store_true', default=False,
+                    help='Pretrain the model on CIFAR dataset?')
+parser.add_argument('--load-ram', action='store_true', default=False,
+                    help='Load data in ram')
+parser.add_argument('--debug', action='store_true', default=True,
+                    help='Debug messages')
+parser.add_argument('--seed', type=int, default=2323,
+                    help='Seeds values to be used')
+parser.add_argument('--log-interval', type=int, default=5, metavar='N',
+                    help='how many batches to wait before logging training status')
+parser.add_argument('--model-type', default="resnet",
+                    help='model type to be used. Example : resnet32, resnet20, densenet, test')
+parser.add_argument('--name', default="noname",
+                    help='Name of the experiment')
+parser.add_argument('--output-dir', default="../",
+                    help='Directory to store the results; a new folder "DDMMYYYY" will be created '
+                         'in the specified directory to save the results.')
+parser.add_argument('--decay', type=float, default=0.00001, help='Weight decay (L2 penalty).')
+parser.add_argument('--epochs', type=int, default=40, help='Number of epochs for each increment')
+parser.add_argument('--dataset', default="document", help='Dataset to be used; example CIFAR, MNIST')
+parser.add_argument('--loader', default="hdd", help='Dataset to be used; example CIFAR, MNIST')
+parser.add_argument("-i", "--data-dirs", nargs='+', default="/Users/khurramjaved96/documentTest64",
+                    help="input Directory of train data")
+parser.add_argument("-v", "--validation-dirs", nargs='+', default="/Users/khurramjaved96/documentTest64",
+                    help="input Directory of val data")
+
+args = parser.parse_args()
+
+# Define an experiment.
+my_experiment = ex.experiment(args.name, args, args.output_dir)
+
+# Add logging support
+logger = utils.utils.setup_logger(my_experiment.path)
+
+args.cuda = not args.no_cuda and torch.cuda.is_available()
+
+dataset = dataprocessor.DatasetFactory.get_dataset(args.data_dirs, args.dataset)
+
+dataset_val = dataprocessor.DatasetFactory.get_dataset(args.validation_dirs, args.dataset)
+
+# Fix the seed.
+seed = args.seed
+torch.manual_seed(seed)
+if args.cuda:
+    torch.cuda.manual_seed(seed)
+
+train_dataset_loader = dataprocessor.LoaderFactory.get_loader(args.loader, dataset.myData,
+                                                              transform=dataset.train_transform,
+                                                              cuda=args.cuda)
+# Loader used for training data
+val_dataset_loader = dataprocessor.LoaderFactory.get_loader(args.loader, dataset_val.myData,
+                                                            transform=dataset.test_transform,
+                                                            cuda=args.cuda)
+kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
+
+# Iterator to iterate over training data.
+train_iterator = torch.utils.data.DataLoader(train_dataset_loader,
+                                             batch_size=args.batch_size, shuffle=True, **kwargs)
+# Iterator to iterate over training data.
+val_iterator = torch.utils.data.DataLoader(val_dataset_loader,
+                                           batch_size=args.batch_size, shuffle=True, **kwargs)
+
+# Get the required model
+myModel = model.ModelFactory.get_model(args.model_type, args.dataset)
+if args.cuda:
+    myModel.cuda()
+
+# Should I pretrain the model on CIFAR?
+if args.pretrain:
+    trainset = dataprocessor.DatasetFactory.get_dataset(None, "CIFAR")
+    train_iterator_cifar = torch.utils.data.DataLoader(trainset, batch_size=32, shuffle=True, num_workers=2)
+
+    # Define the optimizer used in the experiment
+    cifar_optimizer = torch.optim.SGD(myModel.parameters(), args.lr, momentum=args.momentum,
+                                      weight_decay=args.decay, nesterov=True)
+
+    # Trainer object used for training
+    cifar_trainer = trainer.CIFARTrainer(train_iterator_cifar, myModel, args.cuda, cifar_optimizer)
+
+    for epoch in range(0, 70):
+        logger.info("Epoch : %d", epoch)
+        cifar_trainer.update_lr(epoch, [30, 45, 60], args.gammas)
+        cifar_trainer.train(epoch)
+
+    # Freeze the model
+    counter = 0
+    for name, param in myModel.named_parameters():
+        # Getting the length of total layers so I can freeze x% of layers
+        gen_len = sum(1 for _ in myModel.parameters())
+        if counter < int(gen_len * 0.5):
+            param.requires_grad = False
+            logger.warning(name)
+        else:
+            logger.info(name)
+        counter += 1
+
+# Define the optimizer used in the experiment
+optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, myModel.parameters()), args.lr,
+                            momentum=args.momentum,
+                            weight_decay=args.decay, nesterov=True)
+
+# Trainer object used for training
+my_trainer = trainer.Trainer(train_iterator, myModel, args.cuda, optimizer)
+
+# Evaluator
+my_eval = trainer.EvaluatorFactory.get_evaluator("rmse", args.cuda)
+# Running epochs_class epochs
+for epoch in range(0, args.epochs):
+    logger.info("Epoch : %d", epoch)
+    my_trainer.update_lr(epoch, args.schedule, args.gammas)
+    my_trainer.train(epoch)
+    my_eval.evaluate(my_trainer.model, val_iterator)
+
+torch.save(myModel.state_dict(), my_experiment.path + args.dataset + "_" + args.model_type+ ".pb")
+my_experiment.store_json()

Dewarp/trainer/__init__.py

@@ -0,0 +1,2 @@
+from trainer.evaluator import *
+from trainer.trainer import *

Dewarp/trainer/evaluator.py

@@ -0,0 +1,63 @@
+''' Incremental-Classifier Learning 
+ Authors : Khurram Javed, Muhammad Talha Paracha
+ Maintainer : Khurram Javed
+ Lab : TUKL-SEECS R&D Lab
+ Email : 14besekjaved@seecs.edu.pk '''
+
+import logging
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch.autograd import Variable
+from torchnet.meter import confusionmeter
+from tqdm import tqdm
+
+logger = logging.getLogger('iCARL')
+
+
+class EvaluatorFactory():
+    '''
+    This class is used to get different versions of evaluators
+    '''
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_evaluator(testType="rmse", cuda=True):
+        if testType == "rmse":
+            return DocumentMseEvaluator(cuda)
+
+
+
+class DocumentMseEvaluator():
+    '''
+    Evaluator class for softmax classification 
+    '''
+    def __init__(self, cuda):
+        self.cuda = cuda
+
+
+    def evaluate(self, model, iterator):
+        model.eval()
+        lossAvg = None
+        with torch.no_grad():
+            for img, target in tqdm(iterator):
+                if self.cuda:
+                    img, target = img.cuda(), target.cuda()
+
+                response = model(Variable(img))
+                # print (response[0])
+                # print (target[0])
+                loss = F.mse_loss(response, Variable(target.float()))
+                loss = torch.sqrt(loss)
+                if lossAvg is None:
+                    lossAvg = loss
+                else:
+                    lossAvg += loss
+                # logger.debug("Cur loss %s", str(loss))
+
+        lossAvg /= len(iterator)
+        logger.info("Avg Val Loss %s", str((lossAvg).cpu().data.numpy()))
+
+

Dewarp/trainer/trainer.py

@@ -0,0 +1,111 @@
+''' Pytorch Recursive CNN Trainer
+ Authors : Khurram Javed
+ Maintainer : Khurram Javed
+ Lab : TUKL-SEECS R&D Lab
+ Email : 14besekjaved@seecs.edu.pk '''
+
+from __future__ import print_function
+
+import logging
+
+from torch.autograd import Variable
+
+logger = logging.getLogger('iCARL')
+import torch.nn.functional as F
+import torch
+from tqdm import tqdm
+
+
+class GenericTrainer:
+    '''
+    Base class for trainer; to implement a new training routine, inherit from this. 
+    '''
+
+    def __init__(self):
+        pass
+
+
+
+class Trainer(GenericTrainer):
+    def __init__(self, train_iterator, model, cuda, optimizer):
+        super().__init__()
+        self.cuda = cuda
+        self.train_iterator = train_iterator
+        self.model = model
+        self.optimizer = optimizer
+
+    def update_lr(self, epoch, schedule, gammas):
+        for temp in range(0, len(schedule)):
+            if schedule[temp] == epoch:
+                for param_group in self.optimizer.param_groups:
+                    self.current_lr = param_group['lr']
+                    param_group['lr'] = self.current_lr * gammas[temp]
+                    logger.debug("Changing learning rate from %0.9f to %0.9f", self.current_lr,
+                                 self.current_lr * gammas[temp])
+                    self.current_lr *= gammas[temp]
+
+    def train(self, epoch):
+        self.model.train()
+        lossAvg = None
+        for img, target in tqdm(self.train_iterator):
+            if self.cuda:
+                img, target = img.cuda(), target.cuda()
+            self.optimizer.zero_grad()
+            response = self.model(Variable(img))
+            # print (response[0])
+            # print (target[0])
+            loss = F.mse_loss(response, Variable(target.float()))
+            loss = torch.sqrt(loss)
+            if lossAvg is None:
+                lossAvg = loss
+            else:
+                lossAvg += loss
+            # logger.debug("Cur loss %s", str(loss))
+            loss.backward()
+            self.optimizer.step()
+
+        lossAvg /= len(self.train_iterator)
+        logger.info("Avg Loss %s", str((lossAvg).cpu().data.numpy()))
+        return str((lossAvg).cpu().data.numpy())
+
+
+class CIFARTrainer(GenericTrainer):
+    def __init__(self, train_iterator, model, cuda, optimizer):
+        super().__init__()
+        self.cuda = cuda
+        self.train_iterator = train_iterator
+        self.model = model
+        self.optimizer = optimizer
+        self.criterion = torch.nn.CrossEntropyLoss()
+
+    def update_lr(self, epoch, schedule, gammas):
+        for temp in range(0, len(schedule)):
+            if schedule[temp] == epoch:
+                for param_group in self.optimizer.param_groups:
+                    self.current_lr = param_group['lr']
+                    param_group['lr'] = self.current_lr * gammas[temp]
+                    logger.debug("Changing learning rate from %0.9f to %0.9f", self.current_lr,
+                                 self.current_lr * gammas[temp])
+                    self.current_lr *= gammas[temp]
+
+    def train(self, epoch):
+        self.model.train()
+        train_loss = 0
+        correct = 0
+        total = 0
+        for inputs, targets in tqdm(self.train_iterator):
+            if self.cuda:
+                inputs, targets = inputs.cuda(), targets.cuda()
+            self.optimizer.zero_grad()
+            outputs = self.model(Variable(inputs), pretrain=True)
+            loss = self.criterion(outputs, Variable(targets))
+            loss.backward()
+            self.optimizer.step()
+
+            train_loss += loss.item()
+            _, predicted = outputs.max(1)
+            total += targets.size(0)
+            correct += predicted.eq(targets).sum().item()
+
+        logger.info("Accuracy : %s", str((correct * 100) / total))
+        return correct / total

Dewarp/utils/__init__.py

@@ -0,0 +1,2 @@
+from utils import utils
+from utils import colorer

Dewarp/utils/colorer.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+# encoding: utf-8
+import logging
+
+# Source : # http://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output
+
+# now we patch Python code to add color support to logging.StreamHandler
+def add_coloring_to_emit_windows(fn):
+    # add methods we need to the class
+    def _out_handle(self):
+        import ctypes
+        return ctypes.windll.kernel32.GetStdHandle(self.STD_OUTPUT_HANDLE)
+
+    out_handle = property(_out_handle)
+
+    def _set_color(self, code):
+        import ctypes
+        # Constants from the Windows API
+        self.STD_OUTPUT_HANDLE = -11
+        hdl = ctypes.windll.kernel32.GetStdHandle(self.STD_OUTPUT_HANDLE)
+        ctypes.windll.kernel32.SetConsoleTextAttribute(hdl, code)
+
+    setattr(logging.StreamHandler, '_set_color', _set_color)
+
+    def new(*args):
+        FOREGROUND_BLUE = 0x0001  # text color contains blue.
+        FOREGROUND_GREEN = 0x0002  # text color contains green.
+        FOREGROUND_RED = 0x0004  # text color contains red.
+        FOREGROUND_INTENSITY = 0x0008  # text color is intensified.
+        FOREGROUND_WHITE = FOREGROUND_BLUE | FOREGROUND_GREEN | FOREGROUND_RED
+        # winbase.h
+        STD_INPUT_HANDLE = -10
+        STD_OUTPUT_HANDLE = -11
+        STD_ERROR_HANDLE = -12
+
+        # wincon.h
+        FOREGROUND_BLACK = 0x0000
+        FOREGROUND_BLUE = 0x0001
+        FOREGROUND_GREEN = 0x0002
+        FOREGROUND_CYAN = 0x0003
+        FOREGROUND_RED = 0x0004
+        FOREGROUND_MAGENTA = 0x0005
+        FOREGROUND_YELLOW = 0x0006
+        FOREGROUND_GREY = 0x0007
+        FOREGROUND_INTENSITY = 0x0008  # foreground color is intensified.
+
+        BACKGROUND_BLACK = 0x0000
+        BACKGROUND_BLUE = 0x0010
+        BACKGROUND_GREEN = 0x0020
+        BACKGROUND_CYAN = 0x0030
+        BACKGROUND_RED = 0x0040
+        BACKGROUND_MAGENTA = 0x0050
+        BACKGROUND_YELLOW = 0x0060
+        BACKGROUND_GREY = 0x0070
+        BACKGROUND_INTENSITY = 0x0080  # background color is intensified.
+
+        levelno = args[1].levelno
+        if (levelno >= 50):
+            color = BACKGROUND_YELLOW | FOREGROUND_RED | FOREGROUND_INTENSITY | BACKGROUND_INTENSITY
+        elif (levelno >= 40):
+            color = FOREGROUND_RED | FOREGROUND_INTENSITY
+        elif (levelno >= 30):
+            color = FOREGROUND_YELLOW | FOREGROUND_INTENSITY
+        elif (levelno >= 20):
+            color = FOREGROUND_GREEN
+        elif (levelno >= 10):
+            color = FOREGROUND_MAGENTA
+        else:
+            color = FOREGROUND_WHITE
+        args[0]._set_color(color)
+
+        ret = fn(*args)
+        args[0]._set_color(FOREGROUND_WHITE)
+        # print "after"
+        return ret
+
+    return new
+
+
+def add_coloring_to_emit_ansi(fn):
+    # add methods we need to the class
+    def new(*args):
+        levelno = args[1].levelno
+        if (levelno >= 50):
+            color = '\x1b[31m'  # red
+        elif (levelno >= 40):
+            color = '\x1b[31m'  # red
+        elif (levelno >= 30):
+            color = '\x1b[33m'  # yellow
+        elif (levelno >= 20):
+            color = '\x1b[32m'  # green
+        elif (levelno >= 10):
+            color = '\x1b[35m'  # pink
+        else:
+            color = '\x1b[0m'  # normal
+        args[1].msg = color + args[1].msg + '\x1b[0m'  # normal
+        # print "after"
+        return fn(*args)
+
+    return new
+
+
+import platform
+
+if platform.system() == 'Windows':
+    # Windows does not support ANSI escapes and we are using API calls to set the console color
+    logging.StreamHandler.emit = add_coloring_to_emit_windows(logging.StreamHandler.emit)
+else:
+    # all non-Windows platforms are supporting ANSI escapes so we use them
+    logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit)
+    # log = logging.getLogger()
+    # log.addFilter(log_filter())
+    # //hdlr = logging.StreamHandler()
+    # //hdlr.setFormatter(formatter())

Dewarp/utils/utils.py

@@ -0,0 +1,296 @@
+''' Document Localization using Recursive CNN
+ Maintainer : Khurram Javed
+ Email : kjaved@ualberta.ca '''
+
+import random
+
+import cv2
+import numpy as np
+import Polygon
+
+def unison_shuffled_copies(a, b):
+    assert len(a) == len(b)
+    p = np.random.permutation(len(a))
+    return a[p], b[p]
+
+
+def intersection(a, b, img):
+    img1 = np.zeros_like(img)
+
+    cv2.fillConvexPoly(img1, a, (255, 0, 0))
+    img1 = np.sum(img1, axis=2)
+
+    img1 = img1 / 255
+
+    img2 = np.zeros_like(img)
+    cv2.fillConvexPoly(img2, b, (255, 0, 0))
+    img2 = np.sum(img2, axis=2)
+    img2 = img2 / 255
+
+    inte = img1 * img2
+    union = np.logical_or(img1, img2)
+    iou = np.sum(inte) / np.sum(union)
+    print(iou)
+    return iou
+
+
+def intersection_with_correction(a, b, img):
+    img1 = np.zeros_like(img)
+    cv2.fillConvexPoly(img1, a, (255, 0, 0))
+
+    img2 = np.zeros_like(img)
+    cv2.fillConvexPoly(img2, b, (255, 0, 0))
+    min_x = min(a[0][0], a[1][0], a[2][0], a[3][0])
+    min_y = min(a[0][1], a[1][1], a[2][1], a[3][1])
+    max_x = max(a[0][0], a[1][0], a[2][0], a[3][0])
+    max_y = max(a[0][1], a[1][1], a[2][1], a[3][1])
+
+    dst = np.array(((min_x, min_y), (max_x, min_y), (max_x, max_y), (min_x, max_y)))
+    mat = cv2.getPerspectiveTransform(a.astype(np.float32), dst.astype(np.float32))
+    img1 = cv2.warpPerspective(img1, mat, tuple((img.shape[0], img.shape[1])))
+    img2 = cv2.warpPerspective(img2, mat, tuple((img.shape[0], img.shape[1])))
+
+    img1 = np.sum(img1, axis=2)
+    img1 = img1 / 255
+    img2 = np.sum(img2, axis=2)
+    img2 = img2 / 255
+
+    inte = img1 * img2
+    union = np.logical_or(img1, img2)
+    iou = np.sum(inte) / np.sum(union)
+    return iou
+
+def intersection_with_correction_smart_doc_implementation(gt, prediction, img):
+
+    # Reference : https://github.com/jchazalon/smartdoc15-ch1-eval
+
+    gt = sort_gt(gt)
+    prediction = sort_gt(prediction)
+    img1 = np.zeros_like(img)
+    cv2.fillConvexPoly(img1, gt, (255, 0, 0))
+
+    target_width = 2100
+    target_height = 2970
+    # Referential: (0,0) at TL, x > 0 toward right and y > 0 toward bottom
+    # Corner order: TL, BL, BR, TR
+    # object_coord_target = np.float32([[0, 0], [0, target_height], [target_width, target_height], [target_width, 0]])
+    object_coord_target = np.array(np.float32([[0, 0], [target_width, 0], [target_width, target_height],[0, target_height]]))
+    # print (gt, object_coord_target)
+    H = cv2.getPerspectiveTransform(gt.astype(np.float32).reshape(-1, 1, 2), object_coord_target.reshape(-1, 1, 2))
+
+    # 2/ Apply to test result to project in target referential
+    test_coords = cv2.perspectiveTransform(prediction.astype(np.float32).reshape(-1, 1, 2), H)
+
+    # 3/ Compute intersection between target region and test result region
+    # poly = Polygon.Polygon([(0,0),(1,0),(0,1)])
+    poly_target = Polygon.Polygon(object_coord_target.reshape(-1, 2))
+    poly_test = Polygon.Polygon(test_coords.reshape(-1, 2))
+    poly_inter = poly_target & poly_test
+
+    area_target = poly_target.area()
+    area_test = poly_test.area()
+    area_inter = poly_inter.area()
+
+    area_union = area_test + area_target - area_inter
+    # Little hack to cope with float precision issues when dealing with polygons:
+    #   If intersection area is close enough to target area or GT area, but slighlty >,
+    #   then fix it, assuming it is due to rounding issues.
+    area_min = min(area_target, area_test)
+    if area_min < area_inter and area_min * 1.0000000001 > area_inter:
+        area_inter = area_min
+        print("Capping area_inter.")
+
+    jaccard_index = area_inter / area_union
+    return jaccard_index
+
+
+
+def __rotateImage(image, angle):
+    rot_mat = cv2.getRotationMatrix2D((image.shape[1] / 2, image.shape[0] / 2), angle, 1)
+    result = cv2.warpAffine(image, rot_mat, (image.shape[1], image.shape[0]), flags=cv2.INTER_LINEAR)
+    return result, rot_mat
+
+
+def rotate(img, gt, angle):
+    img, mat = __rotateImage(img, angle)
+    gt = gt.astype(np.float64)
+    for a in range(0, 4):
+        gt[a] = np.dot(mat[..., 0:2], gt[a]) + mat[..., 2]
+    return img, gt
+
+
+def random_crop(img, gt):
+    ptr1 = (min(gt[0][0], gt[1][0], gt[2][0], gt[3][0]),
+            min(gt[0][1], gt[1][1], gt[2][1], gt[3][1]))
+
+    ptr2 = ((max(gt[0][0], gt[1][0], gt[2][0], gt[3][0]),
+             max(gt[0][1], gt[1][1], gt[2][1], gt[3][1])))
+
+    start_x = np.random.randint(0, int(max(ptr1[0] - 1, 1)))
+    start_y = np.random.randint(0, int(max(ptr1[1] - 1, 1)))
+
+    end_x = np.random.randint(int(min(ptr2[0] + 1, img.shape[1] - 1)), img.shape[1])
+    end_y = np.random.randint(int(min(ptr2[1] + 1, img.shape[0] - 1)), img.shape[0])
+
+    img = img[start_y:end_y, start_x:end_x]
+
+    myGt = gt - (start_x, start_y)
+    myGt = myGt * (1.0 / img.shape[1], 1.0 / img.shape[0])
+
+    myGtTemp = myGt * myGt
+    sum_array = myGtTemp.sum(axis=1)
+    tl_index = np.argmin(sum_array)
+    tl = myGt[tl_index]
+    tr = myGt[(tl_index + 1) % 4]
+    br = myGt[(tl_index + 2) % 4]
+    bl = myGt[(tl_index + 3) % 4]
+
+    return img, (tl, tr, br, bl)
+
+
+def get_corners(img, gt):
+    gt = gt.astype(int)
+    list_of_points = {}
+    myGt = gt
+
+    myGtTemp = myGt * myGt
+    sum_array = myGtTemp.sum(axis=1)
+
+    tl_index = np.argmin(sum_array)
+    tl = myGt[tl_index]
+    tr = myGt[(tl_index + 1) % 4]
+    br = myGt[(tl_index + 2) % 4]
+    bl = myGt[(tl_index + 3) % 4]
+
+    list_of_points["tr"] = tr
+    list_of_points["tl"] = tl
+    list_of_points["br"] = br
+    list_of_points["bl"] = bl
+    gt_list = []
+    images_list = []
+    for k, v in list_of_points.items():
+
+        if (k == "tl"):
+            cords_x = __get_cords(v[0], 0, list_of_points["tr"][0], buf=10, size=abs(list_of_points["tr"][0] - v[0]))
+            cords_y = __get_cords(v[1], 0, list_of_points["bl"][1], buf=10, size=abs(list_of_points["bl"][1] - v[1]))
+            # print cords_y, cords_x
+            gt = (v[0] - cords_x[0], v[1] - cords_y[0])
+
+            cut_image = img[cords_y[0]:cords_y[1], cords_x[0]:cords_x[1]]
+
+        if (k == "tr"):
+            cords_x = __get_cords(v[0], list_of_points["tl"][0], img.shape[1], buf=10,
+                                  size=abs(list_of_points["tl"][0] - v[0]))
+            cords_y = __get_cords(v[1], 0, list_of_points["br"][1], buf=10, size=abs(list_of_points["br"][1] - v[1]))
+            # print cords_y, cords_x
+            gt = (v[0] - cords_x[0], v[1] - cords_y[0])
+
+            cut_image = img[cords_y[0]:cords_y[1], cords_x[0]:cords_x[1]]
+
+        if (k == "bl"):
+            cords_x = __get_cords(v[0], 0, list_of_points["br"][0], buf=10,
+                                  size=abs(list_of_points["br"][0] - v[0]))
+            cords_y = __get_cords(v[1], list_of_points["tl"][1], img.shape[0], buf=10,
+                                  size=abs(list_of_points["tl"][1] - v[1]))
+            # print cords_y, cords_x
+            gt = (v[0] - cords_x[0], v[1] - cords_y[0])
+
+            cut_image = img[cords_y[0]:cords_y[1], cords_x[0]:cords_x[1]]
+
+        if (k == "br"):
+            cords_x = __get_cords(v[0], list_of_points["bl"][0], img.shape[1], buf=10,
+                                  size=abs(list_of_points["bl"][0] - v[0]))
+            cords_y = __get_cords(v[1], list_of_points["tr"][1], img.shape[0], buf=10,
+                                  size=abs(list_of_points["tr"][1] - v[1]))
+            # print cords_y, cords_x
+            gt = (v[0] - cords_x[0], v[1] - cords_y[0])
+
+            cut_image = img[cords_y[0]:cords_y[1], cords_x[0]:cords_x[1]]
+
+        # cv2.circle(cut_image, gt, 2, (255, 0, 0), 6)
+        mah_size = cut_image.shape
+        cut_image = cv2.resize(cut_image, (300, 300))
+        a = int(gt[0] * 300 / mah_size[1])
+        b = int(gt[1] * 300 / mah_size[0])
+        images_list.append(cut_image)
+        gt_list.append((a, b))
+    return images_list, gt_list
+
+
+def __get_cords(cord, min_start, max_end, size=299, buf=5, random_scale=True):
+    # size = max(abs(cord-min_start), abs(cord-max_end))
+    iter = 0
+    if (random_scale):
+        size /= random.randint(1, 4)
+    while (max_end - min_start) < size:
+        size = size * .9
+    temp = -1
+    while (temp < 1):
+        temp = random.normalvariate(size / 2, size / 6)
+    x_start = max(cord - temp, min_start)
+    x_start = int(x_start)
+    if x_start >= cord:
+        print("XSTART AND CORD", x_start, cord)
+    # assert (x_start < cord)
+    while ((x_start < min_start) or (x_start + size > max_end) or (x_start + size <= cord)):
+        # x_start = random.randint(int(min(max(min_start, int(cord - size + buf)), cord - buf - 1)), cord - buf)
+        temp = -1
+        while (temp < 1):
+            temp = random.normalvariate(size / 2, size / 6)
+        temp = max(temp, 1)
+        x_start = max(cord - temp, min_start)
+        x_start = int(x_start)
+        size = size * .995
+        iter += 1
+        if (iter == 1000):
+            x_start = int(cord - (size / 2))
+            print("Gets here")
+            break
+    assert (x_start >= 0)
+    if x_start >= cord:
+        print("XSTART AND CORD", x_start, cord)
+    # assert (x_start < cord)
+    # assert (x_start + size <= max_end)
+    # assert (x_start + size > cord)
+    return (x_start, int(x_start + size))
+
+
+def setup_logger(path):
+    import logging
+    logger = logging.getLogger('iCARL')
+    logger.setLevel(logging.DEBUG)
+
+    fh = logging.FileHandler(path + ".log")
+    fh.setLevel(logging.DEBUG)
+
+    fh2 = logging.FileHandler("../temp.log")
+    fh2.setLevel(logging.DEBUG)
+
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.DEBUG)
+
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    fh.setFormatter(formatter)
+    fh2.setFormatter(formatter)
+
+    logger.addHandler(fh)
+    logger.addHandler(fh2)
+    logger.addHandler(ch)
+    return logger
+
+
+def sort_gt(gt):
+    '''
+    Sort the ground truth labels so that TL corresponds to the label with smallest distance from O
+    :param gt: 
+    :return: sorted gt
+    '''
+    myGtTemp = gt * gt
+    sum_array = myGtTemp.sum(axis=1)
+    tl_index = np.argmin(sum_array)
+    tl = gt[tl_index]
+    tr = gt[(tl_index + 1) % 4]
+    br = gt[(tl_index + 2) % 4]
+    bl = gt[(tl_index + 3) % 4]
+
+    return np.asarray((tl, tr, br, bl))

Dewarp/video_to_images.py

@@ -0,0 +1,40 @@
+import argparse
+import os
+import uuid
+import datetime
+import cv2
+from tqdm import tqdm
+
+
+def work(video_folder, output_folder, interval):
+    if not os.path.exists(output_folder):
+        os.makedirs(output_folder)
+    video_list = os.listdir(video_folder)
+    num = 1
+    cnt = 1
+    for video in tqdm(video_list):
+        video_path = video_folder + '/' + video
+        vid = cv2.VideoCapture(video_path)
+        time = datetime.datetime.now()
+        t = str(time.year).zfill(4) + '-' + str(time.month).zfill(2) + '-' + str(time.day).zfill(2)
+        while vid.isOpened():
+            is_read, frame = vid.read()
+            if is_read:
+                if num % interval == 0:
+                    file_name = '%04d' % cnt
+                    cv2.imwrite(output_folder + '/' + 'kdan_' + t + '_' + str(file_name) + '_' + str(uuid.uuid1())[0:4] + '.jpg', frame)
+                    # 00000111.jpg 代表第111帧
+                    cv2.waitKey(1)
+                    cnt += 1
+                num += 1
+            else:
+                break
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--video_folder', type=str, default='./video/01')
+    parser.add_argument('--output_folder', type=str, default='./video_frame')
+    parser.add_argument('--interval', type=int, default=10, help='帧率间隔')
+    args = parser.parse_args()
+    work(args.video_folder, args.output_folder, args.interval)

Dewarp/write_json.py

@@ -0,0 +1,20 @@
+import json
+from base64 import b64encode
+
+
+def make_json(img_fold, json_file_path, shapes_list, image_path, image_height, image_width):
+    flags = {}
+    # 读取二进制图片，获得原始字节码
+    with open(str(img_fold) + '\\' + image_path, 'rb') as jpg_file:
+        byte_content = jpg_file.read()
+    jpg_file.close()
+    # 把原始字节码编码成base64字节码
+    base64_bytes = b64encode(byte_content)
+    # 把base64字节码解码成utf-8格式的字符串
+    base64_string = base64_bytes.decode('utf-8')
+    data = {'version': '5.0.1', 'flags': flags, 'shapes': shapes_list, 'imagePath': image_path, 'imageData': base64_string,
+            'imageHeight': image_height, 'imageWidth': image_width}
+    with open(str(img_fold) + '\\' + json_file_path, 'w') as fp:
+        json.dump(data, fp, sort_keys=False, indent=2)
+    fp.close()
+    return