1.3.8

CMU-INF-DIVA · Oct 26, 2020 · 4b0e340 · 4b0e340
1 parent bb67660
commit 4b0e340
Show file tree

Hide file tree

Showing 6 changed files with 135 additions and 38 deletions.
diff --git a/avi_r/reader.py b/avi_r/reader.py
@@ -230,7 +230,7 @@ def __del__(self):
         if hasattr(self, '_container'):
             self._del()
 
-    def _init(self, video_stream_id=0, timeout=None):
+    def _init(self, video_stream_id=0, timeout=20):
         self._container = av.open(
             self.path, metadata_errors='replace', timeout=timeout)
         self._stream = self._container.streams.video[video_stream_id]
@@ -246,6 +246,7 @@ def _del(self):
 
     def _get_frame_gen(self, start_frame_id=0, retry=5, retry_step=120):
         seek_frame_id = start_frame_id
+        retry = min(retry, start_frame_id // retry_step + 1)
         for _ in range(retry):
             if seek_frame_id != start_frame_id:
                 self._logger.info(
@@ -270,9 +271,15 @@ def _get_frame_gen(self, start_frame_id=0, retry=5, retry_step=120):
                     'Failed to seek to frame 0, still trying to read the '
                     'current frame, please check its frame id')
             frame_gen = self._fix_missing(seek_frame_id)
-            frame = next(frame_gen)
+            try:
+                frame = next(frame_gen)
+            except StopIteration:
+                return
         while frame.frame_id < start_frame_id:
-            frame = next(frame_gen)
+            try:
+                frame = next(frame_gen)
+            except StopIteration:
+                return
         yield frame
         while True:
             try:

diff --git a/docs/speed.md b/docs/speed.md
@@ -3,39 +3,40 @@
 Test performed by [tests/speed_test.sh](../tests/speed_test.sh).
 
 ```sh
-./tests/speed_test.sh <video_dir>
+./tests/speed_test.sh <video_dir> | tee log.txt
 ```
 
 ## Test Environment
 
 - CPU: Intel Core i9-9900X
 - Memory: 128GB
 - Disk: SSD
-- ffmpeg: 4.0
+- avi-r: 1.3.8
 - moviepy: 1.0.3
-- opencv: 4.2.0
+- opencv: 4.4.0
+- ffmpeg: 4.3
 
 ## Overall Performance
 
 Loading all frames of 7 videos from the [MEVA dataset](http://mevadata.org). Each video is 5-min long and 1080p at 30 fps.
 
 |                 | `avi_r.AVIReader` | `moviepy.editor.VideoFileClip` | `cv2.VideoCapture` |
-| :-------------: | :---------------: | :-----------------------------: | :----------------: |
-|    User Time    |      331.10s      |             889.68s             |      807.88s       |
-|   System Time   |       0.80s       |             287.64s             |       5.88s        |
-| CPU Utilization |       100%        |              294%               |        270%        |
-|   Total Time    |      331.89s      |             400.31s             |      300.46s       |
+| :-------------: | :---------------: | :----------------------------: | :----------------: |
+|    User Time    |      245.63s      |            868.50s             |      712.73s       |
+|   System Time   |       0.98s       |            270.87s             |       11.47s       |
+| CPU Utilization |        97%        |              296%              |        515%        |
+|   Total Time    |      253.67s      |            263.92s             |      140.55s       |
 
 ## Detailed Results
 
 Loading time and number of frames on each video.
 
 |                   Video Name                   |                    Video Description                    | `avi_r.AVIReader` | `moviepy.editor .VideoFileClip` | `cv2.VideoCapture` |
 | :--------------------------------------------: | :-----------------------------------------------------: | :---------------: | :-----------------------------: | :----------------: |
-| 2018-03-11.16-30-08.16-35-08.hospital.G436.avi |                       No missing                        |    0:44 / 9000    |           0:57 / 9000           |    0:26 / 9000     |
-|  2018-03-07.16-55-06.17-00-06.school.G336.avi  |                  Missing 104-109, 2294                  |    0:54 / 9007    |           0:56 / 9007           |  0:26  / **8980**  |
-|  2018-03-11.11-25-01.11-30-01.school.G424.avi  |                    Missing 7391-7499                    |    0:37 / 9000    |           0:57 / 9000           |  1:32  / **8880**  |
-|  2018-03-11.16-25-00.16-30-00.school.G639.avi  |           Bidirectional frames, missing 1, 4            |    0:54 / 9002    |           0:56 / 9002           |  0:26 / **8989**   |
-|  2018-03-11.11-35-00.11-40-00.school.G299.avi  | Packet id and frame id unsychronized, missing 5789-5797 |    0:49 / 9009    |           0:56 / 9009           |  0:27 / **8989**   |
-|  2018-03-11.11-35-00.11-40-00.school.G330.avi  | Packet id and frame id unsychronized, missing 5755-5761 |    0:49 / 9007    |           0:57 / 9007           |  1:14 / **8985**   |
-| 2018-03-12.10-05-00.10-10-00.hospital.G436.avi |                    First packet fail                    |    0:42 / 9000    |           0:56 / 9000           |    0:26 / 9000     |
+| 2018-03-11.16-30-08.16-35-08.hospital.G436.avi |                       No missing                        |    0:32 / 9000    |           0:56 / 9000           |    0:13 / 9000     |
+|  2018-03-07.16-55-06.17-00-06.school.G336.avi  |                  Missing 104-109, 2294                  |    0:49 / 9007    |           0:55 / 9007           |  0:13  / **9000**  |
+|  2018-03-11.11-25-01.11-30-01.school.G424.avi  |                    Missing 7391-7499                    |    0:26 / 9000    |           0:54 / 9000           |    0:12  / 9000    |
+|  2018-03-11.16-25-00.16-30-00.school.G639.avi  |           Bidirectional frames, missing 1, 4            |    0:42 / 9002    |           0:55 / 9002           |  0:54 / **9000**   |
+|  2018-03-11.11-35-00.11-40-00.school.G299.avi  | Packet id and frame id unsychronized, missing 5789-5797 |    0:37 / 9009    |           0:55 / 9009           |  0:18 / **9000**   |
+|  2018-03-11.11-35-00.11-40-00.school.G330.avi  | Packet id and frame id unsychronized, missing 5755-5761 |    0:38 / 9007    |           0:56 / 9007           |  0:17 / **9000**   |
+| 2018-03-12.10-05-00.10-10-00.hospital.G436.avi |                    First packet fail                    |    0:30 / 9000    |           0:54 / 9000           |    0:13 / 9000     |
diff --git a/docs/version.md b/docs/version.md
@@ -1,5 +1,11 @@
 # Version History
 
+## AVI-R v1.3.8
+
+* Add back timeout.
+* More robust during iteration initialization.
+* Limit seek retries based on frame id.
+
 ## AVI-R v1.3.7
 
 * Remove default timeout.

diff --git a/setup.py b/setup.py
@@ -8,7 +8,7 @@
 
 setuptools.setup(
     name='avi-r',
-    version='1.3.7',
+    version='1.3.8',
     author='Lijun Yu',
     author_email='lijun@lj-y.com',
     description='A robust reader for avi videos.',

diff --git a/tests/log.txt b/tests/log.txt
@@ -0,0 +1,71 @@
+avi_r.AVIReader
+	 2018-03-11.16-30-08.16-35-08.hospital.G436.avi
+	 frame: 9000
+	 32.439380168914795
+	 2018-03-07.16-55-06.17-00-06.school.G336.avi
+	 frame: 9007
+	 49.441840171813965
+	 2018-03-11.11-25-01.11-30-01.school.G424.avi
+	 frame: 9000
+	 25.697554111480713
+	 2018-03-11.16-25-00.16-30-00.school.G639.avi
+	 frame: 9002
+	 41.59361243247986
+	 2018-03-11.11-35-00.11-40-00.school.G299.avi
+	 frame: 9009
+	 37.053707122802734
+	 2018-03-11.11-35-00.11-40-00.school.G330.avi
+	 frame: 9007
+	 37.56602120399475
+	 2018-03-12.10-05-00.10-10-00.hospital.G436.avi
+	 frame: 9000
+	 29.70570683479309
+python -c   245.63s user 0.98s system 97% cpu 4:13.67 total
+
+moviepy.editor.VideoFileClip
+	 2018-03-11.16-30-08.16-35-08.hospital.G436.avi
+	 frame: 9000
+	 55.74726581573486
+	 2018-03-07.16-55-06.17-00-06.school.G336.avi
+	 frame: 9007
+	 54.80690956115723
+	 2018-03-11.11-25-01.11-30-01.school.G424.avi
+	 frame: 9000
+	 54.131500244140625
+	 2018-03-11.16-25-00.16-30-00.school.G639.avi
+	 frame: 9002
+	 54.55790114402771
+	 2018-03-11.11-35-00.11-40-00.school.G299.avi
+	 frame: 9009
+	 54.53456687927246
+	 2018-03-11.11-35-00.11-40-00.school.G330.avi
+	 frame: 9007
+	 55.5817973613739
+	 2018-03-12.10-05-00.10-10-00.hospital.G436.avi
+	 frame: 9000
+	 53.84965968132019
+python -c   868.50s user 270.87s system 296% cpu 6:23.92 total
+
+cv2.VideoCapture
+	 video: 2018-03-11.16-30-08.16-35-08.hospital.G436.avi
+	 frame: 9000
+	 time: 12.745321989059448
+	 video: 2018-03-07.16-55-06.17-00-06.school.G336.avi
+	 frame: 9000
+	 time: 13.280585050582886
+	 video: 2018-03-11.11-25-01.11-30-01.school.G424.avi
+	 frame: 9000
+	 time: 12.488405704498291
+	 video: 2018-03-11.16-25-00.16-30-00.school.G639.avi
+	 frame: 9000
+	 time: 54.35420060157776
+	 video: 2018-03-11.11-35-00.11-40-00.school.G299.avi
+	 frame: 9000
+	 time: 17.511691331863403
+	 video: 2018-03-11.11-35-00.11-40-00.school.G330.avi
+	 frame: 9000
+	 time: 17.46544885635376
+	 video: 2018-03-12.10-05-00.10-10-00.hospital.G436.avi
+	 frame: 9000
+	 time: 12.538724422454834
+python -c   712.73s user 11.47s system 515% cpu 2:20.55 total
diff --git a/tests/test.py b/tests/test.py
@@ -2,9 +2,8 @@
 import sys
 import time
 
-from progressbar import ProgressBar
-
 from avi_r import AVIReader
+from tqdm import tqdm
 
 # Videos from the MEVA dataset (http://mevadata.org)
 VIDEO_LIST = [
@@ -23,54 +22,67 @@ def integrity_test(video_dir, video_list=VIDEO_LIST,
     print('Fix missing with random access')
     start_frame_id, length = random_access_point
     for video_name in video_list:
-        print('\t', video_name, flush=True)
-        bar = ProgressBar().start()
+        print('\t video:', video_name, flush=True)
+        start = time.time()
         v = AVIReader(video_name, video_dir)
-        for i, f in bar(enumerate(v)):
+        for i, f in tqdm(enumerate(v), total=v.num_frames):
             assert f.frame_id == i
-        bar = ProgressBar().start()
-        v = AVIReader(video_name, video_dir)
+        v = AVIReader(video_name, video_dir, silence_warning=False)
         v.seek(start_frame_id)
-        for i, frame in bar(enumerate(v.get_iter(length))):
+        for i, frame in tqdm(enumerate(v.get_iter(length))):
             assert frame.frame_id == start_frame_id + i
     print('No fix missing')
     for video_name in video_list:
-        print('\t', video_name, flush=True)
-        bar = ProgressBar().start()
+        print('\t video:', video_name, flush=True)
         v = AVIReader(video_name, video_dir, fix_missing=False)
-        for i, f in bar(enumerate(v)):
+        for i, f in tqdm(enumerate(v), total=v.num_frames):
             pass
 
 
 def speed_test_opencv(video_dir, video_list=VIDEO_LIST):
     import cv2
     for video_name in video_list:
-        print('\t', video_name, flush=True)
-        bar = ProgressBar().start()
+        print('\t video:', video_name, flush=True)
+        start = time.time()
         cap = cv2.VideoCapture(osp.join(video_dir, video_name))
-        for _ in bar(range(int(cap.get(cv2.CAP_PROP_FRAME_COUNT)))):
+        frame = 0
+        for _ in tqdm(range(int(cap.get(cv2.CAP_PROP_FRAME_COUNT)))):
             r, _ = cap.read()
             if not r:
                 break
+            frame += 1
+        total = time.time() - start
+        print('\t frame:', frame)
+        print('\t time:', total, flush=True)
 
 
 def speed_test_moviepy(video_dir, video_list=VIDEO_LIST):
     from moviepy.editor import VideoFileClip
     for video_name in video_list:
         print('\t', video_name, flush=True)
-        bar = ProgressBar().start()
+        start = time.time()
         clip = VideoFileClip(osp.join(video_dir, video_name))
-        for i in bar(range(int(round(clip.duration * clip.fps)))):
+        frame = 0
+        for i in tqdm(range(int(round(clip.duration * clip.fps)))):
             clip.get_frame(i / clip.fps)
+            frame += 1
+        total = time.time() - start
+        print('\t frame:', frame)
+        print('\t', total, flush=True)
 
 
 def speed_test_avi_r(video_dir, video_list=VIDEO_LIST):
     for video_name in video_list:
         print('\t', video_name, flush=True)
-        bar = ProgressBar().start()
-        video = AVIReader(video_name, video_dir)
-        for _ in bar(range(video.length)):
+        start = time.time()
+        video = AVIReader(video_name, video_dir, silence_warning=False)
+        frame = 0
+        for _ in tqdm(range(video.length)):
             video.read()
+            frame += 1
+        total = time.time() - start
+        print('\t frame:', frame)
+        print('\t', total, flush=True)
 
 
 if __name__ == "__main__":