* Have Frame and FrameConverter implement AutoCloseable to rel…

…ease memory explicitly (issue #1574)

CHANGELOG.md

@@ -1,4 +1,5 @@
 
+ * Have `Frame` and `FrameConverter` implement `AutoCloseable` to release memory explicitly ([issue #1574](https://github.com/bytedeco/javacv/issues/1574))
  * Add new `YOLONet` sample for object detection ([pull #1595](https://github.com/bytedeco/javacv/pull/1595))
  * Fix crash on `FFmpegFrameGrabber.stop()` when in `ImageMode.RAW` ([issue #1568](https://github.com/bytedeco/javacv/issues/1568))
  * Let `FFmpegFrameRecorder.flush()` ignore errors from the encoder ([issue #1563](https://github.com/bytedeco/javacv/issues/1563))

platform/src/test/java/org/bytedeco/javacv/FrameConverterTest.java

@@ -137,6 +137,9 @@ public class FrameConverterTest {
 
         colorFrameIdx.release();
         grayFrameIdx.release();
+        converter.close();
+        colorFrame.close();
+        grayFrame.close();
     }
 
     @Test public void testJava2DFrameConverter() {
@@ -179,6 +182,8 @@ public class FrameConverterTest {
 
                 frameIdx.release();
                 frame2Idx.release();
+                converter.close();
+                frame.close();
             }
         }
 
@@ -205,6 +210,7 @@ public class FrameConverterTest {
                       | ((array2[4 * j + 2] & 0xFF) << 8)  |  (array2[4 * j + 3] & 0xFF);
                 assertEquals(array[j], n);
             }
+            converter.close();
         }
     }
 
@@ -294,6 +300,10 @@ public class FrameConverterTest {
         frame1Idx.release();
         frame2Idx.release();
         frame3Idx.release();
+        converter1.close();
+        converter2.close();
+        converter3.close();
+        frame.close();
     }
 
     @Test public void testLeptonicaFrameConverter() {
@@ -358,5 +368,7 @@ public class FrameConverterTest {
 
         pix2.deallocate();
         pix.deallocate();
+        converter.close();
+        frame.close();
     }
 }

platform/src/test/java/org/bytedeco/javacv/FrameFilterTest.java

@@ -129,6 +129,7 @@ public void testFFmpegFrameFilter() {
             grabber.restart();
             grabber.stop();
             grabber.release();
+            frame.close();
         } catch (Exception e) {
             e.printStackTrace();
             fail("Exception should not have been thrown: " + e);
@@ -215,6 +216,7 @@ public void testFFmpegFrameFilterMultipleInputs() {
             grabber.restart();
             grabber.stop();
             grabber.release();
+            frame.close();
         } catch (Exception e) {
             e.printStackTrace();
             fail("Exception should not have been thrown: " + e);

platform/src/test/java/org/bytedeco/javacv/FrameGrabberChangingResolutionTest.java

@@ -72,6 +72,9 @@ private void makeTestfile() throws Exception {
         }
         recorder.stop();
         recorder.release();
+        for (int n = 0; n < frames.length; n++) {
+            frames[n].close();
+        }
     }
 
     final public void setupUDPSender(final int x, final int y, final int bandwidth, final int count) throws IOException {

platform/src/test/java/org/bytedeco/javacv/FrameGrabberTest.java

@@ -125,12 +125,16 @@ public void testFFmpegFrameGrabber() {
                         m++;
                     }
                 }
+                clone2.close();
             }
             assertEquals(frames.length, n);
             assertEquals(null, grabber.grab());
             grabber.restart();
             grabber.stop();
             grabber.release();
+            for (n = 0; n < frames.length; n++) {
+                frames[n].close();
+            }
         } catch (Exception e) {
             fail("Exception should not have been thrown: " + e);
         } finally {
@@ -229,6 +233,9 @@ public void run() {
                         grabber.restart();
                         grabber.stop();
                         grabber.release();
+                        for (n = 0; n < frames.length; n++) {
+                            frames[n].close();
+                        }
                     } catch (Error | Exception e) {
                         failed[0] = true;
                         fail("Exception should not have been thrown: " + e);
@@ -321,6 +328,7 @@ public void testFFmpegFrameGrabberSeeking() throws IOException {
                         recorder.record(audioFrame);
                     }
                 }
+                frame.close();
             } else {
                 Frame audioFrame = new Frame();
                 ShortBuffer audioBuffer = ShortBuffer.allocate(48000 * 2 * 10000 / 30);

platform/src/test/java/org/bytedeco/javacv/SeekableByteArrayOutputStreamTest.java

@@ -80,6 +80,7 @@ private void createVideo(FFmpegFrameRecorder recorder) throws Exception {
                 }
             }
             recorder.record(frame);
+            frame.close();
         }
         recorder.close();
     }

src/main/java/org/bytedeco/javacv/AndroidFrameConverter.java

@@ -51,6 +51,9 @@ public class AndroidFrameConverter extends FrameConverter<Bitmap> {
     public Frame convert(byte[] data, int width, int height) {
         if (frame == null || frame.imageWidth != width
                 || frame.imageHeight != height || frame.imageChannels != 3) {
+            if (frame != null) {
+                frame.close();
+            }
             frame = new Frame(width, height, Frame.DEPTH_UBYTE, 3);
         }
         ByteBuffer out = (ByteBuffer)frame.image[0];
@@ -112,6 +115,9 @@ public Frame convert(byte[] data, int width, int height) {
 
         if (frame == null || frame.imageWidth != bitmap.getWidth() || frame.imageStride != bitmap.getRowBytes()
                 || frame.imageHeight != bitmap.getHeight() || frame.imageChannels != channels) {
+            if (frame != null) {
+                frame.close();
+            }
             frame = new Frame(bitmap.getWidth(), bitmap.getHeight(), Frame.DEPTH_UBYTE, channels, bitmap.getRowBytes());
         }
 

src/main/java/org/bytedeco/javacv/Frame.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2015-2019 Samuel Audet
+ * Copyright (C) 2015-2021 Samuel Audet
  *
  * Licensed either under the Apache License, Version 2.0, or (at your option)
  * under the terms of the GNU General Public License as published by
@@ -57,7 +57,7 @@
  *
  * @author Samuel Audet
  */
-public class Frame implements Indexable {
+public class Frame implements AutoCloseable, Indexable {
     /** A flag set by a FrameGrabber or a FrameRecorder to indicate a key frame. */
     public boolean keyFrame;
 
@@ -142,7 +142,7 @@ public Frame(int width, int height, int depth, int channels, int imageStride) {
             case DEPTH_DOUBLE: image[0] = buffer.asDoubleBuffer(); break;
             default: throw new UnsupportedOperationException("Unsupported depth value: " + imageDepth);
         }
-        opaque = pointer;
+        opaque = new Pointer[] {pointer.retainReference()};
     }
 
     /** Returns {@code createIndexer(true, 0)}. */
@@ -339,6 +339,9 @@ private static Pointer cloneBufferArray(Buffer[] srcBuffers, Buffer[] clonedBuff
             }
         }
 
+        if (opaque != null) {
+            opaque.retainReference();
+        }
         return opaque;
     }
 
@@ -350,4 +353,16 @@ public EnumSet<Type> getTypes() {
         if (data != null) type.add(Type.DATA);
         return type;
     }
+
+    @Override public void close() {
+        if (opaque instanceof Pointer[]) {
+            for (Pointer p : (Pointer[])opaque) {
+                if (p != null) {
+                    p.releaseReference();
+                    p = null;
+                }
+            }
+            opaque = null;
+        }
+    }
 }

src/main/java/org/bytedeco/javacv/FrameConverter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2015 Samuel Audet
+ * Copyright (C) 2015-2021 Samuel Audet
  *
  * Licensed either under the Apache License, Version 2.0, or (at your option)
  * under the terms of the GNU General Public License as published by
@@ -30,13 +30,20 @@
  * of this, and for performance reasons, any object returned by this class is
  * guaranteed to remain valid only until the next call to {@code convert()},
  * anywhere in a chain of {@code FrameConverter} objects, and only as long as
- * the latter themselves are not garbage collected.
+ * the latter themselves are not closed or garbage collected.
  *
  * @author Samuel Audet
  */
-public abstract class FrameConverter<F> {
+public abstract class FrameConverter<F> implements AutoCloseable {
     protected Frame frame;
 
     public abstract Frame convert(F f);
     public abstract F convert(Frame frame);
+
+    @Override public void close() {
+        if (frame != null) {
+            frame.close();
+            frame = null;
+        }
+    }
 }

src/main/java/org/bytedeco/javacv/Java2DFrameConverter.java

@@ -728,6 +728,9 @@ public Frame getFrame(BufferedImage image, double gamma, boolean flipChannels) {
         }
         if (frame == null || frame.imageWidth != image.getWidth() || frame.imageHeight != image.getHeight()
                 || frame.imageDepth != depth || frame.imageChannels != numChannels) {
+            if (frame != null) {
+                frame.close();
+            }
             frame = new Frame(image.getWidth(), image.getHeight(), depth, numChannels);
         }
         copy(image, frame, gamma, flipChannels, null);

src/main/java/org/bytedeco/javacv/Java2DFrameUtils.java

@@ -17,8 +17,8 @@
  *
  * All created Frame, Mat, IplImages and BufferedImages are cloned internally
  * after creation so that their memory locations remain valid after the
- * converters which created them are garbage collected. This is safer for the
- * called, but may be slower.
+ * converters which created them are closed or garbage collected. This is safer
+ * for the caller, but may be slower.
  *
  * If performance is critical, use the *FrameConverter classes directly, after
  * reading about the image validity constraints (eg, images data is only valid

src/main/java/org/bytedeco/javacv/LeptonicaFrameConverter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2019 Samuel Audet
+ * Copyright (C) 2018-2021 Samuel Audet
  *
  * Licensed either under the Apache License, Version 2.0, or (at your option)
  * under the terms of the GNU General Public License as published by
@@ -26,7 +26,6 @@
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import org.bytedeco.javacpp.BytePointer;
-import org.bytedeco.javacpp.IntPointer;
 import org.bytedeco.javacpp.Loader;
 import org.bytedeco.javacpp.Pointer;
 
@@ -44,6 +43,7 @@ public class LeptonicaFrameConverter extends FrameConverter<PIX> {
     static { Loader.load(org.bytedeco.leptonica.global.lept.class); }
 
     PIX pix;
+    BytePointer frameData, pixData;
     ByteBuffer frameBuffer, pixBuffer;
 
     static boolean isEqual(Frame frame, PIX pix) {
@@ -63,13 +63,22 @@ public PIX convert(Frame frame) {
         } else if (!isEqual(frame, pix)) {
             Pointer data;
             if (ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)) {
-                data = new BytePointer(frame.imageHeight * frame.imageStride);
+                if (pixData == null || pixData.capacity() < frame.imageHeight * frame.imageStride) {
+                    if (pixData != null) {
+                        pixData.releaseReference();
+                    }
+                    pixData = new BytePointer(frame.imageHeight * frame.imageStride).retainReference();
+                }
+                data = pixData;
                 pixBuffer = data.asByteBuffer().order(ByteOrder.BIG_ENDIAN);
             } else {
                 data = new Pointer(frame.image[0].position(0));
             }
+            if (pix != null) {
+                pix.releaseReference();
+            }
             pix = PIX.create(frame.imageWidth, frame.imageHeight, frame.imageChannels * 8, data)
-                     .wpl(frame.imageStride / 4 * Math.abs(frame.imageDepth) / 8);
+                     .wpl(frame.imageStride / 4 * Math.abs(frame.imageDepth) / 8).retainReference();
         }
 
         if (ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)) {
@@ -111,12 +120,23 @@ public Frame convert(PIX pix) {
             frame.imageChannels = pix.d() / 8;
             frame.imageStride = pix.wpl() * 4;
             if (ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN)) {
-                Pointer data = new BytePointer(frame.imageHeight * frame.imageStride);
-                frameBuffer = data.asByteBuffer().order(ByteOrder.LITTLE_ENDIAN);
-                frame.opaque = data;
+                if (frameData == null || frameData.capacity() < frame.imageHeight * frame.imageStride) {
+                    if (frameData != null) {
+                        frameData.releaseReference();
+                    }
+                    frameData = new BytePointer(frame.imageHeight * frame.imageStride).retainReference();
+                }
+                frameBuffer = frameData.asByteBuffer().order(ByteOrder.LITTLE_ENDIAN);
+                frame.opaque = frameData;
                 frame.image = new Buffer[] { frameBuffer };
             } else {
-                frame.opaque = tempPix != null ? pix.clone() : pix;
+                if (tempPix != null) {
+                    if (this.pix != null) {
+                        this.pix.releaseReference();
+                    }
+                    this.pix = pix = pix.clone();
+                }
+                frame.opaque = pix;
                 frame.image = new Buffer[] { pix.createBuffer() };
             }
         }
@@ -131,4 +151,20 @@ public Frame convert(PIX pix) {
         }
         return frame;
     }
+
+    @Override public void close() {
+        super.close();
+        if (pix != null) {
+            pix.releaseReference();
+            pix = null;
+        }
+        if (pixData != null) {
+            pixData.releaseReference();
+            pixData = null;
+        }
+        if (frameData != null) {
+            frameData.releaseReference();
+            frameData = null;
+        }
+    }
 }

src/main/java/org/bytedeco/javacv/OpenCVFrameConverter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2015-2018 Samuel Audet
+ * Copyright (C) 2015-2021 Samuel Audet
  *
  * Licensed either under the Apache License, Version 2.0, or (at your option)
  * under the terms of the GNU General Public License as published by
@@ -100,9 +100,12 @@ public IplImage convertToIplImage(Frame frame) {
             return (IplImage)frame.opaque;
         } else if (!isEqual(frame, img)) {
             int depth = getIplImageDepth(frame.imageDepth);
-            img = depth < 0 ? null : IplImage.create(frame.imageWidth, frame.imageHeight, depth, frame.imageChannels, new Pointer(frame.image[0].position(0)))
+            if (img != null) {
+                img.releaseReference();
+            }
+            img = depth < 0 ? null : (IplImage)IplImage.create(frame.imageWidth, frame.imageHeight, depth, frame.imageChannels, new Pointer(frame.image[0].position(0)))
                     .widthStep(frame.imageStride * Math.abs(frame.imageDepth) / 8)
-                    .imageSize(frame.image[0].capacity() * Math.abs(frame.imageDepth) / 8);
+                    .imageSize(frame.image[0].capacity() * Math.abs(frame.imageDepth) / 8).retainReference();
         }
         return img;
     }
@@ -148,8 +151,11 @@ public Mat convertToMat(Frame frame) {
             return (Mat)frame.opaque;
         } else if (!isEqual(frame, mat)) {
             int depth = getMatDepth(frame.imageDepth);
-            mat = depth < 0 ? null : new Mat(frame.imageHeight, frame.imageWidth, CV_MAKETYPE(depth, frame.imageChannels),
-                    new Pointer(frame.image[0].position(0)), frame.imageStride * Math.abs(frame.imageDepth) / 8);
+            if (mat != null) {
+                mat.releaseReference();
+            }
+            mat = depth < 0 ? null : (Mat)new Mat(frame.imageHeight, frame.imageWidth, CV_MAKETYPE(depth, frame.imageChannels),
+                    new Pointer(frame.image[0].position(0)), frame.imageStride * Math.abs(frame.imageDepth) / 8).retainReference();
         }
         return mat;
     }
@@ -226,4 +232,20 @@ public Frame convert(final org.opencv.core.Mat mat) {
         frame.opaque = mat;
         return frame;
     }
+
+    @Override public void close() {
+        super.close();
+        if (img != null) {
+            img.releaseReference();
+            img = null;
+        }
+        if (mat != null) {
+            mat.releaseReference();
+            mat = null;
+        }
+        if (orgOpenCvCoreMat != null) {
+            orgOpenCvCoreMat.release();
+            orgOpenCvCoreMat = null;
+        }
+    }
 }