Gltimestamp

encoder/src/main/java/com/pedro/encoder/input/video/Camera1ApiManager.java

@@ -189,7 +189,7 @@ private void start() {
       Camera.Parameters parameters = camera.getParameters();
       parameters.setPreviewSize(width, height);
       parameters.setPreviewFormat(imageFormat);
-      int[] range = FpsUtils.INSTANCE.adaptFpsRange(fps, parameters.getSupportedPreviewFpsRange());
+      int[] range = FpsUtils.INSTANCE.adaptFpsRange(fps * 1000, parameters.getSupportedPreviewFpsRange());
       Log.i(TAG, "fps: " + range[0] + " - " + range[1]);
       parameters.setPreviewFpsRange(range[0], range[1]);
 

encoder/src/main/java/com/pedro/encoder/input/video/FpsLimiter.java

@@ -24,10 +24,8 @@
 
 public class FpsLimiter {
 
-  private long startTS = TimeUtils.getCurrentTimeMillis();
-  private long ratioF = 1000 / 30;
-  private long ratio = 1000 / 30;
-  private long frameStartTS = 0;
+  private long interval = 1_000_000_000L / 30;
+  private long time = 0;
   private boolean configured = false;
 
   public void setFPS(int fps) {
@@ -37,26 +35,18 @@ public void setFPS(int fps) {
     } else {
       configured = true;
     }
-    startTS = TimeUtils.getCurrentTimeMillis();
-    ratioF = 1000 / fps;
-    ratio = 1000 / fps;
+    interval = 1_000_000_000L / fps;
   }
 
-  public boolean limitFPS() {
+  public boolean limitFPS(long timestamp) {
     if (!configured) return false;
-    long lastFrameTimestamp = TimeUtils.getCurrentTimeMillis() - startTS;
-    if (ratio < lastFrameTimestamp) {
-      ratio += ratioF;
+    if (time == 0) {
+      time = timestamp;
       return false;
     }
-    return true;
-  }
-
-  public void setFrameStartTs() {
-    frameStartTS = TimeUtils.getCurrentTimeMillis();
-  }
-
-  public long getSleepTime() {
-    return Math.max(0, ratioF - (TimeUtils.getCurrentTimeMillis() - frameStartTS));
+    if (timestamp - time < interval - (interval / 4)) return true;
+    time += interval;
+    if (timestamp - time > interval) time = timestamp;
+    return false;
   }
 }

encoder/src/main/java/com/pedro/encoder/video/VideoEncoder.java

@@ -414,7 +414,7 @@ protected MediaCodecInfo chooseEncoder(String mime) {
   protected Frame getInputFrame() throws InterruptedException {
     Frame frame = queue.take();
     if (frame == null) return null;
-    if (fpsLimiter.limitFPS()) return getInputFrame();
+    if (fpsLimiter.limitFPS(frame.getTimeStamp() * 1000)) return getInputFrame();
     byte[] buffer = frame.getBuffer();
     boolean isYV12 = frame.getFormat() == ImageFormat.YV12;
 

library/src/main/java/com/pedro/library/view/GlStreamInterface.kt

@@ -96,6 +96,8 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener,
   private var previewViewPort: ViewPort? = null
   private var streamViewPort: ViewPort? = null
   private var surfaceHandlerThread: HandlerThread? = null
+  private val sync = Any()
+  private val glTimestamp = GlTimestamp()
 
   private val sensorRotationManager = SensorRotationManager(context, true, true) { orientation, isPortrait ->
     if (autoHandleOrientation && shouldHandleOrientation) {
@@ -185,6 +187,7 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener,
   }
 
   override fun start() {
+    glTimestamp.reset()
     threadQueue.clear()
     executor?.shutdownNow()
     executor = null
@@ -210,11 +213,14 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener,
         mainRender.getSurfaceTexture().setOnFrameAvailableListener(this)
       }
       forceRender.start {
-        executor?.execute {
-          try {
-            draw(true)
-          } catch (e: RuntimeException) {
-            renderErrorCallback?.onRenderError(e) ?: throw e
+        synchronized(sync) {
+          val timestamp = TimeUtils.getCurrentTimeNano()
+          executor?.execute {
+            try {
+              draw(true, timestamp)
+            } catch (e: RuntimeException) {
+              renderErrorCallback?.onRenderError(e) ?: throw e
+            }
           }
         }
       }
@@ -249,9 +255,8 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener,
     mainRender.release()
   }
 
-  private fun draw(forced: Boolean) {
+  private fun draw(forced: Boolean, clockTimestamp: Long) {
     if (!isRunning) return
-    val limitFps = fpsLimiter.limitFPS()
     if (!forced) forceRender.frameAvailable()
 
     if (!filterQueue.isEmpty() && mainRender.isReady()) {
@@ -270,9 +275,9 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener,
       if (!surfaceManager.makeCurrent()) return
       mainRender.updateFrame()
       mainRender.drawSource()
-      surfaceManager.swapBuffer()
     }
-    val timestamp = TimeUtils.getCurrentTimeNano()
+    val timestamp = glTimestamp.getTimestamp(surfaceTexture.timestamp, clockTimestamp)
+    val limitFps = fpsLimiter.limitFPS(timestamp)
 
     val orientation = when (orientationForced) {
       OrientationForced.PORTRAIT -> true
@@ -359,11 +364,14 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener,
 
   override fun onFrameAvailable(surfaceTexture: SurfaceTexture?) {
     if (!isRunning) return
-    executor?.execute {
-      try {
-        draw(false)
-      } catch (e: RuntimeException) {
-        renderErrorCallback?.onRenderError(e) ?: throw e
+    synchronized(sync) {
+      val timestamp = TimeUtils.getCurrentTimeNano()
+      executor?.execute {
+        try {
+          draw(false, timestamp)
+        } catch (e: RuntimeException) {
+          renderErrorCallback?.onRenderError(e) ?: throw e
+        }
       }
     }
   }
@@ -538,6 +546,7 @@ class GlStreamInterface(private val context: Context): OnFrameAvailableListener,
   }
 
   override fun forceFpsLimit(fps: Int) {
+    glTimestamp.setFps(fps)
     fpsLimiter.setFPS(fps)
   }
 

library/src/main/java/com/pedro/library/view/GlTimestamp.kt

@@ -0,0 +1,43 @@
+package com.pedro.library.view
+
+import kotlin.math.min
+
+class GlTimestamp {
+
+  companion object {
+    private const val DEFAULT_DRIFT = 100_000_000L
+  }
+  private var drift = DEFAULT_DRIFT
+  private var lastSourceTimestamp = 0L
+  private var lastClockTimestamp = 0L
+  private var currentTimestamp = 0L
+
+  fun setFps(fps: Int) {
+    drift = if (fps <= 0) DEFAULT_DRIFT else min(DEFAULT_DRIFT, 1000_000_000L / fps)
+  }
+
+  fun getTimestamp(sourceTimestamp: Long, clockTimestamp: Long): Long {
+    if (lastClockTimestamp == 0L) {
+      lastSourceTimestamp = sourceTimestamp
+      lastClockTimestamp = clockTimestamp
+      currentTimestamp = clockTimestamp
+      return currentTimestamp
+    }
+    val delta = if (sourceTimestamp <= lastSourceTimestamp) {
+      clockTimestamp - lastClockTimestamp
+    } else sourceTimestamp - lastSourceTimestamp
+    lastSourceTimestamp = sourceTimestamp
+    lastClockTimestamp = clockTimestamp
+    currentTimestamp += delta
+
+    if (currentTimestamp > clockTimestamp + drift) currentTimestamp = clockTimestamp + drift
+    else if (currentTimestamp < clockTimestamp - drift) currentTimestamp = clockTimestamp - drift
+    return currentTimestamp
+  }
+
+  fun reset() {
+    lastSourceTimestamp = 0
+    lastClockTimestamp = 0
+    currentTimestamp = 0
+  }
+}

library/src/main/java/com/pedro/library/view/OpenGlView.kt

@@ -78,6 +78,8 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf
     private val forceRenderer = ForceRenderer()
     private var renderErrorCallback: RenderErrorCallback? = null
     private var surfaceHandlerThread: HandlerThread? = null
+    private val sync = Any()
+    private val glTimestamp = GlTimestamp()
 
     constructor(context: Context?) : super(context) {
         holder.addCallback(this)
@@ -141,6 +143,7 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf
     }
 
     override fun forceFpsLimit(fps: Int) {
+        glTimestamp.setFps(fps)
         fpsLimiter.setFPS(fps)
     }
 
@@ -212,9 +215,8 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf
         this.takePhotoCallback = takePhotoCallback
     }
 
-    private fun draw(forced: Boolean) {
+    private fun draw(forced: Boolean, clockTimestamp: Long) {
         if (!isRunning) return
-        val limitFps = fpsLimiter.limitFPS()
         if (!forced) forceRenderer.frameAvailable()
 
         if (!filterQueue.isEmpty() && mainRender.isReady()) {
@@ -228,21 +230,24 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf
                 return
             }
         }
-
         if (surfaceManager.isReady && mainRender.isReady()) {
             if (!surfaceManager.makeCurrent()) return
             mainRender.updateFrame()
             mainRender.drawSource()
-            if (!limitFps) {
+        }
+        val timestamp = glTimestamp.getTimestamp(surfaceTexture.timestamp, clockTimestamp)
+        val limitFps = fpsLimiter.limitFPS(timestamp)
+
+        if (surfaceManager.isReady && mainRender.isReady() && !limitFps) {
+            if (surfaceManager.makeCurrent()) {
                 mainRender.drawFilters(true)
                 mainRender.drawScreen(
                     previewWidth, previewHeight, aspectRatioMode, 0,
                     isPreviewVerticalFlip, isPreviewHorizontalFlip, null
                 )
+                surfaceManager.swapBuffer()
             }
-            surfaceManager.swapBuffer()
         }
-        val timestamp = TimeUtils.getCurrentTimeNano()
 
         if (surfaceManagerEncoder.isReady || surfaceManagerEncoderRecord.isReady || surfaceManagerPhoto.isReady) {
             mainRender.drawFilters(false)
@@ -310,6 +315,7 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf
     }
 
     override fun start() {
+        glTimestamp.reset()
         threadQueue.clear()
         executor?.shutdownNow()
         executor = null
@@ -332,11 +338,14 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf
               mainRender.getSurfaceTexture().setOnFrameAvailableListener(this)
             }
             forceRenderer.start {
-                executor?.execute {
-                    try {
-                        draw(true)
-                    } catch (e: RuntimeException) {
-                        renderErrorCallback?.onRenderError(e) ?: throw e
+                synchronized(sync) {
+                    val timestamp = TimeUtils.getCurrentTimeNano()
+                    executor?.execute {
+                        try {
+                            draw(true, timestamp)
+                        } catch (e: RuntimeException) {
+                            renderErrorCallback?.onRenderError(e) ?: throw e
+                        }
                     }
                 }
             }
@@ -367,11 +376,14 @@ open class OpenGlView : SurfaceView, GlInterface, OnFrameAvailableListener, Surf
 
     override fun onFrameAvailable(surfaceTexture: SurfaceTexture) {
         if (!isRunning) return
-        executor?.execute {
-            try {
-                draw(false)
-            } catch (e: RuntimeException) {
-                renderErrorCallback?.onRenderError(e) ?: throw e
+        synchronized(sync) {
+            val timestamp = TimeUtils.getCurrentTimeNano()
+            executor?.execute {
+                try {
+                    draw(false, timestamp)
+                } catch (e: RuntimeException) {
+                    renderErrorCallback?.onRenderError(e) ?: throw e
+                }
             }
         }
     }