How to pan and zoom properly in 2D?

All I want to do is create a really simple pan and zoom feature in 2D with OpenGL through pyglet. As you can see, the zooming is working perfectly after the first jump:( Then again, the dragging (panning) is also working, but it also jumps (and it jumps a pretty big one)..

Here is my simplified code and a video (pyglet_test.mp4) that shows how it behaves:

import pyglet
from pyglet.gl import *

# Zooming constants
ZOOM_IN_FACTOR = 1.2
ZOOM_OUT_FACTOR = 1/ZOOM_IN_FACTOR



class App(pyglet.window.Window):

    def __init__(self, width, height, *args, **kwargs):
        # Create GL configuration
        conf = Config(  sample_buffers=1,
                        samples=4,
                        depth_size=16,
                        double_buffer=True )
        # Initialize parent
        super().__init__( width, height, config=conf, *args, **kwargs )

        # Create Group
        self.group = group = pyglet.graphics.Group()
        # Create Batch
        self.batch = batch = pyglet.graphics.Batch()

        # Create QUAD for testing and add it to batch
        batch.add(
            4, GL_QUADS, group,

            ('v2i', ( -50, -50,
                       50, -50,
                       50,  50,
                      -50,  50 )),

            ('c3B', ( 255,   0,   0,
                      255, 255,   0,
                        0, 255,   0,
                        0,   0, 255 ))
        )

        # Initialize OpenGL
        self.init_gl()

        # Initialize camera values
        self.camera_x = 0
        self.camera_y = 0
        self.camera_zoom = 1

    def init_gl(self):
        # Set clear color
        glClearColor(0/255, 0/255, 0/255, 0/255)

        # Set antialiasing
        glEnable( GL_LINE_SMOOTH )
        glEnable( GL_POLYGON_SMOOTH )
        glHint( GL_LINE_SMOOTH_HINT, GL_NICEST )

        # Set alpha blending
        glEnable( GL_BLEND )
        glBlendFunc( GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA )

        # Set viewport
        glViewport( 0, 0, self.width, self.height )

        # Initialize Projection matrix
        glMatrixMode( GL_PROJECTION )
        glLoadIdentity()

        # Set orthographic projection matrix
        glOrtho( 0, self.width, 0, self.height, 1, -1 )

        # Initialize Modelview matrix
        glMatrixMode( GL_MODELVIEW )
        glLoadIdentity()

        # Save the default modelview matrix
        glPushMatrix()

    def on_resize(self, width, height):
        # Initialize OpenGL for new dimensions
        self.width  = width
        self.height = height
        self.init_gl()

    def camera_matrix(transformations):

        # Create camera setter function
        def set_camera(self):
            # Take saved matrix off the stack and reset it
            glMatrixMode( GL_MODELVIEW )
            glPopMatrix()
            glLoadIdentity()
            # Call wrapped function
            transformations(self)
            # Save default matrix again with camera translation
            glPushMatrix()

        # Return wrapper function
        return set_camera

    @camera_matrix
    def move_camera(self):
        # Move to camera position
        glTranslatef( self.camera_x, self.camera_y, 0 )
        # Scale camera
        glScalef( self.camera_zoom, self.camera_zoom, 1 )

    @camera_matrix
    def zoom_camera(self):
        # Move to camera position
        glTranslatef( self.camera_x, self.camera_y, 0 )
        # Scale camera
        glScalef( self.camera_zoom, self.camera_zoom, 1 )
        # Move back from camera position
        glTranslatef( -self.camera_x, -self.camera_y, 0 )

    def on_mouse_drag(self, x, y, dx, dy, button, modifiers):
        # Move camera
        self.camera_x += dx
        self.camera_y += dy
        self.move_camera()

    def on_mouse_scroll(self, x, y, dx, dy):
        # Get scale factor
        f = ZOOM_IN_FACTOR if dy < 0 else ZOOM_OUT_FACTOR if dy > 0 else 1
        # If zoom_level is in the proper range
        if .2 < self.camera_zoom*f < 5:
            # Zoom camera
            self.camera_x = x
            self.camera_y = y
            self.camera_zoom *= f
            self.zoom_camera()

    def on_draw(self):
        # Clear window with ClearColor
        glClear( GL_COLOR_BUFFER_BIT )

        # Pop default matrix onto current matrix
        glMatrixMode( GL_MODELVIEW )
        glPopMatrix()

        # Save default matrix again
        glPushMatrix()

        # Move to center of the screen
        glTranslatef( self.width/2, self.height/2, 0 )

        # Draw objects
        self.batch.draw()

    def run(self):
        pyglet.app.run()

# Create instance of app and run it
App(500, 500).run()